U
    @6^o                     @   sH   d dl Z d dlZd dlZd dlZd dlmZmZmZ G dd deZ	dS )    N)
SpiderFootSpiderFootPluginSpiderFootEventc                   @   sV   e Zd ZdZddiZddiZdZdZe fddZ	d	d
 Z
dd Zdd Zdd ZdS )sfp_urlscanzhURLScan.io:Footprint,Investigate,Passive:Search Engines::Search URLScan.io cache for domain information.verifyTzCVerify that any hostnames found on the target domain still resolve?NFc                 C   s:   || _ |  | _d| _t| D ]}|| | j|< q"d S )NF)sftempStorageresults
errorStatelistkeysopts)selfsfcuserOptsopt r   6/var/www/spiderfoot.crq.systems/modules/sfp_urlscan.pysetup   s
    
zsfp_urlscan.setupc                 C   s   dgS )NINTERNET_NAMEr   r   r   r   r   watchedEvents'   s    zsfp_urlscan.watchedEventsc                 C   s   ddddddddgS )	NGEOINFOLINKED_URL_INTERNALRAW_RIR_DATADOMAIN_NAMEr   INTERNET_NAME_UNRESOLVEDBGP_AS_MEMBERWEBSERVER_BANNERr   r   r   r   r   producedEvents+   s       zsfp_urlscan.producedEventsc              
   C   s   dd| djddd i}| jjdtj| | jd | jd	 d
}|d dkrj| jdd d| _	d S |d d kr| j
d|  d S zt|d }W n4 tk
r } z| jd W Y d S d }~X Y nX |S )Nqzdomain:raw_unicode_escapeasciireplace)errorsz"https://urlscan.io/api/v1/search/?_fetchtimeout
_useragent)timeout	useragentcode429z)You are being rate-limited by URLScan.io.FTcontentzNo results info found for zError processing JSON response.)encodedecoder   fetchUrlurllibparse	urlencoder   errorr
   infojsonloads	Exceptiondebug)r   qryparamsresresulter   r   r   query1   s(     zsfp_urlscan.queryc                 C   sH  |j }|j}|j}| jrd S | jd| d |  || jkrV| jd| d  d S d| j|< | |}|d krvd S |d}|sd S t	dt
|| j|}| | t }t }	t }
t }t }|D ]}|d}|sq|d	}|sq|  j|dd
sq| | kr |
| |d}|rB|	|dd ddd |d|dfD }|rv|| |d}|r|| |d}|sq|d}|  j| j|dd
r|| qt|D ] }t	d|| j|}| | qt|D ] }t	d|| j|}| | q| jd r`t|
dkr`| jdt
tt|
 d  t|
D ]}| jd r| j|st	d|| j|}| | nt	d|| j|}| | | j|| jd rht	d|| j|}| | qht|	D ] }t	d || j|}| | qt|D ] }t	d!|| j|}| | q"d S )"NzReceived event, z, from z	Skipping z as already mapped.Tr	   r   pagedomain)includeParentsasnAS z, c                 S   s   g | ]}|r|qS r   r   ).0_fr   r   r   
<listcomp>   s      z+sfp_urlscan.handleEvent.<locals>.<listcomp>citycountryservertaskurlr   r   r   r   z
Resolving z domains ...r   r   _internettldsr   r   r   )	eventTypemoduledatar
   r   r7   r	   r=   getr   str__name__notifyListenersr   	getTargetmatcheslowerappendr#   joinurlFQDNsetr   lenr3   resolveHostisDomain)r   event	eventNamesrcModuleName	eventDatarO   r	   evturlsZasnsdomainsZ	locationsserversr:   r>   r?   rA   locationrI   rJ   rK   r   r   r   handleEventL   s    









$




 
zsfp_urlscan.handleEvent)rR   
__module____qualname____doc__r   optdescsr	   r
   dictr   r   r   r=   rg   r   r   r   r   r      s     	r   )
r4   urllib.requestr/   urllib.parseurllib.errorsflibr   r   r   r   r   r   r   r   <module>   s   