3
@6^o                 @   sH   d dl Z d dlZd dlZd dlZd dlmZmZmZ G dd deZ	dS )    N)
SpiderFootSpiderFootPluginSpiderFootEventc               @   sV   e Zd ZdZddiZddiZdZdZe fddZ	d	d
 Z
dd Zdd Zdd ZdS )sfp_urlscanzhURLScan.io:Footprint,Investigate,Passive:Search Engines::Search URLScan.io cache for domain information.verifyTzCVerify that any hostnames found on the target domain still resolve?NFc             C   s>   || _ | j | _d| _x"t|j D ]}|| | j|< q$W d S )NF)sftempStorageresults
errorStatelistkeysopts)selfsfcuserOptsopt r   6/var/www/spiderfoot.crq.systems/modules/sfp_urlscan.pysetup   s
    
zsfp_urlscan.setupc             C   s   dgS )NINTERNET_NAMEr   )r   r   r   r   watchedEvents'   s    zsfp_urlscan.watchedEventsc             C   s   ddddddddgS )	NGEOINFOLINKED_URL_INTERNALRAW_RIR_DATADOMAIN_NAMEr   INTERNET_NAME_UNRESOLVEDBGP_AS_MEMBERWEBSERVER_BANNERr   )r   r   r   r   producedEvents+   s    zsfp_urlscan.producedEventsc             C   s   dd|j djddd i}| jjdtjj| | jd | jd	 d
}|d dkrj| jjdd d| _	d S |d d kr| jj
d|  d S ytj|d }W n, tk
r } z| jjd d S d }~X nX |S )Nqzdomain:raw_unicode_escapeasciireplace)errorsz"https://urlscan.io/api/v1/search/?_fetchtimeout
_useragent)timeout	useragentcode429z)You are being rate-limited by URLScan.io.FTcontentzNo results info found for zError processing JSON response.)encodedecoder   fetchUrlurllibparse	urlencoder   errorr
   infojsonloads	Exceptiondebug)r   qryparamsresresulter   r   r   query1   s"    zsfp_urlscan.queryc             C   sd  |j }|j}|j}| jrd S | jjd| d |  || jkrV| jjd| d  d S d| j|< | j|}|d krvd S |jd}|sd S t	dt
|| j|}| j| t }t }	t }
t }t }x|D ]}|jd}|sq|jd	}|sq| j j|dd
sq|j |j kr$|
j| |jd}|rF|	j|jdd djdd |jd|jdgD }|rz|j| |jd}|r|j| |jd}|sq|jd}| j j| jj|dd
r|j| qW x,t|D ] }t	d|| j|}| j| qW x,t|D ] }t	d|| j|}| j| qW | jd rnt|
dkrn| jjdt
tt|
 d  xt|
D ]}| jd r| jj| rt	d|| j|}| j| nt	d|| j|}| j| | jj|| jd rxt	d|| j|}| j| qxW x,t|	D ] }t	d || j|}| j| qW x,t|D ] }t	d!|| j|}| j| q<W d S )"NzReceived event, z, from z	Skipping z as already mapped.Tr	   r   pagedomain)includeParentsasnAS z, c             S   s   g | ]}|r|qS r   r   ).0_fr   r   r   
<listcomp>   s    z+sfp_urlscan.handleEvent.<locals>.<listcomp>citycountryservertaskurlr   r   r   r   z
Resolving z domains ...r   r   _internettldsr   r   r   )	eventTypemoduledatar
   r   r6   r	   r<   getr   str__name__notifyListenersr   	getTargetmatcheslowerappendr"   joinurlFQDNsetr   lenr2   resolveHostisDomain)r   event	eventNamesrcModuleName	eventDatarN   r	   evturlsZasnsdomainsZ	locationsserversr9   r=   r>   r@   locationrH   rI   rJ   r   r   r   handleEventL   s    








$




 
zsfp_urlscan.handleEvent)rQ   
__module____qualname____doc__r   optdescsr	   r
   dictr   r   r   r<   rf   r   r   r   r   r      s   	r   )
r3   urllib.requestr.   urllib.parseurllib.errorsflibr   r   r   r   r   r   r   r   <module>   s   