3
@6^-                @   s  d dl mZ d dlmZ d dlZd dlZd dlZd dlZd dl	Zd dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Zd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dl m!Z!m"Z" d dl#m$Z$ d dl%Z%e%j&e%j'j( G d	d
 d
Z)G dd de*Z+G dd de*Z,G dd de*Z-G dd de*Z.G dd dZ/e/ a0dS )    )Signal)
ControllerN)nx)
GEXFWriter)datetime)BeautifulSoupSoupStrainer)deepcopyc               @   sP  e Zd ZdZdZeZdZdddZdd Z	dd Z
dd	 ZdddZe fddZg fddZe fddZdd Zdd Zdd ZdddZdddZdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Zd)d* Zd+d, Zd-d. Zdd/d0Zdd1d2Z d3d4 Z!d5d6 Z"d7d8 Z#d9d: Z$d;d< Z%d=d> Z&d?d@ Z'dAdB Z(dCdD Z)dEdF Z*dGdH Z+dIdJ Z,dKdL Z-dMdN Z.dOdP Z/dQdR Z0dSdT Z1dUdV Z2dWdX Z3dYdZ Z4d[d\ Z5d]d^ Z6d_d` Z7dadb Z8dcdd Z9dedf Z:dgdh Z;didj Z<dkdl Z=dmdn Z>dodp Z?dqdr Z@ddtduZAdvdw ZBdxdy ZCdzd{ ZDd|d} ZEdddZFdd ZGeH fddZIeH fddZJdS )
SpiderFootNc             C   s\   || _ t|| _tjdkr"tjt_| jjdddkrXt	j
j }| jd g|_t	j
j| d S )N      	   
_dnsserver )r   r   r   )handler	   optssysversion_infossl_create_unverified_context_create_default_https_contextgetdnsresolverZResolverZnameserversZoverride_system_resolver)selfoptionsr   res r   (/var/www/spiderfoot.crq.systems/sflib.py__init__9   s    


zSpiderFoot.__init__c             C   s
   || _ d S )N)
socksProxy)r   r    r   r   r   updateSocketI   s    zSpiderFoot.updateSocketc             C   s
   d | _ d S )N)r    )r   r   r   r   revertSocketL   s    zSpiderFoot.revertSocketc             C   s   | j d dkrd S yR| jd tj| j d | j d d$}|j  |jtj tj	d W d Q R X W n4 t
k
r } z| jdt|  W Y d d }~X nX d S )	N_socks1typeZTORzRe-circuiting TOR..._socks2addr_torctlport)addressport
   zUnable to re-circuit TOR: )r   infor   Z	from_portZauthenticatesignalr   ZNEWNYMtimesleepBaseExceptionfatalstr)r   
controllerer   r   r   refreshTorIdentP   s    
zSpiderFoot.refreshTorIdentTc             C   sj  |d krd S |j dr|jdd }y^| jd|  t|d}|rp|j }t }x&|D ]}|j|jd qVW n|j }|j	  |S  t
k
r }	 z"|r| jd| d  nd S W Y d d }	~	X nX |j j ds|j j d	rfy:| jd
|  | j }
|
j|}|r|jj S |jS W nJ t
k
rd } z,|rP| jd| d t|  nd S W Y d d }~X nX |S )N@   z!Loading configuration data from: r
zUnable to open option file, .zhttp://zhttps://z%Downloading configuration data from: zUnable to open option URL, z: )
startswithsplitr)   open	readlineslistappendrstripreadcloser-   errorlower
getSessionr   content
splitlinesr/   )r   valr.   Z
splitLinesfnamefarrretxbsessionr   r1   r   r   r   optValueToDatab   s@    





zSpiderFoot.optValueToDatac                s>  t  }t  t  fddx|D ]}|d dksD|d dkrt|dkrv|d |ksh|d dkrd |d	 < nd |d	 < |d	 krt |d	 < |d	  j|d
 |d g q(W xz D ]r}xl| D ]`\}}| kr||kr|j||f qt }||}	x&|	D ]}
||
kr|j||
f qW qW qW |S )Nc                sf   t  }xZ|  D ]N\}}||kr"q| kr6|j| q|j| x||D ]}|j| qLW qW |S )N)r<   r=   )itemZpidsrJ   parentidp)entitiesget_next_parent_entitiesparentsr   r   rT      s    
z;SpiderFoot.buildGraphData.<locals>.get_next_parent_entities   ZENTITYZINTERNALr      Tr4   r      )setdictlenr<   r=   add)r   datafltmappingrowentityrP   rQ   ZppidsZnext_parentsZnext_parentr   )rS   rT   rU   r   buildGraphData   s0    
"



zSpiderFoot.buildGraphDatac             C   s*  | j ||}tj }t }d}x|D ]}	|	\}
}dddg}|
dks$|dkrLq$|
|kr|d }|
|krndddg}|j|
}d|d |d |d di|j|
 d	< |||
< ||kr|d }||krdddg}|j| d|d |d |d di|j| d	< |||< |j||
 q$W t|d
}t|j	dS )Nr   0ROOTr4   255colorr   )r5   grL   Zviz)graphzutf-8)
rb   r   ZGraphrZ   Zadd_nodenodeZadd_edger   r/   encode)r   roottitler]   r^   r_   rh   nodelistncounterpairdstsrccolri   Zgexfr   r   r   buildGraphGexf   s4    



&


&
zSpiderFoot.buildGraphGexfc          	   C   sb  | j ||}t }t |d< t |d< t }d}d}x |D ]}	|	\}
}d}|
dks<|dkr`q<|
|kr|d }|
|kr|d}|d jt|t|
tj jddtj jddd	|d
 |||
< ||kr"||krd}|d }|d jt|t|tj jddtj jddd	|d
 |||< |d }|d jt|t|| t||
 d q<W tj	|S )NZnodesZedgesr   z#000rd   r4   z#f00i  1)rQ   labelrK   ysizerf   )rQ   sourcetarget)
rb   rZ   r<   r=   r/   randomSystemRandomrandintjsondumps)r   rk   r]   r^   r_   rJ   rm   Zecounterrn   ro   rp   rq   rr   r   r   r   buildGraphJson   sN    





zSpiderFoot.buildGraphJsonc             C   s
   || _ d S )N)dbh)r   r   r   r   r   setDbh  s    zSpiderFoot.setDbhc             C   s
   || _ d S )N)GUID)r   uidr   r   r   setGUID#  s    zSpiderFoot.setGUIDc             C   s   t tj jdd j S )N-r   )r/   uuiduuid4r9   upper)r   ZscanNamer   r   r   genScanInstanceGUID'  s    zSpiderFoot.genScanInstanceGUIDc             C   s   | j j| j|||S )N)r   ZscanLogEventr   )r   levelmessageZ	componentr   r   r   _dblog/  s    zSpiderFoot._dblogc             C   s^   | j d sd S | jd kr&td|  n| jd| | j jdrJtd|  |rZtd| d S )N	__loggingz
[Error] %sERROR__logstdoutzInternal Error Encountered: )r   r   printr   r   r-   )r   r   	exceptionr   r   r   rA   3  s    

zSpiderFoot.errorc             C   sB   | j d krtd|  n| jd| tttj  tjd d S )Nz
[Fatal] %sFATALr4   )r   r   r   r/   inspectstackr   exit)r   rA   r   r   r   r.   @  s
    
zSpiderFoot.fatalc             C   sN   | j d sd S | jd kr&td|  n| jd| | j jdrJtd|  d S )Nr   z[Status] %sZSTATUSr   z[*] %s)r   r   r   r   r   )r   r   r   r   r   statusH  s    

zSpiderFoot.statusc             C   s   | j d sd S tj d }tj|d }|d kr6d}n@|jdkrptj d }tj|d }|d krhd}qv|j}n|j}| jd krtd||f  n| jd|| | j jd	rtd
|  d S )Nr   r4   r   Unknownsflibr   z[%s] %sINFOr   z[*] %s)	r   r   r   	getmodule__name__r   r   r   r   )r   r   frmmodmodNamer   r   r   r)   S  s&    


zSpiderFoot.infoc             C   s   | j d sd S | j d sd S tj d }tj|d }|d krDd}n@|jdkr~tj d }tj|d }|d krvd}q|j}n|j}| jd krtd||f  n| jd	|| | j jd
rtd||f  d S )N_debugr   r4   r   r   r   r   z[%s] %sDEBUGr   z	[d:%s] %s)	r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   debugo  s*    



zSpiderFoot.debugc             C   s$   t tdrtjjtjS tjjtS )Nfrozen)hasattrr   ospathdirname
executable__file__)r   r   r   r   myPath  s    
zSpiderFoot.myPathc             C   s0   |}t |ttgkrt|}tj|jdj S )Nraw_unicode_escape)typer<   rZ   r/   hashlibsha256rj   	hexdigest)r   stringsr   r   r   
hashstring  s    zSpiderFoot.hashstringc             C   s&   | j  d }tjj|s"tj| |S )Nz/cache)r   r   r   isdirmkdir)r   r   r   r   r   	cachePath  s    
zSpiderFoot.cachePathc             C   s   t j|jdj }| j d | }t|d}t|tkrxl|D ]:}t|tkrf|j	| |j	d q@|j	|j
dd  q@W n(t|tkr|j	|j
d n
|j	| |j  d S )Nzutf-8/wr6   )r   sha224rj   r   r   r:   r   r<   r/   writedecodebytesr@   )r   ru   r]   	pathLabel	cacheFilefpliner   r   r   cachePut  s    



zSpiderFoot.cachePutc             C   s   t j|jdj }| j d | }yptj|\
}}}}}	}
}}}}|dkrPd S |tj |d  ksl|dkrt|d}|j	 }W d Q R X |S d S W n  t
k
r } zd S d }~X nX d S )Nzutf-8r   r   i  r5   )r   r   rj   r   r   r   statr+   r:   r?   r-   )r   ru   Z
timeoutHrsr   r   midnurg   szatimemtimectimer   ZfileContentsr1   r   r   r   cacheGet  s    zSpiderFoot.cacheGetc             C   s  t  }xt|j D ]}|jdr(|r(qt|| tksHt|| tkrT|| ||< t|| tkr~|| rvd||< nd||< t|| tkrdj|| ||< qW d|kr|S x>|d D ]0}x(|d | d D ]}|jdr|rqt|d | d | tks(t|d | d | tkrH|d | d | ||d | < t|d | d | tkr|d | d | rd||d | < nd||d | < t|d | d | tkrdjd	d
 |d | d | D ||d | < qW qW |S )N__r4   r   ,__modules__r   _:c             s   s   | ]}t |V  qd S )N)r/   ).0rK   r   r   r   	<genexpr>  s   z-SpiderFoot.configSerialize.<locals>.<genexpr>)	rZ   r<   keysr8   r   intr/   booljoin)r   r   filterSystemZ	storeoptsoptr   r   r   r   configSerialize  s:     
 
.zSpiderFoot.configSerializec             C   s4  |}xt |j D ] }|jdr*|r*q||krt|| tkr`|| dkrXd||< nd||< t|| tkrt|| ||< t|| tkrt|| ||< t|| t krt|| d tkr t  ||< xDt|| jdD ]}|| jt| qW qt|| jd||< qW d|kr(|S x|d D ]}x|d | d D ]}|jd	rj|rjqN|d
 | |krNt|d | d | tkr||d
 |  dkrd|d | d |< nd|d | d |< t|d | d | tkrt||d
 |  |d | d |< t|d | d | tkr^t||d
 |  |d | d |< t|d | d | t krNt|d | d | d tkrt  |d | d |< xnt||d
 |  jdD ]$}|d | d | jt| qW n*t||d
 |  jd|d | d |< qNW q4W |S )Nr   rt   TFr   r   r   r   r   r   )	r<   r   r8   r   r   r/   r   r9   r=   )r   r   ZreferencePointr   Z
returnOptsr   rK   r   r   r   r   configUnserialize  sR    


$$" &6zSpiderFoot.configUnserializec          
   C   s   d }ddiddiddiddid	d
iddiddiddiddig	}xD|D ]<}t |j d }tj||tjtjB rDt |j d }P qDW |S )Nz^\d+\.\d+\.\d+\.\d+$
IP_ADDRESSz^\d+\.\d+\.\d+\.\d+/\d+$NETBLOCK_OWNERz^.*@.*$	EMAILADDRz^\+\d+$PHONE_NUMBERz^".*\s+.*"$
HUMAN_NAMEz^".*"$USERNAMEz^\d+$BGP_AS_OWNERz^[0-9a-f:]+$IPV6_ADDRESSzS^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)+([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$INTERNET_NAMEr   )r<   r   rematch
IGNORECASEUNICODEvalues)r   ry   
targetTypeZregexToTypeZrxpairrxr   r   r   r   @  s     

zSpiderFoot.targetTypec             C   s   t  }xt | jd j D ]l}| jd | d d kr6qxN| jd | d D ]8}||krh||krh|j| d|krJ||krJ|j| qJW qW |S )Nr   provides*)r<   r   r   r=   )r   eventsmodlistr   evtyper   r   r   modulesProducingY  s    
zSpiderFoot.modulesProducingc             C   s   t  }xt | jd j D ]z}| jd | d d kr6qd| jd | d kr^||kr^|j| x4| jd | d D ]}||krr||krr|j| qrW qW |S )Nr   consumesr   )r<   r   r   r=   )r   r   r   r   r   r   r   r   modulesConsumingi  s    
zSpiderFoot.modulesConsumingc             C   sj   t  }x^|D ]V}|t | jd j kr| jd | d d k	rx$| jd | d D ]}|j| qPW qW |S )Nr   r   )r<   r   r   r=   )r   modulesevtlistr   evtr   r   r   eventsFromModulesz  s    
zSpiderFoot.eventsFromModulesc             C   sj   t  }x^|D ]V}|t | jd j kr| jd | d d k	rx$| jd | d D ]}|j| qPW qW |S )Nr   r   )r<   r   r   r=   )r   r   r   r   r   r   r   r   eventsToModules  s    
zSpiderFoot.eventsToModulesc             C   st   t  }d|kr|S |jd}xL|D ]D}|dkr\t|dkr<q"d|krRt|dkrRq"|j  q"|j| q"W dj|S )Nz..r   r4   z://   )r<   r9   r[   popr=   r   )r   urlZ	finalBitsbitschunkr   r   r   urlRelativeToAbsolute  s    

z SpiderFoot.urlRelativeToAbsolutec             C   sV   |j d}t|dkr|d S d|kr<|jddk r<|d S dj|d d }|d S )Nr   r   z://r   r4   r   )r9   r[   countr   )r   r   r   baser   r   r   
urlBaseDir  s    
zSpiderFoot.urlBaseDirc             C   s@   d|krt jd|}nt jd|}|d kr2|j S |jdj S )Nz://z(\w+://.[^/:\?]*)[:/\?].*z(.[^/:\?]*)[:/\?]r4   )r   r   rB   group)r   r   r   r   r   r   
urlBaseUrl  s    zSpiderFoot.urlBaseUrlc             C   s.   | j |}d|krd}nd}|jd| j S )Nz://r   r   r   )r  r9   rB   )r   r   Zbaseurlr   r   r   r   urlFQDN  s
    
zSpiderFoot.urlFQDNc             C   sV   dj | j|j |jddd  }|j jd| d}d|krN|jdd S |S d S )Nr7   r4   r   r   )r   
hostDomainrB   r9   replace)r   domaintldListtldrJ   r   r   r   domainKeyword  s
    $zSpiderFoot.domainKeywordc             C   s   t  }xl|D ]d}dj| j|j |jddd  }|j jd| d}d|krf|j|jdd  q|j| qW | jdt|  |S )Nr7   r4   r   z
Keywords: r   )	r<   r   r  rB   r9   r  r=   r   r/   )r   Z
domainListr  rI   r  r  rJ   r   r   r   domainKeywords  s    
$zSpiderFoot.domainKeywordsc             C   s   t |}|j|S )N)PublicSuffixListget_public_suffix)r   hostnamer  psr   r   r   r    s    zSpiderFoot.hostDomainc             C   s>   d|krdS t jd|t js dS t|}|j|dd}|d kS )Nr7   Fz^[a-z0-9-\.]*$T)strict)r   r   r   r  r  )r   r  r  r  Zsfxr   r   r   	validHost  s    zSpiderFoot.validHostc             C   s   t |}|j|}||kS )N)r  r  )r   r  r  r  suffixr   r   r   isDomain  s    
zSpiderFoot.isDomainc             C   s   |sdS t j|S )NF)netaddr
valid_ipv4)r   r&   r   r   r   validIP  s    zSpiderFoot.validIPc             C   s   |sdS t j|S )NF)r  Z
valid_ipv6)r   r&   r   r   r   validIP6  s    zSpiderFoot.validIP6c          	   C   s<   y,dt |kr&tjt |jdkr&dS dS W n
   dS d S )Nr   r   TF)r/   r  	IPNetworkrw   )r   Zcidrr   r   r   validIpNetwork%  s     zSpiderFoot.validIpNetworkc             C   sd   t  }xX|D ]P}t|t krDx>|D ]}t|jd}|j| q"W qt|jd}|j| qW |S )Nr7   )r<   r   r/   r>   r=   )r   r   rJ   addrhostr   r   r   normalizeDNS/  s    

zSpiderFoot.normalizeDNSc          &   C   s   ddddddddd	d
dddddddddddddddddddddd d!d"d#d$d%d&g&}x|D ]}|j  |krVd'S qVW d(|krzd'S |jd%rd'S t|d)k rd'S d*S )+NarL   cr   r1   rH   rg   hr   jklr   r   orR   qr5   r   tr   vr   rK   rv   zrc   rt   23456789r   r7   Fz..r   T)rB   r8   r[   )r   cmdcharsr  r   r   r   sanitiseInput<  s    

zSpiderFoot.sanitiseInputc             C   s   t  }ddddg}x|D ]}y.t| j d | d d}|j }|j  W n6 tk
r } z| jdt|  wW Y d d }~X nX x*|D ]"}|j j	 }d	||j
d
d < qW qW t|j S )Nenglishgermanfrenchspanishz/dicts/ispell/z.dictr5   zCould not read dictionary: Tr   r   )rZ   r:   r   r;   r@   r-   r   r/   striprB   r9   r<   r   )r   wddictsr   wdctdlinesr1   r   r   r   r   	dictwordsP  s    

zSpiderFoot.dictwordsc             C   s   t  }dg}x|D ]}y.t| j d | d d}|j }|j  W n6 tk
rz } z| jdt|  wW Y d d }~X nX x*|D ]"}|j j	 }d||j
dd < qW qW t|j S )	Nnamesz/dicts/ispell/z.dictr5   zCould not read dictionary: Tr   r   )rZ   r:   r   r;   r@   r-   r   r/   r6  rB   r9   r<   r   )r   r7  r8  r   r9  r:  r1   r   r   r   r   	dictnamese  s    

zSpiderFoot.dictnamesc                s    fdd d }xdt |j D ]T}|| d kr0qd}x2t |j D ]"}|| d krTqB||| krBd}qBW |r|}P qW |d kri }n| ||d}|S )Nc                sV   t  }| t |j krd S ||  d kr*d S x&||  D ]}|j| ||d q4W |S )N)namechildren)r<   r   r=   )ZneedleZhaystackrJ   r  )get_childrenr   r   r@  }  s    z6SpiderFoot.dataParentChildToTree.<locals>.get_childrenTF)r>  r?  )r<   r   )r   r]   rk   r   Z	contenderZckfinalr   )r@  r   dataParentChildToTree|  s$    z SpiderFoot.dataParentChildToTreec             C   sp   y,| j tj|}t|dkr*tt|S d S  tk
rj } z"| jdt| d t|  d S d }~X nX d S )Nr   zUnable to resolve z: )	r  socketgethostbyname_exr[   r<   rY   r-   r   r/   )r   r  addrsr1   r   r   r   resolveHost  s    zSpiderFoot.resolveHostc             C   s~   | j d|  y,| jtj|}t|dkr8tt|S d S  tk
rx } z"| j d| d t| d  d S d }~X nX d S )NzPerforming reverse-resolve of r   zUnable to resolve z ())	r   r  rC  gethostbyaddrr[   r<   rY   r-   r/   )r   ipaddrrE  r1   r   r   r   	resolveIP  s    zSpiderFoot.resolveIPc             C   s   y|t  }tj|d tj}x.|D ]&}|d d |kr|j|d d  qW t|dk rXd S | jd| d t|  t t|S  t	k
r } z"| jd| d t| d  d S d }~X nX d S )	NrW   r   r4   z	Resolved z
 to IPv6: zUnable to IPv6 resolve z (rG  )
r<   rC  getaddrinfoAF_INET6r=   r[   r   r/   rY   r-   )r   r  rE  r   r  r1   r   r   r   resolveHost6  s    
zSpiderFoot.resolveHost6c             C   s8   | j |}|d krdS x|D ]}t||krdS qW dS )NFT)rF  r/   )r   r  iprE  r  r   r   r   
validateIP  s    

zSpiderFoot.validateIPc             C   s  t  }|j }|j }|d
kr6| j|}|r6|j| |dkrV| j|}|rV|j| |dkrxtj|D ]}t|}|j	dd dkrqjd|j	dkrqj|j
| | j|}	|	rj|rx8|	D ]$}
| j|
}|r||kr|j
|
 qW qj|j|	 qjW t|d	krt t|S d S )Nr   r   r   r   r7   r   re   rc   r   )r   r   )re   rc   )r<   getTypegetValuerJ  extendrF  r  r  r/   r9   r=   r[   rY   )r   ry   ZvalidateReverserJ   r$  r%  r5   r  rI  r<  r  Zchkr   r   r   resolveTargets  s>    







zSpiderFoot.resolveTargetsc             C   s*   t j|t|ft|}|jt| |S )N)rC  create_connectionr   
settimeout)r   r  r'   timeoutsockr   r   r   
safeSocket  s    zSpiderFoot.safeSocketc             C   s>   t j  }|jt| |j|t|f tj|}|j  |S )N)rC  rU  r   connectr   wrap_socketdo_handshake)r   r  r'   rV  r   rW  r   r   r   safeSSLSocket  s    
zSpiderFoot.safeSSLSocketc             C   s`   t  }xT|j D ]H}|j jdrtjd|tj}| jd|jd  |j	|jd qqW |S )Nz	disallow:zdisallow:\s*(.[^ #]*)z#robots.txt parsing found disallow: r4   )
r<   rE   rB   r8   r   r   r   r   r  r=   )r   ZrobotsTxtDataZ	returnArrr   r   r   r   r   parseRobotsTxt  s    zSpiderFoot.parseRobotsTxtc             C   s   t  }tjd|}xl|D ]d}| jd|  t|dk rB| jd qd|krZ| jd|  qd|krr| jd|  q|j| qW t|S )	Nz:([\%a-zA-Z\.0-9_\-\+]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)zFound possible email:    z%Skipped likely invalid email address.%zSkipped invalid email address: z...z#Skipped incomplete e-mail address: )rY   r   findallr   r[   r\   r<   )r   r]   Zemailsmatchesr   r   r   r   parseEmails&  s    

zSpiderFoot.parseEmailsc             C   s
   t j|S )N)r   DER_cert_to_PEM_cert)r   Zderr   r   r   sslDerToPemA  s    zSpiderFoot.sslDerToPem   c          :   C   s  t  }d|kr|jdd}t|tkr0|jd}ddlm} tjj	||}t
jjt
jj|}t
jjt
jj|}|jddd|d< t|j|d	< t |d
< d|d< d|d< d|d< d|d< t|j|d< y~tj|j jdd}	t|	jd|d< |	jd|d< ttj }
|
|d  }|d |kr,d|d< |d |
krBd|d< W n> tk
r } z | jdt| d d|d< |S d }~X nX yZ|jjtjj}xD|jD ]:}t |tjj!r|d
 j"|jj# jdjddd qW W n6 tk
r } z| j$dt|  W Y d d }~X nX t }yL|jj%tjj&j'j(}t)|dkrf|d jj# }||d
 krf|j"| W n6 tk
r } z| j$dt|  W Y d d }~X nX |r|d r|j# }yd| |d j# kr|j"| x$|d
 D ]}|j"|jdd qW ||d< | j$d | d!  d"j*|j+d"dd  j# }d}x<|D ]4}||krRd}|d#| krdd}||kr@d}q@W |sd|d< W n@ tk
r } z"| jd$t| d d|d< W Y d d }~X nX |S )%Nr   zutf-8r   )backendr  )errorstextissuerZaltnamesFexpiredZexpiringZmismatchZ	certerrorZissuedz%Y%m%d%H%M%SZz%sexpiryz%Y-%m-%d %H:%M:%SZ	expirystriQ Tz&Error processing date in certificate: r   asciiz Problem processing certificate: r4   zcn=zdns:hostszChecking for z in certificate subjectr7   z*.zError processing certificate: ),rZ   r  r   r/   rj   Z$cryptography.hazmat.backends.opensslrg  cryptographyx509Zload_pem_x509_certificateOpenSSLZcryptoZload_certificateZFILETYPE_PEMZdump_certificateZFILETYPE_TEXTr   rj  r<   subjectr   strptimeZget_notAfterr   strftimer+   r-   rA   
extensionsZget_extension_for_classZSubjectAlternativeNamevalue
isinstanceZDNSNamer=   rB   r   Zget_attributes_for_oidoidZNameOIDZCOMMON_NAMEr[   r   r9   )r   ZrawcertZfqdnZexpiringdaysrJ   rg  certZsslcertZsslcert_dumpZnotafternowZwarnexpr1   extrK   Z	certhostsattrsr>  r  Zfqdn_tldfoundchostr   r   r   	parseCertE  s    

.




zSpiderFoot.parseCertc              C   s  t  }g }t|tkr|g}dddddddd}y|jdd }W n& tk
rh } z
d}W Y d d }~X nX |d krvd}|d kst|dkr| jd d S y\xVt|j D ]F}	x@t	|d	t
|	d
j|	D ]$}
|
j||	 r|j|
||	   qW qW W n6 tk
r* } z| jdt| d d S d }~X nX x|D ]~}t|tkrPt|}|j }d }t|dk rnq4|t|d  dks|d dksd|ksd|kr| jd|  q4tjd|r| jd|  q4d|kr| jd|  q4d|kr
tjj|}d|kr|}|jdr2| j|| }|jdrJ|d | }x6|D ].}|d krP|j |j krP|d | }qPW |d kr| j|| }| j|}||d||< q4W |S )NZhrefrq   action)r  ZimgscriptlinkZarear   formr   r   httpz*parseLinks() called with no data to parse.Zlxml)Z
parse_onlyz"Error parsing with BeautifulSoup: Fr4   r7   +zjavascript:z()zunlikely link: z	.*#.[^/]+zin-page link: zmailto:zIgnoring mail link: z%2fz://r   z//)rx   original)rZ   r   r/   r9   r-   r[   r   r<   r   r   r   Zfind_allZhas_attrr=   rA   rB   r   r   urllibparseunquoter8   r  r   r   )r   r   r]   domainsZreturnLinksZurlsRelZtagsprotor1   r$  Zlnkr  ZlinklZabsLinkr  r   r   r   
parseLinks  s|    
$





zSpiderFoot.parseLinksc             C   s   t jddd |S )Nu   [-ÿ]c             S   s   dt | jd S )Nz%%%02xr   )ordr  )r  r   r   r   <lambda>  s    z-SpiderFoot.urlEncodeUnicode.<locals>.<lambda>)r   sub)r   r   r   r   r   urlEncodeUnicode  s    zSpiderFoot.urlEncodeUnicodec             C   s"   t j }| jr| j| jd|_|S )N)r  https)requestsrM   r    proxies)r   rM   r   r   r   rC     s
    zSpiderFoot.getSessionc             C   s:   ddddd}|}x"|D ]}t j||| |t j}qW |S )Nzkey=XXXzpass=XXXzuser=XXXzpassword=XXX)zkey=\S+zpass=\S+zuser=\S+zpassword=\S+)r   r  r   )r   r   patsrJ   patr   r   r   removeUrlCreds  s    
zSpiderFoot.removeUrlCredsFc       &   >   C   s  d d d d |d}|d krd S t  }| jd r2| jd g}dddg}| j|}| jd r^d}nd}||krnd}x|D ]}|j|rtd}qtW x|D ]}|j|rd}qW |r$| jd	|  | jd
| jd  d t| jd   d| jd  d t| jd  d| jd  d t| jd  d}n| jd|  yt  }tj }t|t	krft
j j||d< n||d< |d k	rxFt	|j D ]6}t|| tkrt|| ||< n|| ||< qW |
s|r|s| jd| j| d |d  d t| d  | j j|||||d}t|jjdd}|jjd||d< t|j|d< |rV|S ||
krd|S |d |kr|s| jd| j| d |d  d t| d  | j j|d ||||d}t|jjdd}|jjd|d |d< t|j|d< ||
kr|S |d k	rJ|s| jd| j| d |d  d t| d  n6|s| jd| j| d |d  d t| d  yF|r| j j||||d|||d}n| j j|||d|||d}W n( tjjk
r   td| d Y nX t  |d < xT|jj D ]F\}}t|tkr$t|}t|tkr:t|}||d  |j < qW |
rt|j|
kr| jd! |j|d< t|j|d< |S d"|d  kr y|d  d" j d#d$ }W n8 t!k
r } z| jd%|d  d"   |S d }~X nX | jd&| j|  | j"|||||||||	|
|S |j|d< t|j|d< |	rJ|j|d'< nJy|jj#d(|d'< W n4 t$k
r } z|jj#d)|d'< W Y d d }~X nX |ry|j%  W nJ tjj&k
r } z(| j'd*t|j d+ |j d,  W Y d d }~X nX W n t!k
r } z|sby0| j(d-t| d. | d | j(t)j* d W n" t!k
r` }  z|S d } ~ X nX d |d'< t||d/< |r| j'd*t| d,  W Y d d }~X nX t+j, d$ }!t+j-|!d }"|"j.}#tj }$t|$| }%| jd0tt|d' pd1 d2 | j| d3 |% d4  |S )5N)coder   rD   headersrealurlr#   r$   z192.168.z127.z10.TFzUsing proxy for zProxy set to r   _socks3portz
socks5h://)r  r  zNot using proxy for z
User-AgentzFetching (HEAD only): z [user-agent: z] [timeout: ])r  r  verifyrV  zcontent-lengthr   locationr  r  zFetching (incl. cookies): z
Fetching: )r]   r  r  allow_redirectscookiesrV  r  )r  r  r  r  rV  r  zFailed to connect to %sr  z>Content exceeded size limit, so returning no data just headersZrefreshz;url=r4   z+Refresh header found but was not parsable: z&Refresh header found, re-directing to rD   zutf-8rm  zURL could not be fetched (z / rG  zUnexpected exception (z) occurred fetching: r   zFetched data: r   z (z), took r   )/rZ   r   r  endswithr8   r   r/   r+   r   r<   rz   r{   choicer   r)   r  rC   headr   r  r   Zstatus_codepostr  
exceptionsZRequestException	ExceptionitemsrB   r[   rD   r   r9   r-   fetchUrlr   UnicodeDecodeErrorZraise_for_status	HTTPErrorr.   rA   	traceback
format_excr   r   r   r   )&r   r   r.   r  rV  	useragentr  ZnoLogZpostDataZ
dontMangleZ	sizeLimitZheadOnlyr  resultr  ZneverProxyNamesZneverProxySubnetsr  proxyr   r  headerZbtimer   hdrrw   r   rv  newurlr1   r  rK   rH   r   r   r   r   r$  r   r   r   r  '  s   






&&
,

,


.,




"8(6zSpiderFoot.fetchUrlc                sF   |sdS d dj  fddtdD }| j|d | d krBdS dS )	NFZbcdfghjklmnpqrstvwxyz3456789r   c                s   g | ]}t j j qS r   )rz   r{   r  )r   rK   )randpoolr   r   
<listcomp>  s    z/SpiderFoot.checkDnsWildcard.<locals>.<listcomp>r(   r7   T)r   rangerF  )r   ry   Zrandhostr   )r  r   checkDnsWildcard  s    zSpiderFoot.checkDnsWildcardc       	      C   s   dj |jddd}|d |d d}| j|tjj| |d d	}|d
 dkrb| jddd d S ytj|d }W n" t	k
r   | jddd d S X d|krdddddd}dj |jddtjj|d}dd |d D |d}nd S |S )Nz=https://www.googleapis.com/customsearch/v1?q={search_string}& z%20)search_stringZcse_idapi_key)ZcxkeyrV  )rV  r  200z2Failed to get a valid response from the Google APIF)r   rD   zHthe key 'content' in the Google API response doesn't contain valid json.r  zutf-8r$  zorg.mozilla:en-US:officialz	firefox-a)ZieZoeZaqZrlsclientz8https://www.google.com/search?q={search_string}&{params})r  paramsc             S   s   g | ]}t |d  qS )r  )r/   )r   r   r   r   r   r    s    z,SpiderFoot.googleIterate.<locals>.<listcomp>)urlswebSearchUrl)
formatr  r  r  r  	urlencoderA   r}   loads
ValueError)	r   searchStringr   endpointr  responseresponse_jsonZ
search_urlresultsr   r   r   googleIterate  s8    

zSpiderFoot.googleIteratec             C   s   dj |jddd}d|d d}| j|tjj| |d |d	 d
|d id}|d dkrn| jddd d S ytj|d }W n" t	k
r   | jddd d S X d|krd|d krd|d krdd |d d D |d d d}nd S |S )NzGhttps://api.cognitive.microsoft.com/bing/v7.0/search?q={search_string}&r  z%20)r  ZWebpagesr   )ZresponseFilterr   rV  r  zOcp-Apim-Subscription-Keyr  )rV  r  r  r  r  z0Failed to get a valid response from the bing APIF)r   rD   zFthe key 'content' in the bing API response doesn't contain valid json.ZwebPagesrv  r  c             S   s   g | ]}|d  qS )r   r   )r   r  r   r   r   r  Q  s    z*SpiderFoot.bingIterate.<locals>.<listcomp>)r  r  )
r  r  r  r  r  r  rA   r}   r  r  )r   r  r   r  r  r  r  r  r   r   r   bingIterate0  s0    zSpiderFoot.bingIterate)N)TT)N)T)T)T)Nre  )FNre  r
   NFNFNFF)Kr   
__module____qualname__r   r   rC  Z	savedsockr    r   r!   r"   r2   rN   r<   rb   rs   r   r   r   r   r   rA   r.   r   r)   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r	  r
  r  r  r  r  r  r  r  r1  r;  r=  rB  rF  rJ  rM  rO  rS  rX  r\  r]  rb  rd  r  r  r  rC   r  r  r  rZ   r  r  r   r   r   r   r
   /   s   

.5$5

	
.
A
/&

f`
   
 :8r
   c               @   s   e Zd ZdZe ZdZdZdZ dZdZ	dZ
dZdZdZdd Zdd Zd	d
 Ze fddZdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd Zdd  Zd!d" Zd#d$ Zd%d& Zd'd( Z d)d* Z!d+d, Z"dS )-SpiderFootPluginFNzmodule_name_not_set!r4   c             C   s   d S )Nr   )r   r   r   r   r   v  s    zSpiderFootPlugin.__init__c             C   s
   || _ d S )N)r    )r   r    r   r   r   _updateSocket{  s    zSpiderFootPlugin._updateSocketc             C   s   t  | _d| _d S )NF)r<   _listenerModules_stopScanning)r   r   r   r   clearListeners  s    zSpiderFootPlugin.clearListenersc             C   s   d S )Nr   )r   sfZuserOptsr   r   r   setup  s    zSpiderFootPlugin.setupc             C   s   d S )Nr   )r   ry   r   r   r   enrichTarget  s    zSpiderFootPlugin.enrichTargetc             C   s
   || _ d S )N)_currentTarget)r   ry   r   r   r   	setTarget  s    zSpiderFootPlugin.setTargetc             C   s
   || _ d S )N)__sfdb__)r   r   r   r   r   r     s    zSpiderFootPlugin.setDbhc             C   s
   || _ d S )N)
__scanId__)r   rQ   r   r   r   	setScanId  s    zSpiderFootPlugin.setScanIdc             C   s   | j S )N)r  )r   r   r   r   	getScanId  s    zSpiderFootPlugin.getScanIdc             C   s"   | j d krtd tjd | j S )Nz<Internal Error: Module called getTarget() but no target set.r4   r   )r  r   r   r   )r   r   r   r   	getTarget  s    

zSpiderFootPlugin.getTargetc             C   s   | j j| d S )N)r  r=   )r   listenerr   r   r   registerListener  s    z!SpiderFootPlugin.registerListenerc             C   s
   || _ d S )N)__outputFilter__)r   typesr   r   r   setOutputFilter  s    z SpiderFootPlugin.setOutputFilterc             C   s   t  S )N)rZ   )r   r   r   r   tempStorage  s    zSpiderFootPlugin.tempStoragec             C   s  |j }| jr2|dkr2|| j j kr2|| jkr2d S d}|jd ks\t|jtkr`t|jdkr`d S | j rld S |j	}xF|d k	r|j	d k	r|j	j |j kr|j	jj
 |jj
 krd}P |j	}qtW | jjdd d x| jD ]}||j krd|j krq|od	|jkrq||_| j rd S y.t|jtkr:|jjd
d|_|j| W q tk
r } z^tdd}|jd|j d t| d  tj \}}	}
|jttj||	|
 |j  W Y d d }~X qX qW d S )Nrd   Fr   Tc             S   s   | j S )N)	_priority)r   r   r   r   r    s    z2SpiderFootPlugin.notifyListeners.<locals>.<lambda>)r  r   Z__storzutf-8ignorezsferror.logr  zModule (z) encountered an error: r6   )	eventTyper  r  rP  r]   r   r/   r[   checkForStopsourceEventrB   r  sortwatchedEventsr  _currentEventr   r   handleEventr-   r:   r   r   exc_inforeprr  format_exceptionr@   )r   sfEventZ	eventNameZ	storeOnlyZ	prevEventr  r1   rH   exc_type	exc_valueexc_tracebackr   r   r   notifyListeners  sJ    
&




 z SpiderFootPlugin.notifyListenersc             C   s   t j| jdkrdS dS )NzABORT-REQUESTEDTF)globalScanStatus	getStatusr  )r   r   r   r   r    s    zSpiderFootPlugin.checkForStopc             C   s   | j S )N)r   )r   r   r   r   defaultOpts
  s    zSpiderFootPlugin.defaultOptsc             C   s   dgS )Nr   r   )r   r   r   r   r    s    zSpiderFootPlugin.watchedEventsc             C   s   d S )Nr   )r   r   r   r   producedEvents  s    zSpiderFootPlugin.producedEventsc             C   s   d S )Nr   )r   r  r   r   r   r    s    zSpiderFootPlugin.handleEventc             C   s   d S )Nr   )r   r   r   r   start$  s    zSpiderFootPlugin.start)#r   r  r  r  r<   r  r  r  r  r  Z__dataSource__r  r  Z
errorStater   r  r  rZ   r  r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r   r   r  \  s>   Nr  c            	   @   sz   e Zd Zddddddddd	g	Zd
Zd
Ze Zdd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd ZdddZd
S ) SpiderFootTargetr   r   r   r   r   r   r   r   r   Nc             C   sR   || j kr<|| _t|tkr,t|j | _n|| _t | _ntd t	j
d d S )Nz$Internal Error: Invalid target type.r4   r   )_validTypesr   r   r/   rB   targetValuer<   targetAliasesr   r   r   )r   r  typeNamer   r   r   r   0  s    

zSpiderFootTarget.__init__c             C   s   | j S )N)r   )r   r   r   r   rP  <  s    zSpiderFootTarget.getTypec             C   s   | j S )N)r  )r   r   r   r   rQ  ?  s    zSpiderFootTarget.getValuec             C   s.   ||d| j krd S | j j||j d d S )N)r   rv  )r  r=   rB   )r   rv  r  r   r   r   setAliasH  s    zSpiderFootTarget.setAliasc             C   s   | j S )N)r  )r   r   r   r   
getAliasesP  s    zSpiderFootTarget.getAliasesc             C   s8   t  }x,| jD ]"}|d |kr|j|d j  qW |S )Nr   rv  )r<   r  r=   rB   )r   r  rJ   rO   r   r   r   _getEquivalentsS  s
    z SpiderFootTarget._getEquivalentsc             C   sj   | j d}| jdkr2| jj |kr2|j| jj  t }x,|D ]$}|jt|tkr\|jdn| q>W |S )Nr   r   zutf-8)r   r   )	r  r   r  rB   r=   r<   r   r   r   )r   r1   r<  r>  r   r   r   getNames[  s    

$zSpiderFootTarget.getNamesc             C   sD   | j d}| jdkr |j| j | j d}| jdkr@|j| j |S )Nr   r   )r  r   r=   r  )r   r1   r   r   r   getAddressesf  s    



zSpiderFootTarget.getAddressesFTc             C   s   |j  }t|tkr|jdn|}|d ks2|dkr6dS | jdkrDdS tj|r|| j kr^dS | jdkrtj|tj	| j
krdS | jdkrtj|tj	tj| j
krdS nJxH| j D ]<}||krdS |r|jd| rdS |r|jd| rdS qW d S )Nzutf-8r   Fr   r   r   Tr   r   r   r7   )r   r   r   )r   r   )rB   r   r   r   r   r  r  r   Z	IPAddressr  r  r  r  )r   rv  ZincludeParentsZincludeChildrenr>  r   r   r   ra    s0    



zSpiderFootTarget.matches)FT)r   r  r  r  r   r  r<   r  r   rP  rQ  r  r  r  r  r   ra  r   r   r   r   r  )  s   	r  c               @   sv   e Zd ZdZdZdZdZdZdZdZ	dZ
dZdZdZdZdddZdd Zdd	 Zd
d Zdd Zdd Zdd ZdS )SpiderFootEventNd   r   c             C   s$  || _ tj | _|| _|| _|| _|| _|| _t|t	krxt|t	krxt
dt|  t
d|  t
d|  tjd	 t|t	kr|d krt	|| _n|| _|dkrd| _d S t|tkrt
d|  t
d|  t
d|  tjd
 |j | _| j t	| j | j t	tj jdd | _d S )Nz1FATAL: Only string events are accepted, not '%s'.zFATAL: Offending module: %szFATAL: Offending type: %sr4   rd   zFATAL: Invalid source event: %sr   ir   r   )r  r+   	generated
confidence
visibilityriskmoduler  r   r/   r   r   r   r]   sourceEventHashr  getHashrz   r{   r|   _SpiderFootEvent__id)r   r  r]   r  r  r  r  r  r   r   r   r     s4    



zSpiderFootEvent.__init__c             C   s>   t | j| j| j| jd}| jdkr.d|d< n| jj|d< |S )N)r  r   r]   r  rd   r   rx   )r   r  r  r]   r  r  )r   Zevt_dictr   r   r   asDict  s    


zSpiderFootEvent.asDictc             C   s(   | j dkrdS | jjd}tj|j S )Nrd   r   )r  r
  rj   r   r   r   )r   Z	digestStrr   r   r   r	    s    
zSpiderFootEvent.getHashc             C   s
   || _ d S )N)r  )r   r  r   r   r   setConfidence  s    zSpiderFootEvent.setConfidencec             C   s
   || _ d S )N)r  )r   r  r   r   r   setVisibility  s    zSpiderFootEvent.setVisibilityc             C   s
   || _ d S )N)r  )r   r  r   r   r   setRisk  s    zSpiderFootEvent.setRiskc             C   s
   || _ d S )N)r  )r   ZsrcHashr   r   r   setSourceEventHash  s    z"SpiderFootEvent.setSourceEventHash)r  r  r   )r   r  r  r  r  r  r  r  r  r]   r  r  ZmoduleDataSourceZactualSourcer
  r   r  r	  r  r  r  r  r   r   r   r   r    s&   
$r  c               @   sF   e Zd Zdd Zdd Zdd Zdd Zd	d
 Zdd ZdddZ	dS )r  c             C   s   | j |}| j|| _dS )aq  Reads and parses public suffix list.

        input_file is a file object or another iterable that returns
        lines of a public suffix list file. If input_file is None, an
        UTF-8 encoded file named "publicsuffix.txt" in the same
        directory as this Python module is used.

        The file format is described at http://publicsuffix.org/list/
        N)_build_structure	_simplifyrk   )r   Z
input_datark   r   r   r   r     s    
zPublicSuffixList.__init__c             C   sh   |s|S t |dkr|ji  t |dks.t|\}}|j }|j|d }|s\dg ||< }| j||S )Nr4   r   r   )r[   r=   AssertionErrorr   r   
_find_node)r   rP   partsnegater?  childZ
child_noder   r   r   r    s    
zPublicSuffixList._find_nodec             C   s>   |j drd}|dd  }nd}|jd}|| j||d< d S )N!r4   r   r7   )r8   r9   r  )r   rk   Zruler  r  r   r   r   	_add_rule,  s    

zPublicSuffixList._add_rulec                s>   t |dkr|d S |d t fddt|d j D fS )Nr4   r   c             3   s    | ]\}}| j |fV  qd S )N)r  )r   r   r%  )r   r   r   r   :  s    z-PublicSuffixList._simplify.<locals>.<genexpr>)r[   rZ   r<   r  )r   ri   r   )r   r   r  6  s    zPublicSuffixList._simplifyc             C   sP   dg}xD|D ]<}t |j }|jds| r.q| j||j d jd qW |S )Nr   z//r7   )r/   r6  r8   r  r9   lstrip)r   r   rk   r   r   r   r   r  <  s    
z!PublicSuffixList._build_structurec       	      C   sx   |dkr|}d }n|\}}||| < |t |k rt|rtx>d||  fD ],}|j|d }|d k	rD| j||d || qDW d S )Nr   r4   r   )r   r4   )r[   r   _lookup_node)	r   ra  depthrP   r  r  r?  r>  r  r   r   r   r  H  s    
zPublicSuffixList._lookup_nodeFc             C   s   |j  jdjd}dgt| }|r<|d | jd kr<dS | j|d| j| x6t|D ]*\}}|dk	rX|dkrXdj||d S qXW dS )a[  get_public_suffix("www.example.com") -> "example.com"

        Calling this function with a DNS name will return the
        public suffix for that name.

        Note that for internationalized domains the list at
        http://publicsuffix.org uses decoded names, so it is
        up to the caller to decode any Punycode-encoded names.
        r7   Nr4   r   r   )rB   r  r9   r[   rk   r  	enumerater   )r   r  r  r  hitsr   whatr   r   r   r  W  s    z"PublicSuffixList.get_public_suffixN)F)
r   r  r  r   r  r  r  r  r  r  r   r   r   r   r    s   
r  c               @   s2   e Zd Ze Zej Zdd Zdd Z	dd Z
dS )SpiderFootScanStatusc          
   C   s    | j  || j|< W d Q R X d S )N)lockstatusTable)r   scanIdr   r   r   r   	setStatust  s    zSpiderFootScanStatus.setStatusc          	   C   s*   | j  || jkr| j| S d S Q R X d S )N)r   r!  )r   r"  r   r   r   r  x  s    

zSpiderFootScanStatus.getStatusc          	   C   s   | j  | jS Q R X d S )N)r   r!  )r   r   r   r   getStatusAll~  s    z!SpiderFootScanStatus.getStatusAllN)r   r  r  rZ   r!  	threadingLockr   r#  r  r$  r   r   r   r   r  p  s
   r  )1stemr   Zstem.controlr   r   r   urllib.requestr  urllib.parseurllib.errorr}   r   r   rz   r  rC  r   r   r+   r  r%  r  rq  r   ro  Zdns.resolverr   Znetworkxr   Znetworkx.readwrite.gexfr   r   Zbs4r   r   copyr	   Zurllib3Zdisable_warningsr  ZInsecureRequestWarningr
   objectr  r  r  r  r  r  r   r   r   r   <module>   sb               9 N ]j