U
    @†6^Ù- ã                   @   s”  d dl mZ d dlmZ d dlZd dlZd dlZd dlZd dl	Zd dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Zd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dl m!Z!m"Z" d dl#m$Z$ d dl%Z%e% &e%j'j(¡ G d	d
„ d
ƒZ)G dd„ de*ƒZ+G dd„ de*ƒZ,G dd„ de*ƒZ-G dd„ de*ƒZ.G dd„ dƒZ/e/ƒ a0dS )é    )ÚSignal)Ú
ControllerN)Únx)Ú
GEXFWriter)Údatetime)ÚBeautifulSoupÚSoupStrainer)Údeepcopyc                   @   sP  e Zd ZdZdZeZdZd‡dd„Zdd„ Z	dd„ Z
dd	„ Zdˆdd„Zeƒ fdd„Zg fdd„Zeƒ fdd„Zdd„ Zdd„ Zdd„ Zd‰dd„ZdŠdd„Zdd„ Zdd „ Zd!d"„ Zd#d$„ Zd%d&„ Zd'd(„ Zd)d*„ Zd+d,„ Zd-d.„ Zd‹d/d0„ZdŒd1d2„Z d3d4„ Z!d5d6„ Z"d7d8„ Z#d9d:„ Z$d;d<„ Z%d=d>„ Z&d?d@„ Z'dAdB„ Z(dCdD„ Z)dEdF„ Z*dGdH„ Z+dIdJ„ Z,dKdL„ Z-dMdN„ Z.dOdP„ Z/dQdR„ Z0dSdT„ Z1dUdV„ Z2dWdX„ Z3dYdZ„ Z4d[d\„ Z5d]d^„ Z6d_d`„ Z7dadb„ Z8dcdd„ Z9dedf„ Z:dgdh„ Z;didj„ Z<dkdl„ Z=dmdn„ Z>dodp„ Z?dqdr„ Z@ddtdu„ZAdvdw„ ZBdxdy„ ZCdzd{„ ZDd|d}„ ZEdŽdd€„ZFdd‚„ ZGeHƒ fdƒd„„ZIeHƒ fd…d†„ZJdS )Ú
SpiderFootNc                 C   s\   || _ t|ƒ| _tjdkr"tjt_| j dd¡dkrXt	j
 ¡ }| jd g|_t	j
 |¡ d S )N)é   é   é	   Ú
_dnsserverÚ )Úhandler	   ÚoptsÚsysÚversion_infoÚsslÚ_create_unverified_contextÚ_create_default_https_contextÚgetÚdnsÚresolverZResolverZnameserversZoverride_system_resolver)ÚselfÚoptionsr   Úres© r   ú(/var/www/spiderfoot.crq.systems/sflib.pyÚ__init__9   s    


zSpiderFoot.__init__c                 C   s
   || _ d S ©N©Ú
socksProxy©r   r"   r   r   r   ÚupdateSocketI   s    zSpiderFoot.updateSocketc                 C   s
   d | _ d S r    r!   ©r   r   r   r   ÚrevertSocketL   s    zSpiderFoot.revertSocketc              
   C   sž   | j d dkrd S zR|  d¡ tj| j d | j d d$}| ¡  | tj¡ t 	d¡ W 5 Q R X W n4 t
k
r˜ } z|  dt|ƒ ¡ W 5 d }~X Y nX d S )	NÚ_socks1typeZTORzRe-circuiting TOR...Ú_socks2addrÚ_torctlport)ÚaddressÚporté
   zUnable to re-circuit TOR: )r   Úinfor   Z	from_portZauthenticateÚsignalr   ZNEWNYMÚtimeÚsleepÚBaseExceptionÚfatalÚstr)r   Ú
controllerÚer   r   r   ÚrefreshTorIdentP   s    
ÿzSpiderFoot.refreshTorIdentTc              
   C   sx  |d krd S |  d¡rÆ| d¡d }z\|  d| ¡ t|dƒ}|rl| ¡ }tƒ }|D ]}| | d¡¡ qTn| ¡ }| 	¡  |W S  t
k
rÄ }	 z&|rª|  d| d ¡ n
W Y ¢d S W 5 d }	~	X Y nX | ¡   d¡sä| ¡   d	¡rtz>|  d
| ¡ |  ¡ }
|
 |¡}|r|j ¡ W S |jW S W nP t
k
rr } z0|rX|  d| d t|ƒ ¡ n
W Y ¢d S W 5 d }~X Y nX |S )Nú@é   z!Loading configuration data from: ÚrÚ
zUnable to open option file, Ú.zhttp://zhttps://z%Downloading configuration data from: zUnable to open option URL, ú: )Ú
startswithÚsplitr-   ÚopenÚ	readlinesÚlistÚappendÚrstripÚreadÚcloser1   ÚerrorÚlowerÚ
getSessionr   ÚcontentÚ
splitlinesr3   )r   Úvalr2   Z
splitLinesÚfnameÚfÚarrÚretÚxÚbÚsessionr   r5   r   r   r   ÚoptValueToDatab   s@    


zSpiderFoot.optValueToDatac                    s.  t ƒ }tƒ ‰ tƒ ‰‡ ‡‡fdd„‰|D ]’}|d dksB|d dkr€t|ƒdkrt|d |ksf|d dkr€dˆ |d	 < ndˆ |d	 < |d	 ˆkrštƒ ˆ|d	 < ˆ|d	   |d
 |d g¡ q&ˆ D ]j}ˆ| D ]\\}}|ˆ krò||krð| ||f¡ qÊtƒ }ˆ||ƒ}	|	D ]}
||
kr| ||
f¡ qqÊq¾|S )Nc                    s^   t ƒ }ˆ|  D ]J\}}||kr q|ˆ kr4| |¡ q| |¡ ˆ||ƒD ]}| |¡ qHq|S r    )rA   rB   )ÚitemZpidsrO   ÚparentÚidÚp©ÚentitiesÚget_next_parent_entitiesÚparentsr   r   rZ   •   s    
z;SpiderFoot.buildGraphData.<locals>.get_next_parent_entitiesé   ZENTITYZINTERNALr   é   Tr8   r   é   )ÚsetÚdictÚlenrA   rB   Úadd)r   ÚdataÚfltÚmappingÚrowÚentityrU   rV   ZppidsZnext_parentsZnext_parentr   rX   r   ÚbuildGraphData   s0     

zSpiderFoot.buildGraphDatac                 C   s$  |   ||¡}t ¡ }tƒ }d}|D ]è}	|	\}
}dddg}|
dks"|dkrJq"|
|kr¤|d }|
|krldddg}| |
¡}d|d |d |d dœi|j|
 d	< |||
< ||krþ|d }||krÆdddg}| |¡ d|d |d |d dœi|j| d	< |||< | ||
¡ q"t|d
}t|ƒ 	d¡S )Nr   Ú0ÚROOTr8   Ú255Úcolorr   )r9   ÚgrQ   Zviz)Úgraphúutf-8)
rh   r   ZGraphr`   Zadd_nodeÚnodeZadd_edger   r3   Úencode)r   ÚrootÚtitlerc   rd   re   rn   ÚnodelistÚncounterÚpairÚdstÚsrcÚcolrp   Zgexfr   r   r   ÚbuildGraphGexfÅ   s4    


&

&
zSpiderFoot.buildGraphGexfc              
   C   s\  |   ||¡}tƒ }tƒ |d< tƒ |d< tƒ }d}d}|D ]}	|	\}
}d}|
dks8|dkr\q8|
|kr¼|d }|
|krxd}|d  t|ƒt|
ƒt ¡  dd¡t ¡  dd¡d	|d
œ¡ |||
< ||kr||krÒd}|d }|d  t|ƒt|ƒt ¡  dd¡t ¡  dd¡d	|d
œ¡ |||< |d }|d  t|ƒt|| ƒt||
 ƒdœ¡ q8t 	|¡S )NZnodesZedgesr   z#000rj   r8   z#f00iè  Ú1)rV   ÚlabelrP   ÚyÚsizerl   )rV   ÚsourceÚtarget)
rh   r`   rA   rB   r3   ÚrandomÚSystemRandomÚrandintÚjsonÚdumps)r   rr   rc   rd   re   rO   rt   Zecounterru   rv   rw   rx   ry   r   r   r   ÚbuildGraphJsoné   sT    


û
û

þ
zSpiderFoot.buildGraphJsonc                 C   s
   || _ d S r    )Údbh)r   r   r   r   r   ÚsetDbh  s    zSpiderFoot.setDbhc                 C   s
   || _ d S r    )ÚGUID)r   Úuidr   r   r   ÚsetGUID#  s    zSpiderFoot.setGUIDc                 C   s   t t ¡ ƒ d¡d  ¡ S )Nú-r   )r3   ÚuuidÚuuid4r>   Úupper)r   ZscanNamer   r   r   ÚgenScanInstanceGUID'  s    zSpiderFoot.genScanInstanceGUIDc                 C   s   | j  | j|||¡S r    )r‡   ZscanLogEventr‰   )r   ÚlevelÚmessageZ	componentr   r   r   Ú_dblog/  s    zSpiderFoot._dblogc                 C   s^   | j d sd S | jd kr&td| ƒ n|  d|¡ | j  d¡rJtd| ƒ |rZtd| ƒ‚d S )NÚ	__loggingz
[Error] %sÚERRORÚ__logstdoutzInternal Error Encountered: )r   r‡   Úprintr“   r   r1   )r   r’   Ú	exceptionr   r   r   rF   3  s    

zSpiderFoot.errorc                 C   sB   | j d krtd| ƒ n|  d|¡ ttt ¡ ƒƒ t d¡ d S )Nz
[Fatal] %sÚFATALéÿÿÿÿ)r‡   r—   r“   r3   ÚinspectÚstackr   Úexit)r   rF   r   r   r   r2   @  s
    
zSpiderFoot.fatalc                 C   sN   | j d sd S | jd kr&td| ƒ n|  d|¡ | j  d¡rJtd| ƒ d S )Nr”   z[Status] %sZSTATUSr–   ú[*] %s)r   r‡   r—   r“   r   )r   r’   r   r   r   ÚstatusH  s    

zSpiderFoot.statusc                 C   s¼   | j d sd S t ¡ d }t |d ¡}|d kr6d}n@|jdkrpt ¡ d }t |d ¡}|d krhd}qv|j}n|j}| jd kr’td||f ƒ n|  d||¡ | j  d	¡r¸td
| ƒ d S )Nr”   r8   r   ÚUnknownÚsflibr   ú[%s] %sÚINFOr–   rž   ©	r   r›   rœ   Ú	getmoduleÚ__name__r‡   r—   r“   r   ©r   r’   ÚfrmÚmodÚmodNamer   r   r   r-   S  s&    


zSpiderFoot.infoc                 C   sÎ   | j d sd S | j d sd S t ¡ d }t |d ¡}|d krDd}n@|jdkr~t ¡ d }t |d ¡}|d krvd}q„|j}n|j}| jd kr td||f ƒ n|  d	||¡ | j  d
¡rÊtd||f ƒ d S )NÚ_debugr”   r8   r   r    r¡   r   r¢   ÚDEBUGr–   z	[d:%s] %sr¤   r§   r   r   r   Údebugo  s*    



zSpiderFoot.debugc                 C   s$   t tdƒrtj tj¡S tj t¡S )NÚfrozen)Úhasattrr   ÚosÚpathÚdirnameÚ
executableÚ__file__r%   r   r   r   ÚmyPathŒ  s    
zSpiderFoot.myPathc                 C   s0   |}t |ƒttfkrt|ƒ}t | d¡¡ ¡ S )NÚraw_unicode_escape)ÚtyperA   r`   r3   ÚhashlibÚsha256rq   Ú	hexdigest)r   ÚstringÚsr   r   r   Ú
hashstring•  s    zSpiderFoot.hashstringc                 C   s&   |   ¡ d }tj |¡s"t |¡ |S )Nz/cache)rµ   r°   r±   ÚisdirÚmkdir)r   r±   r   r   r   Ú	cachePath   s    
zSpiderFoot.cachePathc                 C   s°   t  | d¡¡ ¡ }|  ¡ d | }t|dƒ}t|ƒtkr||D ]:}t|ƒtkrd| 	|¡ | 	d¡ q>| 	| 
d¡d ¡ q>n(t|ƒtkrš| 	| 
d¡¡ n
| 	|¡ | ¡  d S )Nro   ú/Úwr:   )r¸   Úsha224rq   rº   rÀ   r?   r·   rA   r3   ÚwriteÚdecodeÚbytesrE   )r   r|   rc   Ú	pathLabelÚ	cacheFileÚfpÚliner   r   r   ÚcachePut§  s    


zSpiderFoot.cachePutc              
   C   sÈ   t  | d¡¡ ¡ }|  ¡ d | }zvt |¡\
}}}}}	}
}}}}|dkrRW d S |t ¡ |d  ksn|dkr’t|dƒ}| 	¡ }W 5 Q R X |W S W d S W n( t
k
rÂ } z
W Y ¢d S d }~X Y nX d S )Nro   rÁ   r   i  r9   )r¸   rÃ   rq   rº   rÀ   r°   Ústatr/   r?   rD   r1   )r   r|   Z
timeoutHrsrÇ   rÈ   ÚmÚiÚdÚnÚurm   ÚszÚatimeÚmtimeÚctimerÉ   ZfileContentsr5   r   r   r   ÚcacheGet¹  s    
zSpiderFoot.cacheGetc                 C   sæ  t ƒ }t| ¡ ƒD ]Œ}| d¡r&|r&qt|| ƒtksFt|| ƒtkrR|| ||< t|| ƒtkr||| rtd||< nd||< t|| ƒtkrd || ¡||< qd|kr¬|S |d D ]*}|d | d D ]}| d¡rà|ràqÊt|d | d | ƒtkst|d | d | ƒtkr<|d | d | ||d | < t|d | d | ƒtkr’|d | d | r‚d||d | < nd||d | < t|d | d | ƒtkrÊd d	d
„ |d | d | D ƒ¡||d | < qÊq´|S )NÚ__r8   r   ú,Ú__modules__r   Ú_ú:c                 s   s   | ]}t |ƒV  qd S r    ©r3   ©Ú.0rP   r   r   r   Ú	<genexpr>ö  s   ÿz-SpiderFoot.configSerialize.<locals>.<genexpr>)	r`   rA   Úkeysr=   r·   Úintr3   ÚboolÚjoin)r   r   ÚfilterSystemZ	storeoptsÚoptr©   r   r   r   ÚconfigSerializeÑ  s>     
ÿ 
ÿzSpiderFoot.configSerializec                 C   s  |}t | ¡ ƒD ]ú}| d¡r$|r$q||krt|| ƒtkrZ|| dkrRd||< nd||< t|| ƒtkrzt|| ƒ||< t|| ƒtkršt|| ƒ||< t|| ƒt krt|| d ƒtkrôt ƒ ||< t|| ƒ d¡D ]}||  t|ƒ¡ qÚqt|| ƒ d¡||< qd|kr|S |d D ]ì}|d | d D ]Ò}| d	¡rT|rTq8|d
 | |kr8t|d | d | ƒtkrÄ||d
 |  dkr°d|d | d |< nd|d | d |< t|d | d | ƒtkrt||d
 |  ƒ|d | d |< t|d | d | ƒtkrHt||d
 |  ƒ|d | d |< t|d | d | ƒt kr8t|d | d | d ƒtkràt ƒ |d | d |< t||d
 |  ƒ d¡D ]$}|d | d |  t|ƒ¡ q¸n*t||d
 |  ƒ d¡|d | d |< q8q"|S )Nr×   r{   TFr   rØ   rÙ   r   rÚ   rÛ   )	rA   rà   r=   r·   râ   r3   rá   r>   rB   )r   r   ZreferencePointrä   Z
returnOptsrå   rP   rª   r   r   r   ÚconfigUnserializeÿ  sX    


ÿÿ"$ÿzSpiderFoot.configUnserializec              
   C   s†   d }ddiddiddiddid	d
iddiddiddiddig	}|D ]>}t | ¡ ƒd }t ||tjtjB ¡rBt | ¡ ƒd } q‚qB|S )Nz^\d+\.\d+\.\d+\.\d+$Ú
IP_ADDRESSz^\d+\.\d+\.\d+\.\d+/\d+$ÚNETBLOCK_OWNERz^.*@.*$Ú	EMAILADDRz^\+\d+$ÚPHONE_NUMBERz^".*\s+.*"$Ú
HUMAN_NAMEz^".*"$ÚUSERNAMEz^\d+$ÚBGP_AS_OWNERz^[0-9a-f:]+$ÚIPV6_ADDRESSzS^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)+([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$ÚINTERNET_NAMEr   )rA   rà   ÚreÚmatchÚ
IGNORECASEÚUNICODEÚvalues)r   r€   Ú
targetTypeZregexToTypeZrxpairÚrxr   r   r   rö   @  s"    ÷zSpiderFoot.targetTypec                 C   s†   t ƒ }t | jd  ¡ ƒD ]h}| jd | d d kr4q| jd | d D ]8}||krd||krd| |¡ d|krF||krF| |¡ qFq|S )NrÙ   ÚprovidesÚ*©rA   r   rà   rB   ©r   ÚeventsÚmodlistr©   Zevtyper   r   r   ÚmodulesProducingY  s    
zSpiderFoot.modulesProducingc                 C   s”   t ƒ }t | jd  ¡ ƒD ]v}| jd | d d kr4qd| jd | d kr\||kr\| |¡ | jd | d D ]}||krn||krn| |¡ qnq|S )NrÙ   Úconsumesrù   rú   rû   r   r   r   ÚmodulesConsumingi  s    
zSpiderFoot.modulesConsumingc                 C   sb   t ƒ }|D ]R}|t | jd  ¡ ƒkr
| jd | d d k	r
| jd | d D ]}| |¡ qLq
|S )NrÙ   rø   rú   ©r   ÚmodulesZevtlistr©   Zevtr   r   r   ÚeventsFromModulesz  s    zSpiderFoot.eventsFromModulesc                 C   sb   t ƒ }|D ]R}|t | jd  ¡ ƒkr
| jd | d d k	r
| jd | d D ]}| |¡ qLq
|S )NrÙ   rÿ   rú   r  r   r   r   ÚeventsToModules†  s    zSpiderFoot.eventsToModulesc                 C   sp   t ƒ }d|kr|S | d¡}|D ]D}|dkrZt|ƒdkr:q d|krPt|ƒdkrPq | ¡  q | |¡ q d |¡S )Nú..rÁ   r8   ú://é   )rA   r>   ra   ÚpoprB   rã   )r   ÚurlZ	finalBitsÚbitsÚchunkr   r   r   ÚurlRelativeToAbsolute•  s    
z SpiderFoot.urlRelativeToAbsolutec                 C   sV   |  d¡}t|ƒdkr|d S d|kr<| d¡dk r<|d S d |d d… ¡}|d S )NrÁ   r   r  r  rš   )r>   ra   Úcountrã   )r   r	  r
  Úbaser   r   r   Ú
urlBaseDir°  s    
zSpiderFoot.urlBaseDirc                 C   s@   d|krt  d|¡}nt  d|¡}|d kr2| ¡ S | d¡ ¡ S )Nr  z(\w+://.[^/:\?]*)[:/\?].*z(.[^/:\?]*)[:/\?]r8   )rñ   rò   rG   Úgroup)r   r	  r
  r   r   r   Ú
urlBaseUrlÅ  s    zSpiderFoot.urlBaseUrlc                 C   s.   |   |¡}d|krd}nd}| d¡|  ¡ S )Nr  r   r   rÁ   )r  r>   rG   )r   r	  Zbaseurlr  r   r   r   ÚurlFQDNÒ  s
    
zSpiderFoot.urlFQDNc                 C   sV   d  |  | ¡ |¡ d¡dd … ¡}| ¡  d| d¡}d|krN| d¡d S |S d S )Nr;   r8   r   rš   )rã   Ú
hostDomainrG   r>   Úreplace)r   ÚdomainÚtldListÚtldrO   r   r   r   ÚdomainKeywordÞ  s
    $zSpiderFoot.domainKeywordc                 C   s†   t ƒ }|D ]d}d |  | ¡ |¡ d¡dd … ¡}| ¡  d| d¡}d|krd| | d¡d ¡ q
| |¡ q
|  dt|ƒ ¡ |S )Nr;   r8   r   rš   z
Keywords: )	rA   rã   r  rG   r>   r  rB   r­   r3   )r   Z
domainListr  rN   r  r  rO   r   r   r   ÚdomainKeywordsë  s    $zSpiderFoot.domainKeywordsc                 C   s   t |ƒ}| |¡S r    ©ÚPublicSuffixListÚget_public_suffix)r   Úhostnamer  Úpsr   r   r   r  ý  s    zSpiderFoot.hostDomainc                 C   s>   d|krdS t  d|t j¡s dS t|ƒ}|j|dd}|d kS )Nr;   Fz^[a-z0-9-\.]*$T)Ústrict)rñ   rò   ró   r  r  )r   r  r  r  Zsfxr   r   r   Ú	validHost  s    zSpiderFoot.validHostc                 C   s   t |ƒ}| |¡}||kS r    r  )r   r  r  r  Úsuffixr   r   r   ÚisDomain  s    
zSpiderFoot.isDomainc                 C   s   |sdS t  |¡S ©NF)ÚnetaddrÚ
valid_ipv4©r   r*   r   r   r   ÚvalidIP  s    zSpiderFoot.validIPc                 C   s   |sdS t  |¡S r#  )r$  Z
valid_ipv6r&  r   r   r   ÚvalidIP6  s    zSpiderFoot.validIP6c                 C   sD   z0dt |ƒkr(t t |ƒ¡jdkr(W dS W dS W n   Y dS X d S )NrÁ   r   TF)r3   r$  Ú	IPNetworkr~   )r   Zcidrr   r   r   ÚvalidIpNetwork%  s     
zSpiderFoot.validIpNetworkc                 C   s\   t ƒ }|D ]L}t|ƒt kr>|D ]}t|ƒ d¡}| |¡ qq
t|ƒ d¡}| |¡ q
|S )Nr;   )rA   r·   r3   rC   rB   )r   r   rO   ÚaddrÚhostr   r   r   ÚnormalizeDNS/  s    zSpiderFoot.normalizeDNSc              &   C   sš   ddddddddd	d
dddddddddddddddddddddd d!d"d#d$d%d&g&}|D ]}|  ¡ |krT d'S qTd(|krxd'S | d%¡r†d'S t|ƒd)k r–d'S d*S )+NÚarQ   ÚcrÏ   r5   rM   rm   ÚhrÎ   ÚjÚkÚlrÍ   rÐ   ÚorW   Úqr9   r¼   ÚtrÑ   ÚvrÂ   rP   r}   Úzri   r{   Ú2Ú3Ú4Ú5Ú6Ú7Ú8Ú9rŒ   r;   Fr  r  T)rG   r=   ra   )r   ÚcmdÚcharsr/  r   r   r   ÚsanitiseInput<  sJ                           þ
zSpiderFoot.sanitiseInputc                 C   s¼   t ƒ }ddddg}|D ]˜}z.t|  ¡ d | d dƒ}| ¡ }| ¡  W n< tk
r„ } z|  dt|ƒ ¡ W Y ¢qW 5 d }~X Y nX |D ]"}| ¡  	¡ }d	|| 
d
¡d < qŠqt| ¡ ƒS )NÚenglishÚgermanÚfrenchÚspanishú/dicts/ispell/ú.dictr9   úCould not read dictionary: TrÁ   r   ©r`   r?   rµ   r@   rE   r1   r­   r3   ÚstriprG   r>   rA   rà   ©r   ÚwdÚdictsrÏ   ZwdctZdlinesr5   rÂ   r   r   r   Ú	dictwordsP  s    zSpiderFoot.dictwordsc                 C   s¶   t ƒ }dg}|D ]˜}z.t|  ¡ d | d dƒ}| ¡ }| ¡  W n< tk
r~ } z|  dt|ƒ ¡ W Y ¢qW 5 d }~X Y nX |D ]"}| ¡  	¡ }d|| 
d¡d < q„qt| ¡ ƒS )	NÚnamesrH  rI  r9   rJ  TrÁ   r   rK  rM  r   r   r   Ú	dictnamese  s    zSpiderFoot.dictnamesc                    s’   ‡ fdd„‰ d }t | ¡ ƒD ]R}|| d kr.qd}t | ¡ ƒD ]"}|| d krPq>||| kr>d}q>|r|} qpq|d kr~i }n|ˆ ||ƒdœ}|S )Nc                    sR   t ƒ }| t | ¡ ƒkrd S ||  d kr*d S ||  D ]}| |ˆ ||ƒdœ¡ q2|S )N©ÚnameÚchildren)rA   rà   rB   )ZneedleZhaystackrO   r/  ©Úget_childrenr   r   rW  }  s    z6SpiderFoot.dataParentChildToTree.<locals>.get_childrenTFrS  )rA   rà   )r   rc   rr   r2  Z	contenderZckÚfinalr   rV  r   ÚdataParentChildToTree|  s$    z SpiderFoot.dataParentChildToTreec              
   C   s|   z0|   t |¡¡}t|ƒdkr,tt|ƒƒW S W d S  tk
rv } z(|  dt|ƒ d t|ƒ ¡ W Y ¢d S d }~X Y nX d S )Nr   úUnable to resolve r<   )	r-  ÚsocketÚgethostbyname_exra   rA   r_   r1   r­   r3   )r   r,  Úaddrsr5   r   r   r   ÚresolveHost«  s    zSpiderFoot.resolveHostc              
   C   sŠ   |   d| ¡ z0|  t |¡¡}t|ƒdkr:tt|ƒƒW S W d S  tk
r„ } z(|   d| d t|ƒ d ¡ W Y ¢d S d }~X Y nX d S )NzPerforming reverse-resolve of r   rZ  ú (ú))	r­   r-  r[  Úgethostbyaddrra   rA   r_   r1   r3   )r   Úipaddrr]  r5   r   r   r   Ú	resolveIP¶  s    zSpiderFoot.resolveIPc              
   C   sÈ   z|t ƒ }t |d tj¡}|D ]&}|d d |kr| |d d ¡ qt|ƒdk rVW d S |  d| d t|ƒ ¡ t t|ƒƒW S  t	k
rÂ } z(|  d| d t|ƒ d ¡ W Y ¢d S d }~X Y nX d S )	Nr]   r   r8   z	Resolved z
 to IPv6: zUnable to IPv6 resolve r_  r`  )
rA   r[  ÚgetaddrinfoÚAF_INET6rB   ra   r­   r3   r_   r1   )r   r  r]  r   r+  r5   r   r   r   ÚresolveHost6Ã  s    zSpiderFoot.resolveHost6c                 C   s6   |   |¡}|d krdS |D ]}t|ƒ|kr dS qdS )NFT)r^  r3   )r   r,  Úipr]  r+  r   r   r   Ú
validateIPÓ  s    
zSpiderFoot.validateIPc                 C   s
  t ƒ }| ¡ }| ¡ }|dkr6|  |¡}|r6| |¡ |dkrV|  |¡}|rV| |¡ |dkrìt |¡D ]‚}t|ƒ}| 	d¡d dkrˆqhd| 	d¡kr˜qh| 
|¡ |  |¡}	|	rh|rà|	D ]$}
|  |
¡}|r¸||kr¸| 
|
¡ q¸qh| |	¡ qht|ƒdkrt t|ƒƒS d S )	N©rè   rï   rð   ré   r;   r  )rk   ri   rk   r   )rA   ÚgetTypeÚgetValuerc  Úextendr^  r$  r)  r3   r>   rB   ra   r_   )r   r€   ZvalidateReverserO   r6  r7  r9   r+  rb  rQ  r,  Zchkr   r   r   ÚresolveTargetsà  s>    






zSpiderFoot.resolveTargetsc                 C   s*   t  |t|ƒft|ƒ¡}| t|ƒ¡ |S r    )r[  Úcreate_connectionrá   Ú
settimeout)r   r,  r+   ÚtimeoutÚsockr   r   r   Ú
safeSocket  s    zSpiderFoot.safeSocketc                 C   s>   t   ¡ }| t|ƒ¡ | |t|ƒf¡ t |¡}| ¡  |S r    )r[  ro  rá   Úconnectr   Úwrap_socketÚdo_handshake)r   r,  r+   rp  r¼   rq  r   r   r   ÚsafeSSLSocket  s    
zSpiderFoot.safeSSLSocketc                 C   s\   t ƒ }| ¡ D ]H}| ¡  d¡rt d|tj¡}|  d| d¡ ¡ | 	| d¡¡ qq|S )Nz	disallow:zdisallow:\s*(.[^ #]*)z#robots.txt parsing found disallow: r8   )
rA   rJ   rG   r=   rñ   rò   ró   r­   r  rB   )r   ZrobotsTxtDataZ	returnArrrÊ   rÍ   r   r   r   ÚparseRobotsTxt  s    zSpiderFoot.parseRobotsTxtc                 C   s„   t ƒ }t d|¡}|D ]d}|  d| ¡ t|ƒdk r@|  d¡ qd|krX|  d| ¡ qd|krp|  d| ¡ q| |¡ qt|ƒS )	Nz:([\%a-zA-Z\.0-9_\-\+]+@[a-zA-Z\.0-9\-]+\.[a-zA-Z\.0-9\-]+)zFound possible email: é   z%Skipped likely invalid email address.ú%zSkipped invalid email address: z...z#Skipped incomplete e-mail address: )r_   rñ   Úfindallr­   ra   rb   rA   )r   rc   ZemailsÚmatchesrò   r   r   r   ÚparseEmails&  s    
zSpiderFoot.parseEmailsc                 C   s
   t  |¡S r    )r   ÚDER_cert_to_PEM_cert)r   Zderr   r   r   ÚsslDerToPemA  s    zSpiderFoot.sslDerToPemé   c              
   C   sÌ  t ƒ }d|kr| dd¡}t|ƒtkr0| d¡}ddlm} tj 	||¡}t
j t
jj|¡}t
j t
jj|¡}|jddd|d< t|jƒ|d	< tƒ |d
< d|d< d|d< d|d< d|d< t|jƒ|d< z~t | ¡  d¡d¡}	t|	 d¡ƒ|d< |	 d¡|d< tt ¡ ƒ}
|
|d  }|d |kr,d|d< |d |
krBd|d< W nH tk
rŒ } z(|  dt|ƒ d¡ d|d< | W Y ¢S d }~X Y nX zV|j tjj¡}|jD ]:}t |tjj!ƒr¦|d
  "|j #¡  d¡jddd¡ q¦W n6 tk
r } z|  $dt|ƒ ¡ W 5 d }~X Y nX tƒ }zL|j %tjj&j'j(¡}t)|ƒdkrl|d j #¡ }||d
 krl| "|¡ W n6 tk
r¤ } z|  $dt|ƒ ¡ W 5 d }~X Y nX |rÈ|d rÈ| #¡ }zÈd| |d  #¡ krà| "|¡ |d
 D ]}| "| dd¡¡ qè||d< |  $d | d! ¡ d" *| +d"¡dd … ¡ #¡ }d}|D ]4}||krRd}|d#| krdd}||kr@d}q@|s„d|d< W n@ tk
rÆ } z |  d$t|ƒ d¡ d|d< W 5 d }~X Y nX |S )%Núr   ro   r   )Úbackendr  )ÚerrorsÚtextÚissuerZaltnamesFÚexpiredZexpiringZmismatchZ	certerrorZissuedz%Y%m%d%H%M%SZz%sÚexpiryz%Y-%m-%d %H:%M:%SZ	expirystri€Q Tz&Error processing date in certificate: r¶   Úasciiz Problem processing certificate: r8   zcn=zdns:ZhostszChecking for z in certificate subjectr;   z*.zError processing certificate: ),r`   r  r·   r3   rq   Z$cryptography.hazmat.backends.opensslr  ÚcryptographyÚx509Zload_pem_x509_certificateÚOpenSSLZcryptoZload_certificateZFILETYPE_PEMZdump_certificateZFILETYPE_TEXTrÅ   r„  rA   Úsubjectr   ÚstrptimeZget_notAfterrá   Ústrftimer/   r1   rF   Ú
extensionsZget_extension_for_classZSubjectAlternativeNameÚvalueÚ
isinstanceZDNSNamerB   rG   r­   Zget_attributes_for_oidÚoidZNameOIDZCOMMON_NAMEra   rã   r>   )r   ZrawcertZfqdnZexpiringdaysrO   r  ÚcertZsslcertZsslcert_dumpZnotafterÚnowZwarnexpr5   ÚextrP   Z	certhostsÚattrsrT  r,  Zfqdn_tldÚfoundÚchostr   r   r   Ú	parseCertE  s’    


,


zSpiderFoot.parseCertc              
   C   s²  t ƒ }g }t|ƒtkr|g}ddddddddœ}z| d¡d }W n& tk
rh } zd}W 5 d }~X Y nX |d krvd}|d ksŠt|ƒdkr˜|  d¡ d S zTt| ¡ ƒD ]B}	t	|d	t
|	ƒd
 |	¡D ]$}
|
 ||	 ¡rÂ| |
||	  ¡ qÂq¦W n> tk
r* } z|  dt|ƒ d¡ W Y ¢d S d }~X Y nX |D ]z}t|ƒtkrLt|ƒ}| ¡ }d }t|ƒdk rjq0|t|ƒd  dks¢|d dks¢d|ks¢d|kr´|  d| ¡ q0t d|¡rÔ|  d| ¡ q0d|krð|  d| ¡ q0d|krtj |¡}d|kr|}| d¡r.|  |¡| }| d¡rF|d | }|D ].}|d krJ| ¡ | ¡ krJ|d | }qJ|d kr’|  |¡| }|  |¡}||dœ||< q0|S )NZhrefrx   Úaction)r.  ZimgÚscriptÚlinkZarear  ÚformrÛ   r   Úhttpz*parseLinks() called with no data to parse.Zlxml)Z
parse_onlyz"Error parsing with BeautifulSoup: Fr8   r;   ú+zjavascript:z()zunlikely link: z	.*#.[^/]+zin-page link: zmailto:zIgnoring mail link: z%2fr  rÁ   ú//)r   Úoriginal)r`   r·   r3   r>   r1   ra   r­   rA   rà   r   r   Zfind_allZhas_attrrB   rF   rG   rñ   rò   ÚurllibÚparseÚunquoter=   r  r  r  )r   r	  rc   ÚdomainsZreturnLinksZurlsRelZtagsÚprotor5   r6  Zlnkr›  ZlinklZabsLinkr  r   r   r   Ú
parseLinks«  sŠ    ù

ÿÿ
$ÿÿ




zSpiderFoot.parseLinksc                 C   s   t  ddd„ |¡S )Nu   [Â€-Ã¿]c                 S   s   dt |  d¡ƒ S )Nz%%%02xr   )Úordr  )r/  r   r   r   Ú<lambda>  ó    z-SpiderFoot.urlEncodeUnicode.<locals>.<lambda>)rñ   Úsub)r   r	  r   r   r   ÚurlEncodeUnicode  s    zSpiderFoot.urlEncodeUnicodec                 C   s"   t  ¡ }| jr| j| jdœ|_|S )N©r  Úhttps)ÚrequestsrR   r"   Úproxies)r   rR   r   r   r   rH     s    þzSpiderFoot.getSessionc                 C   s6   dddddœ}|}|D ]}t  ||| |t j¡}q|S )Nzkey=XXXzpass=XXXzuser=XXXzpassword=XXX)zkey=\S+zpass=\S+zuser=\S+zpassword=\S+)rñ   rª  ró   )r   r	  ZpatsrO   Úpatr   r   r   ÚremoveUrlCreds  s    üzSpiderFoot.removeUrlCredsFc           &      C   s.  d d d d |dœ}|d krd S t ƒ }| jd r*| jd g}dddg}|  |¡}| jd r^d}nd}||krnd}|D ]}| |¡rrd}qr|D ]}| |¡rŠd}qŠ|r|  d	| ¡ |  d
| jd  d t| jd ƒ ¡ d| jd  d t| jd ƒ d| jd  d t| jd ƒ dœ}n|  d| ¡ zÎt ƒ }t ¡ }t|ƒt	kr^t
 ¡  |¡|d< n||d< |d k	r´t	| ¡ ƒD ]6}t|| ƒtkr¤t|| ƒ||< n|| ||< q||
sÀ|r|sö|  d|  |¡ d |d  d t|ƒ d ¡ |  ¡ j|||||d}t|j dd¡ƒ}|j d|¡|d< t|jƒ|d< |rL|W S ||
kr\|W S |d |kr|s |  d|  |¡ d |d  d t|ƒ d ¡ |  ¡ j|d ||||d}t|j dd¡ƒ}|j d|d ¡|d< t|jƒ|d< ||
kr|W S |d k	rD|sz|  d|  |¡ d |d  d t|ƒ d ¡ n6|sz|  d|  |¡ d |d  d t|ƒ d ¡ zF|r¢|  ¡ j||||d|||d}n|  ¡ j|||d|||d}W n( tjjk
rè   td| ƒd ‚Y nX t ƒ |d < |j ¡ D ]F\}}t|ƒtkrt|ƒ}t|ƒtkr2t|ƒ}||d  | ¡ < qþ|
r„t|jƒ|
kr„|  d!¡ |j|d< t|jƒ|d< |W S d"|d  kr&z|d  d"   d#¡d$ }W nD t!k
rð } z$|  d%|d  d"  ¡ | W Y ¢W S d }~X Y nX |  d&|  |¡ ¡ |  "|||||||||	|
|¡W S |j|d< t|jƒ|d< |	rP|j|d'< nJz|j #d(¡|d'< W n4 t$k
r˜ } z|j #d)¡|d'< W 5 d }~X Y nX |røz| %¡  W nJ tjj&k
rö } z&|  'd*t|jƒ d+ |j d, ¡ W 5 d }~X Y nX W nÀ t!k
rº } z |szz0|  (d-t|ƒ d. | d¡ |  (t) *¡ d¡ W n4 t!k
rx }  z| W Y ¢
 W Y ¢DS d } ~ X Y nX d |d'< t|ƒ|d/< |rª|  'd*t|ƒ d, ¡ W 5 d }~X Y nX t+ ,¡ d$ }!t+ -|!d ¡}"|"j.}#t ¡ }$t|$| ƒ}%|  d0tt|d' pd1ƒƒ d2 |  |¡ d3 |% d4 ¡ |S )5N)ÚcoderŸ   rI   ÚheadersÚrealurlr'   r(   z192.168.z127.z10.TFzUsing proxy for zProxy set to rÛ   Ú_socks3portz
socks5h://r¬  zNot using proxy for z
User-AgentzFetching (HEAD only): z [user-agent: z] [timeout: ú])r³  r¯  Úverifyrp  zcontent-lengthr   Úlocationr´  r²  zFetching (incl. cookies): z
Fetching: )rc   r³  r¯  Úallow_redirectsÚcookiesrp  r·  )r³  r¯  r¹  rº  rp  r·  zFailed to connect to %sr³  z>Content exceeded size limit, so returning no data just headersZrefreshz;url=r8   z+Refresh header found but was not parsable: z&Refresh header found, re-directing to rI   ro   r‡  zURL could not be fetched (z / r`  zUnexpected exception (z) occurred fetching: rŸ   zFetched data: r   r_  z), took r¼   )/r`   r   r  Úendswithr=   r­   r3   r/   r·   rA   r   r‚   Úchoicerà   r-   r±  rH   Úheadrá   r³  r   Zstatus_codeÚpostr®  Ú
exceptionsZRequestExceptionÚ	ExceptionÚitemsrG   ra   rI   r	  r>   r1   ÚfetchUrlrÅ   ÚUnicodeDecodeErrorZraise_for_statusÚ	HTTPErrorr2   rF   Ú	tracebackÚ
format_excr›   rœ   r¥   r¦   )&r   r	  r2   rº  rp  Ú	useragentr³  ZnoLogZpostDataZ
dontMangleZ	sizeLimitZheadOnlyr·  Úresultr¯  ZneverProxyNamesZneverProxySubnetsr,  Úproxyr¼   rª  ÚheaderZbtimer2  Úhdrr~   r   r  Únewurlr5   r0  rP   rM   r¨   r©   rÍ   rÓ   r6  r   r   r   rÂ  '  s†   û




&þ
ÿÿÿþþ ÿ
ÿÿÿþþ ÿ

ÿÿÿþþÿÿÿÿ  þ  ÿ



     þ
"8"(ÿÿÿÿÿzSpiderFoot.fetchUrlc                    sF   |sdS d‰ d  ‡ fdd„tdƒD ƒ¡}|  |d | ¡d krBdS dS )	NFZbcdfghjklmnpqrstvwxyz3456789r   c                    s   g | ]}t  ¡  ˆ ¡‘qS r   )r   r‚   r¼  rÝ   ©Zrandpoolr   r   Ú
<listcomp>é  s     z/SpiderFoot.checkDnsWildcard.<locals>.<listcomp>r,   r;   T)rã   Úranger^  )r   r€   Zrandhostr   rÍ  r   ÚcheckDnsWildcardã  s    zSpiderFoot.checkDnsWildcardc           	      C   sò   dj | dd¡d}|d |d dœ}| j|tj |¡ |d d	}|d
 dkrb| jddd d S zt |d ¡}W n$ t	k
r˜   | jddd Y d S X d|krêddddddœ}dj | dd¡tj |¡d}dd„ |d D ƒ|dœ}nd S |S )Nz=https://www.googleapis.com/customsearch/v1?q={search_string}&ú ú%20©Úsearch_stringZcse_idÚapi_key)ZcxÚkeyrp  )rp  r²  Ú200z2Failed to get a valid response from the Google APIF©r˜   rI   zHthe key 'content' in the Google API response doesn't contain valid json.rÁ  ro   r6  zorg.mozilla:en-US:officialz	firefox-a)ZieZoeZaqZrlsÚclientz8https://www.google.com/search?q={search_string}&{params})rÔ  Úparamsc                 S   s   g | ]}t |d  ƒ‘qS )r›  rÜ   )rÞ   r2  r   r   r   rÎ    s     z,SpiderFoot.googleIterate.<locals>.<listcomp>©ÚurlsÚwebSearchUrl©
Úformatr  rÂ  r¡  r¢  Ú	urlencoderF   r„   ÚloadsÚ
ValueError)	r   ÚsearchStringr   ÚendpointrÚ  ÚresponseÚresponse_jsonZ
search_urlÚresultsr   r   r   ÚgoogleIterateø  sD    
ÿþþû

þþzSpiderFoot.googleIteratec                 C   sô   dj | dd¡d}d|d dœ}| j|tj |¡ |d |d	 d
|d id}|d dkrn| jddd d S zt |d ¡}W n$ t	k
r¤   | jddd Y d S X d|krìd|d krìd|d krìdd„ |d d D ƒ|d d dœ}nd S |S )NzGhttps://api.cognitive.microsoft.com/bing/v7.0/search?q={search_string}&rÑ  rÒ  rÓ  ZWebpagesr  )ZresponseFilterr  rp  rÇ  zOcp-Apim-Subscription-KeyrÕ  )rp  rÇ  r³  r²  r×  z0Failed to get a valid response from the bing APIFrØ  rI   zFthe key 'content' in the bing API response doesn't contain valid json.ZwebPagesr  rÝ  c                 S   s   g | ]}|d  ‘qS )r	  r   )rÞ   rÈ  r   r   r   rÎ  Q  s     z*SpiderFoot.bingIterate.<locals>.<listcomp>rÛ  rÞ  )r   rã  r   rä  rÚ  rå  ræ  rç  r   r   r   ÚbingIterate0  s>    
ÿþ
üÿ
þ
ý
þzSpiderFoot.bingIterate)N)TT)N)T)T)T)Nr  )FNr  r
   NFNFNFF)Kr¦   Ú
__module__Ú__qualname__r‡   r‰   r[  Z	savedsockr"   r   r$   r&   r6   rS   rA   rh   rz   r†   rˆ   r‹   r   r“   rF   r2   rŸ   r-   r­   rµ   r½   rÀ   rË   rÖ   ræ   rç   rö   rþ   r   r  r  r  r  r  r  r  r  r  r   r"  r'  r(  r*  r-  rC  rP  rR  rY  r^  rc  rf  rh  rm  rr  rv  rw  r|  r~  r˜  r¦  r«  rH   r±  rÂ  rÐ  r`   rè  ré  r   r   r   r   r
   /   sž   

.5$5

	
.
A
/&

f`
              ý
 =8r
   c                   @   sà   e Zd ZdZeƒ ZdZdZdZ dZdZ	dZ
dZdZdZdd„ Zdd„ Zd	d
„ Zeƒ fdd„Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd „ Zd!d"„ Zd#d$„ Zd%d&„ Zd'd(„ Z d)d*„ Z!d+d,„ Z"dS )-ÚSpiderFootPluginFNzmodule_name_not_set!r8   c                 C   s   d S r    r   r%   r   r   r   r   v  s    zSpiderFootPlugin.__init__c                 C   s
   || _ d S r    r!   r#   r   r   r   Ú_updateSocket{  s    zSpiderFootPlugin._updateSocketc                 C   s   t ƒ | _d| _d S r#  )rA   Ú_listenerModulesÚ_stopScanningr%   r   r   r   ÚclearListeners€  s    zSpiderFootPlugin.clearListenersc                 C   s   d S r    r   )r   ÚsfZuserOptsr   r   r   Úsetup…  s    zSpiderFootPlugin.setupc                 C   s   d S r    r   ©r   r€   r   r   r   ÚenrichTargetŠ  s    zSpiderFootPlugin.enrichTargetc                 C   s
   || _ d S r    )Ú_currentTargetró  r   r   r   Ú	setTargetŽ  s    zSpiderFootPlugin.setTargetc                 C   s
   || _ d S r    )Ú__sfdb__)r   r‡   r   r   r   rˆ   “  s    zSpiderFootPlugin.setDbhc                 C   s
   || _ d S r    ©Ú
__scanId__)r   rV   r   r   r   Ú	setScanId—  s    zSpiderFootPlugin.setScanIdc                 C   s   | j S r    rø  r%   r   r   r   Ú	getScanId›  s    zSpiderFootPlugin.getScanIdc                 C   s"   | j d krtdƒ t d¡ | j S )Nz<Internal Error: Module called getTarget() but no target set.rš   )rõ  r—   r   r   r%   r   r   r   Ú	getTargetŸ  s    

zSpiderFootPlugin.getTargetc                 C   s   | j  |¡ d S r    )rî  rB   )r   Úlistenerr   r   r   ÚregisterListener§  s    z!SpiderFootPlugin.registerListenerc                 C   s
   || _ d S r    )Ú__outputFilter__)r   Útypesr   r   r   ÚsetOutputFilterª  s    z SpiderFootPlugin.setOutputFilterc                 C   s   t ƒ S r    )r`   r%   r   r   r   ÚtempStorage®  s    zSpiderFootPlugin.tempStoragec                 C   sÄ  |j }| jr2|dkr2||  ¡  ¡ kr2|| jkr2d S d}|jd ks\t|jƒtkr`t|jƒdkr`d S |  ¡ rld S |j	}|d k	r¶|j	d k	r®|j	j |j kr®|j	j 
¡ |j 
¡ kr®d}q¶|j	}qr| jjdd„ d | jD ]ð}|| ¡ krìd| ¡ krìqÎ|rüd	|jkrüqÎ||_|  ¡ r d S z.t|jƒtkr4|j d
d¡|_| |¡ W qÎ tk
r¼ } z\tddƒ}| d|j d t|ƒ d ¡ t ¡ \}}	}
| tt ||	|
¡ƒ¡ | ¡  W 5 d }~X Y qÎX qÎd S )Nrj   Fr   Tc                 S   s   | j S r    )Ú	_priority)rÍ   r   r   r   r¨  ß  r©  z2SpiderFootPlugin.notifyListeners.<locals>.<lambda>)rÖ  rù   Z__storro   Úignorezsferror.logr.  zModule (z) encountered an error: r:   )Ú	eventTyperÿ  rü  rj  rc   r·   r3   ra   ÚcheckForStopÚsourceEventrG   rî  ÚsortÚwatchedEventsrê  Ú_currentEventrÆ   rÅ   ÚhandleEventr1   r?   rÄ   r   Úexc_infoÚreprrÅ  Úformat_exceptionrE   )r   ÚsfEventZ	eventNameZ	storeOnlyZ	prevEventrý  r5   rM   Úexc_typeÚ	exc_valueÚexc_tracebackr   r   r   ÚnotifyListeners´  sN    ÿ&
ÿ


 z SpiderFootPlugin.notifyListenersc                 C   s   t  | j¡dkrdS dS )NzABORT-REQUESTEDTF)ÚglobalScanStatusÚ	getStatusrù  r%   r   r   r   r    s    zSpiderFootPlugin.checkForStopc                 C   s   | j S r    )r   r%   r   r   r   ÚdefaultOpts
  s    zSpiderFootPlugin.defaultOptsc                 C   s   dgS )Nrù   r   r%   r   r   r   r	    s    zSpiderFootPlugin.watchedEventsc                 C   s   d S r    r   r%   r   r   r   ÚproducedEvents  s    zSpiderFootPlugin.producedEventsc                 C   s   d S r    r   )r   r  r   r   r   r    s    zSpiderFootPlugin.handleEventc                 C   s   d S r    r   r%   r   r   r   Ústart$  s    zSpiderFootPlugin.start)#r¦   rê  rë  rï  rA   rî  r
  rõ  r÷  rù  Z__dataSource__rÿ  r  Z
errorStater   rí  rð  r`   rò  rô  rö  rˆ   rú  rû  rü  rþ  r  r  r  r  r  r	  r  r  r  r   r   r   r   rì  \  s>   Nrì  c                	   @   sz   e Zd Zddddddddd	g	Zd
Zd
Zeƒ Zdd„ Zdd„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zddd„Zd
S ) ÚSpiderFootTargetrè   rï   ré   rð   rê   rì   rî   rë   rí   Nc                 C   sR   || j kr<|| _t|ƒtkr,t|ƒ ¡ | _n|| _tƒ | _ntdƒ t	 
d¡ d S )Nz$Internal Error: Invalid target type.rš   )Ú_validTypesrö   r·   r3   rG   ÚtargetValuerA   ÚtargetAliasesr—   r   r   )r   r  ÚtypeNamer   r   r   r   0  s    

zSpiderFootTarget.__init__c                 C   s   | j S r    )rö   r%   r   r   r   rj  <  s    zSpiderFootTarget.getTypec                 C   s   | j S r    )r  r%   r   r   r   rk  ?  s    zSpiderFootTarget.getValuec                 C   s.   ||dœ| j krd S | j  || ¡ dœ¡ d S )N)r·   r  )r  rB   rG   )r   r  r  r   r   r   ÚsetAliasH  s
    ÿzSpiderFootTarget.setAliasc                 C   s   | j S r    )r  r%   r   r   r   Ú
getAliasesP  s    zSpiderFootTarget.getAliasesc                 C   s4   t ƒ }| jD ]"}|d |kr| |d  ¡ ¡ q|S )Nr·   r  )rA   r  rB   rG   )r   r  rO   rT   r   r   r   Ú_getEquivalentsS  s
    
z SpiderFootTarget._getEquivalentsc                 C   sf   |   d¡}| jdkr2| j ¡ |kr2| | j ¡ ¡ tƒ }|D ]$}| t|ƒtkrZ| d¡n|¡ q<|S )Nrð   )rð   rê   ro   )	r   rö   r  rG   rB   rA   r·   rÆ   rÅ   )r   r5   rQ  rT  r   r   r   ÚgetNames[  s    
"zSpiderFootTarget.getNamesc                 C   sD   |   d¡}| jdkr | | j¡ |   d¡}| jdkr@| | j¡ |S )Nrè   rï   )r   rö   rB   r  )r   r5   r   r   r   ÚgetAddressesf  s    



zSpiderFootTarget.getAddressesFTc                 C   sþ   |  ¡ }t|ƒtkr| d¡n|}|d ks2|dkr6dS | jdkrDdS t |¡r®||  ¡ kr^dS | jdkr‚t |¡t 	| j
¡kr‚dS | jdkrút |¡t 	t | j
¡¡krúdS nL|  ¡ D ]B}||krÈ dS |rà| d| ¡rà dS |r¶| d| ¡r¶ dS q¶d S )	Nro   r   F)rì   rë   rí   Tré   ri  r;   )rG   r·   rÆ   rÅ   rö   r$  r%  r"  Z	IPAddressr)  r  r!  r»  )r   r  ZincludeParentsZincludeChildrenrT  r   r   r   r{  €  s2    



ÿzSpiderFootTarget.matches)FT)r¦   rê  rë  r  rö   r  rA   r  r   rj  rk  r  r  r   r!  r"  r{  r   r   r   r   r  )  s&       ÿ	r  c                   @   sv   e Zd ZdZdZdZdZdZdZdZ	dZ
dZdZdZdZddd„Zdd„ Zdd	„ Zd
d„ Zdd„ Zdd„ Zdd„ ZdS )ÚSpiderFootEventNéd   r   c                 C   s$  || _ t ¡ | _|| _|| _|| _|| _|| _t|ƒt	krxt|ƒt	krxt
dt|ƒ ƒ t
d| ƒ t
d| ƒ t d¡ t|ƒt	kr˜|d kr˜t	|ƒ| _n|| _|dkr°d| _d S t|ƒtkrêt
d| ƒ t
d| ƒ t
d| ƒ t d¡ | ¡ | _| j t	| jƒ | j t	t ¡  dd¡ƒ | _d S )	Nz1FATAL: Only string events are accepted, not '%s'.zFATAL: Offending module: %szFATAL: Offending type: %srš   rj   zFATAL: Invalid source event: %sr   iÿàõ)r  r/   Ú	generatedÚ
confidenceÚ
visibilityÚriskÚmoduler  r·   r3   r—   r   r   rc   ÚsourceEventHashr#  ÚgetHashr   r‚   rƒ   Ú_SpiderFootEvent__id)r   r  rc   r)  r  r&  r'  r(  r   r   r   r   ·  s6    



ÿzSpiderFootEvent.__init__c                 C   s>   t | jƒ| j| j| jdœ}| jdkr.d|d< n| jj|d< |S )N)r%  r·   rc   r)  rj   r   r   )rá   r%  r  rc   r)  r  )r   Zevt_dictr   r   r   ÚasDictÜ  s    ü

zSpiderFootEvent.asDictc                 C   s(   | j dkrdS | j d¡}t |¡ ¡ S )Nrj   r¶   )r  r,  rq   r¸   r¹   rº   )r   Z	digestStrr   r   r   r+  ì  s    
zSpiderFootEvent.getHashc                 C   s
   || _ d S r    )r&  )r   r&  r   r   r   ÚsetConfidenceó  s    zSpiderFootEvent.setConfidencec                 C   s
   || _ d S r    )r'  )r   r'  r   r   r   ÚsetVisibilityö  s    zSpiderFootEvent.setVisibilityc                 C   s
   || _ d S r    )r(  )r   r(  r   r   r   ÚsetRiskù  s    zSpiderFootEvent.setRiskc                 C   s
   || _ d S r    )r*  )r   ZsrcHashr   r   r   ÚsetSourceEventHashü  s    z"SpiderFootEvent.setSourceEventHash)r$  r$  r   )r¦   rê  rë  r%  r  r&  r'  r(  r)  rc   r  r*  ZmoduleDataSourceZactualSourcer,  r   r-  r+  r.  r/  r0  r1  r   r   r   r   r#  ©  s,        ÿ
%r#  c                   @   sF   e Zd Zdd„ Zdd„ Zdd„ Zdd„ Zd	d
„ Zdd„ Zddd„Z	dS )r  c                 C   s   |   |¡}|  |¡| _dS )aq  Reads and parses public suffix list.

        input_file is a file object or another iterable that returns
        lines of a public suffix list file. If input_file is None, an
        UTF-8 encoded file named "publicsuffix.txt" in the same
        directory as this Python module is used.

        The file format is described at http://publicsuffix.org/list/
        N)Ú_build_structureÚ	_simplifyrr   )r   Z
input_datarr   r   r   r   r     s    
zPublicSuffixList.__init__c                 C   sh   |s|S t |ƒdkr| i ¡ t |ƒdks.t‚|\}}| ¡ }| |d ¡}|s\dg ||< }|  ||¡S )Nr8   r   r   )ra   rB   ÚAssertionErrorr  r   Ú
_find_node)r   rU   ÚpartsÚnegaterU  ÚchildZ
child_noder   r   r   r5    s    
zPublicSuffixList._find_nodec                 C   s>   |  d¡rd}|dd … }nd}| d¡}||  ||¡d< d S )Nú!r8   r   r;   )r=   r>   r5  )r   rr   Zruler7  r6  r   r   r   Ú	_add_rule,  s    

zPublicSuffixList._add_rulec                    s>   t |ƒdkr|d S |d t‡ fdd„t|d  ¡ ƒD ƒƒfS )Nr8   r   c                 3   s    | ]\}}|ˆ   |¡fV  qd S r    )r3  )rÞ   r2  r7  r%   r   r   rß   :  s     z-PublicSuffixList._simplify.<locals>.<genexpr>)ra   r`   rA   rÁ  )r   rp   r   r%   r   r3  6  s    zPublicSuffixList._simplifyc                 C   sJ   dg}|D ]:}t |ƒ ¡ }| d¡s
|s*q
|  || ¡ d  d¡¡ q
|S )Nr   rŸ  r;   )r3   rL  r=   r:  r>   Úlstrip)r   rÉ   rr   rÊ   r   r   r   r2  <  s    z!PublicSuffixList._build_structurec           	      C   st   |dkr|}d }n|\}}||| < |t |ƒk rp|rpd||  fD ],}| |d ¡}|d k	rB|  ||d ||¡ qBd S )N)r   r8   rù   r8   )ra   r   Ú_lookup_node)	r   r{  ÚdepthrU   r6  r7  rU  rT  r8  r   r   r   r<  H  s    
zPublicSuffixList._lookup_nodeFc                 C   sŠ   |  ¡  d¡ d¡}dgt|ƒ }|r<|d | jd kr<dS |  |d| j|¡ t|ƒD ].\}}|dk	rV|dkrVd ||d… ¡  S qVdS )a[  get_public_suffix("www.example.com") -> "example.com"

        Calling this function with a DNS name will return the
        public suffix for that name.

        Note that for internationalized domains the list at
        http://publicsuffix.org uses decoded names, so it is
        up to the caller to decode any Punycode-encoded names.
        r;   Nrš   r8   r   )rG   r;  r>   ra   rr   r<  Ú	enumeraterã   )r   r  r  r6  ÚhitsrÎ   Úwhatr   r   r   r  W  s    z"PublicSuffixList.get_public_suffixN)F)
r¦   rê  rë  r   r5  r:  r3  r2  r<  r  r   r   r   r   r    s   
r  c                   @   s2   e Zd Zeƒ Ze ¡ Zdd„ Zdd„ Z	dd„ Z
dS )ÚSpiderFootScanStatusc              	   C   s    | j  || j|< W 5 Q R X d S r    ©ÚlockÚstatusTable)r   ÚscanIdrŸ   r   r   r   Ú	setStatust  s    zSpiderFootScanStatus.setStatusc              
   C   s@   | j 0 || jkr(| j| W  5 Q R £ S W 5 Q R £ d S Q R X d S r    rB  )r   rE  r   r   r   r  x  s    
zSpiderFootScanStatus.getStatusc              
   C   s$   | j  | jW  5 Q R £ S Q R X d S r    rB  r%   r   r   r   ÚgetStatusAll~  s    z!SpiderFootScanStatus.getStatusAllN)r¦   rê  rë  r`   rD  Ú	threadingÚLockrC  rF  r  rG  r   r   r   r   rA  p  s
   rA  )1Ústemr   Zstem.controlr   r›   r¸   Úurllib.requestr¡  Úurllib.parseÚurllib.errorr„   rñ   r°   r   r®  r[  r   r   r/   r$  rH  rÅ  rŠ  r   rˆ  Zdns.resolverr   Znetworkxr   Znetworkx.readwrite.gexfr   r   Zbs4r   r   Úcopyr	   Zurllib3Zdisable_warningsr¿  ZInsecureRequestWarningr
   Úobjectrì  r  r#  r  rA  r  r   r   r   r   Ú<module>   sb               9 N ]j