U
    @6^                      @   s8   d dl Z d dlZd dlmZmZmZ G dd deZdS )    N)
SpiderFootSpiderFootPluginSpiderFootEventc                   @   sr   e Zd ZdZddddZdddZd	Ze Zd	Z	e fd
dZ
dd Zdd Zdd Zdd Zdd Zdd Zd	S )sfp_s3bucketzAmazon S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential Amazon S3 buckets associated with the target and attempt to list their contents.a  s3.amazonaws.com,s3-external-1.amazonaws.com,s3-us-west-1.amazonaws.com,s3-us-west-2.amazonaws.com,s3.ap-south-1.amazonaws.com,s3-ap-south-1.amazonaws.com,s3.ap-northeast-2.amazonaws.com,s3-ap-northeast-2.amazonaws.com,s3-ap-southeast-1.amazonaws.com,s3-ap-southeast-2.amazonaws.com,s3-ap-northeast-1.amazonaws.com,s3.eu-central-1.amazonaws.com,s3-eu-central-1.amazonaws.com,s3-eu-west-1.amazonaws.com,s3-sa-east-1.amazonaws.comztest,dev,web,beta,bucket,space,files,content,data,prod,staging,production,stage,app,media,development,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data,-prod,-staging,-production,-stage,-app,-media,-development   )	endpointssuffixes_maxthreadszDifferent S3 endpoints to check where buckets may exist, as per http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_regionz;List of suffixes to append to domains tried as bucket names)r   r   Nc                 C   sF   || _ t | _|  | _t | _t|	 D ]}|| | j
|< q.d S )N)sfdict	s3resultstempStorageresults	threadingLocklocklistkeysopts)selfsfcuserOptsopt r   7/var/www/spiderfoot.crq.systems/modules/sfp_s3bucket.pysetup'   s    

zsfp_s3bucket.setupc                 C   s   ddgS )NDOMAIN_NAMELINKED_URL_EXTERNALr   r   r   r   r   watchedEvents1   s    zsfp_s3bucket.watchedEventsc                 C   s   ddgS )NCLOUD_STORAGE_BUCKETCLOUD_STORAGE_BUCKET_OPENr   r   r   r   r   producedEvents7   s    zsfp_s3bucket.producedEventsc              	   C   s   | j j|dddd}|d dkrJ|d d ks8d|d krJ| j d	|  nPd
|d kr~| j |d d| j|< W 5 Q R X n| j d| j|< W 5 Q R X d S )N
   r   T)timeout	useragentnoLogcode)Z301Z302200contentZNoSuchBucketzNot a valid bucket: ZListBucketResultz<Key>r   )r
   fetchUrldebugr   countr   )r   urlresr   r   r   	checkSite:   s    

 zsfp_s3bucket.checkSitec           	      C   s   t  }t | _d}d}g }|D ]T}|  r0 d S | jd|  |tjd| | j	|fd || 
  |d7 }q|rd}t D ]}|jdrd}q|sd}td qt| jS )	NTr   z!Spawning thread to check bucket: Zthread_sfp_s3buckets_)nametargetargs   Fg      ?)r   r   r   checkForStopr
   infoappendr   Threadr/   start	enumerater0   
startswithtimesleep)	r   siteListretrunningitsitefoundrtr   r   r   threadSitesH   s0     
zsfp_s3bucket.threadSitesc                 C   s   d}t  }t  }|D ]|}|| jd kr~| |}|d kr@|  S t | D ]&}|| rL||d t||   qLd}t  }|| |d7 }q|S )Nr   r	   :r3   )r   r   rE   r   r6   str)r   sitesr@   r.   r=   rB   datar>   r   r   r   
batchSitesh   s     


zsfp_s3bucket.batchSitesc                 C   s  |j }|j}|j}|| jkr d S d| j|< | jd| d |  |dkr|d|krx| j|}td|| j|}| 	| d S |
dd| j|| jd	 g}t }|D ]n}	| jd
 dD ]X}
dg| jd d }|D ]8}|  r   d S |	| d |
 }d| }|| qqq| |}|D ]}|d}td|d d |d  | j|}| 	| |d dkr$|d 
dd}td|d |d  d | j|}| 	| q$d S )NTzReceived event, z, from r   z.amazonaws.comr    . _internettldsr   ,r   zhttps://rF   r   r3      0z//r!   z: z files found.)	eventTypemodulerI   r   r
   r+   urlFQDNr   __name__notifyListenersreplacedomainKeywordr   r   splitr4   r6   rJ   )r   event	eventNamesrcModuleName	eventDatabevttargetsurlsrA   er   sr-   r>   bucketZ
bucketnamer   r   r   handleEvent   sH    


 


 
 zsfp_s3bucket.handleEvent)rT   
__module____qualname____doc__r   optdescsr   r   r   r   r   r   r"   r/   rE   rJ   rd   r   r   r   r   r      s$   
 r   )r   r;   sflibr   r   r   r   r   r   r   r   <module>   s   