3
@6^                  @   s8   d dl Z d dlZd dlmZmZmZ G dd deZdS )    N)
SpiderFootSpiderFootPluginSpiderFootEventc               @   sr   e Zd ZdZddddZdddZd	Ze Zd	Z	e fd
dZ
dd Zdd Zdd Zdd Zdd Zdd Zd	S )sfp_s3bucketzAmazon S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential Amazon S3 buckets associated with the target and attempt to list their contents.a  s3.amazonaws.com,s3-external-1.amazonaws.com,s3-us-west-1.amazonaws.com,s3-us-west-2.amazonaws.com,s3.ap-south-1.amazonaws.com,s3-ap-south-1.amazonaws.com,s3.ap-northeast-2.amazonaws.com,s3-ap-northeast-2.amazonaws.com,s3-ap-southeast-1.amazonaws.com,s3-ap-southeast-2.amazonaws.com,s3-ap-northeast-1.amazonaws.com,s3.eu-central-1.amazonaws.com,s3-eu-central-1.amazonaws.com,s3-eu-west-1.amazonaws.com,s3-sa-east-1.amazonaws.comztest,dev,web,beta,bucket,space,files,content,data,prod,staging,production,stage,app,media,development,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data,-prod,-staging,-production,-stage,-app,-media,-development   )	endpointssuffixes_maxthreadszDifferent S3 endpoints to check where buckets may exist, as per http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_regionz;List of suffixes to append to domains tried as bucket names)r   r   Nc             C   sJ   || _ t | _| j | _tj | _x"t|j	 D ]}|| | j
|< q0W d S )N)sfdict	s3resultstempStorageresults	threadingLocklocklistkeysopts)selfsfcuserOptsopt r   7/var/www/spiderfoot.crq.systems/modules/sfp_s3bucket.pysetup'   s    

zsfp_s3bucket.setupc             C   s   ddgS )NDOMAIN_NAMELINKED_URL_EXTERNALr   )r   r   r   r   watchedEvents1   s    zsfp_s3bucket.watchedEventsc             C   s   ddgS )NCLOUD_STORAGE_BUCKETCLOUD_STORAGE_BUCKET_OPENr   )r   r   r   r   producedEvents7   s    zsfp_s3bucket.producedEventsc          
   C   s   | j j|dddd}|d dkrJ|d	 d ks8d
|d	 krJ| j jd|  nPd|d	 kr~| j |d	 jd| j|< W d Q R X n| j d| j|< W d Q R X d S )N
   r   T)timeout	useragentnoLogcode301302200contentZNoSuchBucketzNot a valid bucket: ZListBucketResultz<Key>r   )r'   r(   r)   )r
   fetchUrldebugr   countr   )r   urlresr   r   r   	checkSite:   s     zsfp_s3bucket.checkSitec       	      C   s   t  }t | _d}d}g }xZ|D ]R}| j r0d S | jjd|  |jtjd| | j	|fd || j
  |d7 }q W x@|rd}x tj D ]}|jjdrd}qW |sd}tjd qxW | jS )	NTr   z!Spawning thread to check bucket: Zthread_sfp_s3buckets_)nametargetargs   Fg      ?)r   r   r   checkForStopr
   infoappendr   Threadr0   start	enumerater1   
startswithtimesleep)	r   siteListretrunningitsitefoundrtr   r   r   threadSitesH   s,    
zsfp_s3bucket.threadSitesc             C   s   d}t  }t  }x|D ]|}|| jd kr| j|}|d kr>|S x6t |j D ]&}|| rL|j|d t||   qLW d}t  }|j| |d7 }qW |S )Nr   r	   :r4   )r   r   rF   r   r7   str)r   sitesrA   r/   r>   rC   datar?   r   r   r   
batchSitesh   s     


zsfp_s3bucket.batchSitesc             C   s  |j }|j}|j}|| jkr d S d| j|< | jjd| d |  |dkr|d|krx| jj|}td|| j|}| j	| d S |j
dd| jj|| jd	 g}t }xx|D ]p}	xj| jd
 jdD ]V}
dg| jd jd }x:|D ]2}| j rd S |	| d |
 }d| }|j| qW qW qW | j|}x|D ]}|jd}td|d d |d  | j|}| j	| |d dkr,|d j
dd}td|d |d  d | j|}| j	| q,W d S )NTzReceived event, z, from r   z.amazonaws.comr   . _internettldsr   ,r   zhttps://rG   r   r4      0z//r    z: z files found.)	eventTypemodulerJ   r   r
   r,   urlFQDNr   __name__notifyListenersreplacedomainKeywordr   r   splitr5   r7   rK   )r   event	eventNamesrcModuleName	eventDatabevttargetsurlsrB   er   sr.   r?   bucketZ
bucketnamer   r   r   handleEvent   sD    


 




 

zsfp_s3bucket.handleEvent)rU   
__module____qualname____doc__r   optdescsr   r   r   r   r   r   r!   r0   rF   rK   re   r   r   r   r   r      s    
 r   )r   r<   sflibr   r   r   r   r   r   r   r   <module>   s   