3
@6^                 @   s   d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZma d d
lmZ d dlmZ G dd dZdS )    N)SecureHeaders)_cperror)
itemgetter)deepcopy)TemplateLookup)Template)SpiderFootDb)
SpiderFootglobalScanStatus)SpiderFootScanner)StringIOc               @   s  e Zd ZedgdZe Ze ZdZdZ	dd Z
dd Zdd	 Zd
d ZdaddZdbddZde_dcddZde_ddddZde_dd Zde_deddZde_dfddZde_dd Zde_d d! Zde_d"d# Zde_d$d% Zde_d&d' Zde_d(d) Zde_d*d+ Zde_d,d- Zde_d.d/ Zde_d0d1 Zde_d2d3 Z dgd5d6Z!de!_dhd7d8Z"de"_did9d:Z#de#_d;d< Z$de$_d=d> Z%de%_d?d@ Z&de&_dAdB Z'de'_dCdD Z(de(_dEdF Z)de)_djdGdHZ*de*_dIdJ Z+de+_dkdKdLZ,de,_dldMdNZ-de-_dmdOdPZ.de._dQdR Z/de/_dSdT Z0de0_dUdV Z1de1_dndWdXZ2de2_dodYdZZ3de3_dpd[d\Z4de4_d]d^ Z5de5_d_d` Z6de6_dS )qSpiderFootWebUi )directoriesNc             C   s   t || _t| j}t| j}|j|j | j| _| jd dkrFd}n
| jd }| jd jd| _t	jj
| j| jd t }t	jj
d|j	 d td	 td	 td
 td td| d t| jd  | j  td
 td	 td	 d S )N	__webaddrz0.0.0.0z<IP of this host>	__docroot/)zerror_page.404zrequest.error_responseT)ztools.response_headers.onztools.response_headers.headersr   z=*************************************************************z; Use SpiderFoot by starting your web browser of choice and z browse to http://:	__webport)r   defaultConfigr   r	   configUnserialize	configGetconfigrstripdocrootcherrypyupdateerror_page_404
error_pager   printstr)selfr   dbhsfaddrZsecure_headers r%   */var/www/spiderfoot.crq.systems/sfwebui.py__init__%   s.    



$zSpiderFootWebUi.__init__c             C   s6   dt j_| jd r*tjdtj dt j_ndt j_d S )Ni  _debug)status	tracebackz<html><body>Error</body></html>)r   responser)   r   r   get_error_page
format_excbody)r!   r%   r%   r&   r   J   s    
zSpiderFootWebUi.error_pagec             C   s    t d| jd}|jd| j|dS )Nzdyn/error.tmpl)filenamelookupz	Not Found)messager   r)   )r   r0   renderr   )r!   r)   r1   r*   versiontemplr%   r%   r&   r   R   s    zSpiderFootWebUi.error_page_404c             C   sN   t  }xB|D ]:}tj|d}|jdd}|jddjdd}|j| qW |S )NT'z&quot;z&amp;&")listhtmlescapereplaceappend)r!   Z	inputListretitemcr%   r%   r&   cleanUserInputW   s    
zSpiderFootWebUi.cleanUserInputc             C   sf  d}|||gj ddks,|||gj d dkr0d S |jdr\|jdr\|dt|d  }d}|jdd}|dkr|dkrd}d}t| j}|dkrd n||dkrd n||dkrd n||dkrd n|d}|j|}g }x|D ]}	tj	dtj
|	d	 }
tj|	d }tj|	d
 }|j|
|||	d |	d |	d |	d |	d |	d |	d |	d |	d |	d g qW |S )Nr      r      *%)scan_idtypevalueregexz%Y-%m-%d %H:%M:%Sr                  
               )Nr   )Nr   )count
startswithendswithlenr;   r   r   searchtimestrftime	localtimer9   r:   r<   )r!   id	eventTyperG   rH   r"   criteriadataretdatarowlastseenescapeddata
escapedsrcr%   r%   r&   
searchBasec   s4    



$"zSpiderFootWebUi.searchBaseexcelc             C   s   t | j}|j||}t }tj||d}|jddddddg x~|D ]v}|d d	krXqFtjd
tj	|d }	t
|d jddjdd}
|j|	t
|d t
|d t
|d |d |
g qFW dtjjd< dtjjd< dtjjd< |j jdS )N)dialectUpdatedTypeModuleSourcezF/PDatarP   ROOTz%Y-%m-%d %H:%M:%Sr   rB   z<SFURL>r   z</SFURL>rA   rI   rQ   z#attachment; filename=SpiderFoot.csvzContent-Dispositionzapplication/csvzContent-Typezno-cachePragmazutf-8)r   r   scanResultEventr   csvwriterwriterowrX   rY   rZ   r    r;   r   r+   headersgetvalueencode)r!   r[   rF   rf   r"   r^   fileobjparserr`   ra   	datafieldr%   r%   r&   scaneventresultexport   s    

6z%SpiderFootWebUi.scaneventresultexportTc          	   C   s2  t | j}t }t }x.|jdD ] }|j|||< ||j| }q"W t }tj	||d}|j
ddddddd	g x|D ]}	|	d
 dkrqxtjdtj|	d }
t|	d jddjdd}|j
||	d  d |
t|	d
 t|	d t|	d |	d |g qxW dtjjd< dtjjd< dtjjd< |j jdS )N,)rf   z	Scan Namerg   rh   ri   rj   zF/Prk   rP   rl   z%Y-%m-%d %H:%M:%Sr   rB   z<SFURL>r   z</SFURL>   rA   rI   rQ   z#attachment; filename=SpiderFoot.csvzContent-Dispositionzapplication/csvzContent-Typezno-cacherm   zUtf-8)r   r   dictr8   splitscanInstanceGetrn   r   ro   rp   rq   rX   rY   rZ   r    r;   r   r+   rr   rs   rt   )r!   idsrf   r"   scaninfor^   r[   ru   rv   r`   ra   rw   r%   r%   r&   scaneventresultexportmulti   s(    

(z*SpiderFootWebUi.scaneventresultexportmultic       
      C   s   | j |||}t }tj||d}|jddddddg |s@d S xl|D ]d}|d d	krXqFt|d
 jddjdd}	|j|d t|d t|d t|d |d |	g qFW dtjj	d< dtjj	d< dtjj	d< |j
 jdS )N)rf   rg   rh   ri   rj   zF/Prk   rN   rl   rB   z<SFURL>r   z</SFURL>r   rA   rI   rO   z#attachment; filename=SpiderFoot.csvzContent-Dispositionzapplication/csvzContent-Typezno-cacherm   zUtf-8)rd   r   ro   rp   rq   r    r;   r   r+   rr   rs   rt   )
r!   r[   r\   rG   rf   r^   ru   rv   r`   rw   r%   r%   r&   scansearchresultexport   s    
:z&SpiderFootWebUi.scansearchresultexportc             C   s  t | j}t }x|jdD ]}|j|}|d kr4q|d }x|j|D ]}tjdtj|d }t	|d j
ddj
dd}	t	|d }
t	|d	 }|d
 }|d }|dkrqH|j|	|||
||||d d qHW qW dtjjd< dtjjd< dtjjd< tj|jdS )Nry   r   z%Y-%m-%d %H:%M:%SrB   z<SFURL>r   z</SFURL>rI   rA   rP   rQ   rl   )r^   
event_typemodulesource_datafalse_positiveZ	last_seen	scan_nameZscan_targetz$attachment; filename=SpiderFoot.jsonzContent-Dispositionzapplication/json; charset=utf-8zContent-Typezno-cacherm   zutf-8)r   r   r8   r|   r}   rn   rX   rY   rZ   r    r;   r<   r   r+   rr   jsondumpsrt   )r!   r~   r"   r   r[   scanr   r`   ra   Z
event_datar   Zsource_moduler   r   r%   r%   r&   scanexportjsonmulti   s:    

z#SpiderFootWebUi.scanexportjsonmulti0c             C   s   t | j}t| j}|j|dd}|j|}|d }|dkrpdtjjd< dtjjd< d	tjjd
< |j|gd|S |j	|g|S d S )NT)filterFprB   r   z$attachment; filename=SpiderFoot.gexfzContent-Dispositionzapplication/gexfzContent-Typezno-cacherm   zSpiderFoot Export)
r   r   r	   rn   r}   r   r+   rr   buildGraphGexfbuildGraphJson)r!   r[   gexfr"   r#   r^   r   rootr%   r%   r&   scanviz   s    


zSpiderFootWebUi.scanviz1c             C   s   t | j}t| j}t }t }x8|jdD ]*}||j|dd }|j|j|d  q,W |dkrdtj	j
d< dtj	j
d	< d
tj	j
d< |j|d|S d S d S )Nry   T)r   rB   r   z$attachment; filename=SpiderFoot.gexfzContent-Dispositionzapplication/gexfzContent-Typezno-cacherm   zSpiderFoot Export)r   r   r	   r8   r|   rn   r<   r}   r   r+   rr   r   )r!   r~   r   r"   r#   r^   rootsr[   r%   r%   r&   scanvizmulti  s    

zSpiderFootWebUi.scanvizmultic       
      C   s`  t  }t| j}|j||d< t  |d< xt|d j D ]}d|kr^| jd | |d |< q:|jd\}}|t| jd j krq:|t| jd | d j krq:| jd | d | |d |< q:W |j|}|stj	g S |d dkrt
jd	t
j|d }nd
}|d dkr.t
jd	t
j|d }	nd
}	|d |d |d ||	|d g|d< tj	|S )Nr   Z
configdescr   __globaloptdescs____modules__optdescsrA   r   z%Y-%m-%d %H:%M:%SzNot yetrP   rB   rI   rJ   meta)r{   r   r   scanConfigGetr8   keysr|   r}   r   r   rX   rY   rZ   )
r!   r[   r=   r"   keymodNameZmodOptr   startedfinishedr%   r%   r&   scanopts  s0    

"

$zSpiderFootWebUi.scanoptsc             C   s$  t | j}t }t }t|}t|}|j|}|j|}|d }	|d }
d }t|dkrd| j	dS |d j
d}d|kr|jd |j|
}|d krd}|dkr|
j }
|j|	}t|	|
|||||}|j  x$tj|d krtd tjd qW td	| jd
}|j|t|	| jtj|ddS )Nr   rB   z Something went wrong internally._modulesenabledry   sfp__stor_stdout
HUMAN_NAMEz,[info] Waiting for the scan to initialize...zdyn/scaninfo.tmpl)r/   r0   SCANLIST)r[   namer   r)   pageid)r   r   r{   r8   r	   r   r}   r   rV   errorr|   remove
targetTypelowergenScanInstanceGUIDr   startr
   	getStatusr   rX   sleepr   r0   r2   r    r   )r!   r[   cfgmodoptsmodlistr#   r"   info
scanconfigscanname
scantargetr   newIdtr4   r%   r%   r&   	rerunscanB  s<    








zSpiderFootWebUi.rerunscanc          	   C   s&  t | j}t }t }t|}t|}x|jdD ]}|j|}|j|}	|d }
|d }d }t	|	dkrt| j
dS |	d jd}d|kr|jd |j|}|d kr| j
dS |j|
}t|
|j |||||}|j  x&tj|d kr td	 tjd qW q2W td
| jd}|jd| jddS )Nry   r   rB   z Something went wrong internally.r   r   z/Invalid target type. Could not recognize it as zCa human name, IP address, IP subnet, ASN, domain name or host name.z,[info] Waiting for the scan to initialize...zdyn/scanlist.tmpl)r/   r0   Tr   )Z
rerunscansr   r   zrInvalid target type. Could not recognize it as a human name, IP address, IP subnet, ASN, domain name or host name.)r   r   r{   r8   r	   r   r|   r}   r   rV   r   r   r   r   r   r   r   r
   r   r   rX   r   r   r0   r2   r   )r!   r~   r   r   r   r#   r"   r[   r   r   r   r   r   r   r   r4   r%   r%   r&   rerunscanmultip  s8    






zSpiderFootWebUi.rerunscanmultic          	   C   s@   t | j}|j }td| jd}|jd|| j| jd ddddS )Nzdyn/newscan.tmpl)r/   r0   NEWSCANr   r   )r   typesr   modulesr   selectedmodsr   )r   r   
eventTypesr   r0   r2   r   )r!   r"   r   r4   r%   r%   r&   newscan  s    

zSpiderFootWebUi.newscanc          	   C   s   t | j}t| j}|j }|j|}|j|}|d }|d }d }	|dks`|dks`t|dkrj| jdS |j|}	|	d krd| d }|d j	d}
t
d| jd	}|jd
|| j| jd |
t|t|dS )Nr   rB   r   z Something went wrong internally.z&quot;r   ry   zdyn/newscan.tmpl)r/   r0   r   r   )r   r   r   r   r   r   r   )r	   r   r   r   r}   r   rV   r   r   r|   r   r0   r2   r   r    )r!   r[   r#   r"   r   r   r   r   r   r   r   r4   r%   r%   r&   	clonescan  s&    






zSpiderFootWebUi.clonescanc             C   s   t d| jd}|jd| jdS )Nzdyn/scanlist.tmpl)r/   r0   r   )r   r   )r   r0   r2   r   )r!   r4   r%   r%   r&   index  s    zSpiderFootWebUi.indexc             C   sZ   t | j}|j|}|d kr&| jdS td| jdd}|j|tj|d |d | j	ddS )	NzScan ID not found.zdyn/scaninfo.tmplzutf-8)r/   r0   Zinput_encodingr   rJ   r   )r[   r   r)   r   r   )
r   r   r}   r   r   r0   r2   r9   r:   r   )r!   r[   r"   resr4   r%   r%   r&   r     s    


zSpiderFootWebUi.scaninfoc             C   s8   t d| jd}tj jdd| _|j| jd| j| jdS )Nzdyn/opts.tmpl)r/   r0   r   iSETTINGS)optsr   tokenr   )	r   r0   randomSystemRandomrandintr   r2   r   r   )r!   r4   r%   r%   r&   r     s    zSpiderFootWebUi.optsc             C   s   t | j}|j| j}d}xjt|D ]^}d|ks$|jdr<q$|s^||d t||  d 7 }q$||kr$||d t||  d 7 }q$W dtjjd< dtjjd	< |S )
Nr   z:__=
z%attachment; filename="SpiderFoot.cfg"zContent-Dispositionz
text/plainzContent-Type)	r	   r   configSerializesortedrT   r    r   r+   rr   )r!   patternr#   confcontentoptr%   r%   r&   
optsexport  s    
 zSpiderFootWebUi.optsexportc             C   s   t  }tj jdd| _x| jD ]}|jdr|dkr xpt| jd j D ]Z}xTt| jd | d j D ]6}|jdrzqj| jd | d | |d| d | < qjW qJW q | j| |d	| < q W t	j
d
| j|dgS )Nr   i__r   r   r   zmodule..zglobal.SUCCESS)r   r^   )r{   r   r   r   r   r   rT   r   r   r   r   )r!   r=   r   modmor%   r%   r&   optsraw  s    
 
.zSpiderFootWebUi.optsrawc             C   s   t d| jd}|j|| jdS )Nzdyn/error.tmpl)r/   r0   )r1   r   )r   r0   r2   r   )r!   r1   r4   r%   r%   r&   r     s    zSpiderFootWebUi.errorFc             C   s   t | j}|j|}|d kr8|s*| jdS tjddgS |d k	rj|j| |sZtjdqtjddgS n2t	d| j
d}|j|t|d t t d	| jd
S d S )NzScan ID not found.ERRORr   r   r   zdyn/scandelete.tmpl)r/   r0   r   r   )r[   r   namesr~   r   r   )r   r   r}   r   r   r   scanInstanceDeleter   HTTPRedirectr   r0   r2   r    r8   r   )r!   r[   confirmrawr"   r   r4   r%   r%   r&   
scandelete  s    



zSpiderFootWebUi.scandeletec             C   s   t | j}t }x^|jdD ]P}|j|}|jt|d  |d krV| jd| d S |d dkr| jd	S qW |d k	rx|jdD ]}|j| qW t	j
d
n,td| jd}|jd d |jd|d| jdS d S )Nry   r   zScan ID not found (z).rJ   RUNNINGSTARTINGSTARTEDz You cannot delete running scans.r   zdyn/scandelete.tmpl)r/   r0   r   )r[   r   r~   r   r   r   )r   r   r   )r   r   r8   r|   r}   r<   r    r   r   r   r   r   r0   r2   r   )r!   r~   r   r"   r   r[   r   r4   r%   r%   r&   scandeletemulti%  s     

zSpiderFootWebUi.scandeletemultic          #   C   s  t |t | jkr*| jdt | j d S |r|jr|jj }t|tkrT|jd}ydt }xN|j	dD ]@}d|krvqh|j
 j	d}t|dkrd|d< |d ||d < qhW tj|}W n2 tk
r } z| jd	t | d
 S d }~X nX yt| j}	|dkr|	j  t| j| _nttj|}
t }x.t|
j D ]}| j|
| gd ||< q6W t| j}t| j}|j||| _|	j|j| j W n0 tk
r } z| jdt | S d }~X nX td| jd}tj  j!dd| _|j"| jdd| j#| jdS )NzInvalid token (z).zutf-8r   r   rB   r   r   z?Failed to parse input file. Was it generated from SpiderFoot? ()RESETz.Processing one or more of your inputs failed: zdyn/opts.tmpl)r/   r0   ir   T)r   r   updatedr   r   )$r    r   r   filereadrF   bytesdecoder{   r|   striprV   r   r   BaseExceptionr   r   configClearr   r   loadsr8   r   r@   r	   r   	configSetr   	Exceptionr   r0   r   r   r   r2   r   )r!   alloptsr   Z
configFilecontentstmplineler"   useropts	cleanoptsr   currentoptsr#   r4   r%   r%   r&   savesettings>  sL    

"




zSpiderFootWebUi.savesettingsc       
      C   s  t |t | jkr.tjddt | j d gS yt| j}|dkrX|j  t| j| _nrtj	|}t
 }x,t|j D ]}| j|| gd ||< qvW t| j}t| j}|j||| _|j|j| j W n4 tk
r  }	 ztjddt |	 gS d }	~	X nX tjddgS )	Nr   zInvalid token (z).r   r   z.Processing one or more of your inputs failed: r   r   )r    r   r   r   r   r   r   r   r   r   r{   r8   r   r@   r	   r   r   r   r   )
r!   r   r   r"   r   r   r   r   r#   r   r%   r%   r&   savesettingsraws  s$    



"zSpiderFootWebUi.savesettingsrawc             C   s   t | j}|dkr tjddgS tj|}|s<tjddgS |j|}|d dkr`tjd
dgS |dkr|j||}x*|D ]"}t|d dkrztjd
dgS qzW |j||}	||	 }
|j	||
|}|stjddgS tjddgS d S )Nr   r   r   z$No FP flag set or not set correctly.zNo IDs supplied.rJ   ABORTEDFINISHEDERROR-FAILEDWARNINGz&Scan must be in a finished state when zsetting False Positives.rR   z.You cannot unset an element as False Positive z,if a parent element is still False Positive.zException encountered.r   r   )r   r   )r   r   r   z>Scan must be in a finished state when setting False Positives.zZYou cannot unset an element as False Positive if a parent element is still False Positive.)
r   r   r   r   r   r}   scanElementSourcesDirectr    scanElementChildrenAllscanResultsUpdateFP)r!   r[   Z	resultidsfpr"   r~   r)   r^   r`   childsZallIdsr=   r%   r%   r&   resultsetfp  s.    



 zSpiderFootWebUi.resultsetfpc             C   sV   t | j}|j }t }x"|D ]}|j|d |d g qW t|tdd}tj|S )NrB   r   )r   )	r   r   r   r8   r<   r   r   r   r   )r!   r"   r   r=   rr%   r%   r&   
eventtypes  s    

zSpiderFootWebUi.eventtypesc             C   s`   t | jd j }|j  t  }x4|D ],}d|kr4q&|j|| jd | d d q&W tj|S )Nr   r   descr)r   r  )r8   r   r   sortr<   r   r   )r!   Zmodinfor=   mr%   r%   r&   r     s    
"zSpiderFootWebUi.modulesc             C   s   t jd| jd gS )Nr   __version__)r   r   r   )r!   r%   r%   r&   ping  s    zSpiderFootWebUi.pingc             C   s   d }t | j}dtjjd< |j jds6tjddgS y|j	j
|}|j }W n. tk
r| } ztjdt|gS d }~X nX tj|S )Nzapplication/json; charset=utf-8zContent-Typeselectr   z2Non-SELECTs are unpredictable and not recommended.)r   r   r   r+   rr   r   rT   r   r   r"   executefetchallr   r    )r!   queryr^   r"   r=   r   r%   r%   r&   r    s    
zSpiderFootWebUi.queryc             C   s  t | j}t }t }	t|}
d }| j||g\}}|dksD|dkr`|sR| jdS tjddgS |dkr|dkr|dkr|s| jdS tjddgS |dkr|j	ddj
d}	t|	dko|dkrb|j	ddj
d}|
j|}	t |	}t |}xpt|dkr`xN|
j|D ]@}|
j|g}x,|D ]$}||	kr"|	j| |j| q"W qW t |}t }qW t|	dkr|dkrx@| jd	 D ]2}|d
ks|| jd	 | d kr|	j| qW d|	kr|	jd |	j  |
j|}|d kr
|s| jdS tjddgS d|	kr|	jd |
j|}|dkr@|j	dd}n|j }t|||||	||}|j  x(tj|d krtd tjd qfW |std| jd}|j||| jtj|ddS tjd|gS d S )Nr   zForm incomplete.r   zIncorrect usage.Zmodule_ry   r   type_r   allcatssfp__stor_dbz/Invalid target type. Could not recognize it as zCa human name, IP address, IP subnet, ASN, domain name or host name.zUnrecognised target type.r   r   USERNAMEr7   z,[info] Waiting for the scan to initialize...rB   zdyn/scaninfo.tmpl)r/   r0   r   )r[   r   r   r)   r   r   zrInvalid target type. Could not recognize it as a human name, IP address, IP subnet, ASN, domain name or host name.)r   r  )r   r   r{   r8   r	   r@   r   r   r   r;   r|   rV   modulesProducingeventsToModulesr<   r  r   r   r   r   r   r   r
   r   r   rX   r   r   r0   r2   r   )r!   r   r   Z
modulelistZtypelistZusecaseclir   r   r   r#   r   Ztypesxnewmods	newmodcpyetypexmodsr   scanIdr   r4   r%   r%   r&   	startscan  sx    







"









zSpiderFootWebUi.startscanc       	      C   s&  t | j}t }x|jdD ]}d}|j|}|d kr@| jdS t|d }tj|dksf|d dkr|j	d| d | d	  d
}| rtj|dks|d dkr|j	d| d  d
}| rtj|d kr|j	d| d d | d  d
}|stj
|d qW td| jd}|jdd
|| jdS )Nry   FzInvalid scan ID specified.r   r   rJ   zScan 'z2' is in a finished state. <a href='/scandelete?id=z4&confirm=1'>Maybe you want to delete it instead?</a>Tr   z' is already aborted.z.' is not actually running. A data consistency z=error for this scan probably exists. <a href='/scandelete?id=z(&confirm=1'>Click here to delete it.</a>zABORT-REQUESTEDzdyn/scanlist.tmpl)r/   r0   r   )r   stoppedscanerrorsr   )r   r   r8   r|   r}   r   r    r
   r   r<   	setStatusr   r0   r2   r   )	r!   r~   r"   r   r[   ZerrStater   r   r4   r%   r%   r&   stopscanmultiN  s0    


 zSpiderFootWebUi.stopscanmultic             C   s  t | j}|j|}|d kr8|s*| jdS tjddgS tj|d krj|s\| jd| d S tjddgS tj|dkr|s| jdS tjdd	gS tj|d
ks|s| jdtj| d d S tjddgS tj|d |st	d| j
d}|jdd| jt dS tjddgS d S )NzInvalid scan ID.r   z6That scan is not actually running. A data consistency z=error for this scan probably exists. <a href='/scandelete?id=z(&confirm=1'>Click here to delete it.</a>z"Scan doesn't appear to be running.r   zThe scan is already aborted.zScan already aborted.r   z,The running scan is currently in the state 'z%', please try again later or restart z SpiderFoot.z&Scan in an invalid state for stopping.zABORT-REQUESTEDzdyn/scanlist.tmpl)r/   r0   r   T)r   r  r   r  r   r   zsThat scan is not actually running. A data consistency error for this scan probably exists. <a href='/scandelete?id=)r   r   r}   r   r   r   r
   r   r   r   r0   r2   r   r8   )r!   r[   r  r"   r   r4   r%   r%   r&   stopscanv  s4    



zSpiderFootWebUi.stopscanc       
      C   sz   t | j}|j||||}g }xP|D ]H}tjdtj|d d }	|j|	|d |d tj|d |d g q$W t	j
|S )Nz%Y-%m-%d %H:%M:%Sr   i  rB   rI   rA   rP   )r   r   scanLogsrX   rY   rZ   r<   r9   r:   r   r   )
r!   r[   limitZrowIdreverser"   r^   r_   r`   	generatedr%   r%   r&   scanlog  s    

.zSpiderFootWebUi.scanlogc             C   sn   t | j}|j||}g }xH|D ]@}tjdtj|d d }|j||d tjt	|d g q W t
j|S )Nz%Y-%m-%d %H:%M:%Sr   i  rB   rI   )r   r   
scanErrorsrX   rY   rZ   r<   r9   r:   r    r   r   )r!   r[   r$  r"   r^   r_   r`   r&  r%   r%   r&   
scanerrors  s    

zSpiderFootWebUi.scanerrorsc             C   s   t | j}|j }g }x|D ]}tjdtj|d }|d dkrZtjdtj|d }nd}|d dkrtjdtj|d }nd}|j|d |d |d ||||d	 |d
 g qW tj|S )Nz%Y-%m-%d %H:%M:%SrA   rP   r   zNot yetrJ   rB   rI   rK   rL   )	r   r   scanInstanceListrX   rY   rZ   r<   r   r   )r!   r"   r^   r_   r`   createdr   r   r%   r%   r&   scanlist  s    

2zSpiderFootWebUi.scanlistc             C   s|   t | j}|j|}tjdtj|d }tjdtj|d }tjdtj|d }|d |d ||||d g}tj|S )Nz%Y-%m-%d %H:%M:%SrI   rA   rP   r   rB   rJ   )r   r   r}   rX   rY   rZ   r   r   )r!   r[   r"   r^   r+  r   endedr_   r%   r%   r&   
scanstatus  s    

zSpiderFootWebUi.scanstatusc             C   sz   t | j}|j||}g }xT|D ]L}|d dkr2q tjdtj|d }|j|d |d ||d |d g q W tj|S )Nr   rl   z%Y-%m-%d %H:%M:%SrI   rB   rA   rP   )	r   r   scanResultSummaryrX   rY   rZ   r<   r   r   )r!   r[   byr"   r^   r_   r`   ra   r%   r%   r&   scansummary  s    

(zSpiderFootWebUi.scansummaryc             C   s   t | j}|j|||}g }x~|D ]v}tjdtj|d }tj|d }	tj|d }
|j||	|
|d |d |d |d |d	 |d
 |d |d g q"W t	j
|ddS )Nz%Y-%m-%d %H:%M:%Sr   rB   rI   rA   rJ   rK   rL   rM   rQ   rR   rP   F)ensure_ascii)r   r   rn   rX   rY   rZ   r9   r:   r<   r   r   )r!   r[   r\   filterfpr"   r^   r_   r`   ra   rb   rc   r%   r%   r&   scaneventresults  s    


z SpiderFootWebUi.scaneventresultsc       	      C   s^   t | j}|j|||}g }x2|D ]*}tj|d }|j||d |d g q"W tj|ddS )Nr   rB   rI   F)r2  )r   r   scanResultEventUniquer9   r:   r<   r   r   )	r!   r[   r\   r3  r"   r^   r_   r`   escapedr%   r%   r&   scaneventresultsunique  s    

z&SpiderFootWebUi.scaneventresultsuniquec             C   s   | j |||}tj|ddS )NF)r2  )rd   r   r   )r!   r[   r\   rG   r_   r%   r%   r&   rW     s    zSpiderFootWebUi.searchc             C   s"   t | j}|j|}tj|ddS )NF)r2  )r   r   scanResultHistoryr   r   )r!   r[   r"   r^   r%   r%   r&   scanhistory  s    

zSpiderFootWebUi.scanhistoryc       	      C   sl   t | j}t| j}t }t }|j||}|j||\}}|d= t }|j||d< ||d< tj|ddS )Nrl   treer^   F)r2  )	r	   r   r   r{   rn   scanElementSourcesAlldataParentChildToTreer   r   )	r!   r[   r\   r#   r"   pcdatamapZleafSetr_   r%   r%   r&   scanelementtypediscovery  s    

z(SpiderFootWebUi.scanelementtypediscovery)NNN)re   )re   )NNre   )r   )r   )NF)N)N)N)N)NNN)N)F)F)NNN)7__name__
__module____qualname__r   r0   r{   r   r   r   r   r'   r   r   r@   rd   rx   exposedr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r  r   r  r  r  r!  r"  r'  r)  r,  r.  r1  r4  r7  rW   r9  r?  r%   r%   r%   r&   r      s   %
(


'

",)



2#

[$
)
	




	
r   )r   r   r9   ro   rX   r   securer   r   operatorr   copyr   Zmako.lookupr   Zmako.templater   sfdbr   sflibr	   r
   sfscanr   ior   r   r%   r%   r%   r&   <module>   s    