U
    @†6^“Æ  ã                   @   s>   d dl Z d dlZd dlZd dlmZ dd„ ZG dd„ dƒZdS )é    N)Ú
SpiderFootc              
   C   sT   z"t  | t jt jB ¡}| |¡}W n( tk
rJ } z
W Y ¢dS d }~X Y nX |d k	S )NF)ÚreÚcompileÚ
IGNORECASEÚDOTALLÚmatchÚBaseException)ÚqryÚdataÚrxÚretÚe© r   ú'/var/www/spiderfoot.crq.systems/sfdb.pyÚ__dbregex__   s    r   c                   @   sR  e Zd ZdZdZdZdddddddd	d
dddgZddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{d|d}d~dd€dd‚dƒd„d…d†d‡dˆd‰dŠd‹dŒddŽddd‘d’d“d”d•d–d—d˜d™dšgZd×dœd„ZdždŸ„ Z	d d¡„ Z
dØd¢d£„Zd¤d¥„ ZdÙd¦d§„Zd¨d©„ ZdÚdªd«„Zd¬d­„ ZdÛd¯d°„ZdÜd²d³„ZdÝd´dµ„ZdÞd¶d·„Zdßd¸d¹„Zdºd»„ Zd¼d½„ Zeƒ fd¾d¿„ZdÀdÁ„ ZdÂdÃ„ Zeƒ fdÄdÅ„ZdÆdÇ„ ZdàdÉdÊ„ZdËdÌ„ ZdÍdÎ„ Z dÏdÐ„ Z!dÑdÒ„ Z"dÓdÔ„ Z#dÕdÖ„ Z$dS )áÚSpiderFootDbNzPRAGMA journal_mode=WALzáCREATE TABLE tbl_event_types (             event       VARCHAR NOT NULL PRIMARY KEY,             event_descr VARCHAR NOT NULL,             event_raw   INT NOT NULL DEFAULT 0,             event_type  VARCHAR NOT NULL         )zºCREATE TABLE tbl_config (             scope   VARCHAR NOT NULL,             opt     VARCHAR NOT NULL,             val     VARCHAR NOT NULL,             PRIMARY KEY (scope, opt)         )aR  CREATE TABLE tbl_scan_instance (             guid        VARCHAR NOT NULL PRIMARY KEY,             name        VARCHAR NOT NULL,             seed_target VARCHAR NOT NULL,             created     INT DEFAULT 0,             started     INT DEFAULT 0,             ended       INT DEFAULT 0,             status      VARCHAR NOT NULL         )a+  CREATE TABLE tbl_scan_log (             scan_instance_id    VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid),             generated           INT NOT NULL,             component           VARCHAR,             type                VARCHAR NOT NULL,             message             VARCHAR         )a  CREATE TABLE tbl_scan_config (             scan_instance_id    VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid),             component           VARCHAR NOT NULL,             opt                 VARCHAR NOT NULL,             val                 VARCHAR NOT NULL         )a§  CREATE TABLE tbl_scan_results (             scan_instance_id    VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid),             hash                VARCHAR NOT NULL,             type                VARCHAR NOT NULL REFERENCES tbl_event_types(event),             generated           INT NOT NULL,             confidence          INT NOT NULL DEFAULT 100,             visibility          INT NOT NULL DEFAULT 100,             risk                INT NOT NULL DEFAULT 0,             module              VARCHAR NOT NULL,             data                VARCHAR,             false_positive      INT NOT NULL DEFAULT 0,             source_event_hash  VARCHAR DEFAULT 'ROOT'         )zGCREATE INDEX idx_scan_results_id ON tbl_scan_results (scan_instance_id)zOCREATE INDEX idx_scan_results_type ON tbl_scan_results (scan_instance_id, type)zOCREATE INDEX idx_scan_results_hash ON tbl_scan_results (scan_instance_id, hash)z_CREATE INDEX idx_scan_results_srchash ON tbl_scan_results (scan_instance_id, source_event_hash)z=CREATE INDEX idx_scan_logs ON tbl_scan_log (scan_instance_id)zˆINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ROOT', 'Internal SpiderFoot Root event', 1, 'INTERNAL')z’INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED', 'Account on External Site', 0, 'ENTITY')z©INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')z´INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_EMAILADDR', 'Affiliate - Email Address', 0, 'ENTITY')z”INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')z¬INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME_UNRESOLVED', 'Affiliate - Internet Name - Unresolved', 0, 'ENTITY')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_IPADDR', 'Affiliate - IP Address', 0, 'ENTITY')zŽINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_WEB_CONTENT', 'Affiliate - Web Content', 1, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DOMAIN_NAME', 'Affiliate - Domain Name', 0, 'ENTITY')z’INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_COMPANY_NAME', 'Affiliate - Company Name', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DOMAIN_WHOIS', 'Affiliate - Domain Whois', 1, 'DATA')z¦INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_CATEGORY', 'Affiliate Description - Category', 0, 'DESCRIPTOR')z¦INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_ABSTRACT', 'Affiliate Description - Abstract', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('APPSTORE_ENTRY', 'App Store Entry', 0, 'ENTITY')zŒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET', 'Cloud Storage Bucket', 0, 'ENTITY')zšINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET_OPEN', 'Cloud Storage Bucket Open', 0, 'DESCRIPTOR')z|INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('COMPANY_NAME', 'Company Name', 0, 'ENTITY')z€INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BASE64_DATA', 'Base64-encoded Data', 1, 'DATA')z‚INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_ADDRESS', 'Bitcoin Address', 0, 'ENTITY')z†INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_BALANCE', 'Bitcoin Balance', 0, 'DESCRIPTOR')z€INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_OWNER', 'BGP AS Ownership', 0, 'ENTITY')z‚INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_MEMBER', 'BGP AS Membership', 0, 'ENTITY')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_PEER', 'BGP AS Peer', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_IPADDR', 'Blacklisted IP Address', 0, 'DESCRIPTOR')z¤INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_AFFILIATE_IPADDR', 'Blacklisted Affiliate IP Address', 0, 'DESCRIPTOR')z—INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_SUBNET', 'Blacklisted IP on Same Subnet', 0, 'DESCRIPTOR')zœINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_NETBLOCK', 'Blacklisted IP on Owned Netblock', 0, 'DESCRIPTOR')z€INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE', 'Co-Hosted Site', 0, 'ENTITY')z•INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE_DOMAIN', 'Co-Hosted Site - Domain Name', 0, 'ENTITY')zšINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE_DOMAIN_WHOIS', 'Co-Hosted Site - Domain Whois', 1, 'DATA')zŽINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_URL', 'Darknet Mention URL', 0, 'DESCRIPTOR')z”INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_CONTENT', 'Darknet Mention Web Content', 1, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DATE_HUMAN_DOB', 'Date of Birth', 0, 'ENTITY')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_INTERNET_NAME', 'Defaced', 0, 'DESCRIPTOR')zˆINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_IPADDR', 'Defaced IP Address', 0, 'DESCRIPTOR')z˜INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_INTERNET_NAME', 'Defaced Affiliate', 0, 'DESCRIPTOR')zŒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_COHOST', 'Defaced Co-Hosted Site', 0, 'DESCRIPTOR')zœINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_IPADDR', 'Defaced Affiliate IP Address', 0, 'DESCRIPTOR')z’INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DESCRIPTION_CATEGORY', 'Description - Category', 0, 'DESCRIPTOR')z’INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DESCRIPTION_ABSTRACT', 'Description - Abstract', 0, 'DESCRIPTOR')z~INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEVICE_TYPE', 'Device Type', 0, 'DESCRIPTOR')zxINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_TEXT', 'DNS TXT Record', 0, 'DATA')zwINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_SRV', 'DNS SRV Record', 0, 'DATA')zwINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_SPF', 'DNS SPF Record', 0, 'DATA')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME', 'Domain Name', 0, 'ENTITY')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME_PARENT', 'Domain Name (Parent)', 0, 'ENTITY')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_REGISTRAR', 'Domain Registrar', 0, 'ENTITY')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, 'DATA')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR', 'Email Address', 0, 'ENTITY')z‘INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_COMPROMISED', 'Hacked Email Address', 0, 'DESCRIPTOR')z|INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ERROR_MESSAGE', 'Error Message', 0, 'DATA')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ETHEREUM_ADDRESS', 'Ethereum Address', 0, 'ENTITY')z€INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('GEOINFO', 'Physical Location', 0, 'DESCRIPTOR')z‹INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HASH_COMPROMISED', 'Compromised Password Hash', 0, 'DATA')z{INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HTTP_CODE', 'HTTP Status Code', 0, 'DATA')zxINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HUMAN_NAME', 'Human Name', 0, 'ENTITY')zˆINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE', 'Interesting File', 0, 'DESCRIPTOR')zšINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE_HISTORIC', 'Historic Interesting File', 0, 'DESCRIPTOR')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('JUNK_FILE', 'Junk File', 0, 'DESCRIPTOR')z~INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME', 'Internet Name', 0, 'ENTITY')z–INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME_UNRESOLVED', 'Internet Name - Unresolved', 0, 'ENTITY')zxINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IP_ADDRESS', 'IP Address', 0, 'ENTITY')z|INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IPV6_ADDRESS', 'IPv6 Address', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_INTERNAL', 'Linked URL - Internal', 0, 'SUBENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_EXTERNAL', 'Linked URL - External', 0, 'SUBENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_ASN', 'Malicious AS', 0, 'DESCRIPTOR')zŒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_IPADDR', 'Malicious IP Address', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_COHOST', 'Malicious Co-Hosted Site', 0, 'DESCRIPTOR')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_EMAILADDR', 'Malicious E-mail Address', 0, 'DESCRIPTOR')z–INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_INTERNET_NAME', 'Malicious Internet Name', 0, 'DESCRIPTOR')zœINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_INTERNET_NAME', 'Malicious Affiliate', 0, 'DESCRIPTOR')z INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_IPADDR', 'Malicious Affiliate IP Address', 0, 'DESCRIPTOR')z˜INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_NETBLOCK', 'Malicious IP on Owned Netblock', 0, 'DESCRIPTOR')z”INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_PHONE_NUMBER', 'Malicious Phone Number', 0, 'DESCRIPTOR')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_SUBNET', 'Malicious IP on Same Subnet', 0, 'DESCRIPTOR')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_OWNER', 'Netblock Ownership', 0, 'ENTITY')z†INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_MEMBER', 'Netblock Membership', 0, 'ENTITY')z~INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_WHOIS', 'Netblock Whois', 1, 'DATA')zˆINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('OPERATING_SYSTEM', 'Operating System', 0, 'DESCRIPTOR')z}INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LEAKSITE_URL', 'Leak Site URL', 0, 'ENTITY')zƒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LEAKSITE_CONTENT', 'Leak Site Content', 1, 'DATA')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PASSWORD_COMPROMISED', 'Compromised Password', 0, 'DATA')z|INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHONE_NUMBER', 'Phone Number', 0, 'ENTITY')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHYSICAL_ADDRESS', 'Physical Address', 0, 'ENTITY')zŒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHYSICAL_COORDINATES', 'Physical Coordinates', 0, 'ENTITY')zwINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PGP_KEY', 'PGP Public Key', 0, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_DNS', 'Name Server (DNS ''NS'' Records)', 0, 'ENTITY')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_JAVASCRIPT', 'Externally Hosted Javascript', 0, 'ENTITY')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_MAIL', 'Email Gateway (DNS ''MX'' Records)', 0, 'ENTITY')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_HOSTING', 'Hosting Provider', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_TELCO', 'Telecommunications Provider', 0, 'ENTITY')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PUBLIC_CODE_REPO', 'Public Code Repository', 0, 'ENTITY')z…INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_RIR_DATA', 'Raw Data from RIRs/APIs', 1, 'DATA')z€INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_DNS_RECORDS', 'Raw DNS Records', 1, 'DATA')z†INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_FILE_META_DATA', 'Raw File Meta Data', 1, 'DATA')z—INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SEARCH_ENGINE_WEB_CONTENT', 'Search Engine''s Web Content', 1, 'DATA')z…INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOCIAL_MEDIA', 'Social Media Presence', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN', 'Similar Domain', 0, 'ENTITY')z‹INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN_WHOIS', 'Similar Domain - Whois', 1, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOFTWARE_USED', 'Software Used', 0, 'SUBENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_RAW', 'SSL Certificate - Raw Data', 1, 'DATA')z•INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUED', 'SSL Certificate - Issued to', 0, 'ENTITY')z•INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUER', 'SSL Certificate - Issued by', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_MISMATCH', 'SSL Certificate Host Mismatch', 0, 'DESCRIPTOR')z–INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRED', 'SSL Certificate Expired', 0, 'DESCRIPTOR')z˜INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRING', 'SSL Certificate Expiring', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT', 'Web Content', 1, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT_TYPE', 'Web Content Type', 0, 'DESCRIPTOR')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_COOKIE', 'Cookies', 0, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN', 'Open TCP Port', 0, 'SUBENTITY')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN_BANNER', 'Open TCP Port Banner', 0, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN', 'Open UDP Port', 0, 'SUBENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN_INFO', 'Open UDP Port Information', 0, 'DATA')z–INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_EXTERNAL', 'URL (AdBlocked External)', 0, 'DESCRIPTOR')z–INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_INTERNAL', 'URL (AdBlocked Internal)', 0, 'DESCRIPTOR')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM', 'URL (Form)', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH', 'URL (Uses Flash)', 0, 'DESCRIPTOR')z‹INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT', 'URL (Uses Javascript)', 0, 'DESCRIPTOR')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK', 'URL (Uses a Web Framework)', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET', 'URL (Uses Java Applet)', 0, 'DESCRIPTOR')z…INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, 'DESCRIPTOR')z‹INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, 'DESCRIPTOR')z‡INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, 'DESCRIPTOR')zŒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM_HISTORIC', 'Historic URL (Form)', 0, 'DESCRIPTOR')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH_HISTORIC', 'Historic URL (Uses Flash)', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT_HISTORIC', 'Historic URL (Uses Javascript)', 0, 'DESCRIPTOR')z¥INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK_HISTORIC', 'Historic URL (Uses a Web Framework)', 0, 'DESCRIPTOR')zŸINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET_HISTORIC', 'Historic URL (Uses Java Applet)', 0, 'DESCRIPTOR')z—INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC_HISTORIC', 'Historic URL (Purely Static)', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD_HISTORIC', 'Historic URL (Accepts Passwords)', 0, 'DESCRIPTOR')z™INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD_HISTORIC', 'Historic URL (Accepts Uploads)', 0, 'DESCRIPTOR')ztINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('USERNAME', 'Username', 0, 'ENTITY')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('VULNERABILITY', 'Vulnerability in Public Domain', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEB_ANALYTICS_ID', 'Web Analytics', 0, 'ENTITY')z|INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')zƒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')z‘INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIFI_ACCESS_POINT', 'WiFi Access Point Nearby', 0, 'ENTITY')zŽINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIKIPEDIA_PAGE_EDIT', 'Wikipedia Page Edit', 0, 'DESCRIPTOR')Fc                 C   sH  t |ƒ| _tj| j ¡ d |d  dd}|d krF| j d|d  ¡ t|_|| _| 	¡ | _
z | j
 d¡ | j ddt¡ W nb tjk
rÞ   z|  ¡  d	}W n8 tk
rÖ } z| j d
|jd  ¡ W 5 d }~X Y nX Y d S X |rD| jD ]L}z| j
 |¡ | j ¡  W qì tk
r6 } zW Y ¢qìW 5 d }~X Y qìX qì| j ¡  d S )Nú/Ú
__databaseé
   )Útimeoutz<Could not connect to internal database, and couldn't create z$SELECT COUNT(*) FROM tbl_scan_configÚREGEXPé   Tz<Tried to set up the SpiderFoot database schema, but failed: r   )r   ÚsfÚsqlite3ÚconnectÚmyPathÚfatalÚstrZtext_factoryÚconnZcursorÚdbhÚexecuteZcreate_functionr   ÚErrorÚcreater   ÚerrorÚargsÚcreateTypeQueriesÚcommit)ÚselfÚoptsÚinitr   r   r	   r   r   r   Ú__init__ì   s2    
 
(
zSpiderFootDb.__init__c              
   C   s„   zH| j D ]}| j |¡ q| j ¡  | jD ]}| j |¡ q*| j ¡  W n6 tjk
r~ } ztd|j	d  ƒ‚W 5 d }~X Y nX d S )Nz0SQL error encountered when setting up database: r   )
ÚcreateSchemaQueriesr   r    r   r&   r%   r   r!   r   r$   ©r'   r	   r   r   r   r   r"     s    


zSpiderFootDb.createc                 C   s   | j  ¡  d S )N)r   Úclose)r'   r   r   r   r-   #  s    zSpiderFootDb.closec              
   C   s@  t | ¡ ƒ d ¡dkrdS t ƒ }d}|r0|d7 }| d¡d k	rT|d7 }| |d ¡ | d¡d k	rx|d7 }| |d ¡ | d	¡d k	rª|d
7 }| |d	 ¡ | |d	 ¡ | d¡d k	rÜ|d7 }| |d ¡ | |d ¡ |d7 }z| j ||¡ | j ¡ W S  tj	k
r: } z| j
 d|jd  ¡ W 5 d }~X Y nX d S )Né   Faþ  SELECT ROUND(c.generated) AS generated, c.data,             s.data as 'source_data',             c.module, c.type, c.confidence, c.visibility, c.risk, c.hash,             c.source_event_hash, t.event_descr, t.event_type, c.scan_instance_id,             c.false_positive as 'fp', s.false_positive as 'parent_fp'             FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t             WHERE s.scan_instance_id = c.scan_instance_id AND             t.event = c.type AND c.source_event_hash = s.hash z AND c.false_positive <> 1 Zscan_idzAND c.scan_instance_id = ? Útypez AND c.type = ? Úvaluez& AND (c.data LIKE ? OR s.data LIKE ?) Úregexz* AND (c.data REGEXP ? OR s.data REGEXP ?) ú ORDER BY c.dataz4SQL error encountered when fetching search results: r   )ÚlistÚvaluesÚcountÚgetÚappendr   r    Úfetchallr   r!   r   r#   r$   )r'   ZcriteriaÚfilterFpÚqvarsr	   r   r   r   r   Úsearch-  s4    	zSpiderFootDb.searchc              
   C   s\   d}z| j  |¡ | j  ¡ W S  tjk
rV } z| j d|jd  ¡ W 5 d }~X Y nX d S )NzESELECT event_descr, event, event_raw, event_type FROM tbl_event_typesz2SQL error encountered when retreiving event types:r   ©r   r    r8   r   r!   r   r#   r$   r,   r   r   r   Ú
eventTypes[  s    zSpiderFootDb.eventTypesc              
   C   sœ   |d krd}d}z.| j  ||t ¡ d |||f¡ | j ¡  W nX tjk
r– } z8d|jd ks†d|jd krpntdt	|jd ƒ ƒ W 5 d }~X Y nX dS )	Nr   zINSERT INTO tbl_scan_log             (scan_instance_id, generated, component, type, message)             VALUES (?, ?, ?, ?, ?)éè  Úlockedr   Úthreadz[warning] Couldn't log due to: T)
r   r    Útimer   r&   r   r!   r$   Úprintr   )r'   Ú
instanceIdZclassificationÚmessageÚ	componentr	   r   r   r   r   ÚscanLogEventd  s"     
   ÿ(zSpiderFootDb.scanLogEventc              
   C   sr   d}z.| j  ||||t ¡ d df¡ | j ¡  W n: tjk
rl } z| j d|j	d  ¡ W 5 d }~X Y nX dS )NzwINSERT INTO tbl_scan_instance             (guid, name, seed_target, created, status)             VALUES (?, ?, ?, ?, ?)r>   ÚCREATEDz!Unable to create instance in DB: r   T)
r   r    rA   r   r&   r   r!   r   r   r$   )r'   rC   ÚscanNameZ
scanTargetr	   r   r   r   r   ÚscanInstanceCreate}  s       
 ÿ(zSpiderFootDb.scanInstanceCreatec                 C   s®   t ƒ }d}|d k	r$|d7 }| |¡ |d k	r>|d7 }| |¡ |d k	rX|d7 }| |¡ |d7 }| |¡ z| j ||¡ | j ¡  W n" tjk
r¨   | j 	d¡ Y nX d S )NzUPDATE tbl_scan_instance SET z started = ?,z ended = ?,z status = ?,z guid = guid WHERE guid = ?z0Unable to set information for the scan instance.)
r3   r7   r   r    r   r&   r   r!   r   r   )r'   rC   ÚstartedZendedÚstatusr:   r	   r   r   r   ÚscanInstanceSetŒ  s$    



zSpiderFootDb.scanInstanceSetc              
   C   sd   d}|g}z| j  ||¡ | j  ¡ W S  tjk
r^ } z| j d|jd  ¡ W 5 d }~X Y nX d S )NzºSELECT name, seed_target, ROUND(created/1000) AS created,             ROUND(started/1000) AS started, ROUND(ended/1000) AS ended, status             FROM tbl_scan_instance WHERE guid = ?z4SQL error encountered when retreiving scan instance:r   )r   r    Zfetchoner   r!   r   r#   r$   ©r'   rC   r	   r:   r   r   r   r   ÚscanInstanceGet¨  s    zSpiderFootDb.scanInstanceGetr/   c              
   C   s„   |dkrd}|dkrd}|dkr$d}|g}z| j  ||¡ | j  ¡ W S  tjk
r~ } z| j d|jd  ¡ W 5 d }~X Y nX d S )	Nr/   a'  SELECT r.type, e.event_descr, MAX(ROUND(generated)) AS last_in,                 count(*) AS total, count(DISTINCT r.data) as utotal FROM                 tbl_scan_results r, tbl_event_types e WHERE e.event = r.type                 AND r.scan_instance_id = ? GROUP BY r.type ORDER BY e.event_descrÚmodulea   SELECT r.module, '', MAX(ROUND(generated)) AS last_in,                 count(*) AS total, count(DISTINCT r.data) as utotal FROM                 tbl_scan_results r, tbl_event_types e WHERE e.event = r.type                 AND r.scan_instance_id = ? GROUP BY r.module ORDER BY r.module DESCÚentitya{  SELECT r.data, e.event_descr, MAX(ROUND(generated)) AS last_in,                 count(*) AS total, count(DISTINCT r.data) as utotal FROM                 tbl_scan_results r, tbl_event_types e WHERE e.event = r.type                 AND r.scan_instance_id = ?                 AND e.event_type in ('ENTITY')                 GROUP BY r.data, e.event_descr ORDER BY total DESC limit 50z4SQL error encountered when fetching result summary: r   r<   )r'   rC   Zbyr	   r:   r   r   r   r   ÚscanResultSummary´  s    zSpiderFootDb.scanResultSummaryÚALLc              
   C   s’   d}|g}|dkr$|d7 }|  |¡ |r0|d7 }|d7 }z| j ||¡ | j ¡ W S  tjk
rŒ } z| j d|jd  ¡ W 5 d }~X Y nX d S )Na$  SELECT ROUND(c.generated) AS generated, c.data,             s.data as 'source_data',             c.module, c.type, c.confidence, c.visibility, c.risk, c.hash,             c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id,             c.false_positive as 'fp', s.false_positive as 'parent_fp'             FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t             WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND             s.scan_instance_id = c.scan_instance_id AND             t.event = c.typerR   z AND c.type = ?z AND c.false_positive <> 1r2   z3SQL error encountered when fetching result events: r   ©	r7   r   r    r8   r   r!   r   r#   r$   ©r'   rC   Ú	eventTyper9   r	   r:   r   r   r   r   ÚscanResultEventÑ  s    

zSpiderFootDb.scanResultEventc              
   C   s’   d}|g}|dkr$|d7 }|  |¡ |r0|d7 }|d7 }z| j ||¡ | j ¡ W S  tjk
rŒ } z| j d|jd  ¡ W 5 d }~X Y nX d S )NzaSELECT DISTINCT data, type, COUNT(*) FROM tbl_scan_results             WHERE scan_instance_id = ?rR   z AND type = ?z AND false_positive <> 1z& GROUP BY type, data ORDER BY COUNT(*)z:SQL error encountered when fetching unique result events: r   rS   rT   r   r   r   ÚscanResultEventUniqueî  s    
z"SpiderFootDb.scanResultEventUniquec              
   C   s¶   d}|r|d7 }|d7 }|r&|d7 }n|d7 }|g}|rB|  |¡ |d k	r\|d7 }|  |¡ z| j ||¡ | j ¡ W S  tjk
r° } z| j d|jd  ¡ W 5 d }~X Y nX d S )	NzwSELECT generated AS generated, component,             type, message, rowid FROM tbl_scan_log WHERE scan_instance_id = ?z and rowid > ?z ORDER BY generated ÚASCZDESCú LIMIT ?z/SQL error encountered when fetching scan logs: r   rS   )r'   rC   ÚlimitZ	fromRowIdÚreverser	   r:   r   r   r   r   ÚscanLogs  s$    


zSpiderFootDb.scanLogsc              
   C   s~   d}|g}|d k	r$|d7 }|  |¡ z| j ||¡ | j ¡ W S  tjk
rx } z| j d|jd  ¡ W 5 d }~X Y nX d S )Nz¡SELECT generated AS generated, component,             message FROM tbl_scan_log WHERE scan_instance_id = ?             AND type = 'ERROR' ORDER BY generated DESCrY   z1SQL error encountered when fetching scan errors: r   rS   )r'   rC   rZ   r	   r:   r   r   r   r   Ú
scanErrors  s    
zSpiderFootDb.scanErrorsc              
   C   sœ   d}d}d}d}|g}zF| j  ||¡ | j  ||¡ | j  ||¡ | j  ||¡ | j ¡  W n: tjk
r– } z| j d|jd  ¡ W 5 d }~X Y nX d S )Nz,DELETE FROM tbl_scan_instance WHERE guid = ?z6DELETE FROM tbl_scan_config WHERE scan_instance_id = ?z7DELETE FROM tbl_scan_results WHERE scan_instance_id = ?z3DELETE FROM tbl_scan_log WHERE scan_instance_id = ?z*SQL error encountered when deleting scan: r   ©	r   r    r   r&   r   r!   r   r#   r$   )r'   rC   Zqry1Zqry2Zqry3Zqry4r:   r   r   r   r   ÚscanInstanceDelete/  s    zSpiderFootDb.scanInstanceDeletec                 C   s~   |D ]j}d}|||g}z| j  ||¡ W q tjk
rl } z$| j d|jd  d¡ W Y ¢ dS d }~X Y qX q| j ¡  dS )NzfUPDATE tbl_scan_results SET false_positive = ? WHERE                 scan_instance_id = ? AND hash = ?z)SQL error encountered when updating F/P: r   FT)	r   r    r   r!   r   r#   r$   r   r&   )r'   rC   ZresultHashesZfpFlagZ
resultHashr	   r:   r   r   r   r   ÚscanResultsUpdateFP?  s    

z SpiderFootDb.scanResultsUpdateFPc                 C   sª   d}t | ¡ ƒD ]”}d|kr>| d¡}|d |d || g}nd||| g}z| j ||¡ W n: tjk
r˜ } z| j d|j	d  ¡ W 5 d }~X Y nX | j
 ¡  qd S )Nz:REPLACE INTO tbl_config (scope, opt, val) VALUES (?, ?, ?)ú:r   é   ÚGLOBALú5SQL error encountered when storing config, aborting: ©r3   ÚkeysÚsplitr   r    r   r!   r   r#   r$   r   r&   )r'   ÚoptMapr	   ÚoptÚpartsÚqvalsr   r   r   r   Ú	configSetN  s    
(zSpiderFootDb.configSetc              
   C   s”   d}zPt ƒ }| j |¡ | j ¡ D ],\}}}|dkr>|||< q"|||d | < q"|W S  tjk
rŽ } z| j d|jd  ¡ W 5 d }~X Y nX d S )Nz&SELECT scope, opt, val FROM tbl_configrc   ra   ú3SQL error encountered when fetching configuration: r   ©	Údictr   r    r8   r   r!   r   r#   r$   )r'   r	   ÚretvalÚscoperi   Úvalr   r   r   r   Ú	configGeta  s    
zSpiderFootDb.configGetc              
   C   s^   d}z| j  |¡ | j ¡  W n: tjk
rX } z| j d|jd  ¡ W 5 d }~X Y nX d S )NzDELETE from tbl_configz1Unable to clear configuration from the database: r   r^   r,   r   r   r   ÚconfigClearr  s    zSpiderFootDb.configClearc                 C   s®   d}t | ¡ ƒD ]˜}d|kr@| d¡}||d |d || g}n|d||| g}z| j ||¡ W n: tjk
rœ } z| j d|j	d  ¡ W 5 d }~X Y nX | j
 ¡  qd S )NzhREPLACE INTO tbl_scan_config                 (scan_instance_id, component, opt, val) VALUES (?, ?, ?, ?)ra   r   rb   rc   rd   re   )r'   Úidrh   r	   ri   rj   rk   r   r   r   r   ÚscanConfigSet{  s    
(zSpiderFootDb.scanConfigSetc           	   
   C   sœ   d}|g}zRt ƒ }| j ||¡ | j ¡ D ],\}}}|dkrF|||< q*|||d | < q*|W S  tjk
r– } z| j d|jd  ¡ W 5 d }~X Y nX d S )NzrSELECT component, opt, val FROM tbl_scan_config                 WHERE scan_instance_id = ? ORDER BY component, optrc   ra   rm   r   rn   )	r'   rC   r	   r:   rp   rE   ri   rr   r   r   r   r   ÚscanConfigGet  s    
zSpiderFootDb.scanConfigGetr   c              
   C   sF  d}t |jƒtk	rzt |jƒtkr,t|jƒ}q€zt|jƒ}W q€ tk
rv } z| j dtt |jƒƒ ¡ W 5 d }~X Y q€X n|j}|dkr”|d|… }|jdkrª| j d¡ d}|| ¡ |j|j	|j
|j|j|j||jg
}z| j ||¡ | j ¡  W d S  tjk
r@ } z(| j dt| jƒ d |jd  ¡ W 5 d }~X Y nX d S )	NÚ zUnhandled type detected: r   )rx   Nz5UNABLE TO CREATE RECORD WITH EMPTY SOURCE EVENT HASH!zÏINSERT INTO tbl_scan_results             (scan_instance_id, hash, type, generated, confidence,             visibility, risk, module, data, source_event_hash)             VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)z/SQL error encountered when storing event data (z: )r/   r
   r   r   r   r   ÚsourceEventHashÚgetHashrU   Ú	generatedÚ
confidenceÚ
visibilityÚriskrO   r   r    r   r&   r   r!   r$   )r'   rC   ÚsfEventZtruncateSizeZ	storeDatar   r	   rk   r   r   r   ÚscanEventStore«  s8    .
    þ
zSpiderFootDb.scanEventStorec              
   C   s\   d}z| j  |¡ | j  ¡ W S  tjk
rV } z| j d|jd  ¡ W 5 d }~X Y nX d S )NaŠ  SELECT i.guid, i.name, i.seed_target, ROUND(i.created/1000),             ROUND(i.started)/1000 as started, ROUND(i.ended)/1000, i.status, COUNT(r.type)             FROM tbl_scan_instance i, tbl_scan_results r WHERE i.guid = r.scan_instance_id             AND r.type <> 'ROOT' GROUP BY i.guid             UNION ALL             SELECT i.guid, i.name, i.seed_target, ROUND(i.created/1000),             ROUND(i.started)/1000 as started, ROUND(i.ended)/1000, i.status, '0'             FROM tbl_scan_instance i  WHERE i.guid NOT IN (             SELECT distinct scan_instance_id FROM tbl_scan_results WHERE type <> 'ROOT')             ORDER BY started DESCz/SQL error encountered when fetching scan list: r   r<   r,   r   r   r   ÚscanInstanceListÓ  s    
zSpiderFootDb.scanInstanceListc              
   C   sd   d}|g}z| j  ||¡ | j  ¡ W S  tjk
r^ } z| j d|jd  ¡ W 5 d }~X Y nX d S )Nz¶SELECT STRFTIME('%H:%M %w', generated, 'unixepoch') AS hourmin,                 type, COUNT(*) FROM tbl_scan_results                 WHERE scan_instance_id = ? GROUP BY hourmin, typez2SQL error encountered when fetching scan history: r   r<   rM   r   r   r   ÚscanResultHistoryè  s    zSpiderFootDb.scanResultHistoryc              
   C   s†   d}|g}|D ]}|d | d }q|d7 }z| j  ||¡ | j  ¡ W S  tjk
r€ } z| j d|jd  ¡ W 5 d }~X Y nX d S )Na4  SELECT ROUND(c.generated) AS generated, c.data,             s.data as 'source_data',             c.module, c.type, c.confidence, c.visibility, c.risk, c.hash,             c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id,             c.false_positive as 'fp', s.false_positive as 'parent_fp'             FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t             WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND             s.scan_instance_id = c.scan_instance_id AND             t.event = c.type AND c.hash in (ú'ú',ú'')z7SQL error encountered when getting source element IDs: r   r<   ©r'   rC   ZelementIdListr	   r:   ZhashIdr   r   r   r   ÚscanElementSourcesDirectõ  s    	z%SpiderFootDb.scanElementSourcesDirectc              
   C   s†   d}|g}|D ]}|d | d }q|d7 }z| j  ||¡ | j  ¡ W S  tjk
r€ } z| j d|jd  ¡ W 5 d }~X Y nX d S )Na4  SELECT ROUND(c.generated) AS generated, c.data,             s.data as 'source_data',             c.module, c.type, c.confidence, c.visibility, c.risk, c.hash,             c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id,             c.false_positive as 'fp', s.false_positive as 'parent_fp'             FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t             WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND             s.scan_instance_id = c.scan_instance_id AND             t.event = c.type AND s.hash in (rƒ   r„   r…   z6SQL error encountered when getting child element IDs: r   r<   r†   r   r   r   ÚscanElementChildrenDirect  s    	z&SpiderFootDb.scanElementChildrenDirectc                 C   s  d}t ƒ }tƒ }tƒ }|D ]\}|d }|d }	|||	< ||krZ|	|| krd||  |	¡ n
|	g||< ||kr| |¡ q|r|  ||¡}
t ƒ }d}|
D ]h}|d }|d }	|||	< ||krØ|	|| krâ||  |	¡ n
|	g||< ||krô| |¡ |dkr˜d}q˜qx|||< ||gS )NTé	   é   FÚROOT)r3   ro   r7   r‡   )r'   rC   Z	childDataÚ	keepGoingÚnextIdsÚdatamapÚpcÚrowZparentIdZchildIdZ	parentSetr   r   r   ÚscanElementSourcesAll*  s@    


z"SpiderFootDb.scanElementSourcesAllc                 C   s´   t ƒ }d}t ƒ }|  ||¡}|D ]}| |d ¡ q |D ]}|d |kr8| |d ¡ q8|r°|  ||¡}|d ks|t|ƒdkr‚d}q°|D ]&}| |d ¡ t ƒ }| |d ¡ q†qX|S )NTrŠ   r   F)r3   rˆ   r7   Úlen)r'   rC   Z	parentIdsrŽ   rŒ   r   ZnextSetr   r   r   r   ÚscanElementChildrenAll_  s&    z#SpiderFootDb.scanElementChildrenAll)F)F)N)NNN)r/   )rR   F)rR   F)NNF)N)r   )%Ú__name__Ú
__module__Ú__qualname__r   r   r   r+   r%   r*   r"   r-   r;   r=   rF   rI   rL   rN   rQ   rV   rW   r\   r]   r_   r`   ro   rl   rs   rt   rv   rw   r€   r   r‚   r‡   rˆ   r‘   r“   r   r   r   r   r      sx  	Ë:€ ó 
+

.	






	
(5r   )r   r   rA   Úsflibr   r   r   r   r   r   r   Ú<module>   s
   	