3 @†6^“Æã@s>ddlZddlZddlZddlmZdd„ZGdd„dƒZdS)éN)Ú SpiderFootcCsLy"tj|tjtjBƒ}|j|ƒ}Wn tk rB}zdSd}~XnX|dk S)NF)ÚreÚcompileÚ IGNORECASEÚDOTALLÚmatchÚ BaseException)ÚqryÚdataÚrxÚretÚe©rú'/var/www/spiderfoot.crq.systems/sfdb.pyÚ __dbregex__s rc@sReZdZdZdZdZdddddddd d d d d g Zddddddddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4d5d6d7d8d9d:d;dd?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{d|d}d~dd€dd‚dƒd„d…d†d‡dˆd‰dŠd‹dŒddŽddd‘d’d“d”d•d–d—d˜d™dšgZd×dœd„ZdždŸ„Z d d¡„Z dØd¢d£„Z d¤d¥„Z dÙd¦d§„Z d¨d©„ZdÚdªd«„Zd¬d­„ZdÛd¯d°„ZdÜd²d³„ZdÝd´dµ„ZdÞd¶d·„Zdßd¸d¹„Zdºd»„Zd¼d½„Zeƒfd¾d¿„ZdÀdÁ„ZdÂdÄZeƒfdÄdÅ„ZdÆdÇ„ZdàdÉdÊ„ZdËdÌ„ZdÍd΄Z dÏdЄZ!dÑdÒ„Z"dÓdÔ„Z#dÕdÖ„Z$dS)áÚ SpiderFootDbNzPRAGMA journal_mode=WALzáCREATE TABLE tbl_event_types ( event VARCHAR NOT NULL PRIMARY KEY, event_descr VARCHAR NOT NULL, event_raw INT NOT NULL DEFAULT 0, event_type VARCHAR NOT NULL )zºCREATE TABLE tbl_config ( scope VARCHAR NOT NULL, opt VARCHAR NOT NULL, val VARCHAR NOT NULL, PRIMARY KEY (scope, opt) )aRCREATE TABLE tbl_scan_instance ( guid VARCHAR NOT NULL PRIMARY KEY, name VARCHAR NOT NULL, seed_target VARCHAR NOT NULL, created INT DEFAULT 0, started INT DEFAULT 0, ended INT DEFAULT 0, status VARCHAR NOT NULL )a+CREATE TABLE tbl_scan_log ( scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), generated INT NOT NULL, component VARCHAR, type VARCHAR NOT NULL, message VARCHAR )aCREATE TABLE tbl_scan_config ( scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), component VARCHAR NOT NULL, opt VARCHAR NOT NULL, val VARCHAR NOT NULL )a§CREATE TABLE tbl_scan_results ( scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), hash VARCHAR NOT NULL, type VARCHAR NOT NULL REFERENCES tbl_event_types(event), generated INT NOT NULL, confidence INT NOT NULL DEFAULT 100, visibility INT NOT NULL DEFAULT 100, risk INT NOT NULL DEFAULT 0, module VARCHAR NOT NULL, data VARCHAR, false_positive INT NOT NULL DEFAULT 0, source_event_hash VARCHAR DEFAULT 'ROOT' )zGCREATE INDEX idx_scan_results_id ON tbl_scan_results (scan_instance_id)zOCREATE INDEX idx_scan_results_type ON tbl_scan_results (scan_instance_id, type)zOCREATE INDEX idx_scan_results_hash ON tbl_scan_results (scan_instance_id, hash)z_CREATE INDEX idx_scan_results_srchash ON tbl_scan_results (scan_instance_id, source_event_hash)z=CREATE INDEX idx_scan_logs ON tbl_scan_log (scan_instance_id)zˆINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ROOT', 'Internal SpiderFoot Root event', 1, 'INTERNAL')z’INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED', 'Account on External Site', 0, 'ENTITY')z©INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')z´INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_EMAILADDR', 'Affiliate - Email Address', 0, 'ENTITY')z”INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')z¬INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME_UNRESOLVED', 'Affiliate - Internet Name - Unresolved', 0, 'ENTITY')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_IPADDR', 'Affiliate - IP Address', 0, 'ENTITY')zŽINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_WEB_CONTENT', 'Affiliate - Web Content', 1, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DOMAIN_NAME', 'Affiliate - Domain Name', 0, 'ENTITY')z’INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_COMPANY_NAME', 'Affiliate - Company Name', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DOMAIN_WHOIS', 'Affiliate - Domain Whois', 1, 'DATA')z¦INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_CATEGORY', 'Affiliate Description - Category', 0, 'DESCRIPTOR')z¦INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_ABSTRACT', 'Affiliate Description - Abstract', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('APPSTORE_ENTRY', 'App Store Entry', 0, 'ENTITY')zŒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET', 'Cloud Storage Bucket', 0, 'ENTITY')zšINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET_OPEN', 'Cloud Storage Bucket Open', 0, 'DESCRIPTOR')z|INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('COMPANY_NAME', 'Company Name', 0, 'ENTITY')z€INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BASE64_DATA', 'Base64-encoded Data', 1, 'DATA')z‚INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_ADDRESS', 'Bitcoin Address', 0, 'ENTITY')z†INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_BALANCE', 'Bitcoin Balance', 0, 'DESCRIPTOR')z€INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_OWNER', 'BGP AS Ownership', 0, 'ENTITY')z‚INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_MEMBER', 'BGP AS Membership', 0, 'ENTITY')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_PEER', 'BGP AS Peer', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_IPADDR', 'Blacklisted IP Address', 0, 'DESCRIPTOR')z¤INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_AFFILIATE_IPADDR', 'Blacklisted Affiliate IP Address', 0, 'DESCRIPTOR')z—INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_SUBNET', 'Blacklisted IP on Same Subnet', 0, 'DESCRIPTOR')zœINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_NETBLOCK', 'Blacklisted IP on Owned Netblock', 0, 'DESCRIPTOR')z€INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE', 'Co-Hosted Site', 0, 'ENTITY')z•INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE_DOMAIN', 'Co-Hosted Site - Domain Name', 0, 'ENTITY')zšINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE_DOMAIN_WHOIS', 'Co-Hosted Site - Domain Whois', 1, 'DATA')zŽINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_URL', 'Darknet Mention URL', 0, 'DESCRIPTOR')z”INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_CONTENT', 'Darknet Mention Web Content', 1, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DATE_HUMAN_DOB', 'Date of Birth', 0, 'ENTITY')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_INTERNET_NAME', 'Defaced', 0, 'DESCRIPTOR')zˆINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_IPADDR', 'Defaced IP Address', 0, 'DESCRIPTOR')z˜INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_INTERNET_NAME', 'Defaced Affiliate', 0, 'DESCRIPTOR')zŒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_COHOST', 'Defaced Co-Hosted Site', 0, 'DESCRIPTOR')zœINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_IPADDR', 'Defaced Affiliate IP Address', 0, 'DESCRIPTOR')z’INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DESCRIPTION_CATEGORY', 'Description - Category', 0, 'DESCRIPTOR')z’INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DESCRIPTION_ABSTRACT', 'Description - Abstract', 0, 'DESCRIPTOR')z~INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEVICE_TYPE', 'Device Type', 0, 'DESCRIPTOR')zxINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_TEXT', 'DNS TXT Record', 0, 'DATA')zwINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_SRV', 'DNS SRV Record', 0, 'DATA')zwINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_SPF', 'DNS SPF Record', 0, 'DATA')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME', 'Domain Name', 0, 'ENTITY')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME_PARENT', 'Domain Name (Parent)', 0, 'ENTITY')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_REGISTRAR', 'Domain Registrar', 0, 'ENTITY')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, 'DATA')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR', 'Email Address', 0, 'ENTITY')z‘INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_COMPROMISED', 'Hacked Email Address', 0, 'DESCRIPTOR')z|INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ERROR_MESSAGE', 'Error Message', 0, 'DATA')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ETHEREUM_ADDRESS', 'Ethereum Address', 0, 'ENTITY')z€INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('GEOINFO', 'Physical Location', 0, 'DESCRIPTOR')z‹INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HASH_COMPROMISED', 'Compromised Password Hash', 0, 'DATA')z{INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HTTP_CODE', 'HTTP Status Code', 0, 'DATA')zxINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HUMAN_NAME', 'Human Name', 0, 'ENTITY')zˆINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE', 'Interesting File', 0, 'DESCRIPTOR')zšINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE_HISTORIC', 'Historic Interesting File', 0, 'DESCRIPTOR')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('JUNK_FILE', 'Junk File', 0, 'DESCRIPTOR')z~INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME', 'Internet Name', 0, 'ENTITY')z–INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME_UNRESOLVED', 'Internet Name - Unresolved', 0, 'ENTITY')zxINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IP_ADDRESS', 'IP Address', 0, 'ENTITY')z|INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IPV6_ADDRESS', 'IPv6 Address', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_INTERNAL', 'Linked URL - Internal', 0, 'SUBENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_EXTERNAL', 'Linked URL - External', 0, 'SUBENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_ASN', 'Malicious AS', 0, 'DESCRIPTOR')zŒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_IPADDR', 'Malicious IP Address', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_COHOST', 'Malicious Co-Hosted Site', 0, 'DESCRIPTOR')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_EMAILADDR', 'Malicious E-mail Address', 0, 'DESCRIPTOR')z–INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_INTERNET_NAME', 'Malicious Internet Name', 0, 'DESCRIPTOR')zœINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_INTERNET_NAME', 'Malicious Affiliate', 0, 'DESCRIPTOR')z INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_IPADDR', 'Malicious Affiliate IP Address', 0, 'DESCRIPTOR')z˜INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_NETBLOCK', 'Malicious IP on Owned Netblock', 0, 'DESCRIPTOR')z”INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_PHONE_NUMBER', 'Malicious Phone Number', 0, 'DESCRIPTOR')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_SUBNET', 'Malicious IP on Same Subnet', 0, 'DESCRIPTOR')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_OWNER', 'Netblock Ownership', 0, 'ENTITY')z†INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_MEMBER', 'Netblock Membership', 0, 'ENTITY')z~INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_WHOIS', 'Netblock Whois', 1, 'DATA')zˆINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('OPERATING_SYSTEM', 'Operating System', 0, 'DESCRIPTOR')z}INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LEAKSITE_URL', 'Leak Site URL', 0, 'ENTITY')zƒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LEAKSITE_CONTENT', 'Leak Site Content', 1, 'DATA')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PASSWORD_COMPROMISED', 'Compromised Password', 0, 'DATA')z|INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHONE_NUMBER', 'Phone Number', 0, 'ENTITY')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHYSICAL_ADDRESS', 'Physical Address', 0, 'ENTITY')zŒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHYSICAL_COORDINATES', 'Physical Coordinates', 0, 'ENTITY')zwINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PGP_KEY', 'PGP Public Key', 0, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_DNS', 'Name Server (DNS ''NS'' Records)', 0, 'ENTITY')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_JAVASCRIPT', 'Externally Hosted Javascript', 0, 'ENTITY')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_MAIL', 'Email Gateway (DNS ''MX'' Records)', 0, 'ENTITY')z„INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_HOSTING', 'Hosting Provider', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_TELCO', 'Telecommunications Provider', 0, 'ENTITY')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PUBLIC_CODE_REPO', 'Public Code Repository', 0, 'ENTITY')z…INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_RIR_DATA', 'Raw Data from RIRs/APIs', 1, 'DATA')z€INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_DNS_RECORDS', 'Raw DNS Records', 1, 'DATA')z†INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_FILE_META_DATA', 'Raw File Meta Data', 1, 'DATA')z—INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SEARCH_ENGINE_WEB_CONTENT', 'Search Engine''s Web Content', 1, 'DATA')z…INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOCIAL_MEDIA', 'Social Media Presence', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN', 'Similar Domain', 0, 'ENTITY')z‹INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN_WHOIS', 'Similar Domain - Whois', 1, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOFTWARE_USED', 'Software Used', 0, 'SUBENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_RAW', 'SSL Certificate - Raw Data', 1, 'DATA')z•INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUED', 'SSL Certificate - Issued to', 0, 'ENTITY')z•INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUER', 'SSL Certificate - Issued by', 0, 'ENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_MISMATCH', 'SSL Certificate Host Mismatch', 0, 'DESCRIPTOR')z–INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRED', 'SSL Certificate Expired', 0, 'DESCRIPTOR')z˜INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRING', 'SSL Certificate Expiring', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT', 'Web Content', 1, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT_TYPE', 'Web Content Type', 0, 'DESCRIPTOR')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_COOKIE', 'Cookies', 0, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN', 'Open TCP Port', 0, 'SUBENTITY')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN_BANNER', 'Open TCP Port Banner', 0, 'DATA')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN', 'Open UDP Port', 0, 'SUBENTITY')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN_INFO', 'Open UDP Port Information', 0, 'DATA')z–INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_EXTERNAL', 'URL (AdBlocked External)', 0, 'DESCRIPTOR')z–INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_INTERNAL', 'URL (AdBlocked Internal)', 0, 'DESCRIPTOR')zzINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM', 'URL (Form)', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH', 'URL (Uses Flash)', 0, 'DESCRIPTOR')z‹INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT', 'URL (Uses Javascript)', 0, 'DESCRIPTOR')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK', 'URL (Uses a Web Framework)', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET', 'URL (Uses Java Applet)', 0, 'DESCRIPTOR')z…INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, 'DESCRIPTOR')z‹INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, 'DESCRIPTOR')z‡INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, 'DESCRIPTOR')zŒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM_HISTORIC', 'Historic URL (Form)', 0, 'DESCRIPTOR')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH_HISTORIC', 'Historic URL (Uses Flash)', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT_HISTORIC', 'Historic URL (Uses Javascript)', 0, 'DESCRIPTOR')z¥INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK_HISTORIC', 'Historic URL (Uses a Web Framework)', 0, 'DESCRIPTOR')zŸINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET_HISTORIC', 'Historic URL (Uses Java Applet)', 0, 'DESCRIPTOR')z—INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC_HISTORIC', 'Historic URL (Purely Static)', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD_HISTORIC', 'Historic URL (Accepts Passwords)', 0, 'DESCRIPTOR')z™INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD_HISTORIC', 'Historic URL (Accepts Uploads)', 0, 'DESCRIPTOR')ztINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('USERNAME', 'Username', 0, 'ENTITY')z“INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('VULNERABILITY', 'Vulnerability in Public Domain', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEB_ANALYTICS_ID', 'Web Analytics', 0, 'ENTITY')z|INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')zƒINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')z‘INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')zŠINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')zINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIFI_ACCESS_POINT', 'WiFi Access Point Nearby', 0, 'ENTITY')zŽINSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIKIPEDIA_PAGE_EDIT', 'Wikipedia Page Edit', 0, 'DESCRIPTOR')Fc%CsDt|ƒ|_tj|jjƒd|ddd}|dkrF|jjd|dƒt|_||_|j ƒ|_ y |j j dƒ|jj ddt ƒWn`tjk rÜy|jƒd }Wn8tk rÖ}z|jjd |jd ƒWYdd}~XnXdSX|r@xP|jD]F}y|j j |ƒ|jjƒWqìtk r0}zwìWYdd}~XqìXqìW|jjƒdS) Nú/Ú __databaseé )Útimeoutzt|jƒƒjdƒdkrdStƒ}d}|r0|d7}|jdƒdk rT|d7}|j|dƒ|jdƒdk rx|d7}|j|dƒ|jd ƒdk rª|d 7}|j|d ƒ|j|d ƒ|jd ƒdk rÜ|d 7}|j|d ƒ|j|d ƒ|d 7}y|jj||ƒ|jjƒStj k r8}z|j j d|j dƒWYdd}~XnXdS)NéFaþSELECT ROUND(c.generated) AS generated, c.data, s.data as 'source_data', c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, c.source_event_hash, t.event_descr, t.event_type, c.scan_instance_id, c.false_positive as 'fp', s.false_positive as 'parent_fp' FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t WHERE s.scan_instance_id = c.scan_instance_id AND t.event = c.type AND c.source_event_hash = s.hash z AND c.false_positive <> 1 Zscan_idzAND c.scan_instance_id = ? Útypez AND c.type = ? Úvaluez& AND (c.data LIKE ? OR s.data LIKE ?) Úregexz* AND (c.data REGEXP ? OR s.data REGEXP ?) z ORDER BY c.dataz4SQL error encountered when fetching search results: r) ÚlistÚvaluesÚcountÚgetÚappendrr Úfetchallrr!rr#r$)r'ZcriteriaÚfilterFpÚqvarsr r rrrÚsearch-s4 zSpiderFootDb.searchcCsZd}y|jj|ƒ|jjƒStjk rT}z|jjd|jdƒWYdd}~XnXdS)NzESELECT event_descr, event, event_raw, event_type FROM tbl_event_typesz2SQL error encountered when retreiving event types:r)rr r6rr!rr#r$)r'r r rrrÚ eventTypes[s   zSpiderFootDb.eventTypescCsœ|dkr d}d}y.|jj||tjƒd|||fƒ|jjƒWnXtjk r–}z:d|jdks†d|jdkrpntdt |jdƒƒWYdd}~XnXdS) NrzINSERT INTO tbl_scan_log (scan_instance_id, generated, component, type, message) VALUES (?, ?, ?, ?, ?)ièÚlockedrÚthreadz[warning] Couldn't log due to: T) rr Útimerr&rr!r$Úprintr)r'Ú instanceIdZclassificationÚmessageÚ componentr r rrrÚ scanLogEventds(zSpiderFootDb.scanLogEventcCsrd}y.|jj||||tjƒddfƒ|jjƒWn:tjk rl}z|jjd|j dƒWYdd}~XnXdS)NzwINSERT INTO tbl_scan_instance (guid, name, seed_target, created, status) VALUES (?, ?, ?, ?, ?)ièÚCREATEDz!Unable to create instance in DB: rT) rr r=rr&rr!rrr$)r'r?ÚscanNameZ scanTargetr r rrrÚscanInstanceCreate}s(zSpiderFootDb.scanInstanceCreatec Cs®tƒ}d}|dk r$|d7}|j|ƒ|dk r>|d7}|j|ƒ|dk rX|d7}|j|ƒ|d7}|j|ƒy|jj||ƒ|jjƒWn"tjk r¨|jj dƒYnXdS)NzUPDATE tbl_scan_instance SET z started = ?,z ended = ?,z status = ?,z guid = guid WHERE guid = ?z0Unable to set information for the scan instance.) r1r5rr rr&rr!rr)r'r?ÚstartedZendedÚstatusr8r rrrÚscanInstanceSetŒs$    zSpiderFootDb.scanInstanceSetcCsbd}|g}y|jj||ƒ|jjƒStjk r\}z|jjd|jdƒWYdd}~XnXdS)NzºSELECT name, seed_target, ROUND(created/1000) AS created, ROUND(started/1000) AS started, ROUND(ended/1000) AS ended, status FROM tbl_scan_instance WHERE guid = ?z4SQL error encountered when retreiving scan instance:r)rr Zfetchonerr!rr#r$)r'r?r r8r rrrÚscanInstanceGet¨s zSpiderFootDb.scanInstanceGetr.cCs‚|dkr d}|dkrd}|dkr$d}|g}y|jj||ƒ|jjƒStjk r|}z|jjd|jdƒWYdd}~XnXdS) Nr.a'SELECT r.type, e.event_descr, MAX(ROUND(generated)) AS last_in, count(*) AS total, count(DISTINCT r.data) as utotal FROM tbl_scan_results r, tbl_event_types e WHERE e.event = r.type AND r.scan_instance_id = ? GROUP BY r.type ORDER BY e.event_descrÚmodulea SELECT r.module, '', MAX(ROUND(generated)) AS last_in, count(*) AS total, count(DISTINCT r.data) as utotal FROM tbl_scan_results r, tbl_event_types e WHERE e.event = r.type AND r.scan_instance_id = ? GROUP BY r.module ORDER BY r.module DESCÚentitya{SELECT r.data, e.event_descr, MAX(ROUND(generated)) AS last_in, count(*) AS total, count(DISTINCT r.data) as utotal FROM tbl_scan_results r, tbl_event_types e WHERE e.event = r.type AND r.scan_instance_id = ? AND e.event_type in ('ENTITY') GROUP BY r.data, e.event_descr ORDER BY total DESC limit 50z4SQL error encountered when fetching result summary: r)rr r6rr!rr#r$)r'r?Zbyr r8r rrrÚscanResultSummary´s zSpiderFootDb.scanResultSummaryÚALLcCsd}|g}|dkr$|d7}|j|ƒ|r0|d7}|d7}y|jj||ƒ|jjƒStjk rŠ}z|jjd|jdƒWYdd}~XnXdS)Na$SELECT ROUND(c.generated) AS generated, c.data, s.data as 'source_data', c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id, c.false_positive as 'fp', s.false_positive as 'parent_fp' FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND s.scan_instance_id = c.scan_instance_id AND t.event = c.typerMz AND c.type = ?z AND c.false_positive <> 1z ORDER BY c.dataz3SQL error encountered when fetching result events: r) r5rr r6rr!rr#r$)r'r?Ú eventTyper7r r8r rrrÚscanResultEventÑs   zSpiderFootDb.scanResultEventcCsd}|g}|dkr$|d7}|j|ƒ|r0|d7}|d7}y|jj||ƒ|jjƒStjk rŠ}z|jjd|jdƒWYdd}~XnXdS)NzaSELECT DISTINCT data, type, COUNT(*) FROM tbl_scan_results WHERE scan_instance_id = ?rMz AND type = ?z AND false_positive <> 1z& GROUP BY type, data ORDER BY COUNT(*)z:SQL error encountered when fetching unique result events: r) r5rr r6rr!rr#r$)r'r?rNr7r r8r rrrÚscanResultEventUniqueîs  z"SpiderFootDb.scanResultEventUniquecCs´d}|r|d7}|d7}|r&|d7}n|d7}|g}|rB|j|ƒ|dk r\|d7}|j|ƒy|jj||ƒ|jjƒStjk r®}z|jjd|jdƒWYdd}~XnXdS) NzwSELECT generated AS generated, component, type, message, rowid FROM tbl_scan_log WHERE scan_instance_id = ?z and rowid > ?z ORDER BY generated ÚASCZDESCz LIMIT ?z/SQL error encountered when fetching scan logs: r) r5rr r6rr!rr#r$)r'r?ÚlimitZ fromRowIdÚreverser r8r rrrÚscanLogss$    zSpiderFootDb.scanLogscCs|d}|g}|dk r$|d7}|j|ƒy|jj||ƒ|jjƒStjk rv}z|jjd|jdƒWYdd}~XnXdS)Nz¡SELECT generated AS generated, component, message FROM tbl_scan_log WHERE scan_instance_id = ? AND type = 'ERROR' ORDER BY generated DESCz LIMIT ?z1SQL error encountered when fetching scan errors: r) r5rr r6rr!rr#r$)r'r?rRr r8r rrrÚ scanErrorss  zSpiderFootDb.scanErrorscCsœd}d}d}d}|g}yF|jj||ƒ|jj||ƒ|jj||ƒ|jj||ƒ|jjƒWn:tjk r–}z|jjd|jdƒWYdd}~XnXdS)Nz,DELETE FROM tbl_scan_instance WHERE guid = ?z6DELETE FROM tbl_scan_config WHERE scan_instance_id = ?z7DELETE FROM tbl_scan_results WHERE scan_instance_id = ?z3DELETE FROM tbl_scan_log WHERE scan_instance_id = ?z*SQL error encountered when deleting scan: r) rr rr&rr!rr#r$)r'r?Zqry1Zqry2Zqry3Zqry4r8r rrrÚscanInstanceDelete/szSpiderFootDb.scanInstanceDeletecCsxxh|D]`}d}|||g}y|jj||ƒWqtjk rd}z|jjd|jddƒdSd}~XqXqW|jjƒdS)NzfUPDATE tbl_scan_results SET false_positive = ? WHERE scan_instance_id = ? AND hash = ?z)SQL error encountered when updating F/P: rFT) rr rr!rr#r$rr&)r'r?Z resultHashesZfpFlagZ resultHashr r8r rrrÚscanResultsUpdateFP?s   z SpiderFootDb.scanResultsUpdateFPcCs®d}x¤t|jƒƒD]”}d|kr@|jdƒ}|d|d||g}nd|||g}y|jj||ƒWn:tjk rš}z|jjd|j dƒWYdd}~XnX|j j ƒqWdS)Nz:REPLACE INTO tbl_config (scope, opt, val) VALUES (?, ?, ?)ú:réÚGLOBALz5SQL error encountered when storing config, aborting: ) r1ÚkeysÚsplitrr rr!rr#r$rr&)r'ÚoptMapr ÚoptÚpartsÚqvalsr rrrÚ configSetNs (zSpiderFootDb.configSetcCs–d}yRtƒ}|jj|ƒx:|jjƒD],\}}}|dkr@|||<q$|||d|<q$W|Stjk r}z|jjd|jdƒWYdd}~XnXdS)Nz&SELECT scope, opt, val FROM tbl_configrZrXz3SQL error encountered when fetching configuration: r) Údictrr r6rr!rr#r$)r'r ÚretvalÚscoper^Úvalr rrrÚ configGetas  zSpiderFootDb.configGetcCs^d}y|jj|ƒ|jjƒWn:tjk rX}z|jjd|jdƒWYdd}~XnXdS)NzDELETE from tbl_configz1Unable to clear configuration from the database: r) rr rr&rr!rr#r$)r'r r rrrÚ configClearrs  zSpiderFootDb.configClearcCs²d}x¨t|jƒƒD]˜}d|krB|jdƒ}||d|d||g}n|d|||g}y|jj||ƒWn:tjk rž}z|jjd|j dƒWYdd}~XnX|j j ƒqWdS)NzhREPLACE INTO tbl_scan_config (scan_instance_id, component, opt, val) VALUES (?, ?, ?, ?)rXrrYrZz5SQL error encountered when storing config, aborting: ) r1r[r\rr rr!rr#r$rr&)r'Úidr]r r^r_r`r rrrÚ scanConfigSet{s (zSpiderFootDb.scanConfigSetc Csžd}|g}yTtƒ}|jj||ƒx:|jjƒD],\}}}|dkrH|||<q,|||d|<q,W|Stjk r˜}z|jjd|jdƒWYdd}~XnXdS)NzrSELECT component, opt, val FROM tbl_scan_config WHERE scan_instance_id = ? ORDER BY component, optrZrXz3SQL error encountered when fetching configuration: r) rbrr r6rr!rr#r$) r'r?r r8rcrAr^rer rrrÚ scanConfigGets zSpiderFootDb.scanConfigGetrcCsDd}t|jƒtk rzt|jƒtkr,t|jƒ}q€yt|jƒ}Wq€tk rv}z |jjdtt|jƒƒƒWYdd}~Xq€Xn|j}|dkr”|d|…}|jdkrª|jjdƒd}||jƒ|j|j |j |j |j |j ||jg }y|jj||ƒ|jjƒdStjk r>}z*|jjdt|jƒd|jdƒWYdd}~XnXdS) NÚzUnhandled type detected: rz5UNABLE TO CREATE RECORD WITH EMPTY SOURCE EVENT HASH!zÏINSERT INTO tbl_scan_results (scan_instance_id, hash, type, generated, confidence, visibility, risk, module, data, source_event_hash) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)z/SQL error encountered when storing event data (z: )rkN)r.r rrrrÚsourceEventHashÚgetHashrNÚ generatedÚ confidenceÚ visibilityÚriskrJrr rr&rr!r$)r'r?ÚsfEventZ truncateSizeZ storeDatar r r`rrrÚscanEventStore«s. .     zSpiderFootDb.scanEventStorecCsZd}y|jj|ƒ|jjƒStjk rT}z|jjd|jdƒWYdd}~XnXdS)NaŠSELECT i.guid, i.name, i.seed_target, ROUND(i.created/1000), ROUND(i.started)/1000 as started, ROUND(i.ended)/1000, i.status, COUNT(r.type) FROM tbl_scan_instance i, tbl_scan_results r WHERE i.guid = r.scan_instance_id AND r.type <> 'ROOT' GROUP BY i.guid UNION ALL SELECT i.guid, i.name, i.seed_target, ROUND(i.created/1000), ROUND(i.started)/1000 as started, ROUND(i.ended)/1000, i.status, '0' FROM tbl_scan_instance i WHERE i.guid NOT IN ( SELECT distinct scan_instance_id FROM tbl_scan_results WHERE type <> 'ROOT') ORDER BY started DESCz/SQL error encountered when fetching scan list: r)rr r6rr!rr#r$)r'r r rrrÚscanInstanceListÓs   zSpiderFootDb.scanInstanceListcCsbd}|g}y|jj||ƒ|jjƒStjk r\}z|jjd|jdƒWYdd}~XnXdS)Nz¶SELECT STRFTIME('%H:%M %w', generated, 'unixepoch') AS hourmin, type, COUNT(*) FROM tbl_scan_results WHERE scan_instance_id = ? GROUP BY hourmin, typez2SQL error encountered when fetching scan history: r)rr r6rr!rr#r$)r'r?r r8r rrrÚscanResultHistoryès zSpiderFootDb.scanResultHistorycCsˆd}|g}x|D]}|d|d}qW|d7}y|jj||ƒ|jjƒStjk r‚}z|jjd|jdƒWYdd}~XnXdS)Na4SELECT ROUND(c.generated) AS generated, c.data, s.data as 'source_data', c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id, c.false_positive as 'fp', s.false_positive as 'parent_fp' FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND s.scan_instance_id = c.scan_instance_id AND t.event = c.type AND c.hash in (ú'z',z'')z7SQL error encountered when getting source element IDs: r)rr r6rr!rr#r$)r'r?Ú elementIdListr r8ÚhashIdr rrrÚscanElementSourcesDirectõs   z%SpiderFootDb.scanElementSourcesDirectcCsˆd}|g}x|D]}|d|d}qW|d7}y|jj||ƒ|jjƒStjk r‚}z|jjd|jdƒWYdd}~XnXdS)Na4SELECT ROUND(c.generated) AS generated, c.data, s.data as 'source_data', c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id, c.false_positive as 'fp', s.false_positive as 'parent_fp' FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND s.scan_instance_id = c.scan_instance_id AND t.event = c.type AND s.hash in (rvz',z'')z6SQL error encountered when getting child element IDs: r)rr r6rr!rr#r$)r'r?rwr r8rxr rrrÚscanElementChildrenDirects   z&SpiderFootDb.scanElementChildrenDirectc Cs d}tƒ}tƒ}tƒ}xd|D]\}|d}|d} ||| <||kr\| ||krf||j| ƒn | g||<||kr|j|ƒqWx’|r|j||ƒ} tƒ}d}xp| D]h}|d}|d} ||| <||krà| ||krê||j| ƒn | g||<||krü|j|ƒ|dkr d}q Wq~W|||<||gS)NTé éFÚROOT)r1rbr5ry) r'r?Z childDataÚ keepGoingÚnextIdsÚdatamapÚpcÚrowZparentIdZchildIdZ parentSetrrrÚscanElementSourcesAll*s@         z"SpiderFootDb.scanElementSourcesAllcCsÄtƒ}d}tƒ}|j||ƒ}x|D]}|j|dƒq"Wx&|D]}|d|kr>|j|dƒq>Wx^|r¾|j||ƒ}|dks†t|ƒdkrŒd}Px.|D]&}|j|dƒtƒ}|j|dƒq’WqbW|S)NTr|rF)r1rzr5Úlen)r'r?Z parentIdsr€r~rZnextSetr‚rrrÚscanElementChildrenAll_s&      z#SpiderFootDb.scanElementChildrenAll)F)F)N)NNN)r.)rMF)rMF)NNF)N)r)%Ú__name__Ú __module__Ú __qualname__rrrr+r%r*r"r,r9r:rBrErHrIrLrOrPrTrUrVrWrbrarfrgrirjrsrtruryrzrƒr…rrrrrsp   +  .         ( 5r)rrr=ÚsflibrrrrrrrÚ s