Add Android forensics, IOC threat-intel DB, Compose companion; scrub secrets from configs
This commit is contained in:
SsSnake
@@ -26,12 +26,63 @@ CATEGORY = "osint"
|
||||
class SnoopDecoder:
|
||||
"""Decoder for Snoop Project encoded databases."""
|
||||
|
||||
# Raw GitHub locations of the Snoop Project site databases.
|
||||
SNOOP_DB_URLS = {
|
||||
'BDfull': "https://raw.githubusercontent.com/snooppr/snoop/master/BDfull",
|
||||
'BDdemo': "https://raw.githubusercontent.com/snooppr/snoop/master/BDdemo",
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.sites_db = SitesDatabase()
|
||||
from core.paths import get_data_dir
|
||||
self.data_dir = get_data_dir() / "sites"
|
||||
self.data_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def download_database(self, which: str = "BDfull", dest: str = None) -> str:
|
||||
"""Download a Snoop database from the official GitHub repository.
|
||||
|
||||
Args:
|
||||
which: Which database to fetch ("BDfull" or "BDdemo").
|
||||
dest: Optional destination path. Defaults to
|
||||
data/sites/snoop/<which>.
|
||||
|
||||
Returns:
|
||||
Path to the downloaded file, or None on failure.
|
||||
"""
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
|
||||
which = 'BDfull' if which.lower() == 'bdfull' else ('BDdemo' if which.lower() == 'bddemo' else which)
|
||||
url = self.SNOOP_DB_URLS.get(which)
|
||||
if not url:
|
||||
print(f"{Colors.RED}[X] Unknown Snoop database: {which}{Colors.RESET}")
|
||||
return None
|
||||
|
||||
if dest is None:
|
||||
snoop_dir = self.data_dir / "snoop"
|
||||
snoop_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest_path = snoop_dir / which
|
||||
else:
|
||||
dest_path = Path(dest)
|
||||
dest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"{Colors.CYAN}[*] Downloading {which} from GitHub...{Colors.RESET}")
|
||||
print(f"{Colors.DIM} {url}{Colors.RESET}")
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={'User-Agent': 'AUTARCH-SnoopDecoder'})
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
content = resp.read()
|
||||
except (urllib.error.URLError, urllib.error.HTTPError, OSError) as e:
|
||||
print(f"{Colors.RED}[X] Download failed: {e}{Colors.RESET}")
|
||||
return None
|
||||
|
||||
dest_path.write_bytes(content)
|
||||
size_mb = len(content) / 1024 / 1024
|
||||
print(f"{Colors.GREEN}[+] Downloaded {size_mb:.2f} MB -> {dest_path}{Colors.RESET}")
|
||||
|
||||
return str(dest_path)
|
||||
|
||||
def decode_database(self, filepath: str) -> dict:
|
||||
"""Decode a Snoop database file.
|
||||
|
||||
@@ -103,11 +154,16 @@ class SnoopDecoder:
|
||||
|
||||
return str(output_path)
|
||||
|
||||
def import_to_database(self, data: dict) -> dict:
|
||||
"""Import decoded Snoop data into AUTARCH sites database.
|
||||
def import_to_database(self, data: dict, source: str = 'snoop') -> dict:
|
||||
"""Import decoded Snoop data into the AUTARCH sites database.
|
||||
|
||||
Only sites whose name isn't already in the database are added; the
|
||||
caller should run deduplicate() afterwards to collapse any URL-level
|
||||
duplicates against existing entries.
|
||||
|
||||
Args:
|
||||
data: Decoded site dictionary.
|
||||
source: Source label stored with each imported site.
|
||||
|
||||
Returns:
|
||||
Import statistics.
|
||||
@@ -127,50 +183,61 @@ class SnoopDecoder:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Get error type - handle encoding issues in key name
|
||||
# Snoop's errorType key name is byte-corrupted in the dumps
|
||||
# ("errorTyp" + garbage), so match on the prefix. Values line up
|
||||
# with the sites DB error_type vocabulary directly
|
||||
# (status_code / message / redirection / response_url).
|
||||
error_type = None
|
||||
for key in entry.keys():
|
||||
if 'errorTyp' in key or 'errortype' in key.lower():
|
||||
error_type = entry[key]
|
||||
break
|
||||
|
||||
# Map Snoop error types to detection methods
|
||||
detection_method = 'status'
|
||||
if error_type:
|
||||
if 'message' in str(error_type).lower():
|
||||
detection_method = 'content'
|
||||
elif 'redirect' in str(error_type).lower():
|
||||
detection_method = 'redirect'
|
||||
|
||||
# Get error message pattern
|
||||
error_pattern = None
|
||||
for key in ['errorMsg', 'errorMsg2']:
|
||||
if key in entry and entry[key]:
|
||||
error_pattern = str(entry[key])
|
||||
# First non-empty error message = the "not found" signature string.
|
||||
error_string = None
|
||||
for key in ('errorMsg', 'errorMsg2', 'errorMsg3'):
|
||||
if entry.get(key):
|
||||
error_string = str(entry[key])
|
||||
break
|
||||
|
||||
# bad_site is a non-empty string when Snoop flags a site as
|
||||
# broken/unreliable — import it but leave it disabled.
|
||||
bad = bool(str(entry.get('bad_site', '')).strip())
|
||||
|
||||
sites_to_add.append({
|
||||
'name': name,
|
||||
'url_template': url,
|
||||
'url_main': entry.get('urlMain'),
|
||||
'detection_method': detection_method,
|
||||
'error_pattern': error_pattern,
|
||||
'category': 'other',
|
||||
'source': source,
|
||||
'nsfw': 0,
|
||||
'enabled': 0 if bad else 1,
|
||||
'error_type': error_type,
|
||||
'error_string': error_string,
|
||||
})
|
||||
|
||||
print(f"{Colors.DIM} Valid sites: {len(sites_to_add):,}{Colors.RESET}")
|
||||
print(f"{Colors.DIM} Skipped: {skipped:,}{Colors.RESET}")
|
||||
print(f"{Colors.DIM} Skipped (no template): {skipped:,}{Colors.RESET}")
|
||||
|
||||
# Add to database
|
||||
# Add to database (new names only; existing names are left untouched)
|
||||
stats = self.sites_db.add_sites_bulk(sites_to_add)
|
||||
|
||||
print(f"{Colors.GREEN}[+] Import complete!{Colors.RESET}")
|
||||
print(f"{Colors.DIM} Added: {stats['added']:,}{Colors.RESET}")
|
||||
print(f"{Colors.DIM} Errors: {stats['errors']:,}{Colors.RESET}")
|
||||
print(f"{Colors.DIM} Added (new): {stats['added']:,}{Colors.RESET}")
|
||||
print(f"{Colors.DIM} Already present: {stats['skipped']:,}{Colors.RESET}")
|
||||
print(f"{Colors.DIM} Errors: {stats['errors']:,}{Colors.RESET}")
|
||||
|
||||
return stats
|
||||
|
||||
def deduplicate_database(self) -> dict:
|
||||
"""Collapse duplicate URL templates in the sites database."""
|
||||
print(f"\n{Colors.CYAN}[*] Deduplicating sites database...{Colors.RESET}")
|
||||
stats = self.sites_db.deduplicate()
|
||||
print(f"{Colors.GREEN}[+] Deduplication complete!{Colors.RESET}")
|
||||
print(f"{Colors.DIM} Duplicate groups: {stats['duplicate_groups']:,}{Colors.RESET}")
|
||||
print(f"{Colors.DIM} Rows removed: {stats['removed']:,}{Colors.RESET}")
|
||||
print(f"{Colors.DIM} Sites remaining: {stats['remaining']:,}{Colors.RESET}")
|
||||
return stats
|
||||
|
||||
def show_sample(self, data: dict, count: int = 10):
|
||||
"""Display sample sites from decoded database.
|
||||
|
||||
@@ -243,6 +310,9 @@ def display_menu():
|
||||
{Colors.GREEN}[4]{Colors.RESET} Quick Import (BDfull from snoop-master)
|
||||
{Colors.GREEN}[5]{Colors.RESET} Quick Import (BDdemo from snoop-master)
|
||||
|
||||
{Colors.GREEN}[6]{Colors.RESET} Download Latest BDfull from GitHub + Import + Dedup
|
||||
{Colors.GREEN}[7]{Colors.RESET} Deduplicate Sites Database
|
||||
|
||||
{Colors.RED}[0]{Colors.RESET} Back to OSINT Menu
|
||||
""")
|
||||
|
||||
@@ -323,8 +393,9 @@ def run():
|
||||
if confirm == 'y':
|
||||
# Save first
|
||||
decoder.save_decoded(data, "snoop_imported.json")
|
||||
# Then import
|
||||
# Then import + deduplicate
|
||||
decoder.import_to_database(data)
|
||||
decoder.deduplicate_database()
|
||||
|
||||
# Show final stats
|
||||
db_stats = decoder.sites_db.get_stats()
|
||||
@@ -364,6 +435,7 @@ def run():
|
||||
if confirm == 'y':
|
||||
decoder.save_decoded(data, "snoop_full.json")
|
||||
decoder.import_to_database(data)
|
||||
decoder.deduplicate_database()
|
||||
|
||||
db_stats = decoder.sites_db.get_stats()
|
||||
print(f"\n{Colors.GREEN}AUTARCH Database now has {db_stats['total_sites']:,} sites!{Colors.RESET}")
|
||||
@@ -388,10 +460,37 @@ def run():
|
||||
if confirm == 'y':
|
||||
decoder.save_decoded(data, "snoop_demo.json")
|
||||
decoder.import_to_database(data)
|
||||
decoder.deduplicate_database()
|
||||
|
||||
db_stats = decoder.sites_db.get_stats()
|
||||
print(f"\n{Colors.GREEN}AUTARCH Database now has {db_stats['total_sites']:,} sites!{Colors.RESET}")
|
||||
|
||||
elif choice == '6':
|
||||
# Download latest BDfull from GitHub, decode, import, deduplicate
|
||||
bdpath = decoder.download_database("BDfull")
|
||||
if not bdpath:
|
||||
continue
|
||||
|
||||
data = decoder.decode_database(bdpath)
|
||||
if data:
|
||||
decoder.show_sample(data, 5)
|
||||
|
||||
confirm = input(f"\n{Colors.YELLOW}Import {len(data):,} sites to AUTARCH? (y/n): {Colors.RESET}").strip().lower()
|
||||
if confirm == 'y':
|
||||
decoder.save_decoded(data, "snoop_full.json")
|
||||
decoder.import_to_database(data)
|
||||
decoder.deduplicate_database()
|
||||
|
||||
db_stats = decoder.sites_db.get_stats()
|
||||
print(f"\n{Colors.GREEN}AUTARCH Database now has {db_stats['total_sites']:,} sites!{Colors.RESET}")
|
||||
|
||||
elif choice == '7':
|
||||
# Deduplicate the sites database
|
||||
decoder.deduplicate_database()
|
||||
db_stats = decoder.sites_db.get_stats()
|
||||
print(f"\n{Colors.GREEN}AUTARCH Database now has {db_stats['total_sites']:,} sites!{Colors.RESET}")
|
||||
input(f"\n{Colors.DIM}Press Enter to continue...{Colors.RESET}")
|
||||
|
||||
else:
|
||||
print(f"{Colors.RED}[!] Invalid option{Colors.RESET}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user