autarch/modules/snoop_decoder.py

"""
AUTARCH Snoop Database Decoder Module
Decrypts and imports Snoop Project databases into AUTARCH
"""

import base64
import json
import os
import sys
from pathlib import Path

# Add parent directory to path for imports
sys.path.insert(0, str(Path(__file__).parent.parent))

from core.banner import Colors
from core.sites_db import SitesDatabase

# Module metadata
NAME = "Snoop Decoder"
DESCRIPTION = "Decrypt and import Snoop Project databases"
AUTHOR = "darkHal Security Group"
VERSION = "1.0"
CATEGORY = "osint"


class SnoopDecoder:
    """Decoder for Snoop Project encoded databases."""

    # Raw GitHub locations of the Snoop Project site databases.
    SNOOP_DB_URLS = {
        'BDfull': "https://raw.githubusercontent.com/snooppr/snoop/master/BDfull",
        'BDdemo': "https://raw.githubusercontent.com/snooppr/snoop/master/BDdemo",
    }

    def __init__(self):
        self.sites_db = SitesDatabase()
        from core.paths import get_data_dir
        self.data_dir = get_data_dir() / "sites"
        self.data_dir.mkdir(parents=True, exist_ok=True)

    def download_database(self, which: str = "BDfull", dest: str = None) -> str:
        """Download a Snoop database from the official GitHub repository.

        Args:
            which: Which database to fetch ("BDfull" or "BDdemo").
            dest: Optional destination path. Defaults to
                  data/sites/snoop/<which>.

        Returns:
            Path to the downloaded file, or None on failure.
        """
        import urllib.request
        import urllib.error

        which = 'BDfull' if which.lower() == 'bdfull' else ('BDdemo' if which.lower() == 'bddemo' else which)
        url = self.SNOOP_DB_URLS.get(which)
        if not url:
            print(f"{Colors.RED}[X] Unknown Snoop database: {which}{Colors.RESET}")
            return None

        if dest is None:
            snoop_dir = self.data_dir / "snoop"
            snoop_dir.mkdir(parents=True, exist_ok=True)
            dest_path = snoop_dir / which
        else:
            dest_path = Path(dest)
            dest_path.parent.mkdir(parents=True, exist_ok=True)

        print(f"{Colors.CYAN}[*] Downloading {which} from GitHub...{Colors.RESET}")
        print(f"{Colors.DIM}    {url}{Colors.RESET}")

        try:
            req = urllib.request.Request(url, headers={'User-Agent': 'AUTARCH-SnoopDecoder'})
            with urllib.request.urlopen(req, timeout=60) as resp:
                content = resp.read()
        except (urllib.error.URLError, urllib.error.HTTPError, OSError) as e:
            print(f"{Colors.RED}[X] Download failed: {e}{Colors.RESET}")
            return None

        dest_path.write_bytes(content)
        size_mb = len(content) / 1024 / 1024
        print(f"{Colors.GREEN}[+] Downloaded {size_mb:.2f} MB -> {dest_path}{Colors.RESET}")

        return str(dest_path)

    def decode_database(self, filepath: str) -> dict:
        """Decode a Snoop database file.

        Args:
            filepath: Path to the encoded database file (BDdemo, BDfull, etc.)

        Returns:
            Decoded dictionary of sites.
        """
        print(f"{Colors.CYAN}[*] Reading encoded database...{Colors.RESET}")

        with open(filepath, 'r', encoding='utf8') as f:
            db = f.read().strip()

        original_size = len(db)
        print(f"{Colors.DIM}    Original size: {original_size:,} chars{Colors.RESET}")

        # Step 1: Decode base32
        print(f"{Colors.CYAN}[*] Decoding base32...{Colors.RESET}")
        try:
            db_bytes = base64.b32decode(db)
        except Exception as e:
            print(f"{Colors.RED}[X] Base32 decode failed: {e}{Colors.RESET}")
            return None

        print(f"{Colors.DIM}    After base32: {len(db_bytes):,} bytes{Colors.RESET}")

        # Step 2: Reverse bytes
        print(f"{Colors.CYAN}[*] Reversing byte order...{Colors.RESET}")
        db_bytes = db_bytes[::-1]

        # Step 3: Decode UTF-8 with error handling
        print(f"{Colors.CYAN}[*] Decoding UTF-8...{Colors.RESET}")
        content = db_bytes.decode('utf-8', errors='replace')

        # Step 4: Reverse string
        print(f"{Colors.CYAN}[*] Reversing string...{Colors.RESET}")
        content = content[::-1]

        # Step 5: Parse JSON
        print(f"{Colors.CYAN}[*] Parsing JSON...{Colors.RESET}")
        try:
            data = json.loads(content)
        except json.JSONDecodeError as e:
            print(f"{Colors.RED}[X] JSON parse failed: {e}{Colors.RESET}")
            return None

        print(f"{Colors.GREEN}[+] Successfully decoded {len(data):,} sites!{Colors.RESET}")
        return data

    def save_decoded(self, data: dict, output_name: str = "snoop_decoded.json") -> str:
        """Save decoded database to JSON file.

        Args:
            data: Decoded site dictionary.
            output_name: Output filename.

        Returns:
            Path to saved file.
        """
        output_path = self.data_dir / output_name

        with open(output_path, 'w', encoding='utf8') as f:
            json.dump(data, f, indent=2, ensure_ascii=False)

        size_mb = output_path.stat().st_size / 1024 / 1024
        print(f"{Colors.GREEN}[+] Saved to: {output_path}{Colors.RESET}")
        print(f"{Colors.DIM}    File size: {size_mb:.2f} MB{Colors.RESET}")

        return str(output_path)

    def import_to_database(self, data: dict, source: str = 'snoop') -> dict:
        """Import decoded Snoop data into the AUTARCH sites database.

        Only sites whose name isn't already in the database are added; the
        caller should run deduplicate() afterwards to collapse any URL-level
        duplicates against existing entries.

        Args:
            data: Decoded site dictionary.
            source: Source label stored with each imported site.

        Returns:
            Import statistics.
        """
        print(f"\n{Colors.CYAN}[*] Importing to AUTARCH database...{Colors.RESET}")

        sites_to_add = []
        skipped = 0

        for name, entry in data.items():
            if not isinstance(entry, dict):
                skipped += 1
                continue

            url = entry.get('url', '')
            if not url or '{}' not in url:
                skipped += 1
                continue

            # Snoop's errorType key name is byte-corrupted in the dumps
            # ("errorTyp" + garbage), so match on the prefix. Values line up
            # with the sites DB error_type vocabulary directly
            # (status_code / message / redirection / response_url).
            error_type = None
            for key in entry.keys():
                if 'errorTyp' in key or 'errortype' in key.lower():
                    error_type = entry[key]
                    break

            # First non-empty error message = the "not found" signature string.
            error_string = None
            for key in ('errorMsg', 'errorMsg2', 'errorMsg3'):
                if entry.get(key):
                    error_string = str(entry[key])
                    break

            # bad_site is a non-empty string when Snoop flags a site as
            # broken/unreliable — import it but leave it disabled.
            bad = bool(str(entry.get('bad_site', '')).strip())

            sites_to_add.append({
                'name': name,
                'url_template': url,
                'category': 'other',
                'source': source,
                'nsfw': 0,
                'enabled': 0 if bad else 1,
                'error_type': error_type,
                'error_string': error_string,
            })

        print(f"{Colors.DIM}    Valid sites: {len(sites_to_add):,}{Colors.RESET}")
        print(f"{Colors.DIM}    Skipped (no template): {skipped:,}{Colors.RESET}")

        # Add to database (new names only; existing names are left untouched)
        stats = self.sites_db.add_sites_bulk(sites_to_add)

        print(f"{Colors.GREEN}[+] Import complete!{Colors.RESET}")
        print(f"{Colors.DIM}    Added (new):     {stats['added']:,}{Colors.RESET}")
        print(f"{Colors.DIM}    Already present: {stats['skipped']:,}{Colors.RESET}")
        print(f"{Colors.DIM}    Errors:          {stats['errors']:,}{Colors.RESET}")

        return stats

    def deduplicate_database(self) -> dict:
        """Collapse duplicate URL templates in the sites database."""
        print(f"\n{Colors.CYAN}[*] Deduplicating sites database...{Colors.RESET}")
        stats = self.sites_db.deduplicate()
        print(f"{Colors.GREEN}[+] Deduplication complete!{Colors.RESET}")
        print(f"{Colors.DIM}    Duplicate groups: {stats['duplicate_groups']:,}{Colors.RESET}")
        print(f"{Colors.DIM}    Rows removed:     {stats['removed']:,}{Colors.RESET}")
        print(f"{Colors.DIM}    Sites remaining:  {stats['remaining']:,}{Colors.RESET}")
        return stats

    def show_sample(self, data: dict, count: int = 10):
        """Display sample sites from decoded database.

        Args:
            data: Decoded site dictionary.
            count: Number of samples to show.
        """
        print(f"\n{Colors.CYAN}Sample Sites ({count}):{Colors.RESET}")
        print("-" * 60)

        for i, (name, info) in enumerate(list(data.items())[:count]):
            url = info.get('url', 'N/A')
            country = info.get('country', '')
            print(f"  {country} {Colors.GREEN}{name}{Colors.RESET}")
            print(f"     {Colors.DIM}{url[:55]}...{Colors.RESET}" if len(url) > 55 else f"     {Colors.DIM}{url}{Colors.RESET}")

    def get_stats(self, data: dict) -> dict:
        """Get statistics about decoded database.

        Args:
            data: Decoded site dictionary.

        Returns:
            Statistics dictionary.
        """
        stats = {
            'total_sites': len(data),
            'by_country': {},
            'detection_methods': {'status_code': 0, 'message': 0, 'redirection': 0, 'other': 0},
        }

        for name, info in data.items():
            # Country stats
            country = info.get('country_klas', 'Unknown')
            stats['by_country'][country] = stats['by_country'].get(country, 0) + 1

            # Detection method stats
            error_type = None
            for key in info.keys():
                if 'errorTyp' in key:
                    error_type = str(info[key]).lower()
                    break

            if error_type:
                if 'status' in error_type:
                    stats['detection_methods']['status_code'] += 1
                elif 'message' in error_type:
                    stats['detection_methods']['message'] += 1
                elif 'redirect' in error_type:
                    stats['detection_methods']['redirection'] += 1
                else:
                    stats['detection_methods']['other'] += 1
            else:
                stats['detection_methods']['other'] += 1

        return stats


def display_menu():
    """Display the Snoop Decoder menu."""
    print(f"""
{Colors.CYAN}  Snoop Database Decoder{Colors.RESET}
{Colors.DIM}  Decrypt and import Snoop Project databases{Colors.RESET}
{Colors.DIM}{'─' * 50}{Colors.RESET}

  {Colors.GREEN}[1]{Colors.RESET} Decode Snoop Database File
  {Colors.GREEN}[2]{Colors.RESET} Decode & Import to AUTARCH
  {Colors.GREEN}[3]{Colors.RESET} View Current Sites Database Stats

  {Colors.GREEN}[4]{Colors.RESET} Quick Import (BDfull from snoop-master)
  {Colors.GREEN}[5]{Colors.RESET} Quick Import (BDdemo from snoop-master)

  {Colors.GREEN}[6]{Colors.RESET} Download Latest BDfull from GitHub + Import + Dedup
  {Colors.GREEN}[7]{Colors.RESET} Deduplicate Sites Database

  {Colors.RED}[0]{Colors.RESET} Back to OSINT Menu
""")


def get_file_path() -> str:
    """Prompt user for file path."""
    print(f"\n{Colors.CYAN}Enter path to Snoop database file:{Colors.RESET}")
    print(f"{Colors.DIM}(e.g., /path/to/BDfull or /path/to/BDdemo){Colors.RESET}")

    filepath = input(f"\n{Colors.GREEN}Path: {Colors.RESET}").strip()

    if not filepath:
        return None

    if not os.path.exists(filepath):
        print(f"{Colors.RED}[X] File not found: {filepath}{Colors.RESET}")
        return None

    return filepath


def run():
    """Main entry point for the module."""
    decoder = SnoopDecoder()

    # Common paths for Snoop databases
    from core.paths import get_app_dir, get_data_dir
    _app = get_app_dir()
    _data = get_data_dir()
    snoop_paths = {
        'bdfull': _app / "snoop" / "snoop-master" / "BDfull",
        'bddemo': _app / "snoop" / "snoop-master" / "BDdemo",
        'bdfull_alt': _data / "snoop" / "BDfull",
        'bddemo_alt': _data / "snoop" / "BDdemo",
    }

    while True:
        display_menu()

        choice = input(f"{Colors.GREEN}Select option: {Colors.RESET}").strip()

        if choice == '0':
            break

        elif choice == '1':
            # Decode only
            filepath = get_file_path()
            if not filepath:
                continue

            data = decoder.decode_database(filepath)
            if data:
                decoder.show_sample(data)

                stats = decoder.get_stats(data)
                print(f"\n{Colors.CYAN}Database Statistics:{Colors.RESET}")
                print(f"  Total sites: {stats['total_sites']:,}")
                print(f"  Detection methods: {stats['detection_methods']}")
                print(f"  Top countries: {dict(sorted(stats['by_country'].items(), key=lambda x: -x[1])[:10])}")

                # Ask to save
                save = input(f"\n{Colors.YELLOW}Save decoded JSON? (y/n): {Colors.RESET}").strip().lower()
                if save == 'y':
                    name = input(f"{Colors.GREEN}Output filename [snoop_decoded.json]: {Colors.RESET}").strip()
                    decoder.save_decoded(data, name if name else "snoop_decoded.json")

        elif choice == '2':
            # Decode and import
            filepath = get_file_path()
            if not filepath:
                continue

            data = decoder.decode_database(filepath)
            if data:
                decoder.show_sample(data, 5)

                confirm = input(f"\n{Colors.YELLOW}Import {len(data):,} sites to AUTARCH? (y/n): {Colors.RESET}").strip().lower()
                if confirm == 'y':
                    # Save first
                    decoder.save_decoded(data, "snoop_imported.json")
                    # Then import + deduplicate
                    decoder.import_to_database(data)
                    decoder.deduplicate_database()

                    # Show final stats
                    db_stats = decoder.sites_db.get_stats()
                    print(f"\n{Colors.GREEN}AUTARCH Database now has {db_stats['total_sites']:,} sites!{Colors.RESET}")

        elif choice == '3':
            # View current stats
            stats = decoder.sites_db.get_stats()
            print(f"\n{Colors.CYAN}AUTARCH Sites Database:{Colors.RESET}")
            print(f"  Total sites: {stats['total_sites']:,}")
            print(f"  NSFW sites: {stats['nsfw_sites']:,}")
            print(f"  Database size: {stats['db_size_mb']:.2f} MB")
            print(f"\n  {Colors.CYAN}By Source:{Colors.RESET}")
            for source, count in sorted(stats['by_source'].items(), key=lambda x: -x[1]):
                print(f"    {source}: {count:,}")
            input(f"\n{Colors.DIM}Press Enter to continue...{Colors.RESET}")

        elif choice == '4':
            # Quick import BDfull
            bdpath = None
            for key in ['bdfull', 'bdfull_alt']:
                if snoop_paths[key].exists():
                    bdpath = str(snoop_paths[key])
                    break

            if not bdpath:
                print(f"{Colors.RED}[X] BDfull not found in known locations{Colors.RESET}")
                print(f"{Colors.DIM}    Checked: {snoop_paths['bdfull']}{Colors.RESET}")
                print(f"{Colors.DIM}    Checked: {snoop_paths['bdfull_alt']}{Colors.RESET}")
                continue

            print(f"{Colors.GREEN}[+] Found BDfull: {bdpath}{Colors.RESET}")

            data = decoder.decode_database(bdpath)
            if data:
                confirm = input(f"\n{Colors.YELLOW}Import {len(data):,} sites? (y/n): {Colors.RESET}").strip().lower()
                if confirm == 'y':
                    decoder.save_decoded(data, "snoop_full.json")
                    decoder.import_to_database(data)
                    decoder.deduplicate_database()

                    db_stats = decoder.sites_db.get_stats()
                    print(f"\n{Colors.GREEN}AUTARCH Database now has {db_stats['total_sites']:,} sites!{Colors.RESET}")

        elif choice == '5':
            # Quick import BDdemo
            bdpath = None
            for key in ['bddemo', 'bddemo_alt']:
                if snoop_paths[key].exists():
                    bdpath = str(snoop_paths[key])
                    break

            if not bdpath:
                print(f"{Colors.RED}[X] BDdemo not found in known locations{Colors.RESET}")
                continue

            print(f"{Colors.GREEN}[+] Found BDdemo: {bdpath}{Colors.RESET}")

            data = decoder.decode_database(bdpath)
            if data:
                confirm = input(f"\n{Colors.YELLOW}Import {len(data):,} sites? (y/n): {Colors.RESET}").strip().lower()
                if confirm == 'y':
                    decoder.save_decoded(data, "snoop_demo.json")
                    decoder.import_to_database(data)
                    decoder.deduplicate_database()

                    db_stats = decoder.sites_db.get_stats()
                    print(f"\n{Colors.GREEN}AUTARCH Database now has {db_stats['total_sites']:,} sites!{Colors.RESET}")

        elif choice == '6':
            # Download latest BDfull from GitHub, decode, import, deduplicate
            bdpath = decoder.download_database("BDfull")
            if not bdpath:
                continue

            data = decoder.decode_database(bdpath)
            if data:
                decoder.show_sample(data, 5)

                confirm = input(f"\n{Colors.YELLOW}Import {len(data):,} sites to AUTARCH? (y/n): {Colors.RESET}").strip().lower()
                if confirm == 'y':
                    decoder.save_decoded(data, "snoop_full.json")
                    decoder.import_to_database(data)
                    decoder.deduplicate_database()

                    db_stats = decoder.sites_db.get_stats()
                    print(f"\n{Colors.GREEN}AUTARCH Database now has {db_stats['total_sites']:,} sites!{Colors.RESET}")

        elif choice == '7':
            # Deduplicate the sites database
            decoder.deduplicate_database()
            db_stats = decoder.sites_db.get_stats()
            print(f"\n{Colors.GREEN}AUTARCH Database now has {db_stats['total_sites']:,} sites!{Colors.RESET}")
            input(f"\n{Colors.DIM}Press Enter to continue...{Colors.RESET}")

        else:
            print(f"{Colors.RED}[!] Invalid option{Colors.RESET}")


if __name__ == "__main__":
    run()