Add Android forensics, IOC threat-intel DB, Compose companion; scrub secrets from configs
This commit is contained in:
SsSnake
184
core/sites_db.py
184
core/sites_db.py
@@ -49,6 +49,12 @@ class SitesDatabase:
|
||||
self._conn = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Ensure the schema exists, then seed from the master dh.json the first
|
||||
# time (or whenever the cache is empty). dh.json is the source of truth;
|
||||
# dh_sites.db is a fast local query cache rebuilt from it automatically.
|
||||
self._ensure_schema()
|
||||
self._auto_populate()
|
||||
|
||||
def _get_connection(self) -> sqlite3.Connection:
|
||||
"""Get thread-safe database connection."""
|
||||
if self._conn is None:
|
||||
@@ -56,6 +62,48 @@ class SitesDatabase:
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
return self._conn
|
||||
|
||||
def _ensure_schema(self) -> None:
|
||||
"""Create the sites table and indexes if they don't already exist."""
|
||||
with self._lock:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS sites (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
url_template TEXT NOT NULL,
|
||||
category TEXT DEFAULT 'other',
|
||||
source TEXT DEFAULT 'dh',
|
||||
nsfw INTEGER DEFAULT 0,
|
||||
enabled INTEGER DEFAULT 1,
|
||||
error_type TEXT,
|
||||
error_code INTEGER,
|
||||
error_string TEXT,
|
||||
match_code INTEGER,
|
||||
match_string TEXT
|
||||
)
|
||||
""")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_sites_category ON sites(category)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_sites_source ON sites(source)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_sites_enabled ON sites(enabled)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_sites_nsfw ON sites(nsfw)")
|
||||
conn.commit()
|
||||
|
||||
def _auto_populate(self) -> None:
|
||||
"""Seed the cache from the master dh.json when the sites table is empty."""
|
||||
with self._lock:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT COUNT(*) FROM sites")
|
||||
count = cursor.fetchone()[0]
|
||||
|
||||
if count > 0:
|
||||
return
|
||||
|
||||
json_path = self.data_dir / "dh.json"
|
||||
if json_path.exists():
|
||||
self.load_from_json(json_path)
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get database statistics."""
|
||||
with self._lock:
|
||||
@@ -390,6 +438,142 @@ class SitesDatabase:
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def add_sites_bulk(self, sites: List[Dict], overwrite: bool = False) -> Dict[str, int]:
|
||||
"""Bulk-insert sites in a single transaction.
|
||||
|
||||
By default only sites with a NOT-yet-seen name are added; existing
|
||||
names are left untouched (INSERT OR IGNORE). Pass overwrite=True to
|
||||
replace existing rows instead.
|
||||
|
||||
Args:
|
||||
sites: List of site dicts. Recognised keys: name, url_template
|
||||
(or url), category, source, nsfw, enabled, error_type,
|
||||
error_code, error_string, match_code, match_string.
|
||||
overwrite: Replace existing rows sharing the same name.
|
||||
|
||||
Returns:
|
||||
Stats dict: {'total', 'added', 'skipped', 'errors'}.
|
||||
"""
|
||||
stats = {'total': len(sites), 'added': 0, 'skipped': 0, 'errors': 0}
|
||||
verb = "INSERT OR REPLACE" if overwrite else "INSERT OR IGNORE"
|
||||
|
||||
with self._lock:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
for site in sites:
|
||||
name = site.get('name')
|
||||
url = site.get('url_template') or site.get('url')
|
||||
if not name or not url:
|
||||
stats['errors'] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
cursor.execute(f"""
|
||||
{verb} INTO sites
|
||||
(name, url_template, category, source, nsfw, enabled,
|
||||
error_type, error_code, error_string, match_code, match_string)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
name,
|
||||
url,
|
||||
site.get('category', 'other'),
|
||||
site.get('source', 'custom'),
|
||||
1 if site.get('nsfw') else 0,
|
||||
0 if site.get('enabled', 1) in (0, False) else 1,
|
||||
site.get('error_type'),
|
||||
site.get('error_code'),
|
||||
site.get('error_string'),
|
||||
site.get('match_code'),
|
||||
site.get('match_string'),
|
||||
))
|
||||
if cursor.rowcount > 0:
|
||||
stats['added'] += 1
|
||||
else:
|
||||
stats['skipped'] += 1
|
||||
except Exception:
|
||||
stats['errors'] += 1
|
||||
|
||||
conn.commit()
|
||||
|
||||
return stats
|
||||
|
||||
@staticmethod
|
||||
def _normalize_url(url: str) -> str:
|
||||
"""Normalise a URL template for duplicate detection.
|
||||
|
||||
Strips scheme, leading www., case, and trailing slash so that
|
||||
http/https, www/non-www and case variants of the same profile URL
|
||||
collapse to one key.
|
||||
"""
|
||||
if not url:
|
||||
return ''
|
||||
u = url.strip().lower()
|
||||
for scheme in ('https://', 'http://'):
|
||||
if u.startswith(scheme):
|
||||
u = u[len(scheme):]
|
||||
break
|
||||
if u.startswith('www.'):
|
||||
u = u[4:]
|
||||
return u.rstrip('/')
|
||||
|
||||
def deduplicate(self) -> Dict[str, int]:
|
||||
"""Remove duplicate sites that share the same (normalised) URL template.
|
||||
|
||||
Names are already unique via the schema constraint; this collapses rows
|
||||
whose URL templates point at the same profile page. For each duplicate
|
||||
group the best row is kept (prefer enabled, then having a detection
|
||||
pattern, then the oldest/lowest id) and the rest are deleted.
|
||||
|
||||
Returns:
|
||||
Stats dict: {'duplicate_groups', 'removed', 'remaining'}.
|
||||
"""
|
||||
with self._lock:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute(
|
||||
"SELECT id, url_template, error_type, error_string, match_string, enabled FROM sites"
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
groups: Dict[str, list] = {}
|
||||
for r in rows:
|
||||
key = self._normalize_url(r['url_template'])
|
||||
if not key:
|
||||
continue
|
||||
groups.setdefault(key, []).append(r)
|
||||
|
||||
def score(r):
|
||||
has_detection = bool(r['error_string'] or r['match_string'] or r['error_type'])
|
||||
# Higher tuple wins; -id keeps the lowest (oldest) id.
|
||||
return (1 if r['enabled'] else 0, 1 if has_detection else 0, -r['id'])
|
||||
|
||||
to_delete = []
|
||||
dup_groups = 0
|
||||
for members in groups.values():
|
||||
if len(members) < 2:
|
||||
continue
|
||||
dup_groups += 1
|
||||
members_sorted = sorted(members, key=score, reverse=True)
|
||||
to_delete.extend(r['id'] for r in members_sorted[1:])
|
||||
|
||||
for i in range(0, len(to_delete), 500):
|
||||
chunk = to_delete[i:i + 500]
|
||||
placeholders = ','.join('?' * len(chunk))
|
||||
cursor.execute(f"DELETE FROM sites WHERE id IN ({placeholders})", chunk)
|
||||
|
||||
conn.commit()
|
||||
|
||||
cursor.execute("SELECT COUNT(*) FROM sites")
|
||||
remaining = cursor.fetchone()[0]
|
||||
|
||||
return {
|
||||
'duplicate_groups': dup_groups,
|
||||
'removed': len(to_delete),
|
||||
'remaining': remaining,
|
||||
}
|
||||
|
||||
def update_detection(
|
||||
self,
|
||||
name: str,
|
||||
|
||||
Reference in New Issue
Block a user