Add Android forensics, IOC threat-intel DB, Compose companion; scrub secrets from configs
This commit is contained in:
SsSnake
@@ -80,7 +80,7 @@ PARAMS: {{"question": "your question"}}
|
||||
self,
|
||||
llm: LLM = None,
|
||||
tools: ToolRegistry = None,
|
||||
max_steps: int = 20,
|
||||
max_steps: int = 200,
|
||||
verbose: bool = True
|
||||
):
|
||||
"""Initialize the agent.
|
||||
|
||||
871
core/android_forensics.py
Normal file
871
core/android_forensics.py
Normal file
@@ -0,0 +1,871 @@
|
||||
"""
|
||||
Autarch Android Forensics — forensic acquisition and IOC scanning for Android devices.
|
||||
by: SsSnake
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
_manager = None
|
||||
|
||||
|
||||
def get_android_forensics_manager():
|
||||
global _manager
|
||||
if _manager is None:
|
||||
_manager = AutarchAndroidManager()
|
||||
return _manager
|
||||
|
||||
|
||||
class AutarchAndroidManager:
|
||||
"""Forensic acquisition and IOC scanning for Android devices."""
|
||||
|
||||
IOC_DATA_DIR = Path(__file__).parent.parent / 'data' / 'ioc'
|
||||
ACQ_DATA_DIR = Path(__file__).parent.parent / 'data' / 'android_forensics'
|
||||
|
||||
def __init__(self):
|
||||
self._ioc_db = None # lazy-loaded
|
||||
|
||||
# ── Helpers ─────────────────────────────────────────────────────
|
||||
|
||||
def _hw(self):
|
||||
from core.hardware import get_hardware_manager
|
||||
return get_hardware_manager()
|
||||
|
||||
def _adb(self, serial, cmd, timeout=60):
|
||||
"""Run an ADB shell command, return stdout string."""
|
||||
result = self._hw().adb_shell_raw(serial, cmd, timeout=timeout)
|
||||
if isinstance(result, dict):
|
||||
return result.get('output', '') or ''
|
||||
return str(result or '')
|
||||
|
||||
def _acq_dir(self, serial, acq_id):
|
||||
d = self.ACQ_DATA_DIR / serial / acq_id
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
def _new_acq_id(self):
|
||||
return datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
|
||||
# ── IOC Database ────────────────────────────────────────────────
|
||||
|
||||
def load_ioc_database(self, force=False):
|
||||
"""Lazy-load and merge all IOC data from data/ioc/. Returns merged dict."""
|
||||
if self._ioc_db is not None and not force:
|
||||
return self._ioc_db
|
||||
|
||||
db = {
|
||||
'packages': {}, # pkg_name -> {name, source, severity}
|
||||
'certs': {}, # sha1/sha256 -> {name, source}
|
||||
'domains': {}, # domain -> {name, source}
|
||||
'ips': set(),
|
||||
'file_paths': {}, # path -> {name, source}
|
||||
'processes': {}, # proc_name -> {name, source}
|
||||
'hashes': {}, # sha256 -> {name, source}
|
||||
}
|
||||
|
||||
# 1. Stalkerware ioc.yaml + watchware.yaml
|
||||
for fname in ('ioc.yaml', 'watchware.yaml'):
|
||||
fpath = self.IOC_DATA_DIR / 'stalkerware' / fname
|
||||
if fpath.exists():
|
||||
self._parse_stalkerware_yaml(fpath, db)
|
||||
|
||||
# 2. Amnesty investigations
|
||||
amnesty_dir = self.IOC_DATA_DIR / 'spyware' / 'amnesty'
|
||||
if amnesty_dir.exists():
|
||||
for campaign_dir in sorted(amnesty_dir.iterdir()):
|
||||
if campaign_dir.is_dir():
|
||||
self._parse_investigation(campaign_dir, db)
|
||||
|
||||
# 3. MVT indicators
|
||||
mvt_dir = self.IOC_DATA_DIR / 'spyware' / 'mvt'
|
||||
if mvt_dir.exists():
|
||||
for campaign_dir in sorted(mvt_dir.iterdir()):
|
||||
if campaign_dir.is_dir():
|
||||
self._parse_investigation(campaign_dir, db)
|
||||
|
||||
# 4. Citizen Lab campaigns
|
||||
cl_dir = self.IOC_DATA_DIR / 'spyware' / 'citizen_lab'
|
||||
if cl_dir.exists():
|
||||
for campaign_dir in sorted(cl_dir.iterdir()):
|
||||
if campaign_dir.is_dir():
|
||||
self._parse_investigation(campaign_dir, db)
|
||||
|
||||
# 5. iMazing indicators (recursive)
|
||||
imazing_dir = self.IOC_DATA_DIR / 'spyware' / 'imazing'
|
||||
if imazing_dir.exists():
|
||||
for item in imazing_dir.rglob('*'):
|
||||
if item.is_dir():
|
||||
self._parse_investigation(item, db)
|
||||
|
||||
# 6. Existing stalkerware_signatures.json
|
||||
sigs_path = Path(__file__).parent.parent / 'data' / 'stalkerware_signatures.json'
|
||||
if sigs_path.exists():
|
||||
self._parse_sigs_json(sigs_path, db)
|
||||
|
||||
# Convert ips set to list for JSON serialisation
|
||||
db['ips'] = list(db['ips'])
|
||||
|
||||
self._ioc_db = db
|
||||
self._save_index(db)
|
||||
return db
|
||||
|
||||
def _parse_stalkerware_yaml(self, path, db):
|
||||
"""Parse stalkerware/watchware ioc.yaml into db."""
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
return
|
||||
try:
|
||||
with open(path, encoding='utf-8', errors='ignore') as f:
|
||||
data = yaml.safe_load(f)
|
||||
if not isinstance(data, dict):
|
||||
return
|
||||
for vendor, info in data.items():
|
||||
if not isinstance(info, dict):
|
||||
continue
|
||||
source = f'stalkerware/{path.name}:{vendor}'
|
||||
for pkg in info.get('packages', []) or []:
|
||||
if pkg:
|
||||
db['packages'][pkg] = {'name': vendor, 'source': source, 'severity': 'high'}
|
||||
for cert in info.get('certificates', []) or []:
|
||||
if cert:
|
||||
db['certs'][cert.lower()] = {'name': vendor, 'source': source}
|
||||
for domain in (info.get('domains', []) or []) + (info.get('c2', []) or []) + (info.get('website', []) or []):
|
||||
if domain:
|
||||
db['domains'][domain.lower()] = {'name': vendor, 'source': source}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _parse_investigation(self, campaign_dir, db):
|
||||
"""Parse a campaign directory for IOC text files and STIX2."""
|
||||
name = campaign_dir.name
|
||||
source = f'spyware/{campaign_dir.parent.name}/{name}'
|
||||
|
||||
for f in campaign_dir.iterdir():
|
||||
if not f.is_file():
|
||||
continue
|
||||
fname = f.name.lower()
|
||||
|
||||
if fname in ('domains.txt', 'domains'):
|
||||
self._load_txt_lines(f, db['domains'], name, source)
|
||||
elif fname in ('package_names.txt', 'package_names'):
|
||||
self._load_txt_lines(f, db['packages'], name, source, extra={'severity': 'critical'})
|
||||
elif fname in ('processes.txt', 'processes'):
|
||||
self._load_txt_lines(f, db['processes'], name, source)
|
||||
elif fname in ('file_paths.txt', 'file_paths'):
|
||||
self._load_txt_lines(f, db['file_paths'], name, source)
|
||||
elif fname in ('ips.txt', 'ip_addresses.txt', 'ip-addresses.txt'):
|
||||
self._load_txt_set(f, db['ips'])
|
||||
elif fname.endswith('.stix2') or fname.endswith('_stix.json'):
|
||||
self._parse_stix(f, db, name, source)
|
||||
elif fname in ('sha256.txt', 'sha256.csv'):
|
||||
self._load_hashes(f, db['hashes'], name, source)
|
||||
elif fname == 'package_cert_hashes.txt':
|
||||
self._load_txt_lines(f, db['certs'], name, source)
|
||||
elif fname.endswith('.json') and 'ioc' in fname:
|
||||
self._parse_ioc_json(f, db, name, source)
|
||||
elif fname.endswith('.csv') and 'ioc' in fname:
|
||||
self._parse_ioc_csv(f, db, name, source)
|
||||
|
||||
def _load_txt_lines(self, path, target_dict, name, source, extra=None):
|
||||
"""Load one-indicator-per-line text file into a dict."""
|
||||
try:
|
||||
with open(path, encoding='utf-8', errors='ignore') as f:
|
||||
for line in f:
|
||||
line = line.strip().lower()
|
||||
if line and not line.startswith('#'):
|
||||
entry = {'name': name, 'source': source}
|
||||
if extra:
|
||||
entry.update(extra)
|
||||
target_dict[line] = entry
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _load_txt_set(self, path, target_set):
|
||||
try:
|
||||
with open(path, encoding='utf-8', errors='ignore') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#'):
|
||||
target_set.add(line)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _load_hashes(self, path, target_dict, name, source):
|
||||
try:
|
||||
with open(path, encoding='utf-8', errors='ignore') as f:
|
||||
for line in f:
|
||||
parts = line.strip().split(',')
|
||||
h = parts[0].strip().lower()
|
||||
if h and len(h) in (32, 40, 64):
|
||||
target_dict[h] = {'name': name, 'source': source}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _parse_stix(self, path, db, name, source):
|
||||
"""Parse a STIX2 JSON file for indicators."""
|
||||
try:
|
||||
with open(path, encoding='utf-8', errors='ignore') as f:
|
||||
data = json.load(f)
|
||||
objects = data.get('objects', []) if isinstance(data, dict) else []
|
||||
for obj in objects:
|
||||
if obj.get('type') != 'indicator':
|
||||
continue
|
||||
pattern = obj.get('pattern', '')
|
||||
for m in re.findall(r"domain-name:value\s*=\s*'([^']+)'", pattern):
|
||||
db['domains'][m.lower()] = {'name': name, 'source': source}
|
||||
for m in re.findall(r"file:hashes\.'[^']+'\s*=\s*'([a-fA-F0-9]+)'", pattern):
|
||||
db['hashes'][m.lower()] = {'name': name, 'source': source}
|
||||
for m in re.findall(r"process:name\s*=\s*'([^']+)'", pattern):
|
||||
db['processes'][m.lower()] = {'name': name, 'source': source}
|
||||
for m in re.findall(r"(?:file:name|directory:path)\s*=\s*'([^']+)'", pattern):
|
||||
db['file_paths'][m] = {'name': name, 'source': source}
|
||||
for m in re.findall(r"ipv[46]-addr:value\s*=\s*'([^']+)'", pattern):
|
||||
db['ips'].add(m)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _parse_ioc_json(self, path, db, name, source):
|
||||
try:
|
||||
with open(path, encoding='utf-8', errors='ignore') as f:
|
||||
data = json.load(f)
|
||||
items = data.get('items', data.get('iocs', [])) if isinstance(data, dict) else data
|
||||
if not isinstance(items, list):
|
||||
return
|
||||
for item in items:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
itype = item.get('type', '').lower()
|
||||
val = str(item.get('value', item.get('indicator', ''))).strip().lower()
|
||||
if not val:
|
||||
continue
|
||||
if 'domain' in itype:
|
||||
db['domains'][val] = {'name': name, 'source': source}
|
||||
elif 'ip' in itype:
|
||||
db['ips'].add(val)
|
||||
elif 'hash' in itype or 'sha' in itype or 'md5' in itype:
|
||||
db['hashes'][val] = {'name': name, 'source': source}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _parse_ioc_csv(self, path, db, name, source):
|
||||
try:
|
||||
import csv
|
||||
with open(path, encoding='utf-8', errors='ignore', newline='') as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
val = (row.get('value') or row.get('indicator') or row.get('domain') or '').strip().lower()
|
||||
itype = (row.get('type') or row.get('indicator_type') or '').lower()
|
||||
if not val:
|
||||
continue
|
||||
if 'domain' in itype:
|
||||
db['domains'][val] = {'name': name, 'source': source}
|
||||
elif 'hash' in itype or 'sha256' in itype:
|
||||
db['hashes'][val] = {'name': name, 'source': source}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _parse_sigs_json(self, path, db):
|
||||
"""Merge existing stalkerware_signatures.json into db."""
|
||||
try:
|
||||
with open(path, encoding='utf-8') as f:
|
||||
data = json.load(f)
|
||||
for vendor, info in data.get('stalkerware', {}).items():
|
||||
source = f'stalkerware_signatures.json:{vendor}'
|
||||
sev = info.get('severity', 'high')
|
||||
for pkg in info.get('packages', []):
|
||||
if pkg and pkg not in db['packages']:
|
||||
db['packages'][pkg] = {'name': vendor, 'source': source, 'severity': sev}
|
||||
for spyware in data.get('government_spyware', []):
|
||||
name = spyware.get('name', 'unknown')
|
||||
source = f'stalkerware_signatures.json:{name}'
|
||||
for ind in spyware.get('indicators', []):
|
||||
itype = ind.get('type', '')
|
||||
val = ind.get('value', '')
|
||||
if not val:
|
||||
continue
|
||||
if itype == 'process':
|
||||
db['processes'][val.lower()] = {'name': name, 'source': source}
|
||||
elif itype == 'domain':
|
||||
db['domains'][val.lower()] = {'name': name, 'source': source, 'severity': 'critical'}
|
||||
elif itype == 'file_path':
|
||||
db['file_paths'][val] = {'name': name, 'source': source}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _save_index(self, db):
|
||||
try:
|
||||
def _count_dirs(d):
|
||||
return sum(1 for p in d.iterdir() if p.is_dir()) if d.exists() else 0
|
||||
|
||||
ips = db.get('ips', [])
|
||||
index = {
|
||||
'generated': datetime.now().isoformat(),
|
||||
'counts': {
|
||||
'packages': len(db.get('packages', {})),
|
||||
'certs': len(db.get('certs', {})),
|
||||
'domains': len(db.get('domains', {})),
|
||||
'ips': len(ips) if not isinstance(ips, set) else len(ips),
|
||||
'file_paths': len(db.get('file_paths', {})),
|
||||
'processes': len(db.get('processes', {})),
|
||||
'hashes': len(db.get('hashes', {})),
|
||||
},
|
||||
'sources': {
|
||||
'stalkerware_ioc_yaml': (self.IOC_DATA_DIR / 'stalkerware' / 'ioc.yaml').exists(),
|
||||
'amnesty_investigations': _count_dirs(self.IOC_DATA_DIR / 'spyware' / 'amnesty'),
|
||||
'mvt_campaigns': _count_dirs(self.IOC_DATA_DIR / 'spyware' / 'mvt'),
|
||||
'citizen_lab_campaigns': _count_dirs(self.IOC_DATA_DIR / 'spyware' / 'citizen_lab'),
|
||||
'yara_rules': sum(1 for _ in (self.IOC_DATA_DIR / 'yara').rglob('*.yar*'))
|
||||
if (self.IOC_DATA_DIR / 'yara').exists() else 0,
|
||||
},
|
||||
}
|
||||
self.IOC_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
with open(self.IOC_DATA_DIR / 'index.json', 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_ioc_stats(self):
|
||||
"""Return IOC database statistics (from index.json if available)."""
|
||||
idx_path = self.IOC_DATA_DIR / 'index.json'
|
||||
if idx_path.exists():
|
||||
try:
|
||||
with open(idx_path, encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
pass
|
||||
db = self.load_ioc_database()
|
||||
return {'counts': {k: len(v) for k, v in db.items() if isinstance(v, (dict, list))}}
|
||||
|
||||
def reload_ioc_database(self):
|
||||
"""Force reload all IOC data from disk."""
|
||||
self._ioc_db = None
|
||||
return self.load_ioc_database(force=True)
|
||||
|
||||
# ── Acquisition Methods ─────────────────────────────────────────
|
||||
|
||||
def acquire_packages(self, serial):
|
||||
"""Get list of installed packages with paths and system/user classification."""
|
||||
try:
|
||||
all_out = self._adb(serial, 'pm list packages -f', timeout=30)
|
||||
sys_out = self._adb(serial, 'pm list packages -s', timeout=30)
|
||||
|
||||
# Build system package set from -s output (format: "package:com.pkg.name")
|
||||
sys_pkgs = set()
|
||||
for line in sys_out.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith('package:'):
|
||||
sys_pkgs.add(line[8:].strip())
|
||||
|
||||
# Parse -f output (format: "package:/path/to/base.apk=com.pkg.name")
|
||||
packages = []
|
||||
for line in all_out.splitlines():
|
||||
line = line.strip()
|
||||
if not line.startswith('package:'):
|
||||
continue
|
||||
line = line[8:] # strip "package:"
|
||||
# Split on last '=' to handle paths that may contain '='
|
||||
eq_pos = line.rfind('=')
|
||||
if eq_pos == -1:
|
||||
pkg_name = line
|
||||
apk_path = ''
|
||||
else:
|
||||
apk_path = line[:eq_pos]
|
||||
pkg_name = line[eq_pos + 1:]
|
||||
pkg_name = pkg_name.strip()
|
||||
if pkg_name:
|
||||
packages.append({
|
||||
'package': pkg_name,
|
||||
'path': apk_path.strip(),
|
||||
'system': pkg_name in sys_pkgs,
|
||||
})
|
||||
return {'ok': True, 'packages': packages, 'total': len(packages)}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'packages': []}
|
||||
|
||||
def acquire_processes(self, serial):
|
||||
"""Get list of running processes."""
|
||||
try:
|
||||
out = self._adb(serial, 'ps -A', timeout=30)
|
||||
lines = [l for l in out.splitlines() if l.strip()]
|
||||
return {'ok': True, 'output': out, 'count': max(0, len(lines) - 1)}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'output': ''}
|
||||
|
||||
def acquire_getprop(self, serial):
|
||||
"""Get all system properties via getprop."""
|
||||
try:
|
||||
out = self._adb(serial, 'getprop', timeout=30)
|
||||
props = {}
|
||||
for line in out.splitlines():
|
||||
m = re.match(r'\[(.+)\]:\s*\[(.*)\]\s*$', line)
|
||||
if m:
|
||||
props[m.group(1)] = m.group(2)
|
||||
return {'ok': True, 'props': props, 'output': out}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'props': {}}
|
||||
|
||||
def acquire_settings(self, serial):
|
||||
"""Get global, secure, and system settings."""
|
||||
try:
|
||||
result = {}
|
||||
for ns in ('global', 'secure', 'system'):
|
||||
out = self._adb(serial, f'settings list {ns}', timeout=30)
|
||||
settings = {}
|
||||
for line in out.splitlines():
|
||||
if '=' in line:
|
||||
k, _, v = line.partition('=')
|
||||
settings[k.strip()] = v.strip()
|
||||
result[ns] = settings
|
||||
return {'ok': True, 'settings': result}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'settings': {}}
|
||||
|
||||
def acquire_services(self, serial):
|
||||
"""List all running Android services."""
|
||||
try:
|
||||
out = self._adb(serial, 'service list', timeout=30)
|
||||
return {'ok': True, 'output': out, 'count': len(out.splitlines())}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'output': ''}
|
||||
|
||||
def acquire_dumpsys(self, serial):
|
||||
"""Full dumpsys output (may be large)."""
|
||||
try:
|
||||
out = self._adb(serial, 'dumpsys', timeout=180)
|
||||
return {'ok': True, 'output': out, 'size': len(out)}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'output': ''}
|
||||
|
||||
def acquire_logcat(self, serial):
|
||||
"""Recent logcat entries (last 1000 lines)."""
|
||||
try:
|
||||
out = self._adb(serial, 'logcat -d -t 1000', timeout=60)
|
||||
return {'ok': True, 'output': out, 'lines': len(out.splitlines())}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'output': ''}
|
||||
|
||||
def acquire_tmp_files(self, serial):
|
||||
"""List files in /data/local/tmp/."""
|
||||
try:
|
||||
out = self._adb(serial, 'ls -la /data/local/tmp/ 2>/dev/null', timeout=15)
|
||||
return {'ok': True, 'output': out, 'lines': len(out.splitlines())}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'output': ''}
|
||||
|
||||
def acquire_files(self, serial, max_depth=5):
|
||||
"""Recursive file listing from common paths."""
|
||||
try:
|
||||
paths = ['/data/local/tmp', '/sdcard/Android/data', '/data/data',
|
||||
'/system/app', '/data/app']
|
||||
results = []
|
||||
for base in paths:
|
||||
out = self._adb(
|
||||
serial,
|
||||
f'find {base} -maxdepth {max_depth} -type f 2>/dev/null | head -500',
|
||||
timeout=30,
|
||||
)
|
||||
for line in out.splitlines():
|
||||
line = line.strip()
|
||||
if line:
|
||||
results.append({'path': line})
|
||||
return {'ok': True, 'files': results, 'total': len(results)}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'files': []}
|
||||
|
||||
def acquire_logs(self, serial):
|
||||
"""Full logcat with thread-time format (last 2000 lines)."""
|
||||
try:
|
||||
out = self._adb(serial, 'logcat -d -v threadtime -t 2000', timeout=90)
|
||||
return {'ok': True, 'output': out, 'lines': len(out.splitlines())}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'output': ''}
|
||||
|
||||
def acquire_bugreport_info(self, serial):
|
||||
"""Collect bugreport header info (device state, battery, memory) without full zip."""
|
||||
try:
|
||||
sections = {}
|
||||
for name, cmd in [
|
||||
('memory', 'cat /proc/meminfo'),
|
||||
('cpuinfo', 'cat /proc/cpuinfo | head -40'),
|
||||
('uptime', 'cat /proc/uptime'),
|
||||
('mounts', 'cat /proc/mounts'),
|
||||
]:
|
||||
sections[name] = self._adb(serial, cmd, timeout=15)
|
||||
return {'ok': True, 'sections': sections}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'sections': {}}
|
||||
|
||||
# ── IOC Scanning ────────────────────────────────────────────────
|
||||
|
||||
def scan_packages(self, serial, packages_data=None):
|
||||
"""Match installed packages against IOC package names."""
|
||||
db = self.load_ioc_database()
|
||||
if packages_data is None:
|
||||
result = self.acquire_packages(serial)
|
||||
if not result.get('ok'):
|
||||
return {'ok': False, 'error': result.get('error'), 'findings': []}
|
||||
packages_data = result['packages']
|
||||
|
||||
findings = []
|
||||
for pkg_info in packages_data:
|
||||
pkg = pkg_info.get('package', '').strip()
|
||||
if pkg and pkg in db['packages']:
|
||||
match = db['packages'][pkg]
|
||||
findings.append({
|
||||
'package': pkg,
|
||||
'path': pkg_info.get('path', ''),
|
||||
'system': pkg_info.get('system', False),
|
||||
'threat_name': match['name'],
|
||||
'source': match['source'],
|
||||
'severity': match.get('severity', 'high'),
|
||||
'type': 'package_match',
|
||||
})
|
||||
return {
|
||||
'ok': True,
|
||||
'total_scanned': len(packages_data),
|
||||
'findings': findings,
|
||||
'clean': len(findings) == 0,
|
||||
}
|
||||
|
||||
def scan_certificates(self, serial):
|
||||
"""Check APK signing cert hashes against known-bad certs in IOC database."""
|
||||
db = self.load_ioc_database()
|
||||
findings = []
|
||||
if not db.get('certs'):
|
||||
return {'ok': True, 'findings': [], 'note': 'No cert IOCs loaded'}
|
||||
try:
|
||||
pkg_result = self.acquire_packages(serial)
|
||||
# Only check user-installed (non-system) packages; limit to 100 for speed
|
||||
packages = [p for p in pkg_result.get('packages', []) if not p.get('system')][:100]
|
||||
for pkg_info in packages:
|
||||
pkg = pkg_info.get('package', '')
|
||||
if not pkg:
|
||||
continue
|
||||
out = self._adb(serial, f'pm dump {pkg} 2>/dev/null | grep -A2 "Signing"', timeout=15)
|
||||
for line in out.splitlines():
|
||||
m = re.search(r'([a-fA-F0-9]{40,64})', line)
|
||||
if m:
|
||||
cert_hash = m.group(1).lower()
|
||||
if cert_hash in db['certs']:
|
||||
match = db['certs'][cert_hash]
|
||||
findings.append({
|
||||
'package': pkg,
|
||||
'cert_hash': cert_hash,
|
||||
'threat_name': match['name'],
|
||||
'source': match['source'],
|
||||
'severity': 'critical',
|
||||
'type': 'cert_match',
|
||||
})
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'findings': findings}
|
||||
return {'ok': True, 'findings': findings, 'clean': len(findings) == 0}
|
||||
|
||||
def scan_network_indicators(self, serial):
|
||||
"""Check DNS/proxy settings and active TCP connections against IOC domains/IPs."""
|
||||
db = self.load_ioc_database()
|
||||
findings = []
|
||||
try:
|
||||
props = self.acquire_getprop(serial).get('props', {})
|
||||
for key, val in props.items():
|
||||
if 'dns' in key.lower() and val and val.lower() in db['domains']:
|
||||
match = db['domains'][val.lower()]
|
||||
findings.append({
|
||||
'type': 'dns_property', 'key': key, 'value': val,
|
||||
'threat_name': match['name'], 'source': match['source'], 'severity': 'high',
|
||||
})
|
||||
|
||||
ioc_ips = set(db.get('ips', []))
|
||||
if ioc_ips:
|
||||
tcp_out = self._adb(serial, 'cat /proc/net/tcp 2>/dev/null', timeout=15)
|
||||
for line in tcp_out.splitlines()[1:]:
|
||||
parts = line.strip().split()
|
||||
if len(parts) < 3:
|
||||
continue
|
||||
try:
|
||||
remote_hex = parts[2]
|
||||
ip_hex, port_hex = remote_hex.split(':')
|
||||
# /proc/net/tcp stores IPs in little-endian hex
|
||||
ip = '.'.join(str(int(ip_hex[i:i+2], 16)) for i in (6, 4, 2, 0))
|
||||
port = int(port_hex, 16)
|
||||
if ip not in ('0.0.0.0', '127.0.0.1') and ip in ioc_ips:
|
||||
findings.append({
|
||||
'type': 'tcp_connection', 'remote_ip': ip, 'remote_port': port,
|
||||
'severity': 'critical', 'source': 'network_scan',
|
||||
'threat_name': 'Known malicious IP',
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'findings': findings}
|
||||
return {'ok': True, 'findings': findings, 'clean': len(findings) == 0}
|
||||
|
||||
def scan_file_indicators(self, serial):
|
||||
"""Check device for known spyware artifact file paths."""
|
||||
db = self.load_ioc_database()
|
||||
if not db.get('file_paths'):
|
||||
return {'ok': True, 'findings': [], 'note': 'No file path IOCs loaded'}
|
||||
findings = []
|
||||
try:
|
||||
for fpath, match in list(db['file_paths'].items())[:200]:
|
||||
out = self._adb(serial, f'ls "{fpath}" 2>/dev/null', timeout=5)
|
||||
if out.strip() and 'No such file' not in out:
|
||||
findings.append({
|
||||
'type': 'file_path',
|
||||
'path': fpath,
|
||||
'threat_name': match['name'],
|
||||
'source': match['source'],
|
||||
'severity': 'critical',
|
||||
})
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'findings': findings}
|
||||
return {'ok': True, 'findings': findings, 'clean': len(findings) == 0}
|
||||
|
||||
def scan_process_indicators(self, serial):
|
||||
"""Cross-reference running processes against known spyware process names."""
|
||||
db = self.load_ioc_database()
|
||||
if not db.get('processes'):
|
||||
return {'ok': True, 'findings': [], 'note': 'No process IOCs loaded'}
|
||||
findings = []
|
||||
try:
|
||||
proc_output = self.acquire_processes(serial).get('output', '').lower()
|
||||
for proc_name, match in db['processes'].items():
|
||||
if proc_name and proc_name in proc_output:
|
||||
findings.append({
|
||||
'type': 'process_match',
|
||||
'process': proc_name,
|
||||
'threat_name': match['name'],
|
||||
'source': match['source'],
|
||||
'severity': 'critical',
|
||||
})
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e), 'findings': findings}
|
||||
return {'ok': True, 'findings': findings, 'clean': len(findings) == 0}
|
||||
|
||||
# ── SLM Heuristic Scanner ────────────────────────────────────────
|
||||
|
||||
def _heuristic_scan(self, data_type, items, threshold=0.65):
|
||||
"""Run SLM heuristic pass on a list of items via the local LLM.
|
||||
data_type: 'packages' | 'processes' | 'network' | 'files'
|
||||
Returns list of {item, score, reason, category} for items scoring >= threshold."""
|
||||
if not items:
|
||||
return []
|
||||
try:
|
||||
from core.llm import get_llm, LLMError
|
||||
llm = get_llm()
|
||||
if not llm or not llm.is_loaded:
|
||||
return []
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
prompts = {
|
||||
'packages': (
|
||||
'Classify each Android package name for stalkerware/spyware/adware risk. '
|
||||
'Score 0.0-1.0. High risk: names impersonating system apps, random strings, '
|
||||
'unknown publishers with device-admin-sounding names. '
|
||||
'Return only a JSON array: [{"item":"...","score":0.0,"reason":"...","category":"stalkerware|adware|suspicious|clean"}]'
|
||||
),
|
||||
'processes': (
|
||||
'Classify each Android process name for malware risk. Score 0.0-1.0. '
|
||||
'Suspicious: random hex strings, names mimicking system processes with slight misspellings. '
|
||||
'Return only a JSON array: [{"item":"...","score":0.0,"reason":"...","category":"malware|suspicious|clean"}]'
|
||||
),
|
||||
'network': (
|
||||
'Classify each domain or IP for C2/malware beacon risk. Score 0.0-1.0. '
|
||||
'High risk: DGA-looking names, dynamic DNS (duckdns, no-ip, ddns), unusual TLDs, high-entropy subdomains. '
|
||||
'Return only a JSON array: [{"item":"...","score":0.0,"reason":"...","category":"c2_beacon|dga_domain|suspicious|clean"}]'
|
||||
),
|
||||
'files': (
|
||||
'Classify each Android file path for spyware/stalkerware artifact risk. Score 0.0-1.0. '
|
||||
'High risk: hidden directories, paths in /data/local/tmp, surveillance-sounding names. '
|
||||
'Return only a JSON array: [{"item":"...","score":0.0,"reason":"...","category":"spyware|suspicious|clean"}]'
|
||||
),
|
||||
}
|
||||
base_prompt = prompts.get(data_type, prompts['packages'])
|
||||
|
||||
findings = []
|
||||
# Process in chunks of 50 (on-device SLM token budget)
|
||||
for i in range(0, len(items), 50):
|
||||
chunk = items[i:i + 50]
|
||||
prompt = f'{base_prompt}\n\nItems:\n' + '\n'.join(str(x) for x in chunk)
|
||||
try:
|
||||
raw = get_llm().generate(prompt, max_tokens=800)
|
||||
# Extract JSON array from response
|
||||
m = re.search(r'\[.*\]', raw, re.DOTALL)
|
||||
if not m:
|
||||
continue
|
||||
parsed = json.loads(m.group(0))
|
||||
for entry in parsed:
|
||||
if isinstance(entry, dict) and float(entry.get('score', 0)) >= threshold:
|
||||
findings.append({
|
||||
'item': str(entry.get('item', '')),
|
||||
'score': float(entry.get('score', 0)),
|
||||
'reason': str(entry.get('reason', '')),
|
||||
'category': str(entry.get('category', 'suspicious')),
|
||||
'confidence': float(entry.get('score', 0)),
|
||||
'heuristic': True,
|
||||
'data_type': data_type,
|
||||
})
|
||||
except Exception:
|
||||
continue
|
||||
return findings
|
||||
|
||||
# ── Composite Operations ─────────────────────────────────────────
|
||||
|
||||
def full_acquisition(self, serial):
|
||||
"""Run all acquisition modules and all IOC scans. Save results to disk."""
|
||||
acq_id = self._new_acq_id()
|
||||
acq_dir = self._acq_dir(serial, acq_id)
|
||||
|
||||
report = {
|
||||
'acq_id': acq_id,
|
||||
'serial': serial,
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'modules': {},
|
||||
'ioc_findings': [],
|
||||
}
|
||||
|
||||
module_map = {
|
||||
'packages': self.acquire_packages,
|
||||
'processes': self.acquire_processes,
|
||||
'getprop': self.acquire_getprop,
|
||||
'settings': self.acquire_settings,
|
||||
'services': self.acquire_services,
|
||||
'logcat': self.acquire_logcat,
|
||||
'tmp_files': self.acquire_tmp_files,
|
||||
'files': self.acquire_files,
|
||||
'logs': self.acquire_logs,
|
||||
'bugreport': self.acquire_bugreport_info,
|
||||
}
|
||||
|
||||
for mod_name, func in module_map.items():
|
||||
try:
|
||||
result = func(serial)
|
||||
report['modules'][mod_name] = result
|
||||
with open(acq_dir / f'{mod_name}.json', 'w', encoding='utf-8') as f:
|
||||
json.dump(result, f, indent=2, default=str)
|
||||
except Exception as e:
|
||||
report['modules'][mod_name] = {'ok': False, 'error': str(e)}
|
||||
|
||||
# IOC scans — reuse already-acquired package list
|
||||
packages_data = report['modules'].get('packages', {}).get('packages', [])
|
||||
all_findings = []
|
||||
for scan_name, scan_func in [
|
||||
('packages', lambda s: self.scan_packages(s, packages_data)),
|
||||
('network', self.scan_network_indicators),
|
||||
('files', self.scan_file_indicators),
|
||||
('processes', self.scan_process_indicators),
|
||||
]:
|
||||
try:
|
||||
scan_result = scan_func(serial)
|
||||
for finding in scan_result.get('findings', []):
|
||||
finding['scan_type'] = scan_name
|
||||
all_findings.append(finding)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
report['ioc_findings'] = all_findings
|
||||
report['scan_summary'] = {
|
||||
'total_findings': len(all_findings),
|
||||
'critical': sum(1 for f in all_findings if f.get('severity') == 'critical'),
|
||||
'high': sum(1 for f in all_findings if f.get('severity') == 'high'),
|
||||
'clean': len(all_findings) == 0,
|
||||
}
|
||||
|
||||
with open(acq_dir / 'report.json', 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, indent=2, default=str)
|
||||
|
||||
return report
|
||||
|
||||
def get_acquisition_list(self, serial):
|
||||
"""List past acquisitions for a device."""
|
||||
device_dir = self.ACQ_DATA_DIR / serial
|
||||
if not device_dir.exists():
|
||||
return []
|
||||
acquisitions = []
|
||||
for acq_dir in sorted(device_dir.iterdir(), reverse=True):
|
||||
if not acq_dir.is_dir():
|
||||
continue
|
||||
report_path = acq_dir / 'report.json'
|
||||
entry = {'acq_id': acq_dir.name, 'has_report': report_path.exists()}
|
||||
if report_path.exists():
|
||||
try:
|
||||
with open(report_path, encoding='utf-8') as f:
|
||||
r = json.load(f)
|
||||
entry['timestamp'] = r.get('timestamp', acq_dir.name)
|
||||
entry['modules'] = list(r.get('modules', {}).keys())
|
||||
entry['findings'] = r.get('scan_summary', {}).get('total_findings', 0)
|
||||
except Exception:
|
||||
pass
|
||||
acquisitions.append(entry)
|
||||
return acquisitions
|
||||
|
||||
def export_report(self, serial, acq_id):
|
||||
"""Load a saved acquisition report from disk."""
|
||||
report_path = self.ACQ_DATA_DIR / serial / acq_id / 'report.json'
|
||||
if not report_path.exists():
|
||||
return {'ok': False, 'error': 'Report not found'}
|
||||
try:
|
||||
with open(report_path, encoding='utf-8') as f:
|
||||
return {'ok': True, 'report': json.load(f), 'path': str(report_path)}
|
||||
except Exception as e:
|
||||
return {'ok': False, 'error': str(e)}
|
||||
|
||||
# ── Direct (WebUSB) Mode ─────────────────────────────────────────
|
||||
|
||||
def get_direct_commands(self, operation):
|
||||
"""Return ADB shell commands for browser-side WebUSB execution."""
|
||||
commands = {
|
||||
'packages': 'pm list packages -f',
|
||||
'packages_sys': 'pm list packages -s',
|
||||
'processes': 'ps -A',
|
||||
'getprop': 'getprop',
|
||||
'settings_global': 'settings list global',
|
||||
'settings_secure': 'settings list secure',
|
||||
'settings_system': 'settings list system',
|
||||
'services': 'service list',
|
||||
'logcat': 'logcat -d -t 1000',
|
||||
'tmp_files': 'ls -la /data/local/tmp/ 2>/dev/null',
|
||||
'tcp_connections': 'cat /proc/net/tcp 2>/dev/null',
|
||||
}
|
||||
if operation == 'full':
|
||||
return commands
|
||||
cmd = commands.get(operation)
|
||||
return {operation: cmd} if cmd else {}
|
||||
|
||||
def parse_direct_output(self, operation, raw_output):
|
||||
"""Parse raw ADB output returned from a WebUSB browser client."""
|
||||
if operation == 'packages':
|
||||
packages = []
|
||||
for line in raw_output.splitlines():
|
||||
line = line.strip()
|
||||
if not line.startswith('package:'):
|
||||
continue
|
||||
line = line[8:]
|
||||
eq_pos = line.rfind('=')
|
||||
if eq_pos == -1:
|
||||
pkg_name, apk_path = line, ''
|
||||
else:
|
||||
apk_path, pkg_name = line[:eq_pos], line[eq_pos + 1:]
|
||||
pkg_name = pkg_name.strip()
|
||||
if pkg_name:
|
||||
packages.append({'package': pkg_name, 'path': apk_path.strip(), 'system': False})
|
||||
return {'ok': True, 'packages': packages, 'total': len(packages)}
|
||||
|
||||
elif operation == 'processes':
|
||||
lines = [l for l in raw_output.splitlines() if l.strip()]
|
||||
return {'ok': True, 'output': raw_output, 'count': max(0, len(lines) - 1)}
|
||||
|
||||
elif operation == 'getprop':
|
||||
props = {}
|
||||
for line in raw_output.splitlines():
|
||||
m = re.match(r'\[(.+)\]:\s*\[(.*)\]\s*$', line)
|
||||
if m:
|
||||
props[m.group(1)] = m.group(2)
|
||||
return {'ok': True, 'props': props, 'output': raw_output}
|
||||
|
||||
else:
|
||||
return {'ok': True, 'output': raw_output}
|
||||
@@ -405,10 +405,18 @@ class AutonomyDaemon:
|
||||
return
|
||||
|
||||
try:
|
||||
# Read step limit from config based on the active LLM backend
|
||||
from core.config import Config
|
||||
cfg = Config()
|
||||
backend = cfg.get('autarch', 'llm_backend', fallback='local')
|
||||
step_key = f'{backend}_max_steps'
|
||||
agent_settings = cfg.get_agent_settings()
|
||||
max_steps = agent_settings.get(step_key, agent_settings.get('local_max_steps', 200))
|
||||
|
||||
agent = Agent(
|
||||
llm=llm_inst,
|
||||
tools=get_tool_registry(),
|
||||
max_steps=15,
|
||||
max_steps=max_steps,
|
||||
verbose=False,
|
||||
)
|
||||
result = agent.run(task)
|
||||
|
||||
@@ -140,17 +140,17 @@ class Config:
|
||||
},
|
||||
'agents': {
|
||||
'backend': 'local',
|
||||
'local_max_steps': '20',
|
||||
'local_max_steps': '200',
|
||||
'local_verbose': 'true',
|
||||
'claude_enabled': 'false',
|
||||
'claude_model': 'claude-sonnet-4-6',
|
||||
'claude_max_tokens': '16384',
|
||||
'claude_max_steps': '30',
|
||||
'claude_max_steps': '200',
|
||||
'openai_enabled': 'false',
|
||||
'openai_model': 'gpt-4o',
|
||||
'openai_base_url': 'https://api.openai.com/v1',
|
||||
'openai_max_tokens': '16384',
|
||||
'openai_max_steps': '30',
|
||||
'openai_max_steps': '200',
|
||||
},
|
||||
'hal_memory': {
|
||||
'max_bytes': str(4 * 1024 * 1024 * 1024), # 4GB default
|
||||
@@ -539,17 +539,17 @@ class Config:
|
||||
"""Get agent configuration settings."""
|
||||
return {
|
||||
'backend': self.get('agents', 'backend', 'local'),
|
||||
'local_max_steps': self.get_int('agents', 'local_max_steps', 20),
|
||||
'local_max_steps': self.get_int('agents', 'local_max_steps', 200),
|
||||
'local_verbose': self.get_bool('agents', 'local_verbose', True),
|
||||
'claude_enabled': self.get_bool('agents', 'claude_enabled', False),
|
||||
'claude_model': self.get('agents', 'claude_model', 'claude-sonnet-4-6'),
|
||||
'claude_max_tokens': self.get_int('agents', 'claude_max_tokens', 16384),
|
||||
'claude_max_steps': self.get_int('agents', 'claude_max_steps', 30),
|
||||
'claude_max_steps': self.get_int('agents', 'claude_max_steps', 200),
|
||||
'openai_enabled': self.get_bool('agents', 'openai_enabled', False),
|
||||
'openai_model': self.get('agents', 'openai_model', 'gpt-4o'),
|
||||
'openai_base_url': self.get('agents', 'openai_base_url', 'https://api.openai.com/v1'),
|
||||
'openai_max_tokens': self.get_int('agents', 'openai_max_tokens', 16384),
|
||||
'openai_max_steps': self.get_int('agents', 'openai_max_steps', 30),
|
||||
'openai_max_steps': self.get_int('agents', 'openai_max_steps', 200),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
|
||||
258
core/cve_db.py
Normal file
258
core/cve_db.py
Normal file
@@ -0,0 +1,258 @@
|
||||
"""
|
||||
Autarch CVE Database — CVE lookup for Android packages via OSV.dev.
|
||||
by: SsSnake
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
_db_instance = None
|
||||
_db_lock = threading.Lock()
|
||||
|
||||
|
||||
def get_cve_db():
|
||||
global _db_instance
|
||||
if _db_instance is None:
|
||||
with _db_lock:
|
||||
if _db_instance is None:
|
||||
_db_instance = CveDatabase()
|
||||
return _db_instance
|
||||
|
||||
|
||||
class CveDatabase:
|
||||
"""CVE lookup for Android packages. Uses OSV.dev as primary data source."""
|
||||
|
||||
DATA_DIR = Path(__file__).parent.parent / 'data' / 'ioc' / 'cve'
|
||||
OSV_QUERY_URL = 'https://api.osv.dev/v1/query'
|
||||
OSV_BATCH_URL = 'https://api.osv.dev/v1/querybatch'
|
||||
|
||||
def __init__(self):
|
||||
self.DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
self._cache = {} # package_name -> [vuln_dicts]
|
||||
self._stats_cache = None
|
||||
self._lock = threading.Lock()
|
||||
self._last_query = None
|
||||
|
||||
def get_cve_for_package(self, package_name, version=None) -> list:
|
||||
"""Query OSV.dev for CVEs affecting an Android package."""
|
||||
with self._lock:
|
||||
if package_name in self._cache:
|
||||
return self._cache[package_name]
|
||||
|
||||
try:
|
||||
body = {
|
||||
"package": {
|
||||
"name": package_name,
|
||||
"ecosystem": "Android"
|
||||
}
|
||||
}
|
||||
if version is not None:
|
||||
body["version"] = version
|
||||
|
||||
response = self._query_osv(body)
|
||||
vulns = response.get("vulns", [])
|
||||
|
||||
results = []
|
||||
for vuln in vulns:
|
||||
parsed = self._parse_vuln(vuln, package_name)
|
||||
results.append(parsed)
|
||||
|
||||
with self._lock:
|
||||
self._cache[package_name] = results
|
||||
self._last_query = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
return results
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def get_cve_for_android_sdk(self, sdk_version) -> list:
|
||||
"""Query OSV.dev for CVEs affecting a specific Android SDK version."""
|
||||
package_name = f"android:{sdk_version}"
|
||||
try:
|
||||
body = {
|
||||
"package": {
|
||||
"name": package_name,
|
||||
"ecosystem": "Android"
|
||||
}
|
||||
}
|
||||
|
||||
response = self._query_osv(body)
|
||||
vulns = response.get("vulns", [])
|
||||
|
||||
results = []
|
||||
for vuln in vulns:
|
||||
parsed = self._parse_vuln(vuln, package_name)
|
||||
results.append(parsed)
|
||||
|
||||
with self._lock:
|
||||
self._cache[package_name] = results
|
||||
self._last_query = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
return results
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def check_packages_batch(self, packages: list) -> dict:
|
||||
"""Query OSV.dev for CVEs affecting multiple Android packages at once."""
|
||||
if not packages:
|
||||
return {}
|
||||
|
||||
try:
|
||||
results = {}
|
||||
# Split into chunks of 50
|
||||
chunk_size = 50
|
||||
for i in range(0, len(packages), chunk_size):
|
||||
chunk = packages[i:i + chunk_size]
|
||||
queries = [
|
||||
{"package": {"name": p["name"], "ecosystem": "Android"}}
|
||||
for p in chunk
|
||||
]
|
||||
body = {"queries": queries}
|
||||
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
self.OSV_BATCH_URL,
|
||||
data=data,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "Autarch/1.0"
|
||||
},
|
||||
method="POST"
|
||||
)
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
response = json.loads(resp.read().decode("utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
batch_results = response.get("results", [])
|
||||
for idx, result in enumerate(batch_results):
|
||||
if idx >= len(chunk):
|
||||
break
|
||||
pkg_name = chunk[idx]["name"]
|
||||
vulns = result.get("vulns", [])
|
||||
parsed_vulns = [self._parse_vuln(v, pkg_name) for v in vulns]
|
||||
results[pkg_name] = parsed_vulns
|
||||
|
||||
with self._lock:
|
||||
self._cache[pkg_name] = parsed_vulns
|
||||
|
||||
with self._lock:
|
||||
if results:
|
||||
self._last_query = datetime.now(timezone.utc).isoformat()
|
||||
|
||||
return results
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
"""Return basic stats about the CVE database cache."""
|
||||
with self._lock:
|
||||
return {
|
||||
"total_cached": len(self._cache),
|
||||
"last_query": self._last_query,
|
||||
"data_dir": str(self.DATA_DIR)
|
||||
}
|
||||
|
||||
def _query_osv(self, body: dict) -> dict:
|
||||
"""POST a JSON body to the OSV.dev query endpoint. Returns parsed JSON or {}."""
|
||||
try:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
req = urllib.request.Request(
|
||||
self.OSV_QUERY_URL,
|
||||
data=data,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "Autarch/1.0"
|
||||
},
|
||||
method="POST"
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
return json.loads(resp.read().decode("utf-8"))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def _parse_vuln(self, vuln: dict, package_name: str) -> dict:
|
||||
"""Extract standardised CVE fields from an OSV vulnerability object."""
|
||||
# cve_id: prefer alias starting with "CVE-", else use vuln id
|
||||
cve_id = vuln.get("id", "UNKNOWN")
|
||||
for alias in vuln.get("aliases", []):
|
||||
if alias.startswith("CVE-"):
|
||||
cve_id = alias
|
||||
break
|
||||
|
||||
# severity: try database_specific first, then severity array score, normalise
|
||||
raw_severity = (
|
||||
vuln.get("database_specific", {}).get("severity")
|
||||
or vuln.get("severity", [{}])[0].get("score", "UNKNOWN")
|
||||
)
|
||||
severity = self._normalise_severity(raw_severity)
|
||||
|
||||
# description: summary or details, capped at 500 chars
|
||||
description = vuln.get("summary", vuln.get("details", ""))[:500]
|
||||
|
||||
# patched_version: look for a "fixed" event in affected ranges
|
||||
patched_version = "unknown"
|
||||
affected_list = vuln.get("affected", [{}])
|
||||
if affected_list:
|
||||
ranges = affected_list[0].get("ranges", [])
|
||||
for r in ranges:
|
||||
for event in r.get("events", []):
|
||||
if "fixed" in event:
|
||||
patched_version = event["fixed"]
|
||||
break
|
||||
if patched_version != "unknown":
|
||||
break
|
||||
|
||||
return {
|
||||
"package": package_name,
|
||||
"cve_id": cve_id,
|
||||
"severity": severity,
|
||||
"description": description,
|
||||
"patched_version": patched_version,
|
||||
"source": "osv.dev"
|
||||
}
|
||||
|
||||
def _normalise_severity(self, raw) -> str:
|
||||
"""Map a raw severity string or CVSS score to critical/high/medium/low."""
|
||||
if raw is None or raw == "UNKNOWN":
|
||||
return "medium"
|
||||
|
||||
raw_str = str(raw).strip().upper()
|
||||
|
||||
# Direct label match
|
||||
label_map = {
|
||||
"CRITICAL": "critical",
|
||||
"HIGH": "high",
|
||||
"MEDIUM": "medium",
|
||||
"MODERATE": "medium",
|
||||
"LOW": "low",
|
||||
"NONE": "low"
|
||||
}
|
||||
if raw_str in label_map:
|
||||
return label_map[raw_str]
|
||||
|
||||
# Try numeric CVSS score
|
||||
try:
|
||||
score = float(raw_str)
|
||||
if score >= 9.0:
|
||||
return "critical"
|
||||
elif score >= 7.0:
|
||||
return "high"
|
||||
elif score >= 4.0:
|
||||
return "medium"
|
||||
else:
|
||||
return "low"
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
# CVSS vector strings often start with digits after the last colon
|
||||
# e.g. "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"
|
||||
# Fall back to medium if we can't parse it
|
||||
return "medium"
|
||||
259
core/keystore.py
Normal file
259
core/keystore.py
Normal file
@@ -0,0 +1,259 @@
|
||||
"""
|
||||
AUTARCH Encrypted Keystore
|
||||
Per-user encrypted storage for API keys, passwords, and credentials.
|
||||
|
||||
Stores secrets in keystore.xml in the user data directory using AES-256-CBC
|
||||
encryption from the existing vault module. Secrets are separated by user
|
||||
account so multi-user deployments don't leak across accounts.
|
||||
|
||||
The keystore file format is XML for human-inspectability of the structure
|
||||
(though all values are encrypted). Each entry has:
|
||||
- A unique key name
|
||||
- The encrypted value (base64-encoded)
|
||||
- The user who owns it
|
||||
- A timestamp
|
||||
- An optional category (api_key, password, token, certificate, etc.)
|
||||
|
||||
Usage:
|
||||
from core.keystore import get_keystore
|
||||
|
||||
ks = get_keystore()
|
||||
|
||||
# Store a secret for the current user
|
||||
ks.set('claude_api_key', 'sk-ant-...', category='api_key')
|
||||
|
||||
# Retrieve it
|
||||
key = ks.get('claude_api_key') # Returns '' if not set
|
||||
|
||||
# List all keys for the current user
|
||||
ks.keys() # ['claude_api_key', 'openai_api_key']
|
||||
|
||||
# Store for a specific user
|
||||
ks.set('shodan_key', 'abc123', user='admin', category='api_key')
|
||||
|
||||
# Delete
|
||||
ks.delete('old_key')
|
||||
"""
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
from xml.dom import minidom
|
||||
|
||||
_log = logging.getLogger('autarch.keystore')
|
||||
|
||||
# Import encryption functions from the existing vault
|
||||
from core.vault import _derive_key, _encrypt, _decrypt
|
||||
|
||||
|
||||
def _get_default_user() -> str:
|
||||
"""Get the current user identity for key ownership."""
|
||||
from core.paths import get_user_profile_path
|
||||
profile_path = get_user_profile_path()
|
||||
if profile_path.exists():
|
||||
try:
|
||||
with open(profile_path) as f:
|
||||
profile = json.load(f)
|
||||
return profile.get('username', 'default')
|
||||
except Exception:
|
||||
pass
|
||||
return 'default'
|
||||
|
||||
|
||||
class Keystore:
|
||||
"""Encrypted per-user secrets keystore backed by XML."""
|
||||
|
||||
def __init__(self, keystore_path: Path = None, master_password: str = ''):
|
||||
from core.paths import get_keystore_path
|
||||
self._path = keystore_path or get_keystore_path()
|
||||
self._master = master_password
|
||||
self._salt = None
|
||||
self._entries: Dict[str, Dict] = {} # key -> {value, user, category, timestamp, salt}
|
||||
self._load()
|
||||
|
||||
def _load(self):
|
||||
"""Load and decrypt the keystore from disk."""
|
||||
if not self._path.exists():
|
||||
_log.info(f'No keystore found at {self._path}, starting fresh')
|
||||
return
|
||||
|
||||
try:
|
||||
tree = ET.parse(str(self._path))
|
||||
root = tree.getroot()
|
||||
|
||||
self._salt = base64.b64decode(root.get('salt', ''))
|
||||
|
||||
for entry in root.findall('entry'):
|
||||
key_name = entry.get('name', '')
|
||||
if not key_name:
|
||||
continue
|
||||
|
||||
self._entries[key_name] = {
|
||||
'encrypted_value': entry.get('value', ''),
|
||||
'user': entry.get('user', 'default'),
|
||||
'category': entry.get('category', ''),
|
||||
'timestamp': entry.get('timestamp', ''),
|
||||
'iv': entry.get('iv', ''),
|
||||
}
|
||||
|
||||
_log.info(f'Loaded keystore with {len(self._entries)} entries')
|
||||
except Exception as e:
|
||||
_log.error(f'Failed to load keystore: {e}')
|
||||
|
||||
def _save(self):
|
||||
"""Encrypt and save the keystore to disk."""
|
||||
try:
|
||||
root = ET.Element('keystore')
|
||||
root.set('version', '1')
|
||||
|
||||
if self._salt is None:
|
||||
self._salt = os.urandom(32)
|
||||
root.set('salt', base64.b64encode(self._salt).decode())
|
||||
root.set('updated', str(int(time.time())))
|
||||
|
||||
for key_name, data in sorted(self._entries.items()):
|
||||
entry = ET.SubElement(root, 'entry')
|
||||
entry.set('name', key_name)
|
||||
entry.set('value', data.get('encrypted_value', ''))
|
||||
entry.set('iv', data.get('iv', ''))
|
||||
entry.set('user', data.get('user', 'default'))
|
||||
entry.set('category', data.get('category', ''))
|
||||
entry.set('timestamp', data.get('timestamp', ''))
|
||||
|
||||
# Pretty-print the XML
|
||||
rough = ET.tostring(root, encoding='unicode')
|
||||
parsed = minidom.parseString(rough)
|
||||
pretty = parsed.toprettyxml(indent=' ', encoding='utf-8')
|
||||
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self._path, 'wb') as f:
|
||||
f.write(pretty)
|
||||
|
||||
_log.info(f'Saved keystore with {len(self._entries)} entries')
|
||||
except Exception as e:
|
||||
_log.error(f'Failed to save keystore: {e}')
|
||||
|
||||
def set(self, key: str, value: str, user: str = None, category: str = ''):
|
||||
"""Store an encrypted secret."""
|
||||
if user is None:
|
||||
user = _get_default_user()
|
||||
|
||||
if self._salt is None:
|
||||
self._salt = os.urandom(32)
|
||||
|
||||
# Derive encryption key from salt + master password
|
||||
enc_key = _derive_key(self._salt, self._master)
|
||||
|
||||
# Encrypt the value
|
||||
iv, ciphertext = _encrypt(value.encode('utf-8'), enc_key)
|
||||
|
||||
self._entries[key] = {
|
||||
'encrypted_value': base64.b64encode(ciphertext).decode(),
|
||||
'iv': base64.b64encode(iv).decode(),
|
||||
'user': user,
|
||||
'category': category,
|
||||
'timestamp': str(int(time.time())),
|
||||
}
|
||||
|
||||
self._save()
|
||||
_log.info(f'Stored secret: {key} (user={user}, category={category})')
|
||||
|
||||
def get(self, key: str, user: str = None) -> str:
|
||||
"""Retrieve a decrypted secret. Returns '' if not found."""
|
||||
if user is None:
|
||||
user = _get_default_user()
|
||||
|
||||
entry = self._entries.get(key)
|
||||
if not entry:
|
||||
return ''
|
||||
|
||||
# Check user ownership
|
||||
if entry['user'] != user and entry['user'] != 'shared':
|
||||
_log.warning(f'Access denied: {key} belongs to {entry["user"]}, not {user}')
|
||||
return ''
|
||||
|
||||
try:
|
||||
enc_key = _derive_key(self._salt, self._master)
|
||||
iv = base64.b64decode(entry['iv'])
|
||||
ciphertext = base64.b64decode(entry['encrypted_value'])
|
||||
plaintext = _decrypt(iv, ciphertext, enc_key)
|
||||
return plaintext.decode('utf-8')
|
||||
except Exception as e:
|
||||
_log.error(f'Failed to decrypt {key}: {e}')
|
||||
return ''
|
||||
|
||||
def delete(self, key: str, user: str = None):
|
||||
"""Delete a secret."""
|
||||
if user is None:
|
||||
user = _get_default_user()
|
||||
|
||||
entry = self._entries.get(key)
|
||||
if entry and (entry['user'] == user or entry['user'] == 'shared'):
|
||||
del self._entries[key]
|
||||
self._save()
|
||||
_log.info(f'Deleted secret: {key}')
|
||||
|
||||
def keys(self, user: str = None, category: str = None) -> List[str]:
|
||||
"""List key names for a user, optionally filtered by category."""
|
||||
if user is None:
|
||||
user = _get_default_user()
|
||||
|
||||
result = []
|
||||
for key, data in self._entries.items():
|
||||
if data['user'] != user and data['user'] != 'shared':
|
||||
continue
|
||||
if category and data.get('category', '') != category:
|
||||
continue
|
||||
result.append(key)
|
||||
return sorted(result)
|
||||
|
||||
def get_all_for_user(self, user: str = None) -> Dict[str, str]:
|
||||
"""Get all decrypted secrets for a user as a dict."""
|
||||
if user is None:
|
||||
user = _get_default_user()
|
||||
return {k: self.get(k, user) for k in self.keys(user)}
|
||||
|
||||
def get_metadata(self, key: str) -> Optional[Dict]:
|
||||
"""Get metadata (user, category, timestamp) for a key without decrypting."""
|
||||
entry = self._entries.get(key)
|
||||
if not entry:
|
||||
return None
|
||||
return {
|
||||
'user': entry['user'],
|
||||
'category': entry.get('category', ''),
|
||||
'timestamp': entry.get('timestamp', ''),
|
||||
}
|
||||
|
||||
def migrate_from_vault(self):
|
||||
"""Import secrets from the old vault.enc into the new keystore."""
|
||||
from core.vault import get_vault
|
||||
vault = get_vault()
|
||||
migrated = 0
|
||||
for key in vault.keys():
|
||||
value = vault.get(key)
|
||||
if value and key not in self._entries:
|
||||
# Guess category from key name
|
||||
category = 'api_key' if 'api_key' in key else 'credential'
|
||||
self.set(key, value, user='default', category=category)
|
||||
migrated += 1
|
||||
if migrated:
|
||||
_log.info(f'Migrated {migrated} secrets from vault to keystore')
|
||||
return migrated
|
||||
|
||||
|
||||
# Singleton
|
||||
_keystore_instance: Optional[Keystore] = None
|
||||
|
||||
|
||||
def get_keystore(master_password: str = '') -> Keystore:
|
||||
"""Get the global keystore instance."""
|
||||
global _keystore_instance
|
||||
if _keystore_instance is None:
|
||||
_keystore_instance = Keystore(master_password=master_password)
|
||||
return _keystore_instance
|
||||
280
core/local_agent.py
Normal file
280
core/local_agent.py
Normal file
@@ -0,0 +1,280 @@
|
||||
"""
|
||||
AUTARCH Local Agent
|
||||
Lightweight client that runs on the user's machine and executes
|
||||
commands locally, streaming results back to the AUTARCH server.
|
||||
|
||||
This makes the web UI feel like AUTARCH is running on the user's
|
||||
machine — scans, tool output, file operations all happen locally.
|
||||
|
||||
Usage:
|
||||
python3 autarch-agent.py --server https://10.0.0.1:8181
|
||||
|
||||
The agent:
|
||||
1. Connects to the AUTARCH server via WebSocket
|
||||
2. Authenticates with the user's token
|
||||
3. Receives command requests from the server
|
||||
4. Executes them locally (subprocess)
|
||||
5. Streams stdout/stderr back in real-time
|
||||
6. Reports system info (OS, arch, hostname, IP) on connect
|
||||
|
||||
Protocol:
|
||||
Server -> Agent: {"type": "exec", "id": "xxx", "command": "nmap -sV 10.0.0.1"}
|
||||
Agent -> Server: {"type": "output", "id": "xxx", "stream": "stdout", "data": "..."}
|
||||
Agent -> Server: {"type": "output", "id": "xxx", "stream": "stderr", "data": "..."}
|
||||
Agent -> Server: {"type": "done", "id": "xxx", "exit_code": 0}
|
||||
Agent -> Server: {"type": "sysinfo", "hostname": "...", "os": "...", ...}
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import websockets
|
||||
except ImportError:
|
||||
print("Installing websockets...")
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "websockets", "-q"])
|
||||
import websockets
|
||||
|
||||
|
||||
def get_system_info():
|
||||
"""Collect local system information to send on connect."""
|
||||
info = {
|
||||
"hostname": socket.gethostname(),
|
||||
"os": platform.system(),
|
||||
"os_version": platform.version(),
|
||||
"arch": platform.machine(),
|
||||
"python": platform.python_version(),
|
||||
"username": os.getenv("USER", os.getenv("USERNAME", "unknown")),
|
||||
"home": str(Path.home()),
|
||||
"cwd": os.getcwd(),
|
||||
"pid": os.getpid(),
|
||||
}
|
||||
|
||||
# Network interfaces
|
||||
try:
|
||||
info["ip"] = socket.gethostbyname(socket.gethostname())
|
||||
except Exception:
|
||||
info["ip"] = "unknown"
|
||||
|
||||
# Available tools
|
||||
tools = {}
|
||||
for tool in ["nmap", "tcpdump", "wireshark", "python3", "git", "curl",
|
||||
"wget", "ssh", "netstat", "ss", "ip", "ifconfig", "arp",
|
||||
"dig", "nslookup", "traceroute", "ping", "whois",
|
||||
"openssl", "nikto", "sqlmap", "hydra", "john", "hashcat"]:
|
||||
path = shutil.which(tool)
|
||||
if path:
|
||||
tools[tool] = path
|
||||
info["tools"] = tools
|
||||
|
||||
# Disk usage
|
||||
try:
|
||||
usage = shutil.disk_usage("/")
|
||||
info["disk_total"] = usage.total
|
||||
info["disk_free"] = usage.free
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return info
|
||||
|
||||
|
||||
async def execute_command(command, cmd_id, websocket):
|
||||
"""Execute a command locally and stream output back."""
|
||||
try:
|
||||
process = await asyncio.create_subprocess_shell(
|
||||
command,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
|
||||
# Stream stdout
|
||||
async def read_stream(stream, name):
|
||||
while True:
|
||||
line = await stream.readline()
|
||||
if not line:
|
||||
break
|
||||
await websocket.send(json.dumps({
|
||||
"type": "output",
|
||||
"id": cmd_id,
|
||||
"stream": name,
|
||||
"data": line.decode("utf-8", errors="replace"),
|
||||
}))
|
||||
|
||||
# Run both streams concurrently
|
||||
await asyncio.gather(
|
||||
read_stream(process.stdout, "stdout"),
|
||||
read_stream(process.stderr, "stderr"),
|
||||
)
|
||||
|
||||
exit_code = await process.wait()
|
||||
|
||||
await websocket.send(json.dumps({
|
||||
"type": "done",
|
||||
"id": cmd_id,
|
||||
"exit_code": exit_code,
|
||||
}))
|
||||
|
||||
except Exception as e:
|
||||
await websocket.send(json.dumps({
|
||||
"type": "error",
|
||||
"id": cmd_id,
|
||||
"message": str(e),
|
||||
}))
|
||||
|
||||
|
||||
async def read_file(path, cmd_id, websocket):
|
||||
"""Read a file and send its contents back."""
|
||||
try:
|
||||
p = Path(path).expanduser()
|
||||
if not p.exists():
|
||||
await websocket.send(json.dumps({
|
||||
"type": "error", "id": cmd_id,
|
||||
"message": f"File not found: {path}"
|
||||
}))
|
||||
return
|
||||
|
||||
if p.is_dir():
|
||||
# List directory
|
||||
entries = []
|
||||
for entry in sorted(p.iterdir()):
|
||||
stat = entry.stat()
|
||||
entries.append({
|
||||
"name": entry.name,
|
||||
"is_dir": entry.is_dir(),
|
||||
"size": stat.st_size,
|
||||
"modified": stat.st_mtime,
|
||||
})
|
||||
await websocket.send(json.dumps({
|
||||
"type": "dirlist", "id": cmd_id,
|
||||
"path": str(p), "entries": entries,
|
||||
}))
|
||||
else:
|
||||
# Read file (limit to 10MB)
|
||||
size = p.stat().st_size
|
||||
if size > 10 * 1024 * 1024:
|
||||
await websocket.send(json.dumps({
|
||||
"type": "error", "id": cmd_id,
|
||||
"message": f"File too large: {size} bytes"
|
||||
}))
|
||||
return
|
||||
|
||||
content = p.read_text(encoding="utf-8", errors="replace")
|
||||
await websocket.send(json.dumps({
|
||||
"type": "file", "id": cmd_id,
|
||||
"path": str(p), "content": content,
|
||||
"size": size,
|
||||
}))
|
||||
|
||||
except Exception as e:
|
||||
await websocket.send(json.dumps({
|
||||
"type": "error", "id": cmd_id,
|
||||
"message": str(e),
|
||||
}))
|
||||
|
||||
|
||||
async def agent_loop(server_url, token):
|
||||
"""Main agent loop - connect to server and process commands."""
|
||||
# Allow self-signed certs
|
||||
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
||||
ssl_context.check_hostname = False
|
||||
ssl_context.verify_mode = ssl.CERT_NONE
|
||||
|
||||
ws_url = server_url.replace("https://", "wss://").replace("http://", "ws://")
|
||||
ws_url = f"{ws_url}/ws/agent"
|
||||
|
||||
print(f"Connecting to {ws_url}...")
|
||||
|
||||
while True:
|
||||
try:
|
||||
async with websockets.connect(ws_url, ssl=ssl_context,
|
||||
extra_headers={"Authorization": f"Bearer {token}"}) as ws:
|
||||
print("Connected to AUTARCH server")
|
||||
|
||||
# Send system info on connect
|
||||
sysinfo = get_system_info()
|
||||
sysinfo["type"] = "sysinfo"
|
||||
await ws.send(json.dumps(sysinfo))
|
||||
|
||||
# Send heartbeat periodically
|
||||
async def heartbeat():
|
||||
while True:
|
||||
await asyncio.sleep(30)
|
||||
try:
|
||||
await ws.send(json.dumps({"type": "heartbeat", "time": time.time()}))
|
||||
except Exception:
|
||||
break
|
||||
|
||||
heartbeat_task = asyncio.create_task(heartbeat())
|
||||
|
||||
# Process commands
|
||||
try:
|
||||
async for message in ws:
|
||||
try:
|
||||
msg = json.loads(message)
|
||||
msg_type = msg.get("type", "")
|
||||
cmd_id = msg.get("id", "")
|
||||
|
||||
if msg_type == "exec":
|
||||
command = msg.get("command", "")
|
||||
if command:
|
||||
print(f" Executing: {command[:80]}")
|
||||
asyncio.create_task(execute_command(command, cmd_id, ws))
|
||||
|
||||
elif msg_type == "read":
|
||||
path = msg.get("path", "")
|
||||
if path:
|
||||
asyncio.create_task(read_file(path, cmd_id, ws))
|
||||
|
||||
elif msg_type == "ping":
|
||||
await ws.send(json.dumps({"type": "pong", "id": cmd_id}))
|
||||
|
||||
elif msg_type == "kill":
|
||||
# Server wants us to disconnect
|
||||
print("Server requested disconnect")
|
||||
break
|
||||
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
finally:
|
||||
heartbeat_task.cancel()
|
||||
|
||||
except (websockets.exceptions.ConnectionClosed,
|
||||
ConnectionRefusedError, OSError) as e:
|
||||
print(f"Connection lost: {e}. Reconnecting in 5s...")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nAgent stopped")
|
||||
break
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="AUTARCH Local Agent")
|
||||
parser.add_argument("--server", "-s", required=True,
|
||||
help="AUTARCH server URL (e.g. https://10.0.0.1:8181)")
|
||||
parser.add_argument("--token", "-t", default="",
|
||||
help="Authentication token")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("AUTARCH Local Agent")
|
||||
print(f"Server: {args.server}")
|
||||
print(f"System: {platform.system()} {platform.machine()}")
|
||||
print(f"Host: {socket.gethostname()}")
|
||||
print()
|
||||
|
||||
asyncio.run(agent_loop(args.server, args.token))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -76,6 +76,49 @@ def get_data_dir() -> Path:
|
||||
return d
|
||||
|
||||
|
||||
def get_user_data_dir() -> Path:
|
||||
"""Return the platform-specific user data directory.
|
||||
Created as a hidden folder on first call.
|
||||
|
||||
Windows: %APPDATA%/Autarch/
|
||||
Linux: ~/.autarch/
|
||||
macOS: ~/Library/Application Support/Autarch/
|
||||
"""
|
||||
system = platform.system()
|
||||
if system == 'Windows':
|
||||
base = os.environ.get('APPDATA', '')
|
||||
if base:
|
||||
d = Path(base) / 'Autarch'
|
||||
else:
|
||||
d = Path.home() / 'AppData' / 'Roaming' / 'Autarch'
|
||||
elif system == 'Darwin':
|
||||
d = Path.home() / 'Library' / 'Application Support' / 'Autarch'
|
||||
else:
|
||||
# Linux and others - hidden dotfolder in home
|
||||
d = Path.home() / '.autarch'
|
||||
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def get_keystore_path() -> Path:
|
||||
"""Return the path to the encrypted keystore file.
|
||||
Stored in the user data directory, separate from the app install."""
|
||||
return get_user_data_dir() / 'keystore.xml'
|
||||
|
||||
|
||||
def get_user_profile_path() -> Path:
|
||||
"""Return the path to the user profile file."""
|
||||
return get_user_data_dir() / 'profile.json'
|
||||
|
||||
|
||||
def get_user_sessions_dir() -> Path:
|
||||
"""Return the directory for user session data (chat history, agent memory)."""
|
||||
d = get_user_data_dir() / 'sessions'
|
||||
d.mkdir(parents=True, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
def get_config_path() -> Path:
|
||||
"""Return config path. Writable copy lives next to the exe;
|
||||
falls back to the bundled default if the writable copy doesn't exist yet."""
|
||||
|
||||
184
core/sites_db.py
184
core/sites_db.py
@@ -49,6 +49,12 @@ class SitesDatabase:
|
||||
self._conn = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# Ensure the schema exists, then seed from the master dh.json the first
|
||||
# time (or whenever the cache is empty). dh.json is the source of truth;
|
||||
# dh_sites.db is a fast local query cache rebuilt from it automatically.
|
||||
self._ensure_schema()
|
||||
self._auto_populate()
|
||||
|
||||
def _get_connection(self) -> sqlite3.Connection:
|
||||
"""Get thread-safe database connection."""
|
||||
if self._conn is None:
|
||||
@@ -56,6 +62,48 @@ class SitesDatabase:
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
return self._conn
|
||||
|
||||
def _ensure_schema(self) -> None:
|
||||
"""Create the sites table and indexes if they don't already exist."""
|
||||
with self._lock:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("""
|
||||
CREATE TABLE IF NOT EXISTS sites (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
url_template TEXT NOT NULL,
|
||||
category TEXT DEFAULT 'other',
|
||||
source TEXT DEFAULT 'dh',
|
||||
nsfw INTEGER DEFAULT 0,
|
||||
enabled INTEGER DEFAULT 1,
|
||||
error_type TEXT,
|
||||
error_code INTEGER,
|
||||
error_string TEXT,
|
||||
match_code INTEGER,
|
||||
match_string TEXT
|
||||
)
|
||||
""")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_sites_category ON sites(category)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_sites_source ON sites(source)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_sites_enabled ON sites(enabled)")
|
||||
cursor.execute("CREATE INDEX IF NOT EXISTS idx_sites_nsfw ON sites(nsfw)")
|
||||
conn.commit()
|
||||
|
||||
def _auto_populate(self) -> None:
|
||||
"""Seed the cache from the master dh.json when the sites table is empty."""
|
||||
with self._lock:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
cursor.execute("SELECT COUNT(*) FROM sites")
|
||||
count = cursor.fetchone()[0]
|
||||
|
||||
if count > 0:
|
||||
return
|
||||
|
||||
json_path = self.data_dir / "dh.json"
|
||||
if json_path.exists():
|
||||
self.load_from_json(json_path)
|
||||
|
||||
def get_stats(self) -> Dict[str, Any]:
|
||||
"""Get database statistics."""
|
||||
with self._lock:
|
||||
@@ -390,6 +438,142 @@ class SitesDatabase:
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def add_sites_bulk(self, sites: List[Dict], overwrite: bool = False) -> Dict[str, int]:
|
||||
"""Bulk-insert sites in a single transaction.
|
||||
|
||||
By default only sites with a NOT-yet-seen name are added; existing
|
||||
names are left untouched (INSERT OR IGNORE). Pass overwrite=True to
|
||||
replace existing rows instead.
|
||||
|
||||
Args:
|
||||
sites: List of site dicts. Recognised keys: name, url_template
|
||||
(or url), category, source, nsfw, enabled, error_type,
|
||||
error_code, error_string, match_code, match_string.
|
||||
overwrite: Replace existing rows sharing the same name.
|
||||
|
||||
Returns:
|
||||
Stats dict: {'total', 'added', 'skipped', 'errors'}.
|
||||
"""
|
||||
stats = {'total': len(sites), 'added': 0, 'skipped': 0, 'errors': 0}
|
||||
verb = "INSERT OR REPLACE" if overwrite else "INSERT OR IGNORE"
|
||||
|
||||
with self._lock:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
for site in sites:
|
||||
name = site.get('name')
|
||||
url = site.get('url_template') or site.get('url')
|
||||
if not name or not url:
|
||||
stats['errors'] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
cursor.execute(f"""
|
||||
{verb} INTO sites
|
||||
(name, url_template, category, source, nsfw, enabled,
|
||||
error_type, error_code, error_string, match_code, match_string)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
name,
|
||||
url,
|
||||
site.get('category', 'other'),
|
||||
site.get('source', 'custom'),
|
||||
1 if site.get('nsfw') else 0,
|
||||
0 if site.get('enabled', 1) in (0, False) else 1,
|
||||
site.get('error_type'),
|
||||
site.get('error_code'),
|
||||
site.get('error_string'),
|
||||
site.get('match_code'),
|
||||
site.get('match_string'),
|
||||
))
|
||||
if cursor.rowcount > 0:
|
||||
stats['added'] += 1
|
||||
else:
|
||||
stats['skipped'] += 1
|
||||
except Exception:
|
||||
stats['errors'] += 1
|
||||
|
||||
conn.commit()
|
||||
|
||||
return stats
|
||||
|
||||
@staticmethod
|
||||
def _normalize_url(url: str) -> str:
|
||||
"""Normalise a URL template for duplicate detection.
|
||||
|
||||
Strips scheme, leading www., case, and trailing slash so that
|
||||
http/https, www/non-www and case variants of the same profile URL
|
||||
collapse to one key.
|
||||
"""
|
||||
if not url:
|
||||
return ''
|
||||
u = url.strip().lower()
|
||||
for scheme in ('https://', 'http://'):
|
||||
if u.startswith(scheme):
|
||||
u = u[len(scheme):]
|
||||
break
|
||||
if u.startswith('www.'):
|
||||
u = u[4:]
|
||||
return u.rstrip('/')
|
||||
|
||||
def deduplicate(self) -> Dict[str, int]:
|
||||
"""Remove duplicate sites that share the same (normalised) URL template.
|
||||
|
||||
Names are already unique via the schema constraint; this collapses rows
|
||||
whose URL templates point at the same profile page. For each duplicate
|
||||
group the best row is kept (prefer enabled, then having a detection
|
||||
pattern, then the oldest/lowest id) and the rest are deleted.
|
||||
|
||||
Returns:
|
||||
Stats dict: {'duplicate_groups', 'removed', 'remaining'}.
|
||||
"""
|
||||
with self._lock:
|
||||
conn = self._get_connection()
|
||||
cursor = conn.cursor()
|
||||
|
||||
cursor.execute(
|
||||
"SELECT id, url_template, error_type, error_string, match_string, enabled FROM sites"
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
groups: Dict[str, list] = {}
|
||||
for r in rows:
|
||||
key = self._normalize_url(r['url_template'])
|
||||
if not key:
|
||||
continue
|
||||
groups.setdefault(key, []).append(r)
|
||||
|
||||
def score(r):
|
||||
has_detection = bool(r['error_string'] or r['match_string'] or r['error_type'])
|
||||
# Higher tuple wins; -id keeps the lowest (oldest) id.
|
||||
return (1 if r['enabled'] else 0, 1 if has_detection else 0, -r['id'])
|
||||
|
||||
to_delete = []
|
||||
dup_groups = 0
|
||||
for members in groups.values():
|
||||
if len(members) < 2:
|
||||
continue
|
||||
dup_groups += 1
|
||||
members_sorted = sorted(members, key=score, reverse=True)
|
||||
to_delete.extend(r['id'] for r in members_sorted[1:])
|
||||
|
||||
for i in range(0, len(to_delete), 500):
|
||||
chunk = to_delete[i:i + 500]
|
||||
placeholders = ','.join('?' * len(chunk))
|
||||
cursor.execute(f"DELETE FROM sites WHERE id IN ({placeholders})", chunk)
|
||||
|
||||
conn.commit()
|
||||
|
||||
cursor.execute("SELECT COUNT(*) FROM sites")
|
||||
remaining = cursor.fetchone()[0]
|
||||
|
||||
return {
|
||||
'duplicate_groups': dup_groups,
|
||||
'removed': len(to_delete),
|
||||
'remaining': remaining,
|
||||
}
|
||||
|
||||
def update_detection(
|
||||
self,
|
||||
name: str,
|
||||
|
||||
159
core/user_profile.py
Normal file
159
core/user_profile.py
Normal file
@@ -0,0 +1,159 @@
|
||||
"""
|
||||
AUTARCH User Profile Manager
|
||||
Handles user identity, preferences, and session management.
|
||||
|
||||
On first launch, creates a default profile in the user data directory.
|
||||
Supports multiple users for shared deployments.
|
||||
|
||||
The profile stores non-sensitive user info (username, display name, role, etc.)
|
||||
Sensitive data (API keys, passwords) goes in the keystore.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, Optional
|
||||
|
||||
_log = logging.getLogger('autarch.user_profile')
|
||||
|
||||
|
||||
class UserProfile:
|
||||
"""User profile and preferences."""
|
||||
|
||||
def __init__(self, profile_path: Path = None):
|
||||
from core.paths import get_user_profile_path
|
||||
self._path = profile_path or get_user_profile_path()
|
||||
self._data: Dict = {}
|
||||
self._load()
|
||||
|
||||
def _load(self):
|
||||
"""Load profile from disk, or create default."""
|
||||
if self._path.exists():
|
||||
try:
|
||||
with open(self._path) as f:
|
||||
self._data = json.load(f)
|
||||
_log.info(f'Loaded profile for {self._data.get("username", "unknown")}')
|
||||
return
|
||||
except Exception as e:
|
||||
_log.error(f'Failed to load profile: {e}')
|
||||
|
||||
# Create default profile
|
||||
self._data = self._create_default()
|
||||
self._save()
|
||||
_log.info('Created default user profile')
|
||||
|
||||
def _create_default(self) -> Dict:
|
||||
"""Create the default user profile."""
|
||||
username = os.getenv('USER', os.getenv('USERNAME', 'operator'))
|
||||
return {
|
||||
'username': username,
|
||||
'display_name': username.capitalize(),
|
||||
'role': 'admin',
|
||||
'created': int(time.time()),
|
||||
'last_login': int(time.time()),
|
||||
'preferences': {
|
||||
'theme': 'dark',
|
||||
'llm_backend': 'local',
|
||||
'hal_auto_analyze': True,
|
||||
'hal_max_steps': 200,
|
||||
'verbose': False,
|
||||
'default_scan_target': 'local', # 'local' scans user device, 'server' scans server
|
||||
'chat_memory_enabled': True,
|
||||
},
|
||||
'session': {
|
||||
'last_module': '',
|
||||
'last_target': '',
|
||||
'last_scan_type': '',
|
||||
},
|
||||
}
|
||||
|
||||
def _save(self):
|
||||
"""Save profile to disk."""
|
||||
try:
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(self._path, 'w') as f:
|
||||
json.dump(self._data, f, indent=2)
|
||||
except Exception as e:
|
||||
_log.error(f'Failed to save profile: {e}')
|
||||
|
||||
@property
|
||||
def username(self) -> str:
|
||||
return self._data.get('username', 'default')
|
||||
|
||||
@property
|
||||
def display_name(self) -> str:
|
||||
return self._data.get('display_name', self.username)
|
||||
|
||||
@property
|
||||
def role(self) -> str:
|
||||
return self._data.get('role', 'user')
|
||||
|
||||
def get_preference(self, key: str, default=None):
|
||||
"""Get a user preference value."""
|
||||
return self._data.get('preferences', {}).get(key, default)
|
||||
|
||||
def set_preference(self, key: str, value):
|
||||
"""Set a user preference and save."""
|
||||
if 'preferences' not in self._data:
|
||||
self._data['preferences'] = {}
|
||||
self._data['preferences'][key] = value
|
||||
self._save()
|
||||
|
||||
def get_session(self, key: str, default='') -> str:
|
||||
"""Get a session value (non-persistent state hints)."""
|
||||
return self._data.get('session', {}).get(key, default)
|
||||
|
||||
def set_session(self, key: str, value: str):
|
||||
"""Set a session value."""
|
||||
if 'session' not in self._data:
|
||||
self._data['session'] = {}
|
||||
self._data['session'][key] = value
|
||||
self._save()
|
||||
|
||||
def update_last_login(self):
|
||||
"""Record login timestamp."""
|
||||
self._data['last_login'] = int(time.time())
|
||||
self._save()
|
||||
|
||||
def to_dict(self) -> Dict:
|
||||
"""Export profile as a dict (for API/display)."""
|
||||
return {
|
||||
'username': self.username,
|
||||
'display_name': self.display_name,
|
||||
'role': self.role,
|
||||
'created': self._data.get('created', 0),
|
||||
'last_login': self._data.get('last_login', 0),
|
||||
'preferences': self._data.get('preferences', {}),
|
||||
}
|
||||
|
||||
|
||||
# Singleton
|
||||
_profile_instance: Optional[UserProfile] = None
|
||||
|
||||
|
||||
def get_profile() -> UserProfile:
|
||||
"""Get the global user profile instance."""
|
||||
global _profile_instance
|
||||
if _profile_instance is None:
|
||||
_profile_instance = UserProfile()
|
||||
return _profile_instance
|
||||
|
||||
|
||||
def init_profile_on_launch():
|
||||
"""Called at app startup to ensure profile exists and update login time."""
|
||||
profile = get_profile()
|
||||
profile.update_last_login()
|
||||
|
||||
# Migrate old vault secrets to new keystore if needed
|
||||
from core.paths import get_keystore_path
|
||||
ks_path = get_keystore_path()
|
||||
if not ks_path.exists():
|
||||
from core.keystore import get_keystore
|
||||
ks = get_keystore()
|
||||
migrated = ks.migrate_from_vault()
|
||||
if migrated:
|
||||
_log.info(f'Migrated {migrated} secrets from vault to keystore on first run')
|
||||
|
||||
return profile
|
||||
Reference in New Issue
Block a user