AUTARCH v1.9 — remote monitoring, SSH manager, daemon, vault, cleanup
- Add Remote Monitoring Station with PIAP device profile system - Add SSH/SSHD manager with fail2ban integration - Add privileged daemon architecture for safe root operations - Add encrypted vault, HAL memory, HAL auto-analyst - Add network security suite, module creator, codex training - Add start.sh launcher script and GTK3 desktop launcher - Remove Output/ build artifacts, installer files, loose docs - Update .gitignore for runtime data and build artifacts - Update README for v1.9 with new launch method, screenshots, and features Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
SsSnake
21
scripts/autarch-daemon.service
Normal file
21
scripts/autarch-daemon.service
Normal file
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=AUTARCH Privileged Daemon
|
||||
Documentation=file:///home/snake/autarch/GUIDE.md
|
||||
Before=autarch-web.service
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
Group=root
|
||||
ExecStart=/home/snake/autarch/venv/bin/python /home/snake/autarch/core/daemon.py
|
||||
ExecStop=/bin/kill -TERM $MAINPID
|
||||
Restart=on-failure
|
||||
RestartSec=3
|
||||
PIDFile=/var/run/autarch-daemon.pid
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=autarch-daemon
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -6,21 +6,27 @@ Wants=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=snake
|
||||
Group=snake
|
||||
User=root
|
||||
Group=root
|
||||
WorkingDirectory=/home/snake/autarch
|
||||
ExecStart=/usr/bin/python3 /home/snake/autarch/autarch.py --web --no-banner
|
||||
|
||||
# Use venv python if available, fall back to system python
|
||||
ExecStart=/bin/bash -c 'if [ -x /home/snake/autarch/venv/bin/python ]; then exec /home/snake/autarch/venv/bin/python /home/snake/autarch/autarch.py --web --no-banner; else exec /usr/bin/python3 /home/snake/autarch/autarch.py --web --no-banner; fi'
|
||||
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=autarch-web
|
||||
|
||||
# Security hardening
|
||||
# Security — run as root for raw sockets, iptables, hardware access
|
||||
NoNewPrivileges=false
|
||||
ProtectHome=false
|
||||
PrivateTmp=true
|
||||
|
||||
# Capabilities needed when not running as root (future: drop root)
|
||||
# AmbientCapabilities=CAP_NET_RAW CAP_NET_ADMIN CAP_NET_BIND_SERVICE
|
||||
|
||||
# Environment
|
||||
Environment=PYTHONUNBUFFERED=1
|
||||
|
||||
|
||||
10
scripts/autarch.desktop
Normal file
10
scripts/autarch.desktop
Normal file
@@ -0,0 +1,10 @@
|
||||
[Desktop Entry]
|
||||
Name=AUTARCH
|
||||
Comment=Security Platform by darkHal Security Group
|
||||
Exec=/home/snake/autarch/venv/bin/python /home/snake/autarch/launcher.py
|
||||
Icon=/home/snake/autarch/icon.svg
|
||||
Terminal=false
|
||||
Type=Application
|
||||
Categories=System;Security;Network;
|
||||
Keywords=security;hacking;pentest;network;
|
||||
StartupWMClass=autarch
|
||||
542
scripts/build_codex.py
Normal file
542
scripts/build_codex.py
Normal file
@@ -0,0 +1,542 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
AUTARCH Codex Generator
|
||||
Scans the codebase and generates a structured knowledge document
|
||||
that LLM agents use to understand how to create modules, routes,
|
||||
and features for AUTARCH.
|
||||
|
||||
Run: python scripts/build_codex.py
|
||||
Output: data/codex/autarch_codex.md
|
||||
|
||||
This should be re-run after any significant codebase changes.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
FRAMEWORK_DIR = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(FRAMEWORK_DIR))
|
||||
|
||||
OUTPUT_PATH = FRAMEWORK_DIR / 'data' / 'codex' / 'autarch_codex.md'
|
||||
|
||||
|
||||
def extract_module_metadata(filepath: Path) -> dict:
|
||||
"""Extract module-level metadata from a Python file using AST."""
|
||||
try:
|
||||
source = filepath.read_text(encoding='utf-8', errors='ignore')
|
||||
tree = ast.parse(source)
|
||||
except (SyntaxError, UnicodeDecodeError):
|
||||
return None
|
||||
|
||||
meta = {'file': str(filepath.relative_to(FRAMEWORK_DIR)), 'functions': [], 'classes': []}
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Assign):
|
||||
for target in node.targets:
|
||||
if isinstance(target, ast.Name) and isinstance(node.value, ast.Constant):
|
||||
meta[target.id] = node.value.value
|
||||
elif isinstance(node, ast.FunctionDef):
|
||||
doc = ast.get_docstring(node) or ''
|
||||
args = [a.arg for a in node.args.args if a.arg != 'self']
|
||||
meta['functions'].append({
|
||||
'name': node.name,
|
||||
'args': args,
|
||||
'doc': doc.split('\n')[0] if doc else '',
|
||||
'line': node.lineno,
|
||||
})
|
||||
elif isinstance(node, ast.ClassDef):
|
||||
doc = ast.get_docstring(node) or ''
|
||||
methods = []
|
||||
for item in node.body:
|
||||
if isinstance(item, ast.FunctionDef):
|
||||
mdoc = ast.get_docstring(item) or ''
|
||||
methods.append(item.name)
|
||||
meta['classes'].append({
|
||||
'name': node.name,
|
||||
'doc': doc.split('\n')[0] if doc else '',
|
||||
'methods': methods,
|
||||
'line': node.lineno,
|
||||
})
|
||||
|
||||
return meta
|
||||
|
||||
|
||||
def extract_route_info(filepath: Path) -> list:
|
||||
"""Extract Flask route decorators and handler info."""
|
||||
routes = []
|
||||
try:
|
||||
source = filepath.read_text(encoding='utf-8', errors='ignore')
|
||||
tree = ast.parse(source)
|
||||
except (SyntaxError, UnicodeDecodeError):
|
||||
return routes
|
||||
|
||||
for node in ast.iter_child_nodes(tree):
|
||||
if isinstance(node, ast.FunctionDef):
|
||||
for deco in node.decorator_list:
|
||||
route_path = None
|
||||
methods = ['GET']
|
||||
if isinstance(deco, ast.Call) and hasattr(deco, 'func'):
|
||||
func = deco.func
|
||||
if hasattr(func, 'attr') and func.attr == 'route':
|
||||
if deco.args and isinstance(deco.args[0], ast.Constant):
|
||||
route_path = deco.args[0].value
|
||||
for kw in deco.keywords:
|
||||
if kw.arg == 'methods' and isinstance(kw.value, ast.List):
|
||||
methods = [e.value for e in kw.value.elts if isinstance(e, ast.Constant)]
|
||||
if route_path:
|
||||
doc = ast.get_docstring(node) or ''
|
||||
routes.append({
|
||||
'path': route_path,
|
||||
'methods': methods,
|
||||
'handler': node.name,
|
||||
'doc': doc.split('\n')[0] if doc else '',
|
||||
'line': node.lineno,
|
||||
})
|
||||
return routes
|
||||
|
||||
|
||||
def extract_template_blocks(filepath: Path) -> dict:
|
||||
"""Extract basic template structure info."""
|
||||
try:
|
||||
content = filepath.read_text(encoding='utf-8', errors='ignore')
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
info = {'file': filepath.name, 'size': len(content)}
|
||||
if '{% extends' in content:
|
||||
import re
|
||||
m = re.search(r'{%\s*extends\s*["\'](.+?)["\']\s*%}', content)
|
||||
if m:
|
||||
info['extends'] = m.group(1)
|
||||
if '{% block content %}' in content:
|
||||
info['has_content_block'] = True
|
||||
return info
|
||||
|
||||
|
||||
def get_example_module(modules_dir: Path) -> str:
|
||||
"""Get a clean, representative module example."""
|
||||
# Pick a simple, well-structured module
|
||||
for candidate in ['geoip.py', 'dossier.py', 'loadtest.py', 'ipcapture.py']:
|
||||
path = modules_dir / candidate
|
||||
if path.exists():
|
||||
source = path.read_text(encoding='utf-8', errors='ignore')
|
||||
# Truncate to first 80 lines if long
|
||||
lines = source.split('\n')
|
||||
if len(lines) > 80:
|
||||
source = '\n'.join(lines[:80]) + '\n# ... (truncated)'
|
||||
return source
|
||||
return '# No example found'
|
||||
|
||||
|
||||
def get_example_route(routes_dir: Path) -> str:
|
||||
"""Get a representative route example."""
|
||||
for candidate in ['ipcapture.py', 'loadtest.py']:
|
||||
path = routes_dir / candidate
|
||||
if path.exists():
|
||||
source = path.read_text(encoding='utf-8', errors='ignore')
|
||||
lines = source.split('\n')
|
||||
if len(lines) > 80:
|
||||
source = '\n'.join(lines[:80]) + '\n# ... (truncated)'
|
||||
return source
|
||||
return '# No example found'
|
||||
|
||||
|
||||
def build_codex():
|
||||
"""Generate the full codex document."""
|
||||
print("[codex] Scanning codebase...")
|
||||
|
||||
# Scan modules
|
||||
modules_dir = FRAMEWORK_DIR / 'modules'
|
||||
modules = {}
|
||||
for f in sorted(modules_dir.glob('*.py')):
|
||||
if f.name == '__init__.py':
|
||||
continue
|
||||
meta = extract_module_metadata(f)
|
||||
if meta:
|
||||
modules[f.stem] = meta
|
||||
|
||||
# Scan core
|
||||
core_dir = FRAMEWORK_DIR / 'core'
|
||||
core_modules = {}
|
||||
for f in sorted(core_dir.glob('*.py')):
|
||||
if f.name == '__init__.py':
|
||||
continue
|
||||
meta = extract_module_metadata(f)
|
||||
if meta:
|
||||
core_modules[f.stem] = meta
|
||||
|
||||
# Scan routes
|
||||
routes_dir = FRAMEWORK_DIR / 'web' / 'routes'
|
||||
all_routes = {}
|
||||
for f in sorted(routes_dir.glob('*.py')):
|
||||
if f.name == '__init__.py':
|
||||
continue
|
||||
routes = extract_route_info(f)
|
||||
if routes:
|
||||
all_routes[f.stem] = routes
|
||||
|
||||
# Scan templates
|
||||
templates_dir = FRAMEWORK_DIR / 'web' / 'templates'
|
||||
templates = {}
|
||||
for f in sorted(templates_dir.glob('*.html')):
|
||||
info = extract_template_blocks(f)
|
||||
if info:
|
||||
templates[f.stem] = info
|
||||
|
||||
# Read config defaults
|
||||
from core.config import Config
|
||||
config_defaults = Config.DEFAULT_CONFIG
|
||||
|
||||
# Build the document
|
||||
sections = []
|
||||
|
||||
# Header
|
||||
sections.append(f"""# AUTARCH Codex
|
||||
## Codebase Knowledge Reference for AI Agents
|
||||
Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}
|
||||
|
||||
This document is auto-generated by `scripts/build_codex.py` and provides
|
||||
structured knowledge about the AUTARCH codebase for LLM agents to use
|
||||
when creating modules, routes, templates, and features.
|
||||
|
||||
---
|
||||
""")
|
||||
|
||||
# Module system
|
||||
categories = {}
|
||||
for name, meta in modules.items():
|
||||
cat = meta.get('CATEGORY', 'core')
|
||||
categories.setdefault(cat, []).append(name)
|
||||
|
||||
sections.append("""## 1. Module System
|
||||
|
||||
AUTARCH modules are Python files in the `modules/` directory. Each module:
|
||||
- Has a `run()` function as the entry point
|
||||
- Declares metadata: `DESCRIPTION`, `AUTHOR`, `VERSION`, `CATEGORY`
|
||||
- Is auto-discovered by `core/menu.py` at startup
|
||||
- Can be run via CLI (`python autarch.py -m <name>`) or from the web UI
|
||||
|
||||
### Required Module Attributes
|
||||
|
||||
```python
|
||||
DESCRIPTION = "Short description of what the module does"
|
||||
AUTHOR = "Your Name"
|
||||
VERSION = "1.0"
|
||||
CATEGORY = "defense" # One of: defense, offense, counter, analyze, osint, simulate, core, hardware
|
||||
```
|
||||
|
||||
### Module Template
|
||||
|
||||
```python
|
||||
\"\"\"
|
||||
Module description here.
|
||||
\"\"\"
|
||||
|
||||
DESCRIPTION = "Short description"
|
||||
AUTHOR = "darkHal"
|
||||
VERSION = "1.0"
|
||||
CATEGORY = "defense"
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
from core.banner import Colors, clear_screen, display_banner
|
||||
|
||||
|
||||
def run():
|
||||
\"\"\"Main entry point — REQUIRED.\"\"\"
|
||||
clear_screen()
|
||||
display_banner()
|
||||
print(f"{Colors.BOLD}Module Name{Colors.RESET}")
|
||||
print(f"{Colors.DIM}{'─' * 50}{Colors.RESET}\\n")
|
||||
|
||||
# Module logic here
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
```
|
||||
|
||||
### Categories and Module Counts
|
||||
|
||||
""")
|
||||
for cat in ['defense', 'offense', 'counter', 'analyze', 'osint', 'simulate', 'core', 'hardware']:
|
||||
mods = categories.get(cat, [])
|
||||
sections.append(f"- **{cat}** ({len(mods)}): {', '.join(mods[:10])}")
|
||||
if len(mods) > 10:
|
||||
sections.append(f" ... and {len(mods) - 10} more")
|
||||
sections.append(f"\n**Total modules: {len(modules)}**\n")
|
||||
|
||||
# Core API reference
|
||||
sections.append("""
|
||||
---
|
||||
|
||||
## 2. Core API Reference
|
||||
|
||||
The `core/` directory contains the framework backbone. Key modules:
|
||||
|
||||
""")
|
||||
for name, meta in sorted(core_modules.items()):
|
||||
doc = meta.get('__doc__', '') or ''
|
||||
classes = meta.get('classes', [])
|
||||
functions = [f for f in meta.get('functions', [])
|
||||
if not f['name'].startswith('_') and f['name'] != 'run']
|
||||
if not classes and not functions:
|
||||
continue
|
||||
|
||||
sections.append(f"### core/{name}.py\n")
|
||||
if classes:
|
||||
for cls in classes[:3]:
|
||||
sections.append(f"- **class `{cls['name']}`** — {cls['doc']}")
|
||||
if cls['methods']:
|
||||
public = [m for m in cls['methods'] if not m.startswith('_')][:8]
|
||||
if public:
|
||||
sections.append(f" - Methods: `{'`, `'.join(public)}`")
|
||||
if functions:
|
||||
for func in functions[:8]:
|
||||
args_str = ', '.join(func['args'][:4])
|
||||
sections.append(f"- `{func['name']}({args_str})` — {func['doc']}")
|
||||
sections.append("")
|
||||
|
||||
# Key imports
|
||||
sections.append("""
|
||||
### Common Imports for Modules
|
||||
|
||||
```python
|
||||
# Colors and display
|
||||
from core.banner import Colors, clear_screen, display_banner
|
||||
|
||||
# Configuration
|
||||
from core.config import get_config
|
||||
|
||||
# LLM access
|
||||
from core.llm import get_llm, LLMError
|
||||
|
||||
# Agent tools
|
||||
from core.tools import get_tool_registry
|
||||
|
||||
# File paths
|
||||
from core.paths import get_app_dir, get_data_dir, find_tool
|
||||
|
||||
# Hardware (ADB/Fastboot)
|
||||
from core.hardware import get_hardware_manager
|
||||
|
||||
# Available Colors
|
||||
Colors.RED, Colors.GREEN, Colors.YELLOW, Colors.BLUE,
|
||||
Colors.MAGENTA, Colors.CYAN, Colors.WHITE, Colors.BOLD,
|
||||
Colors.DIM, Colors.RESET
|
||||
```
|
||||
|
||||
""")
|
||||
|
||||
# Web route patterns
|
||||
sections.append("""---
|
||||
|
||||
## 3. Web Route Patterns
|
||||
|
||||
Routes live in `web/routes/`. Each file defines a Flask Blueprint.
|
||||
|
||||
### Blueprint Template
|
||||
|
||||
```python
|
||||
from flask import Blueprint, render_template, request, jsonify
|
||||
from web.auth import login_required
|
||||
|
||||
myfeature_bp = Blueprint('myfeature', __name__, url_prefix='/myfeature')
|
||||
|
||||
|
||||
@myfeature_bp.route('/')
|
||||
@login_required
|
||||
def index():
|
||||
return render_template('myfeature.html')
|
||||
|
||||
|
||||
@myfeature_bp.route('/action', methods=['POST'])
|
||||
@login_required
|
||||
def action():
|
||||
data = request.get_json(silent=True) or {}
|
||||
# Process...
|
||||
return jsonify({'ok': True, 'result': ...})
|
||||
```
|
||||
|
||||
### Registration
|
||||
|
||||
In `web/app.py`, add:
|
||||
```python
|
||||
from web.routes.myfeature import myfeature_bp
|
||||
app.register_blueprint(myfeature_bp)
|
||||
```
|
||||
|
||||
### Existing Routes
|
||||
|
||||
""")
|
||||
for name, routes in sorted(all_routes.items()):
|
||||
sections.append(f"**{name}** ({len(routes)} routes)")
|
||||
for r in routes[:5]:
|
||||
methods = ','.join(r['methods'])
|
||||
sections.append(f" - `{methods} {r['path']}` → `{r['handler']}`")
|
||||
if len(routes) > 5:
|
||||
sections.append(f" - ... and {len(routes) - 5} more")
|
||||
sections.append("")
|
||||
|
||||
# Template patterns
|
||||
sections.append("""---
|
||||
|
||||
## 4. Template Patterns
|
||||
|
||||
Templates live in `web/templates/` and use Jinja2 extending `base.html`.
|
||||
|
||||
### Template Structure
|
||||
|
||||
```html
|
||||
{%% extends "base.html" %%}
|
||||
{%% block title %%}Feature Name - AUTARCH{%% endblock %%}
|
||||
|
||||
{%% block content %%}
|
||||
<div class="page-header">
|
||||
<h1>Feature Name</h1>
|
||||
</div>
|
||||
|
||||
<div class="section">
|
||||
<h2>Section Title</h2>
|
||||
<!-- Content here -->
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// JS for this page
|
||||
</script>
|
||||
{%% endblock %%}
|
||||
```
|
||||
|
||||
### CSS Variables Available
|
||||
```
|
||||
--bg-main, --bg-card, --bg-secondary, --bg-input
|
||||
--text-primary, --text-secondary, --text-muted
|
||||
--accent (green), --danger (red), --border
|
||||
--radius (border radius), --success (green)
|
||||
```
|
||||
|
||||
### Common UI Patterns
|
||||
- Tab bar: `<div class="tab-bar"><button class="tab active">Tab 1</button></div>`
|
||||
- Card: `<div style="border:1px solid var(--border);background:var(--bg-card);border-radius:var(--radius);padding:0.85rem 1rem">`
|
||||
- Table: `<table class="data-table"><thead>...</thead><tbody>...</tbody></table>`
|
||||
- Button: `<button class="btn btn-primary btn-sm">Action</button>`
|
||||
- Form: `<div class="form-group"><label>...</label><input ...><small>Help text</small></div>`
|
||||
|
||||
""")
|
||||
sections.append(f"### Templates ({len(templates)} total)\n")
|
||||
for name, info in sorted(templates.items()):
|
||||
extends = info.get('extends', 'none')
|
||||
sections.append(f"- `{info['file']}` (extends: {extends})")
|
||||
|
||||
# Config system
|
||||
sections.append(f"""
|
||||
|
||||
---
|
||||
|
||||
## 5. Configuration System
|
||||
|
||||
Config is managed by `core/config.py` using Python's configparser.
|
||||
File: `autarch_settings.conf` (INI format).
|
||||
|
||||
### Config Sections
|
||||
|
||||
""")
|
||||
for section, defaults in config_defaults.items():
|
||||
keys = ', '.join(list(defaults.keys())[:8])
|
||||
more = f" ... +{len(defaults) - 8} more" if len(defaults) > 8 else ""
|
||||
sections.append(f"- **[{section}]**: {keys}{more}")
|
||||
|
||||
sections.append("""
|
||||
### Usage in Code
|
||||
|
||||
```python
|
||||
from core.config import get_config
|
||||
config = get_config()
|
||||
|
||||
# Read values
|
||||
val = config.get('section', 'key', 'default')
|
||||
num = config.get_int('section', 'key', 0)
|
||||
flt = config.get_float('section', 'key', 0.0)
|
||||
bol = config.get_bool('section', 'key', False)
|
||||
|
||||
# Write values
|
||||
config.set('section', 'key', 'value')
|
||||
config.save()
|
||||
|
||||
# Typed getters
|
||||
config.get_llama_settings() # dict
|
||||
config.get_claude_settings() # dict
|
||||
config.get_mcp_settings() # dict
|
||||
config.get_agents_settings() # dict
|
||||
config.get_autonomy_settings() # dict
|
||||
```
|
||||
|
||||
""")
|
||||
|
||||
# Sidebar navigation
|
||||
sections.append("""---
|
||||
|
||||
## 6. Adding to the Navigation
|
||||
|
||||
Edit `web/templates/base.html`. The sidebar has sections:
|
||||
- Top (Dashboard, Port Scanner, Targets)
|
||||
- Categories (Defense, Offense, Counter, Analyze, OSINT, Simulate)
|
||||
- Network (Network Security, Wireshark, Net Mapper)
|
||||
- Tools (Create Module, Enc Modules, Hardware, exploits, Shield, etc.)
|
||||
- System (UPnP, WireGuard, MSF Console, DNS, Settings, etc.)
|
||||
|
||||
Add a nav item:
|
||||
```html
|
||||
<li><a href="{{ url_for('myfeature.index') }}"
|
||||
class="{% if request.blueprint == 'myfeature' %}active{% endif %}">
|
||||
My Feature</a></li>
|
||||
```
|
||||
|
||||
Sub-items use: `style="padding-left:1.5rem;font-size:0.85rem"` with `└` prefix.
|
||||
|
||||
""")
|
||||
|
||||
# MCP tools
|
||||
sections.append("""---
|
||||
|
||||
## 7. MCP Tool System
|
||||
|
||||
Tools exposed via Model Context Protocol (MCP) are defined in `core/mcp_server.py`.
|
||||
To add a new MCP tool:
|
||||
|
||||
```python
|
||||
# In create_mcp_server(), add:
|
||||
@mcp.tool()
|
||||
def my_tool(param1: str, param2: int = 10) -> str:
|
||||
\"\"\"Description of what the tool does.\"\"\"
|
||||
return execute_tool('my_tool', {'param1': param1, 'param2': param2})
|
||||
|
||||
# In execute_tool(), add the handler:
|
||||
elif name == 'my_tool':
|
||||
return _run_my_tool(arguments)
|
||||
|
||||
# Implement the handler:
|
||||
def _run_my_tool(args: dict) -> str:
|
||||
# ... implementation
|
||||
return json.dumps({'result': ...})
|
||||
```
|
||||
|
||||
""")
|
||||
|
||||
# Write output
|
||||
content = '\n'.join(sections)
|
||||
OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
OUTPUT_PATH.write_text(content, encoding='utf-8')
|
||||
print(f"[codex] Written {len(content):,} bytes to {OUTPUT_PATH}")
|
||||
print(f"[codex] Scanned: {len(modules)} modules, {len(core_modules)} core files, "
|
||||
f"{sum(len(r) for r in all_routes.values())} routes, {len(templates)} templates")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
build_codex()
|
||||
295
scripts/build_training_data.py
Normal file
295
scripts/build_training_data.py
Normal file
@@ -0,0 +1,295 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
AUTARCH LoRA Training Data Generator
|
||||
Extracts instruction/input/output triplets from the codebase
|
||||
for fine-tuning LLMs on AUTARCH module creation patterns.
|
||||
|
||||
Run: python scripts/build_training_data.py
|
||||
Output: data/codex/autarch_training.jsonl
|
||||
|
||||
Generates training pairs for:
|
||||
- Module creation (description → code)
|
||||
- Route creation (feature description → Flask blueprint)
|
||||
- Config patterns (section description → config code)
|
||||
- Template patterns (feature → Jinja2 template)
|
||||
"""
|
||||
|
||||
import ast
|
||||
import json
|
||||
import sys
|
||||
import re
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
FRAMEWORK_DIR = Path(__file__).parent.parent
|
||||
sys.path.insert(0, str(FRAMEWORK_DIR))
|
||||
|
||||
OUTPUT_PATH = FRAMEWORK_DIR / 'data' / 'codex' / 'autarch_training.jsonl'
|
||||
|
||||
|
||||
def extract_module_pair(filepath: Path) -> dict:
|
||||
"""Extract a training pair from a module file."""
|
||||
try:
|
||||
source = filepath.read_text(encoding='utf-8', errors='ignore')
|
||||
tree = ast.parse(source)
|
||||
except (SyntaxError, UnicodeDecodeError):
|
||||
return None
|
||||
|
||||
description = None
|
||||
category = None
|
||||
author = None
|
||||
version = None
|
||||
docstring = ast.get_docstring(tree) or ''
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Assign):
|
||||
for target in node.targets:
|
||||
if isinstance(target, ast.Name) and isinstance(node.value, ast.Constant):
|
||||
if target.id == 'DESCRIPTION':
|
||||
description = node.value.value
|
||||
elif target.id == 'CATEGORY':
|
||||
category = node.value.value
|
||||
elif target.id == 'AUTHOR':
|
||||
author = node.value.value
|
||||
elif target.id == 'VERSION':
|
||||
version = node.value.value
|
||||
|
||||
if not description or not category:
|
||||
return None
|
||||
|
||||
# Build the instruction
|
||||
instruction = (
|
||||
f"Create an AUTARCH module in the '{category}' category that {description.lower().rstrip('.')}. "
|
||||
f"The module should follow AUTARCH conventions with DESCRIPTION, AUTHOR, VERSION, CATEGORY "
|
||||
f"attributes and a run() entry point function."
|
||||
)
|
||||
|
||||
return {
|
||||
'instruction': instruction,
|
||||
'input': f"Module name: {filepath.stem}\nCategory: {category}\nDescription: {description}",
|
||||
'output': source,
|
||||
'type': 'module_creation',
|
||||
'category': category,
|
||||
'source_file': str(filepath.relative_to(FRAMEWORK_DIR)),
|
||||
}
|
||||
|
||||
|
||||
def extract_route_pair(filepath: Path) -> dict:
|
||||
"""Extract a training pair from a route file."""
|
||||
try:
|
||||
source = filepath.read_text(encoding='utf-8', errors='ignore')
|
||||
tree = ast.parse(source)
|
||||
except (SyntaxError, UnicodeDecodeError):
|
||||
return None
|
||||
|
||||
docstring = ast.get_docstring(tree) or ''
|
||||
|
||||
# Find blueprint name and prefix
|
||||
bp_name = None
|
||||
bp_prefix = None
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Assign):
|
||||
for target in node.targets:
|
||||
if isinstance(target, ast.Name) and isinstance(node.value, ast.Call):
|
||||
if hasattr(node.value, 'func'):
|
||||
func_name = ''
|
||||
if hasattr(node.value.func, 'id'):
|
||||
func_name = node.value.func.id
|
||||
elif hasattr(node.value.func, 'attr'):
|
||||
func_name = node.value.func.attr
|
||||
if func_name == 'Blueprint':
|
||||
bp_name = target.id
|
||||
for kw in node.value.keywords:
|
||||
if kw.arg == 'url_prefix' and isinstance(kw.value, ast.Constant):
|
||||
bp_prefix = kw.value.value
|
||||
|
||||
if not bp_name:
|
||||
return None
|
||||
|
||||
# Count routes
|
||||
routes = []
|
||||
for node in ast.iter_child_nodes(tree):
|
||||
if isinstance(node, ast.FunctionDef):
|
||||
for deco in node.decorator_list:
|
||||
if isinstance(deco, ast.Call) and hasattr(deco, 'func'):
|
||||
if hasattr(deco.func, 'attr') and deco.func.attr == 'route':
|
||||
doc = ast.get_docstring(node) or ''
|
||||
routes.append({
|
||||
'handler': node.name,
|
||||
'doc': doc.split('\n')[0] if doc else '',
|
||||
})
|
||||
|
||||
feature_name = filepath.stem.replace('_', ' ').title()
|
||||
instruction = (
|
||||
f"Create a Flask blueprint route file for AUTARCH's '{feature_name}' feature. "
|
||||
f"It should have a blueprint with url_prefix='{bp_prefix or '/' + filepath.stem}', "
|
||||
f"use @login_required on all routes, and follow AUTARCH web route conventions. "
|
||||
f"It needs {len(routes)} route handlers."
|
||||
)
|
||||
|
||||
return {
|
||||
'instruction': instruction,
|
||||
'input': f"Feature: {feature_name}\nBlueprint: {bp_name}\nPrefix: {bp_prefix}\nRoutes: {len(routes)}",
|
||||
'output': source,
|
||||
'type': 'route_creation',
|
||||
'source_file': str(filepath.relative_to(FRAMEWORK_DIR)),
|
||||
}
|
||||
|
||||
|
||||
def extract_template_pair(filepath: Path) -> dict:
|
||||
"""Extract a training pair from a template file."""
|
||||
try:
|
||||
source = filepath.read_text(encoding='utf-8', errors='ignore')
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
if '{% extends' not in source or '{% block content %}' not in source:
|
||||
return None
|
||||
|
||||
# Count sections, tabs, buttons, forms
|
||||
sections = source.count('class="section"') + source.count("class='section'")
|
||||
tabs = source.count('class="tab"') + source.count("class='tab'")
|
||||
forms = source.count('<form') + source.count('fetch(')
|
||||
has_script = '<script>' in source
|
||||
|
||||
feature_name = filepath.stem.replace('_', ' ').title()
|
||||
instruction = (
|
||||
f"Create an AUTARCH web template for the '{feature_name}' page. "
|
||||
f"It should extend base.html, have a page header, and use AUTARCH's "
|
||||
f"CSS variables and UI patterns (sections, tab bars, data tables, buttons)."
|
||||
)
|
||||
|
||||
return {
|
||||
'instruction': instruction,
|
||||
'input': (
|
||||
f"Template: {filepath.name}\n"
|
||||
f"Sections: {sections}\nTabs: {tabs}\nForms/API calls: {forms}\n"
|
||||
f"Has JavaScript: {has_script}"
|
||||
),
|
||||
'output': source,
|
||||
'type': 'template_creation',
|
||||
'source_file': str(filepath.relative_to(FRAMEWORK_DIR)),
|
||||
}
|
||||
|
||||
|
||||
def extract_core_api_pairs(filepath: Path) -> list:
|
||||
"""Extract training pairs showing how to use core APIs."""
|
||||
pairs = []
|
||||
try:
|
||||
source = filepath.read_text(encoding='utf-8', errors='ignore')
|
||||
tree = ast.parse(source)
|
||||
except (SyntaxError, UnicodeDecodeError):
|
||||
return pairs
|
||||
|
||||
for node in ast.iter_child_nodes(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||||
if node.name.startswith('_'):
|
||||
continue
|
||||
doc = ast.get_docstring(node) or ''
|
||||
if not doc:
|
||||
continue
|
||||
|
||||
# Extract the function source
|
||||
lines = source.split('\n')
|
||||
start = node.lineno - 1
|
||||
end = node.end_lineno if hasattr(node, 'end_lineno') else start + 20
|
||||
func_source = '\n'.join(lines[start:end])
|
||||
|
||||
args = [a.arg for a in node.args.args if a.arg != 'self']
|
||||
module_name = filepath.stem
|
||||
|
||||
pairs.append({
|
||||
'instruction': f"Show how to implement the `{node.name}` function in core/{module_name}.py",
|
||||
'input': f"Function: {node.name}({', '.join(args)})\nDocstring: {doc.split(chr(10))[0]}",
|
||||
'output': func_source,
|
||||
'type': 'api_reference',
|
||||
'source_file': f"core/{filepath.name}",
|
||||
})
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def build_training_data():
|
||||
"""Generate training data from the codebase."""
|
||||
print("[training] Scanning codebase for training pairs...")
|
||||
|
||||
pairs = []
|
||||
|
||||
# Module pairs
|
||||
modules_dir = FRAMEWORK_DIR / 'modules'
|
||||
for f in sorted(modules_dir.glob('*.py')):
|
||||
if f.name == '__init__.py':
|
||||
continue
|
||||
pair = extract_module_pair(f)
|
||||
if pair:
|
||||
pairs.append(pair)
|
||||
|
||||
module_count = len(pairs)
|
||||
print(f" Modules: {module_count} pairs")
|
||||
|
||||
# Route pairs
|
||||
routes_dir = FRAMEWORK_DIR / 'web' / 'routes'
|
||||
for f in sorted(routes_dir.glob('*.py')):
|
||||
if f.name == '__init__.py':
|
||||
continue
|
||||
pair = extract_route_pair(f)
|
||||
if pair:
|
||||
pairs.append(pair)
|
||||
|
||||
route_count = len(pairs) - module_count
|
||||
print(f" Routes: {route_count} pairs")
|
||||
|
||||
# Template pairs
|
||||
templates_dir = FRAMEWORK_DIR / 'web' / 'templates'
|
||||
for f in sorted(templates_dir.glob('*.html')):
|
||||
pair = extract_template_pair(f)
|
||||
if pair:
|
||||
pairs.append(pair)
|
||||
|
||||
template_count = len(pairs) - module_count - route_count
|
||||
print(f" Templates: {template_count} pairs")
|
||||
|
||||
# Core API pairs
|
||||
core_dir = FRAMEWORK_DIR / 'core'
|
||||
api_start = len(pairs)
|
||||
for f in sorted(core_dir.glob('*.py')):
|
||||
if f.name == '__init__.py':
|
||||
continue
|
||||
api_pairs = extract_core_api_pairs(f)
|
||||
pairs.extend(api_pairs)
|
||||
|
||||
api_count = len(pairs) - api_start
|
||||
print(f" Core API: {api_count} pairs")
|
||||
|
||||
# Write JSONL
|
||||
OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(OUTPUT_PATH, 'w', encoding='utf-8') as f:
|
||||
for pair in pairs:
|
||||
f.write(json.dumps(pair, ensure_ascii=False) + '\n')
|
||||
|
||||
total_size = OUTPUT_PATH.stat().st_size
|
||||
print(f"\n[training] Written {len(pairs)} training pairs ({total_size:,} bytes) to {OUTPUT_PATH}")
|
||||
print(f"[training] Breakdown: {module_count} modules, {route_count} routes, "
|
||||
f"{template_count} templates, {api_count} core API functions")
|
||||
|
||||
# Also output a summary
|
||||
summary_path = OUTPUT_PATH.with_suffix('.summary.json')
|
||||
summary = {
|
||||
'generated': datetime.now().isoformat(),
|
||||
'total_pairs': len(pairs),
|
||||
'modules': module_count,
|
||||
'routes': route_count,
|
||||
'templates': template_count,
|
||||
'core_api': api_count,
|
||||
'output_bytes': total_size,
|
||||
'types': {},
|
||||
}
|
||||
for p in pairs:
|
||||
t = p['type']
|
||||
summary['types'][t] = summary['types'].get(t, 0) + 1
|
||||
summary_path.write_text(json.dumps(summary, indent=2), encoding='utf-8')
|
||||
print(f"[training] Summary: {summary_path}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
build_training_data()
|
||||
17
scripts/com.darkhal.autarch.policy
Normal file
17
scripts/com.darkhal.autarch.policy
Normal file
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE policyconfig PUBLIC
|
||||
"-//freedesktop//DTD PolicyKit Policy Configuration 1.0//EN"
|
||||
"http://www.freedesktop.org/standards/PolicyKit/1/policyconfig.dtd">
|
||||
<policyconfig>
|
||||
<action id="com.darkhal.autarch.daemon">
|
||||
<description>Run AUTARCH privileged daemon</description>
|
||||
<message>AUTARCH needs root access for network security tools (iptables, WiFi scanning, etc.)</message>
|
||||
<defaults>
|
||||
<allow_any>auth_admin</allow_any>
|
||||
<allow_inactive>auth_admin</allow_inactive>
|
||||
<allow_active>auth_admin_keep</allow_active>
|
||||
</defaults>
|
||||
<annotate key="org.freedesktop.policykit.exec.path">/home/snake/autarch/venv/bin/python</annotate>
|
||||
<annotate key="org.freedesktop.policykit.exec.allow_gui">true</annotate>
|
||||
</action>
|
||||
</policyconfig>
|
||||
113
scripts/setup-venv.sh
Executable file
113
scripts/setup-venv.sh
Executable file
@@ -0,0 +1,113 @@
|
||||
#!/bin/bash
|
||||
# AUTARCH Virtual Environment Setup
|
||||
# Creates a project-local venv with all dependencies.
|
||||
# The venv is accessible by any user (including root), eliminating
|
||||
# the need for sys.path hacks or system-wide pip installs.
|
||||
#
|
||||
# Usage:
|
||||
# bash scripts/setup-venv.sh # Create venv and install deps
|
||||
# bash scripts/setup-venv.sh --force # Recreate from scratch
|
||||
#
|
||||
# After setup, run AUTARCH with:
|
||||
# sudo ./venv/bin/python autarch.py --web
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
VENV_DIR="$PROJECT_DIR/venv"
|
||||
REQ_FILE="$PROJECT_DIR/requirements.txt"
|
||||
|
||||
RED='\033[0;31m'; GREEN='\033[0;32m'; CYAN='\033[0;36m'; YELLOW='\033[1;33m'; RESET='\033[0m'
|
||||
|
||||
echo -e "${CYAN}╔═══════════════════════════════════════════╗${RESET}"
|
||||
echo -e "${CYAN}║ AUTARCH Virtual Environment Setup ║${RESET}"
|
||||
echo -e "${CYAN}╚═══════════════════════════════════════════╝${RESET}"
|
||||
echo ""
|
||||
|
||||
# Check Python version
|
||||
PYTHON=""
|
||||
for p in python3.13 python3.12 python3.11 python3; do
|
||||
if command -v "$p" &>/dev/null; then
|
||||
PYTHON="$p"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$PYTHON" ]; then
|
||||
echo -e "${RED}[X] Python 3.10+ not found. Install python3 first.${RESET}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PY_VERSION=$($PYTHON -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
|
||||
echo -e "${GREEN}[+] Using: $PYTHON (Python $PY_VERSION)${RESET}"
|
||||
|
||||
# Check for --force flag
|
||||
if [ "$1" = "--force" ] && [ -d "$VENV_DIR" ]; then
|
||||
echo -e "${YELLOW}[!] Removing existing venv...${RESET}"
|
||||
rm -rf "$VENV_DIR"
|
||||
fi
|
||||
|
||||
# Create venv if it doesn't exist
|
||||
if [ ! -d "$VENV_DIR" ]; then
|
||||
echo -e "${CYAN}[*] Creating virtual environment at $VENV_DIR${RESET}"
|
||||
# Try python -m venv first, fall back to virtualenv (doesn't need python3-venv package)
|
||||
if $PYTHON -m venv "$VENV_DIR" --system-site-packages 2>/dev/null; then
|
||||
echo -e "${GREEN}[+] venv created (venv module)${RESET}"
|
||||
else
|
||||
echo -e "${YELLOW}[!] python3-venv not available, using virtualenv...${RESET}"
|
||||
$PYTHON -m pip install --user virtualenv --quiet 2>/dev/null
|
||||
$PYTHON -m virtualenv "$VENV_DIR" --system-site-packages
|
||||
echo -e "${GREEN}[+] venv created (virtualenv)${RESET}"
|
||||
fi
|
||||
else
|
||||
echo -e "${GREEN}[+] venv already exists at $VENV_DIR${RESET}"
|
||||
fi
|
||||
|
||||
# Upgrade pip
|
||||
echo -e "${CYAN}[*] Upgrading pip...${RESET}"
|
||||
"$VENV_DIR/bin/pip" install --upgrade pip --quiet
|
||||
|
||||
# Install requirements
|
||||
if [ -f "$REQ_FILE" ]; then
|
||||
echo -e "${CYAN}[*] Installing requirements from $REQ_FILE${RESET}"
|
||||
"$VENV_DIR/bin/pip" install -r "$REQ_FILE" 2>&1 | while read line; do
|
||||
if echo "$line" | grep -q "^Successfully\|^Requirement already"; then
|
||||
echo -e " ${GREEN}$line${RESET}"
|
||||
elif echo "$line" | grep -qi "error\|fail"; then
|
||||
echo -e " ${YELLOW}$line${RESET}"
|
||||
fi
|
||||
done
|
||||
echo -e "${GREEN}[+] Requirements installed${RESET}"
|
||||
else
|
||||
echo -e "${YELLOW}[!] No requirements.txt found at $REQ_FILE${RESET}"
|
||||
fi
|
||||
|
||||
# Make venv accessible when running as root via sudo
|
||||
# This is the key: root can use venv/bin/python directly
|
||||
chmod -R a+rX "$VENV_DIR"
|
||||
|
||||
# Verify key packages
|
||||
echo ""
|
||||
echo -e "${CYAN}[*] Verifying key packages:${RESET}"
|
||||
for pkg in flask anthropic openai llama_cpp transformers scapy mcp; do
|
||||
if "$VENV_DIR/bin/python" -c "import $pkg" 2>/dev/null; then
|
||||
ver=$("$VENV_DIR/bin/python" -c "import $pkg; print(getattr($pkg, '__version__', 'ok'))" 2>/dev/null)
|
||||
echo -e " ${GREEN}✓ $pkg ($ver)${RESET}"
|
||||
else
|
||||
echo -e " ${YELLOW}✗ $pkg (not installed or failed)${RESET}"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}╔═══════════════════════════════════════════╗${RESET}"
|
||||
echo -e "${GREEN}║ Setup complete! ║${RESET}"
|
||||
echo -e "${GREEN}╚═══════════════════════════════════════════╝${RESET}"
|
||||
echo ""
|
||||
echo -e "Start AUTARCH with:"
|
||||
echo -e " ${CYAN}sudo $VENV_DIR/bin/python $PROJECT_DIR/autarch.py --web${RESET}"
|
||||
echo ""
|
||||
echo -e "Or activate the venv first:"
|
||||
echo -e " ${CYAN}source $VENV_DIR/bin/activate${RESET}"
|
||||
echo -e " ${CYAN}sudo -E python autarch.py --web${RESET}"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user