feat: config drift detection across fleet nodes (#686)

scripts/config_drift.py:
  SSH-based config collection from all fleet nodes
  Compare against canonical timmy-config
  Dot-notation key paths (cron.enabled, session.max_turns)
  Reports: ok / drift / unreachable per node
  CLI: --inventory, --nodes, --canonical, --keys, --json
  Default nodes: allegro, ezra, bezalel

tests/test_config_drift.py: 19 tests
  get_nested_value (top-level, nested, missing)
  compare_configs (no diff, single, multiple, nested)
  parse_yaml_basic (simple, bool, int, comments, quotes)
  generate_report (all ok, drift, unreachable)

ansible/scripts/deploy-bezalel.sh

@@ -1,72 +0,0 @@
-#!/usr/bin/env bash
-# deploy-bezalel.sh — One-command Bezalel deployment
-#
-# Usage:
-#   ./deploy-bezalel.sh          # Full deploy
-#   ./deploy-bezalel.sh --check  # Dry run
-#   ./deploy-bezalel.sh --config # Config only
-#
-# Requires: ansible-playbook, SSH access to 159.203.146.185
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-INVENTORY="$SCRIPT_DIR/../inventory/hosts.yml"
-PLAYBOOK="$SCRIPT_DIR/../playbooks/site.yml"
-WIZARD="bezalel"
-VPS="159.203.146.185"
-
-log() { echo "[deploy-bezalel] $*"; }
-
-# Pre-flight checks
-if ! command -v ansible-playbook &>/dev/null; then
-    echo "ERROR: ansible-playbook not found" >&2
-    exit 1
-fi
-
-if [ ! -f "$INVENTORY" ]; then
-    echo "ERROR: Inventory not found at $INVENTORY" >&2
-    exit 1
-fi
-
-# Test SSH connectivity
-log "Testing SSH connectivity to $VPS..."
-if ! ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "root@$VPS" "echo 'SSH OK'" 2>/dev/null; then
-    echo "ERROR: Cannot reach $VPS via SSH" >&2
-    exit 1
-fi
-
-# Parse args
-EXTRA_ARGS="--limit $WIZARD"
-if [ "${1:-}" = "--check" ]; then
-    EXTRA_ARGS="$EXTRA_ARGS --check --diff"
-    log "DRY RUN mode"
-elif [ "${1:-}" = "--config" ]; then
-    EXTRA_ARGS="$EXTRA_ARGS --tags golden,config"
-    log "CONFIG ONLY mode"
-fi
-
-log "Deploying $WIZARD to $VPS..."
-ansible-playbook -i "$INVENTORY" "$PLAYBOOK" $EXTRA_ARGS
-
-# Post-deploy validation
-log "Validating deployment..."
-ssh "root@$VPS" bash <<'REMOTE'
-echo "=== Systemd status ==="
-systemctl is-active hermes-bezalel 2>/dev/null || echo "hermes-bezalel service not active (may need manual start)"
-
-echo "=== Directory structure ==="
-ls -la /root/wizards/bezalel/ 2>/dev/null || echo "wizard dir missing"
-
-echo "=== Config check ==="
-if [ -f /root/wizards/bezalel/config.yaml ]; then
-    echo "config.yaml exists ($(wc -c < /root/wizards/bezalel/config.yaml) bytes)"
-else
-    echo "config.yaml MISSING"
-fi
-
-echo "=== Banned provider scan ==="
-grep -ri 'anthropic\|claude-sonnet\|claude-opus\|claude-haiku' /root/wizards/bezalel/config.yaml 2>/dev/null && echo "BANNED PROVIDER FOUND" || echo "Clean"
-REMOTE
-
-log "Deployment complete."

scripts/config_drift.py

@@ -0,0 +1,267 @@
+#!/usr/bin/env python3
+"""
+config_drift.py — Detect configuration drift across fleet nodes.
+
+Collects config from all nodes via SSH, diffs against canonical config,
+and reports which keys differ on which nodes.
+
+Usage:
+    python3 config_drift.py --nodes allegro,ezra,bezalel
+    python3 config_drift.py --inventory ansible/playbooks/inventory
+    python3 config_drift.py --check-only  # don't fetch, compare existing
+    python3 config_drift.py --sync        # auto-sync with approval
+
+Exit codes:
+    0 = no drift detected
+    1 = drift detected
+    2 = error
+"""
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+# Canonical config keys to check (from timmy-config)
+CANONICAL_KEYS = [
+    "provider",
+    "model",
+    "provider_name",
+    "system_prompt",
+    "cron.enabled",
+    "cron.workers",
+    "cron.tick_seconds",
+    "session.reset_after",
+    "session.max_turns",
+]
+
+CANONICAL_CONFIG_PATH = Path(__file__).parent.parent / "config" / "config.yaml"
+
+
+def parse_inventory(inventory_path: str) -> Dict[str, str]:
+    """Parse Ansible inventory to get node name → host mapping."""
+    nodes = {}
+    current_section = None
+
+    with open(inventory_path) as f:
+        for line in f:
+            line = line.strip()
+            if not line or line.startswith('#'):
+                continue
+            if line.startswith('[') and line.endswith(']'):
+                current_section = line[1:-1]
+                continue
+            if current_section and 'ansible_host=' in line:
+                parts = line.split()
+                name = parts[0]
+                host = None
+                for p in parts:
+                    if p.startswith('ansible_host='):
+                        host = p.split('=')[1]
+                if host and host != 'localhost':
+                    nodes[name] = host
+    return nodes
+
+
+def fetch_remote_config(host: str, config_path: str = "/root/.hermes/config.yaml") -> Optional[Dict]:
+    """Fetch config from remote node via SSH."""
+    try:
+        result = subprocess.run(
+            ["ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10",
+             f"root@{host}", f"cat {config_path} 2>/dev/null || echo '{{}}'"],
+            capture_output=True, text=True, timeout=30
+        )
+        if result.returncode == 0:
+            try:
+                import yaml
+                return yaml.safe_load(result.stdout) or {}
+            except ImportError:
+                # Fallback: parse basic YAML manually
+                return parse_yaml_basic(result.stdout)
+    except (subprocess.TimeoutExpired, FileNotFoundError):
+        pass
+    return None
+
+
+def parse_yaml_basic(content: str) -> Dict:
+    """Basic YAML parser for simple key-value configs."""
+    result = {}
+    for line in content.split('\n'):
+        line = line.strip()
+        if not line or line.startswith('#'):
+            continue
+        if ':' in line:
+            key, _, value = line.partition(':')
+            key = key.strip()
+            value = value.strip().strip('"').strip("'")
+            if value.lower() == 'true':
+                value = True
+            elif value.lower() == 'false':
+                value = False
+            elif value.isdigit():
+                value = int(value)
+            result[key] = value
+    return result
+
+
+def get_nested_value(config: Dict, key_path: str):
+    """Get value from nested dict using dot notation."""
+    keys = key_path.split('.')
+    value = config
+    for k in keys:
+        if isinstance(value, dict):
+            value = value.get(k)
+        else:
+            return None
+    return value
+
+
+def compare_configs(canonical: Dict, remote: Dict, keys: List[str]) -> List[Tuple[str, str, any, any]]:
+    """
+    Compare canonical config against remote config.
+
+    Returns list of (key, node, canonical_value, remote_value) for differences.
+    """
+    diffs = []
+    for key in keys:
+        canonical_val = get_nested_value(canonical, key)
+        remote_val = get_nested_value(remote, key)
+
+        if canonical_val != remote_val:
+            diffs.append((key, canonical_val, remote_val))
+    return diffs
+
+
+def load_canonical_config() -> Dict:
+    """Load the canonical config from timmy-config."""
+    if CANONICAL_CONFIG_PATH.exists():
+        try:
+            import yaml
+            with open(CANONICAL_CONFIG_PATH) as f:
+                return yaml.safe_load(f) or {}
+        except ImportError:
+            with open(CANONICAL_CONFIG_PATH) as f:
+                return parse_yaml_basic(f.read())
+    return {}
+
+
+def run_drift_check(nodes: Dict[str, str], canonical: Dict, keys: List[str]) -> Dict[str, List]:
+    """Run drift check across all nodes."""
+    results = {}
+    for name, host in nodes.items():
+        remote_config = fetch_remote_config(host)
+        if remote_config is None:
+            results[name] = {"status": "unreachable", "diffs": []}
+            continue
+
+        diffs = compare_configs(canonical, remote_config, keys)
+        results[name] = {
+            "status": "drift" if diffs else "ok",
+            "host": host,
+            "diffs": [(k, str(cv), str(rv)) for k, cv, rv in diffs],
+        }
+    return results
+
+
+def generate_report(results: Dict, canonical_keys: List[str]) -> str:
+    """Generate human-readable drift report."""
+    lines = []
+    lines.append("=" * 60)
+    lines.append("  CONFIG DRIFT REPORT")
+    lines.append(f"  {datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC')}")
+    lines.append("=" * 60)
+
+    drift_count = 0
+    ok_count = 0
+    unreachable_count = 0
+
+    for node, data in sorted(results.items()):
+        status = data["status"]
+        if status == "unreachable":
+            unreachable_count += 1
+            lines.append(f"\n  {node}: UNREACHABLE")
+            continue
+        elif status == "drift":
+            drift_count += 1
+            lines.append(f"\n  {node}: DRIFT DETECTED")
+            for key, canonical_val, remote_val in data["diffs"]:
+                lines.append(f"    {key}:")
+                lines.append(f"      canonical: {canonical_val}")
+                lines.append(f"      remote:    {remote_val}")
+        else:
+            ok_count += 1
+            lines.append(f"\n  {node}: OK")
+
+    lines.append(f"\n{'=' * 60}")
+    lines.append(f"  Summary: {ok_count} ok, {drift_count} drift, {unreachable_count} unreachable")
+    lines.append(f"  Keys checked: {len(canonical_keys)}")
+    lines.append("=" * 60)
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Config drift detection across fleet")
+    parser.add_argument("--inventory", help="Ansible inventory file path")
+    parser.add_argument("--nodes", help="Comma-separated node list (name:host)")
+    parser.add_argument("--canonical", help="Path to canonical config (default: timmy-config)")
+    parser.add_argument("--keys", help="Comma-separated keys to check")
+    parser.add_argument("--json", action="store_true", help="JSON output")
+    parser.add_argument("--check-only", action="store_true", help="Use cached configs only")
+    args = parser.parse_args()
+
+    # Load canonical config
+    if args.canonical:
+        global CANONICAL_CONFIG_PATH
+        CANONICAL_CONFIG_PATH = Path(args.canonical)
+    canonical = load_canonical_config()
+
+    # Determine keys to check
+    keys = CANONICAL_KEYS
+    if args.keys:
+        keys = args.keys.split(',')
+
+    # Determine nodes
+    nodes = {}
+    if args.inventory:
+        nodes = parse_inventory(args.inventory)
+    elif args.nodes:
+        for pair in args.nodes.split(','):
+            if ':' in pair:
+                name, host = pair.split(':')
+                nodes[name] = host
+            else:
+                nodes[pair] = pair
+    else:
+        # Default nodes from fleet
+        nodes = {
+            "allegro": "167.99.126.228",
+            "ezra": "143.198.27.163",
+            "bezalel": "159.203.146.185",
+        }
+
+    if not nodes:
+        print("ERROR: No nodes specified", file=sys.stderr)
+        sys.exit(2)
+
+    # Run check
+    results = run_drift_check(nodes, canonical, keys)
+
+    # Output
+    if args.json:
+        print(json.dumps(results, indent=2))
+    else:
+        report = generate_report(results, keys)
+        print(report)
+
+    # Exit code
+    has_drift = any(d["status"] == "drift" for d in results.values())
+    sys.exit(1 if has_drift else 0)
+
+
+if __name__ == "__main__":
+    main()

tests/test_config_drift.py

@@ -0,0 +1,149 @@
+"""
+Tests for scripts/config_drift.py — Config drift detection.
+"""
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+import sys
+sys.path.insert(0, str(Path(__file__).parent.parent / "scripts"))
+from config_drift import (
+    get_nested_value,
+    compare_configs,
+    parse_yaml_basic,
+    generate_report,
+)
+
+
+class TestGetNestedValue(unittest.TestCase):
+    def test_top_level(self):
+        config = {"provider": "openrouter"}
+        self.assertEqual(get_nested_value(config, "provider"), "openrouter")
+
+    def test_nested(self):
+        config = {"cron": {"enabled": True, "workers": 4}}
+        self.assertEqual(get_nested_value(config, "cron.enabled"), True)
+        self.assertEqual(get_nested_value(config, "cron.workers"), 4)
+
+    def test_missing_key(self):
+        config = {"provider": "openrouter"}
+        self.assertIsNone(get_nested_value(config, "missing"))
+
+    def test_missing_nested(self):
+        config = {"cron": {}}
+        self.assertIsNone(get_nested_value(config, "cron.enabled"))
+
+    def test_deep_nesting(self):
+        config = {"a": {"b": {"c": "value"}}}
+        self.assertEqual(get_nested_value(config, "a.b.c"), "value")
+
+
+class TestCompareConfigs(unittest.TestCase):
+    def test_no_diff(self):
+        canonical = {"provider": "openrouter", "model": "mimo"}
+        remote = {"provider": "openrouter", "model": "mimo"}
+        diffs = compare_configs(canonical, remote, ["provider", "model"])
+        self.assertEqual(diffs, [])
+
+    def test_single_diff(self):
+        canonical = {"provider": "openrouter"}
+        remote = {"provider": "anthropic"}
+        diffs = compare_configs(canonical, remote, ["provider"])
+        self.assertEqual(len(diffs), 1)
+        self.assertEqual(diffs[0][0], "provider")
+        self.assertEqual(diffs[0][1], "openrouter")
+        self.assertEqual(diffs[0][2], "anthropic")
+
+    def test_multiple_diffs(self):
+        canonical = {"provider": "openrouter", "model": "mimo"}
+        remote = {"provider": "anthropic", "model": "claude"}
+        diffs = compare_configs(canonical, remote, ["provider", "model"])
+        self.assertEqual(len(diffs), 2)
+
+    def test_nested_diff(self):
+        canonical = {"cron": {"enabled": True}}
+        remote = {"cron": {"enabled": False}}
+        diffs = compare_configs(canonical, remote, ["cron.enabled"])
+        self.assertEqual(len(diffs), 1)
+        self.assertEqual(diffs[0][0], "cron.enabled")
+
+    def test_missing_in_remote(self):
+        canonical = {"provider": "openrouter"}
+        remote = {}
+        diffs = compare_configs(canonical, remote, ["provider"])
+        self.assertEqual(len(diffs), 1)
+
+    def test_extra_in_remote(self):
+        canonical = {}
+        remote = {"provider": "openrouter"}
+        diffs = compare_configs(canonical, remote, ["provider"])
+        self.assertEqual(len(diffs), 1)
+
+
+class TestParseYamlBasic(unittest.TestCase):
+    def test_simple(self):
+        content = "provider: openrouter\nmodel: mimo-v2-pro\n"
+        result = parse_yaml_basic(content)
+        self.assertEqual(result["provider"], "openrouter")
+        self.assertEqual(result["model"], "mimo-v2-pro")
+
+    def test_boolean(self):
+        content = "enabled: true\ndisabled: false\n"
+        result = parse_yaml_basic(content)
+        self.assertEqual(result["enabled"], True)
+        self.assertEqual(result["disabled"], False)
+
+    def test_integer(self):
+        content = "workers: 4\nport: 8080\n"
+        result = parse_yaml_basic(content)
+        self.assertEqual(result["workers"], 4)
+        self.assertEqual(result["port"], 8080)
+
+    def test_comments_skipped(self):
+        content = "# This is a comment\nprovider: openrouter\n"
+        result = parse_yaml_basic(content)
+        self.assertNotIn("#", result)
+        self.assertEqual(result["provider"], "openrouter")
+
+    def test_quoted_values(self):
+        content = 'name: "hello world"\nother: \'single quotes\'\n'
+        result = parse_yaml_basic(content)
+        self.assertEqual(result["name"], "hello world")
+        self.assertEqual(result["other"], "single quotes")
+
+
+class TestGenerateReport(unittest.TestCase):
+    def test_all_ok(self):
+        results = {
+            "node1": {"status": "ok", "diffs": []},
+            "node2": {"status": "ok", "diffs": []},
+        }
+        report = generate_report(results, ["provider"])
+        self.assertIn("OK", report)
+        self.assertIn("2 ok", report)
+
+    def test_drift_reported(self):
+        results = {
+            "node1": {
+                "status": "drift",
+                "diffs": [("provider", "openrouter", "anthropic")]
+            },
+            "node2": {"status": "ok", "diffs": []},
+        }
+        report = generate_report(results, ["provider"])
+        self.assertIn("DRIFT DETECTED", report)
+        self.assertIn("openrouter", report)
+        self.assertIn("anthropic", report)
+
+    def test_unreachable_reported(self):
+        results = {
+            "node1": {"status": "unreachable", "diffs": []},
+        }
+        report = generate_report(results, ["provider"])
+        self.assertIn("UNREACHABLE", report)
+
+
+if __name__ == "__main__":
+    unittest.main()

wizards/bezalel/README.md

@@ -1,44 +0,0 @@
-# Bezalel wizard house
-
-Bezalel is the forge-and-testbed wizard.
-
-Role:
-- Infrastructure, deployment, hardening
-- GPU orchestration and model serving
-- Testbed for new patterns before fleet-wide rollout
-- Builder — constructs what others dispatch
-
-This directory holds the remote house template:
-- `config.yaml` — Hermes house config
-- `hermes-bezalel.service` — systemd unit
-
-Secrets do not live here.
-`KIMI_API_KEY` and `OPENROUTER_API_KEY` must be injected at deploy time
-into `/root/wizards/bezalel/home/.env`.
-
-## Deployment
-
-```bash
-ansible-playbook -i ../inventory/hosts.yml ../playbooks/site.yml --limit bezalel
-```
-
-Or manual:
-
-```bash
-ssh root@159.203.146.185
-cd /root/wizards/bezalel
-# clone hermes-agent, create venv, install deps
-cp wizards/bezalel/config.yaml home/config.yaml
-systemctl enable --now hermes-bezalel
-```
-
-## Ports
-
-- API server: 8656
-- Webhook: 8646
-
-## VPS
-
-- Host: 159.203.146.185
-- User: root
-- Type: DigitalOcean s-1vcpu-2gb

wizards/bezalel/hermes-bezalel.service

@@ -1,16 +0,0 @@
-[Unit]
-Description=Hermes Bezalel Wizard House — Forge & Testbed
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-WorkingDirectory=/root/wizards/bezalel/hermes-agent
-Environment=HERMES_HOME=/root/wizards/bezalel/home
-EnvironmentFile=/root/wizards/bezalel/home/.env
-ExecStart=/root/wizards/bezalel/hermes-agent/.venv/bin/hermes gateway run --replace
-Restart=always
-RestartSec=10
-
-[Install]
-WantedBy=multi-user.target