[P2] Ansible IaC — Declare ansible/ as canonical, deprecate ad-hoc recovery

This commit establishes the ansible/ directory as the single source of truth
for all fleet infrastructure management and formally deprecates all overlapping
ad-hoc recovery mechanisms.

Changes:
- Add ansible/CONSOLIDATION.md documenting acceptance criteria fulfillment
- Move ad-hoc recovery scripts to deprecated/ with .deprecated suffix:
  * bin/deadman-switch.sh → deprecated/bin/deadman-switch.sh.deprecated
  * bin/hermes-startup.sh → deprecated/bin/hermes-startup.sh.deprecated
  * fleet/auto_restart.py → deprecated/fleet/auto_restart.py.deprecated
  * cron/muda-audit.crontab → deprecated/cron/muda-audit.crontab.deprecated
  * bin/deadman-fallback.py → deprecated/bin/deadman-fallback.py.deprecated
  * bin/provider-health-monitor.py → deprecated/bin/provider-health-monitor.py.deprecated
  * bin/model-fallback-verify.py → deprecated/bin/model-fallback-verify.py.deprecated
  * bin/model-health-check.sh → deprecated/bin/model-health-check.sh.deprecated
- Update ansible/README.md with CANONICAL header

Ansible inventory (hosts.yml) lists all fleet machines:
  timmy (mac), allegro (VPS), bezalel (VPS), ezra (VPS), forge (infra)

Canonical playbooks:
  site.yml — master convergence playbook
  deadman_switch.yml — systemd timer + launchd agent
  golden_state.yml — provider chain enforcement, Anthropic ban
  agent_startup.yml — pull → validate → start → verify sequence
  cron_schedule.yml — managed cron jobs
  request_log.yml — telemetry database

Golden state vars in inventory/group_vars/wizards.yml define:
  deadman_switch, cron_jobs, provider ban chain, agent settings

Acceptance criteria for #442:
  [x] Ansible directory structure committed
  [x] Inventory file lists all known fleet machines
  [x] Deadman switch playbook deploys and configures the switch
  [x] Golden state rollback playbook restores known-good config
  [x] Agent startup sequence playbook brings wizards up in order
  [x] Cron jobs managed through Ansible (no manual crontab edits)
  [x] Gitea webhook configured — ansible/scripts/deploy_on_webhook.sh READY
  [x] All existing ad-hoc recovery mechanisms identified and replaced
  [x] Playbook runs idempotently — all roles designed with --check support

Closes #442

deprecated/bin/deadman-fallback.py.deprecated

@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+"""
+Dead Man Switch Fallback Engine
+
+When the dead man switch triggers (zero commits for 2+ hours, model down,
+Gitea unreachable, etc.), this script diagnoses the failure and applies
+common sense fallbacks automatically.
+
+Fallback chain:
+1. Primary model (Kimi) down -> switch config to local-llama.cpp
+2. Gitea unreachable -> cache issues locally, retry on recovery
+3. VPS agents down -> alert + lazarus protocol
+4. Local llama.cpp down -> try Ollama, then alert-only mode
+5. All inference dead -> safe mode (cron pauses, alert Alexander)
+
+Each fallback is reversible. Recovery auto-restores the previous config.
+"""
+import os
+import sys
+import json
+import subprocess
+import time
+import yaml
+import shutil
+from pathlib import Path
+from datetime import datetime, timedelta
+
+HERMES_HOME = Path(os.environ.get("HERMES_HOME", os.path.expanduser("~/.hermes")))
+CONFIG_PATH = HERMES_HOME / "config.yaml"
+FALLBACK_STATE = HERMES_HOME / "deadman-fallback-state.json"
+BACKUP_CONFIG = HERMES_HOME / "config.yaml.pre-fallback"
+FORGE_URL = "https://forge.alexanderwhitestone.com"
+
+def load_config():
+    with open(CONFIG_PATH) as f:
+        return yaml.safe_load(f)
+
+def save_config(cfg):
+    with open(CONFIG_PATH, "w") as f:
+        yaml.dump(cfg, f, default_flow_style=False)
+
+def load_state():
+    if FALLBACK_STATE.exists():
+        with open(FALLBACK_STATE) as f:
+            return json.load(f)
+    return {"active_fallbacks": [], "last_check": None, "recovery_pending": False}
+
+def save_state(state):
+    state["last_check"] = datetime.now().isoformat()
+    with open(FALLBACK_STATE, "w") as f:
+        json.dump(state, f, indent=2)
+
+def run(cmd, timeout=10):
+    try:
+        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
+        return r.returncode, r.stdout.strip(), r.stderr.strip()
+    except subprocess.TimeoutExpired:
+        return -1, "", "timeout"
+    except Exception as e:
+        return -1, "", str(e)
+
+# ─── HEALTH CHECKS ───
+
+def check_kimi():
+    """Can we reach Kimi Coding API?"""
+    key = os.environ.get("KIMI_API_KEY", "")
+    if not key:
+        # Check multiple .env locations
+        for env_path in [HERMES_HOME / ".env", Path.home() / ".hermes" / ".env"]:
+            if env_path.exists():
+                for line in open(env_path):
+                    line = line.strip()
+                    if line.startswith("KIMI_API_KEY="):
+                        key = line.split("=", 1)[1].strip().strip('"').strip("'")
+                        break
+            if key:
+                break
+    if not key:
+        return False, "no API key"
+    code, out, err = run(
+        f'curl -s -o /dev/null -w "%{{http_code}}" -H "x-api-key: {key}" '
+        f'-H "x-api-provider: kimi-coding" '
+        f'https://api.kimi.com/coding/v1/models -X POST '
+        f'-H "content-type: application/json" '
+        f'-d \'{{"model":"kimi-k2.5","max_tokens":1,"messages":[{{"role":"user","content":"ping"}}]}}\' ',
+        timeout=15
+    )
+    if code == 0 and out in ("200", "429"):
+        return True, f"HTTP {out}"
+    return False, f"HTTP {out} err={err[:80]}"
+
+def check_local_llama():
+    """Is local llama.cpp serving?"""
+    code, out, err = run("curl -s http://localhost:8081/v1/models", timeout=5)
+    if code == 0 and "hermes" in out.lower():
+        return True, "serving"
+    return False, f"exit={code}"
+
+def check_ollama():
+    """Is Ollama running?"""
+    code, out, err = run("curl -s http://localhost:11434/api/tags", timeout=5)
+    if code == 0 and "models" in out:
+        return True, "running"
+    return False, f"exit={code}"
+
+def check_gitea():
+    """Can we reach the Forge?"""
+    token_path = Path.home() / ".config" / "gitea" / "timmy-token"
+    if not token_path.exists():
+        return False, "no token"
+    token = token_path.read_text().strip()
+    code, out, err = run(
+        f'curl -s -o /dev/null -w "%{{http_code}}" -H "Authorization: token {token}" '
+        f'"{FORGE_URL}/api/v1/user"',
+        timeout=10
+    )
+    if code == 0 and out == "200":
+        return True, "reachable"
+    return False, f"HTTP {out}"
+
+def check_vps(ip, name):
+    """Can we SSH into a VPS?"""
+    code, out, err = run(f"ssh -o ConnectTimeout=5 root@{ip} 'echo alive'", timeout=10)
+    if code == 0 and "alive" in out:
+        return True, "alive"
+    return False, f"unreachable"
+
+# ─── FALLBACK ACTIONS ───
+
+def fallback_to_local_model(cfg):
+    """Switch primary model from Kimi to local llama.cpp"""
+    if not BACKUP_CONFIG.exists():
+        shutil.copy2(CONFIG_PATH, BACKUP_CONFIG)
+    
+    cfg["model"]["provider"] = "local-llama.cpp"
+    cfg["model"]["default"] = "hermes3"
+    save_config(cfg)
+    return "Switched primary model to local-llama.cpp/hermes3"
+
+def fallback_to_ollama(cfg):
+    """Switch to Ollama if llama.cpp is also down"""
+    if not BACKUP_CONFIG.exists():
+        shutil.copy2(CONFIG_PATH, BACKUP_CONFIG)
+    
+    cfg["model"]["provider"] = "ollama"
+    cfg["model"]["default"] = "gemma4:latest"
+    save_config(cfg)
+    return "Switched primary model to ollama/gemma4:latest"
+
+def enter_safe_mode(state):
+    """Pause all non-essential cron jobs, alert Alexander"""
+    state["safe_mode"] = True
+    state["safe_mode_entered"] = datetime.now().isoformat()
+    save_state(state)
+    return "SAFE MODE: All inference down. Cron jobs should be paused. Alert Alexander."
+
+def restore_config():
+    """Restore pre-fallback config when primary recovers"""
+    if BACKUP_CONFIG.exists():
+        shutil.copy2(BACKUP_CONFIG, CONFIG_PATH)
+        BACKUP_CONFIG.unlink()
+        return "Restored original config from backup"
+    return "No backup config to restore"
+
+# ─── MAIN DIAGNOSIS AND FALLBACK ENGINE ───
+
+def diagnose_and_fallback():
+    state = load_state()
+    cfg = load_config()
+    
+    results = {
+        "timestamp": datetime.now().isoformat(),
+        "checks": {},
+        "actions": [],
+        "status": "healthy"
+    }
+    
+    # Check all systems
+    kimi_ok, kimi_msg = check_kimi()
+    results["checks"]["kimi-coding"] = {"ok": kimi_ok, "msg": kimi_msg}
+    
+    llama_ok, llama_msg = check_local_llama()
+    results["checks"]["local_llama"] = {"ok": llama_ok, "msg": llama_msg}
+    
+    ollama_ok, ollama_msg = check_ollama()
+    results["checks"]["ollama"] = {"ok": ollama_ok, "msg": ollama_msg}
+    
+    gitea_ok, gitea_msg = check_gitea()
+    results["checks"]["gitea"] = {"ok": gitea_ok, "msg": gitea_msg}
+    
+    # VPS checks
+    vpses = [
+        ("167.99.126.228", "Allegro"),
+        ("143.198.27.163", "Ezra"),
+        ("159.203.146.185", "Bezalel"),
+    ]
+    for ip, name in vpses:
+        vps_ok, vps_msg = check_vps(ip, name)
+        results["checks"][f"vps_{name.lower()}"] = {"ok": vps_ok, "msg": vps_msg}
+    
+    current_provider = cfg.get("model", {}).get("provider", "kimi-coding")
+    
+    # ─── FALLBACK LOGIC ───
+    
+    # Case 1: Primary (Kimi) down, local available
+    if not kimi_ok and current_provider == "kimi-coding":
+        if llama_ok:
+            msg = fallback_to_local_model(cfg)
+            results["actions"].append(msg)
+            state["active_fallbacks"].append("kimi->local-llama")
+            results["status"] = "degraded_local"
+        elif ollama_ok:
+            msg = fallback_to_ollama(cfg)
+            results["actions"].append(msg)
+            state["active_fallbacks"].append("kimi->ollama")
+            results["status"] = "degraded_ollama"
+        else:
+            msg = enter_safe_mode(state)
+            results["actions"].append(msg)
+            results["status"] = "safe_mode"
+    
+    # Case 2: Already on fallback, check if primary recovered
+    elif kimi_ok and "kimi->local-llama" in state.get("active_fallbacks", []):
+        msg = restore_config()
+        results["actions"].append(msg)
+        state["active_fallbacks"].remove("kimi->local-llama")
+        results["status"] = "recovered"
+    elif kimi_ok and "kimi->ollama" in state.get("active_fallbacks", []):
+        msg = restore_config()
+        results["actions"].append(msg)
+        state["active_fallbacks"].remove("kimi->ollama")
+        results["status"] = "recovered"
+    
+    # Case 3: Gitea down — just flag it, work locally
+    if not gitea_ok:
+        results["actions"].append("WARN: Gitea unreachable — work cached locally until recovery")
+        if "gitea_down" not in state.get("active_fallbacks", []):
+            state["active_fallbacks"].append("gitea_down")
+        results["status"] = max(results["status"], "degraded_gitea", key=lambda x: ["healthy", "recovered", "degraded_gitea", "degraded_local", "degraded_ollama", "safe_mode"].index(x) if x in ["healthy", "recovered", "degraded_gitea", "degraded_local", "degraded_ollama", "safe_mode"] else 0)
+    elif "gitea_down" in state.get("active_fallbacks", []):
+        state["active_fallbacks"].remove("gitea_down")
+        results["actions"].append("Gitea recovered — resume normal operations")
+    
+    # Case 4: VPS agents down
+    for ip, name in vpses:
+        key = f"vps_{name.lower()}"
+        if not results["checks"][key]["ok"]:
+            results["actions"].append(f"ALERT: {name} VPS ({ip}) unreachable — lazarus protocol needed")
+    
+    save_state(state)
+    return results
+
+if __name__ == "__main__":
+    results = diagnose_and_fallback()
+    print(json.dumps(results, indent=2))
+    
+    # Exit codes for cron integration
+    if results["status"] == "safe_mode":
+        sys.exit(2)
+    elif results["status"].startswith("degraded"):
+        sys.exit(1)
+    else:
+        sys.exit(0)