feat: add fleet cost report for #520

docs/BEZALEL_TAILSCALE_BOOTSTRAP.md

@@ -1,96 +0,0 @@
-# Bezalel Tailscale Bootstrap
-
-Refs #535
-
-This is the repo-side operator packet for installing Tailscale on the Bezalel VPS and verifying the internal network path for federation work.
-
-Important truth:
-- issue #535 names `104.131.15.18`
-- older Bezalel control-plane docs also mention `159.203.146.185`
-- the current source of truth in this repo is `ansible/inventory/hosts.ini`, which currently resolves `bezalel` to `67.205.155.108`
-
-Because of that drift, `scripts/bezalel_tailscale_bootstrap.py` now resolves the target host from `ansible/inventory/hosts.ini` by default instead of trusting a stale hardcoded IP.
-
-## What the script does
-
-`python3 scripts/bezalel_tailscale_bootstrap.py`
-
-Safe by default:
-- builds the remote bootstrap script
-- writes it locally to `/tmp/bezalel_tailscale_bootstrap.sh`
-- prints the SSH command needed to run it
-- does **not** touch the VPS unless `--apply` is passed
-
-When applied, the remote script does all of the issue’s repo-side bootstrap steps:
-- installs Tailscale
-- runs `tailscale up --ssh --hostname bezalel`
-- appends the provided Mac SSH public key to `~/.ssh/authorized_keys`
-- prints `tailscale status --json`
-- pings the expected peer targets:
-  - Mac: `100.124.176.28`
-  - Ezra: `100.126.61.75`
-
-## Required secrets / inputs
-
-- Tailscale auth key
-- Mac SSH public key
-
-Provide them either directly or through files:
-- `--auth-key` or `--auth-key-file`
-- `--ssh-public-key` or `--ssh-public-key-file`
-
-## Dry-run example
-
-```bash
-python3 scripts/bezalel_tailscale_bootstrap.py \
-  --auth-key-file ~/.config/tailscale/auth_key \
-  --ssh-public-key-file ~/.ssh/id_ed25519.pub \
-  --json
-```
-
-This prints:
-- resolved host
-- host source (`inventory:<path>` when pulled from `ansible/inventory/hosts.ini`)
-- local script path
-- SSH command to execute
-- peer targets
-
-## Apply example
-
-```bash
-python3 scripts/bezalel_tailscale_bootstrap.py \
-  --auth-key-file ~/.config/tailscale/auth_key \
-  --ssh-public-key-file ~/.ssh/id_ed25519.pub \
-  --apply \
-  --json
-```
-
-## Verifying success after apply
-
-The script now parses the remote stdout into structured verification data:
-- `verification.tailscale.self.tailscale_ips`
-- `verification.tailscale.self.dns_name`
-- `verification.peers`
-- `verification.ping_ok`
-
-A successful run should show:
-- at least one Bezalel Tailscale IP under `tailscale_ips`
-- `ping_ok.mac = 100.124.176.28`
-- `ping_ok.ezra = 100.126.61.75`
-
-## Expected remote install commands
-
-```bash
-curl -fsSL https://tailscale.com/install.sh | sh
-tailscale up --ssh --hostname bezalel
-install -d -m 700 ~/.ssh
-touch ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys
-tailscale status --json
-```
-
-## Why this PR does not claim live completion
-
-This repo can safely ship the bootstrap script, host resolution logic, structured proof parsing, and operator packet.
-It cannot honestly claim that Bezalel was actually joined to the tailnet unless a human/operator runs the script with a real auth key and real SSH access to the VPS.
-
-That means the correct PR language for #535 is advancement, not pretend closure.

docs/RUNBOOK_INDEX.md

@@ -14,7 +14,6 @@ Quick-reference index for common operational tasks across the Timmy Foundation i
 | Agent scorecard | fleet-ops | `python3 scripts/agent_scorecard.py` |
 | View fleet manifest | fleet-ops | `cat manifest.yaml` |
 | Run nightly codebase genome pass | timmy-home | `python3 scripts/codebase_genome_nightly.py --dry-run` |
-| Prepare Bezalel Tailscale bootstrap | timmy-home | `python3 scripts/bezalel_tailscale_bootstrap.py --auth-key-file <path> --ssh-public-key-file <path> --json` |
 
 ## the-nexus (Frontend + Brain)
 

scripts/bezalel_tailscale_bootstrap.py

@@ -16,14 +16,11 @@ import argparse
 import json
 import shlex
 import subprocess
-import re
-from json import JSONDecoder
 from pathlib import Path
 from typing import Any
 
-DEFAULT_HOST = "67.205.155.108"
+DEFAULT_HOST = "159.203.146.185"
 DEFAULT_HOSTNAME = "bezalel"
-DEFAULT_INVENTORY_PATH = Path(__file__).resolve().parents[1] / "ansible" / "inventory" / "hosts.ini"
 DEFAULT_PEERS = {
     "mac": "100.124.176.28",
     "ezra": "100.126.61.75",
@@ -69,37 +66,6 @@ def parse_tailscale_status(payload: dict[str, Any]) -> dict[str, Any]:
     }
 
 
-def resolve_host(host: str | None, inventory_path: Path = DEFAULT_INVENTORY_PATH, hostname: str = DEFAULT_HOSTNAME) -> tuple[str, str]:
-    if host:
-        return host, "explicit"
-    if inventory_path.exists():
-        pattern = re.compile(rf"^{re.escape(hostname)}\s+.*ansible_host=([^\s]+)")
-        for line in inventory_path.read_text().splitlines():
-            match = pattern.search(line.strip())
-            if match:
-                return match.group(1), f"inventory:{inventory_path}"
-    return DEFAULT_HOST, "default"
-
-
-def parse_apply_output(stdout: str) -> dict[str, Any]:
-    result: dict[str, Any] = {"tailscale": None, "ping_ok": {}}
-    text = stdout or ""
-    start = text.find("{")
-    if start != -1:
-        try:
-            payload, _ = JSONDecoder().raw_decode(text[start:])
-            if isinstance(payload, dict):
-                result["tailscale"] = parse_tailscale_status(payload)
-        except Exception:
-            pass
-
-    for line in text.splitlines():
-        if line.startswith("PING_OK:"):
-            _, name, ip = line.split(":", 2)
-            result["ping_ok"][name] = ip
-    return result
-
-
 def build_ssh_command(host: str, remote_script_path: str = "/tmp/bezalel_tailscale_bootstrap.sh") -> list[str]:
     return ["ssh", host, f"bash {shlex.quote(remote_script_path)}"]
 
@@ -123,9 +89,8 @@ def parse_peer_args(items: list[str]) -> dict[str, str]:
 
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Prepare or execute Tailscale bootstrap for the Bezalel VPS.")
-    parser.add_argument("--host")
+    parser.add_argument("--host", default=DEFAULT_HOST)
     parser.add_argument("--hostname", default=DEFAULT_HOSTNAME)
-    parser.add_argument("--inventory-path", type=Path, default=DEFAULT_INVENTORY_PATH)
     parser.add_argument("--auth-key", help="Tailscale auth key")
     parser.add_argument("--auth-key-file", type=Path, help="Path to file containing the Tailscale auth key")
     parser.add_argument("--ssh-public-key", help="SSH public key to append to authorized_keys")
@@ -151,7 +116,6 @@ def main() -> None:
     auth_key = _read_secret(args.auth_key, args.auth_key_file)
     ssh_public_key = _read_secret(args.ssh_public_key, args.ssh_public_key_file)
     peers = parse_peer_args(args.peer)
-    resolved_host, host_source = resolve_host(args.host, args.inventory_path, args.hostname)
 
     if not auth_key:
         raise SystemExit("Missing Tailscale auth key. Use --auth-key or --auth-key-file.")
@@ -162,31 +126,28 @@ def main() -> None:
     write_script(args.script_out, script)
 
     payload: dict[str, Any] = {
-        "host": resolved_host,
-        "host_source": host_source,
+        "host": args.host,
         "hostname": args.hostname,
-        "inventory_path": str(args.inventory_path),
         "script_out": str(args.script_out),
         "remote_script_path": args.remote_script_path,
-        "ssh_command": build_ssh_command(resolved_host, args.remote_script_path),
+        "ssh_command": build_ssh_command(args.host, args.remote_script_path),
         "peer_targets": peers,
         "applied": False,
     }
 
     if args.apply:
-        result = run_remote(resolved_host, args.remote_script_path)
+        result = run_remote(args.host, args.remote_script_path)
         payload["applied"] = True
         payload["exit_code"] = result.returncode
         payload["stdout"] = result.stdout
         payload["stderr"] = result.stderr
-        payload["verification"] = parse_apply_output(result.stdout)
 
     if args.json:
         print(json.dumps(payload, indent=2))
         return
 
     print("--- Bezalel Tailscale Bootstrap ---")
-    print(f"Host: {resolved_host} ({host_source})")
+    print(f"Host: {args.host}")
     print(f"Local script: {args.script_out}")
     print("SSH command: " + " ".join(payload["ssh_command"]))
     if args.apply:

scripts/fleet_cost_report.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+"""Fleet cost report generator.
+
+Reads Timmy's sovereignty metrics database and estimates paid API spend by
+agent/provider lane. Default output targets the local timmy-config reports
+folder so the cost report can be filed from the sidecar repo.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sqlite3
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Iterable
+
+DB_PATH = Path.home() / ".timmy" / "metrics" / "model_metrics.db"
+
+
+AGENT_LANES = (
+    {
+        "agent": "Timmy Cloud Lane",
+        "provider": "OpenRouter",
+        "patterns": ("openrouter/", "google/", "deepseek/", "x-ai/", "mistral/"),
+        "notes": "Cloud fallback and external reasoning routed through OpenRouter-compatible lanes.",
+    },
+    {
+        "agent": "Ezra",
+        "provider": "Anthropic",
+        "patterns": ("claude-", "anthropic/claude"),
+        "notes": "Archivist / long-form reasoning house on Claude-family models.",
+    },
+    {
+        "agent": "Bezalel",
+        "provider": "OpenAI",
+        "patterns": ("gpt-", "openai/", "codex"),
+        "notes": "Forge / implementation house on Codex/OpenAI-backed execution lanes.",
+    },
+    {
+        "agent": "Allegro",
+        "provider": "Kimi / Moonshot",
+        "patterns": ("kimi", "moonshot"),
+        "notes": "Tempo-and-dispatch house on Kimi / Moonshot direct API lanes.",
+    },
+)
+
+
+def default_report_path(report_date: str | None = None) -> Path:
+    if report_date is None:
+        report_date = datetime.now().strftime("%Y-%m-%d")
+    return Path.home() / "code" / "timmy-config" / "reports" / "production" / f"{report_date}-fleet-cost-report.md"
+
+
+def match_lane(model: str) -> dict | None:
+    lowered = (model or "").lower()
+    for lane in AGENT_LANES:
+        if any(pattern in lowered for pattern in lane["patterns"]):
+            return lane
+    return None
+
+
+def load_cost_rows(days: int = 30, db_path: Path = DB_PATH) -> list[tuple[str, int, int, int, float]]:
+    if not db_path.exists():
+        return []
+    cutoff = (datetime.now() - timedelta(days=days)).timestamp()
+    with sqlite3.connect(str(db_path)) as conn:
+        rows = conn.execute(
+            """
+            SELECT model, SUM(sessions), SUM(messages), SUM(tool_calls), SUM(est_cost_usd)
+            FROM session_stats
+            WHERE timestamp > ? AND is_local = 0
+            GROUP BY model
+            ORDER BY SUM(est_cost_usd) DESC, model ASC
+            """,
+            (cutoff,),
+        ).fetchall()
+    return [
+        (model, int(sessions or 0), int(messages or 0), int(tool_calls or 0), float(cost or 0.0))
+        for model, sessions, messages, tool_calls, cost in rows
+    ]
+
+
+def summarize_rows(rows: Iterable[tuple[str, int, int, int, float]], days: int = 30) -> dict:
+    rows = list(rows)
+    agents: dict[str, dict] = {}
+    providers_seen: set[str] = set()
+    inventory = [
+        {
+            "agent": lane["agent"],
+            "provider": lane["provider"],
+            "notes": lane["notes"],
+        }
+        for lane in AGENT_LANES
+    ]
+
+    for lane in AGENT_LANES:
+        agents[lane["agent"]] = {
+            "provider": lane["provider"],
+            "models": [],
+            "sessions": 0,
+            "messages": 0,
+            "tool_calls": 0,
+            "monthly_cost_usd": 0.0,
+            "daily_cost_usd": 0.0,
+            "notes": lane["notes"],
+        }
+
+    unassigned = {
+        "provider": "Unassigned",
+        "models": [],
+        "sessions": 0,
+        "messages": 0,
+        "tool_calls": 0,
+        "monthly_cost_usd": 0.0,
+        "daily_cost_usd": 0.0,
+        "notes": "Observed paid-model spend not yet mapped to a named wizard house.",
+    }
+
+    for model, sessions, messages, tool_calls, monthly_cost in rows:
+        lane = match_lane(model)
+        if lane is None:
+            bucket = unassigned
+        else:
+            bucket = agents[lane["agent"]]
+            providers_seen.add(lane["provider"])
+        bucket["models"].append(
+            {
+                "model": model,
+                "sessions": sessions,
+                "messages": messages,
+                "tool_calls": tool_calls,
+                "monthly_cost_usd": round(monthly_cost, 4),
+            }
+        )
+        bucket["sessions"] += sessions
+        bucket["messages"] += messages
+        bucket["tool_calls"] += tool_calls
+        bucket["monthly_cost_usd"] += monthly_cost
+
+    for bucket in list(agents.values()) + [unassigned]:
+        bucket["monthly_cost_usd"] = round(bucket["monthly_cost_usd"], 4)
+        bucket["daily_cost_usd"] = round(bucket["monthly_cost_usd"] / max(days, 1), 4)
+
+    if unassigned["models"]:
+        agents["Unassigned"] = unassigned
+        providers_seen.add("Unassigned")
+
+    total_monthly = round(sum(item["monthly_cost_usd"] for item in agents.values()), 4)
+    total_daily = round(sum(item["daily_cost_usd"] for item in agents.values()), 4)
+
+    provider_order = sorted(providers_seen)
+    if "Unassigned" in provider_order:
+        provider_order = [p for p in provider_order if p != "Unassigned"] + ["Unassigned"]
+
+    return {
+        "days": days,
+        "providers": provider_order,
+        "inventory": inventory,
+        "agents": agents,
+        "total_monthly_cost_usd": total_monthly,
+        "total_daily_cost_usd": total_daily,
+    }
+
+
+def render_markdown(summary: dict, report_date: str | None = None) -> str:
+    if report_date is None:
+        report_date = datetime.now().strftime("%Y-%m-%d")
+    lines = [
+        f"# Fleet Cost Report — {report_date}",
+        "",
+        f"Window: last {summary['days']} days of paid-model session stats from `~/.timmy/metrics/model_metrics.db`.",
+        "",
+        "## Paid API inventory",
+        "",
+        "| Agent | Provider | Notes |",
+        "| --- | --- | --- |",
+    ]
+    for item in summary["inventory"]:
+        lines.append(f"| {item['agent']} | {item['provider']} | {item['notes']} |")
+
+    lines.extend(
+        [
+            "",
+            "## Estimated cost per agent per day",
+            "",
+            "| Agent | Provider | Daily cost | Monthly estimate | Sessions | Messages | Tool calls |",
+            "| --- | --- | ---: | ---: | ---: | ---: | ---: |",
+        ]
+    )
+    for agent, data in summary["agents"].items():
+        lines.append(
+            f"| {agent} | {data['provider']} | ${data['daily_cost_usd']:.2f} | ${data['monthly_cost_usd']:.2f} | {data['sessions']} | {data['messages']} | {data['tool_calls']} |"
+        )
+
+    lines.extend(
+        [
+            "",
+            f"Total estimated daily paid spend: ${summary['total_daily_cost_usd']:.2f}",
+            f"Total estimated monthly paid spend: ${summary['total_monthly_cost_usd']:.2f}",
+            "",
+            "## Model evidence",
+            "",
+        ]
+    )
+    for agent, data in summary["agents"].items():
+        lines.append(f"### {agent}")
+        if not data["models"]:
+            lines.append("- No paid-model sessions observed in the selected window.")
+        else:
+            for model in data["models"]:
+                lines.append(
+                    f"- `{model['model']}` — {model['sessions']} sessions / {model['messages']} messages / {model['tool_calls']} tool calls / ${model['monthly_cost_usd']:.2f} est."
+                )
+        lines.append("")
+
+    lines.append("Generated by `python3 scripts/fleet_cost_report.py --days 30`. Default output path targets the local timmy-config report lane.")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def write_report(output_path: Path, summary: dict, report_date: str | None = None) -> Path:
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(render_markdown(summary, report_date=report_date), encoding="utf-8")
+    return output_path
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Estimate paid API spend per fleet agent")
+    parser.add_argument("--days", type=int, default=30, help="Lookback window in days")
+    parser.add_argument("--db-path", default=str(DB_PATH), help="Path to model_metrics.db")
+    parser.add_argument("--output", help="Optional markdown output path")
+    parser.add_argument("--date", help="Override report date (YYYY-MM-DD)")
+    args = parser.parse_args()
+
+    rows = load_cost_rows(days=args.days, db_path=Path(args.db_path).expanduser())
+    summary = summarize_rows(rows, days=args.days)
+    report_date = args.date or datetime.now().strftime("%Y-%m-%d")
+    output_path = Path(args.output).expanduser() if args.output else default_report_path(report_date)
+    write_report(output_path, summary, report_date=report_date)
+    print(output_path)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tests/test_bezalel_tailscale_bootstrap.py

@@ -2,12 +2,9 @@ from scripts.bezalel_tailscale_bootstrap import (
     DEFAULT_PEERS,
     build_remote_script,
     build_ssh_command,
-    parse_apply_output,
     parse_peer_args,
     parse_tailscale_status,
-    resolve_host,
 )
-from pathlib import Path
 
 
 def test_build_remote_script_contains_install_up_and_key_append():
@@ -81,46 +78,3 @@ def test_parse_peer_args_merges_overrides_into_defaults():
         "ezra": "100.126.61.76",
         "forge": "100.70.0.9",
     }
-
-
-def test_resolve_host_prefers_inventory_over_stale_default(tmp_path: Path):
-    inventory = tmp_path / "hosts.ini"
-    inventory.write_text(
-        "[fleet]\n"
-        "ezra ansible_host=143.198.27.163 ansible_user=root\n"
-        "bezalel ansible_host=67.205.155.108 ansible_user=root\n"
-    )
-
-    host, source = resolve_host(None, inventory)
-
-    assert host == "67.205.155.108"
-    assert source == f"inventory:{inventory}"
-
-
-def test_parse_apply_output_extracts_status_and_ping_markers():
-    stdout = (
-        '{"Self": {"HostName": "bezalel", "DNSName": "bezalel.tailnet.ts.net", "TailscaleIPs": ["100.90.0.10"]}, '
-        '"Peer": {"node-1": {"HostName": "ezra", "TailscaleIPs": ["100.126.61.75"]}}}'
-        "\nPING_OK:mac:100.124.176.28\n"
-        "PING_OK:ezra:100.126.61.75\n"
-    )
-
-    result = parse_apply_output(stdout)
-
-    assert result["tailscale"]["self"]["tailscale_ips"] == ["100.90.0.10"]
-    assert result["ping_ok"] == {"mac": "100.124.176.28", "ezra": "100.126.61.75"}
-
-
-def test_runbook_doc_exists_and_mentions_inventory_auth_and_peer_checks():
-    doc = Path("docs/BEZALEL_TAILSCALE_BOOTSTRAP.md")
-    assert doc.exists(), "missing docs/BEZALEL_TAILSCALE_BOOTSTRAP.md"
-    text = doc.read_text()
-    assert "ansible/inventory/hosts.ini" in text
-    assert "tailscale up" in text
-    assert "authorized_keys" in text
-    assert "100.124.176.28" in text
-    assert "100.126.61.75" in text
-
-    runbook = Path("docs/RUNBOOK_INDEX.md").read_text()
-    assert "Prepare Bezalel Tailscale bootstrap" in runbook
-    assert "scripts/bezalel_tailscale_bootstrap.py" in runbook

tests/test_fleet_cost_report.py

@@ -0,0 +1,77 @@
+from importlib.util import module_from_spec, spec_from_file_location
+from pathlib import Path
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parent.parent
+SCRIPT_PATH = ROOT / "scripts" / "fleet_cost_report.py"
+
+
+def load_module():
+    spec = spec_from_file_location("fleet_cost_report", SCRIPT_PATH)
+    module = module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(module)
+    return module
+
+
+class TestFleetCostReport(unittest.TestCase):
+    def test_default_output_targets_timmy_config_report_path(self):
+        module = load_module()
+        output_path = module.default_report_path("2026-04-22")
+        self.assertIn("timmy-config", str(output_path))
+        self.assertTrue(str(output_path).endswith("2026-04-22-fleet-cost-report.md"))
+
+    def test_summary_groups_paid_costs_by_agent_and_provider(self):
+        module = load_module()
+        rows = [
+            ("claude-sonnet-4-6", 12, 120, 24, 6.0),
+            ("gpt-5.4", 6, 60, 12, 3.0),
+            ("openrouter/google/gemini-2.5-pro", 4, 40, 8, 2.0),
+            ("kimi-k2", 2, 20, 4, 1.0),
+        ]
+        summary = module.summarize_rows(rows, days=30)
+
+        self.assertEqual(summary["providers"], ["Anthropic", "Kimi / Moonshot", "OpenAI", "OpenRouter"])
+        self.assertAlmostEqual(summary["agents"]["Ezra"]["monthly_cost_usd"], 6.0)
+        self.assertAlmostEqual(summary["agents"]["Bezalel"]["monthly_cost_usd"], 3.0)
+        self.assertAlmostEqual(summary["agents"]["Timmy Cloud Lane"]["monthly_cost_usd"], 2.0)
+        self.assertAlmostEqual(summary["agents"]["Allegro"]["monthly_cost_usd"], 1.0)
+        self.assertAlmostEqual(summary["agents"]["Ezra"]["daily_cost_usd"], 0.2)
+
+    def test_report_render_mentions_inventory_and_agent_costs(self):
+        module = load_module()
+        rows = [
+            ("claude-sonnet-4-6", 12, 120, 24, 6.0),
+            ("gpt-5.4", 6, 60, 12, 3.0),
+            ("openrouter/google/gemini-2.5-pro", 4, 40, 8, 2.0),
+        ]
+        summary = module.summarize_rows(rows, days=30)
+        report = module.render_markdown(summary, report_date="2026-04-22")
+
+        self.assertIn("# Fleet Cost Report — 2026-04-22", report)
+        self.assertIn("## Paid API inventory", report)
+        self.assertIn("Anthropic", report)
+        self.assertIn("OpenRouter", report)
+        self.assertIn("OpenAI", report)
+        self.assertIn("## Estimated cost per agent per day", report)
+        self.assertIn("Timmy Cloud Lane", report)
+        self.assertIn("Ezra", report)
+        self.assertIn("Bezalel", report)
+
+    def test_write_report_creates_markdown_file(self):
+        module = load_module()
+        rows = [("claude-sonnet-4-6", 1, 10, 2, 0.5)]
+        summary = module.summarize_rows(rows, days=30)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            dest = Path(tmpdir) / "fleet-cost.md"
+            module.write_report(dest, summary, report_date="2026-04-22")
+            self.assertTrue(dest.exists())
+            text = dest.read_text()
+            self.assertIn("Fleet Cost Report", text)
+            self.assertIn("Ezra", text)
+
+
+if __name__ == "__main__":
+    unittest.main()