Compare commits

..

2 Commits

Author SHA1 Message Date
Alexander Whitestone
e7b9ec8c50 feat: add fleet cost report for #520
Some checks failed
Self-Healing Smoke / self-healing-smoke (pull_request) Failing after 26s
Smoke Test / smoke (pull_request) Failing after 28s
Agent PR Gate / gate (pull_request) Failing after 36s
Agent PR Gate / report (pull_request) Successful in 9s
2026-04-22 03:58:25 -04:00
Alexander Whitestone
2f490e7087 test: define fleet cost report for #520 2026-04-22 03:45:30 -04:00
12 changed files with 322 additions and 306 deletions

View File

@@ -1,22 +0,0 @@
---
# ansible/playbooks/deploy_mempalace.yml — Deploy MemPalace v3.0.0 to fleet wizards.
#
# Usage:
# ansible-playbook -i inventory/hosts.ini playbooks/deploy_mempalace.yml --limit ezra
# ansible-playbook -i inventory/hosts.ini playbooks/deploy_mempalace.yml
#
# Refs: Issue #570
- name: Deploy MemPalace v3.0.0 to wizard hosts
hosts: fleet
become: false
gather_facts: false
vars:
mempalace_hermes_home: "{{ ansible_env.HOME }}/.hermes"
mempalace_sessions_dir: "{{ mempalace_hermes_home }}/sessions"
mempalace_palace_path: "{{ ansible_env.HOME }}/.mempalace/palace"
mempalace_wing: "{{ inventory_hostname }}_home"
roles:
- role: ../roles/mempalace
vars:
mempalace_venv_path: "{{ ansible_env.HOME }}/.mempalace-venv"

View File

@@ -1,16 +0,0 @@
---
# MemPalace role defaults
mempalace_package_spec: "mempalace==3.0.0"
mempalace_hermes_home: "{{ ansible_env.HOME }}/.hermes"
mempalace_sessions_dir: "{{ mempalace_hermes_home }}/sessions"
mempalace_palace_path: "{{ ansible_env.HOME }}/.mempalace/palace"
mempalace_wing: "{{ inventory_hostname }}_home"
mempalace_wakeup_dir: "{{ mempalace_hermes_home }}/wakeups"
mempalace_wakeup_file: "{{ mempalace_wakeup_dir }}/{{ mempalace_wing }}.txt"
mempalace_venv_path: "{{ ansible_env.HOME }}/.mempalace-venv"
mempalace_config_path: "{{ mempalace_hermes_home }}/mempalace.yaml"
mempalace_mcp_config_path: "{{ mempalace_hermes_home }}/hermes-mcp-mempalace.yaml"
mempalace_session_hook_path: "{{ mempalace_hermes_home }}/session-start-mempalace.sh"
mempalace_run_mining: true
mempalace_run_search_test: true
mempalace_run_wake_up: true

View File

@@ -1,2 +0,0 @@
---
dependencies: []

View File

@@ -1,119 +0,0 @@
---
# MemPalace v3.0.0 deployment role for fleet wizards.
# Refs: Issue #570
- name: Ensure mempalace venv directory exists
ansible.builtin.file:
path: "{{ mempalace_venv_path }}"
state: directory
mode: '0750'
- name: Create mempalace virtual environment
ansible.builtin.command:
cmd: "python3 -m venv {{ mempalace_venv_path }}"
creates: "{{ mempalace_venv_path }}/bin/python"
- name: Install mempalace package
ansible.builtin.pip:
name: "{{ mempalace_package_spec }}"
virtualenv: "{{ mempalace_venv_path }}"
virtualenv_command: "{{ mempalace_venv_path }}/bin/python -m venv"
- name: Ensure Hermes home directory exists
ansible.builtin.file:
path: "{{ mempalace_hermes_home }}"
state: directory
mode: '0750'
- name: Ensure sessions directory exists
ansible.builtin.file:
path: "{{ mempalace_sessions_dir }}"
state: directory
mode: '0750'
- name: Ensure wakeup directory exists
ansible.builtin.file:
path: "{{ mempalace_wakeup_dir }}"
state: directory
mode: '0750'
- name: Ensure palace directory exists
ansible.builtin.file:
path: "{{ mempalace_palace_path }}"
state: directory
mode: '0750'
- name: Deploy mempalace.yaml configuration
ansible.builtin.template:
src: mempalace.yaml.j2
dest: "{{ mempalace_config_path }}"
mode: '0640'
- name: Deploy Hermes MCP mempalace config
ansible.builtin.template:
src: hermes-mcp-mempalace.yaml.j2
dest: "{{ mempalace_mcp_config_path }}"
mode: '0640'
- name: Deploy session-start wake-up hook
ansible.builtin.template:
src: session-start-mempalace.sh.j2
dest: "{{ mempalace_session_hook_path }}"
mode: '0750'
- name: Mine Hermes home directory
ansible.builtin.shell: |
set -euo pipefail
echo "" | {{ mempalace_venv_path }}/bin/mempalace mine {{ mempalace_hermes_home }} --config {{ mempalace_config_path }}
args:
executable: /bin/bash
when: mempalace_run_mining | bool
register: mine_home_result
changed_when: mine_home_result.rc == 0
- name: Mine session history
ansible.builtin.shell: |
set -euo pipefail
echo "" | {{ mempalace_venv_path }}/bin/mempalace mine {{ mempalace_sessions_dir }} --mode convos --config {{ mempalace_config_path }}
args:
executable: /bin/bash
when: mempalace_run_mining | bool
register: mine_sessions_result
changed_when: mine_sessions_result.rc == 0
- name: Run search test
ansible.builtin.shell: |
set -euo pipefail
{{ mempalace_venv_path }}/bin/mempalace search "common queries" --config {{ mempalace_config_path }} | head -20
args:
executable: /bin/bash
when: mempalace_run_search_test | bool
register: search_test_result
changed_when: false
- name: Generate wake-up context
ansible.builtin.shell: |
set -euo pipefail
{{ mempalace_venv_path }}/bin/mempalace wake-up --config {{ mempalace_config_path }} > {{ mempalace_wakeup_file }}
export HERMES_MEMPALACE_WAKEUP_FILE="{{ mempalace_wakeup_file }}"
printf '[MemPalace] wake-up context refreshed: %s\n' "$HERMES_MEMPALACE_WAKEUP_FILE"
args:
executable: /bin/bash
when: mempalace_run_wake_up | bool
register: wake_up_result
changed_when: wake_up_result.rc == 0
- name: Report MemPalace deployment summary
ansible.builtin.debug:
msg:
- "MemPalace deployed for {{ inventory_hostname }}"
- "Package: {{ mempalace_package_spec }}"
- "Config: {{ mempalace_config_path }}"
- "Palace: {{ mempalace_palace_path }}"
- "Wake-up: {{ mempalace_wakeup_file }}"
- "MCP config: {{ mempalace_mcp_config_path }}"
- "Session hook: {{ mempalace_session_hook_path }}"
- "Home mine: {{ 'OK' if mine_home_result.rc | default(1) == 0 else 'SKIPPED' }}"
- "Sessions mine: {{ 'OK' if mine_sessions_result.rc | default(1) == 0 else 'SKIPPED' }}"
- "Search test: {{ 'OK' if search_test_result.rc | default(1) == 0 else 'SKIPPED' }}"
- "Wake-up: {{ 'OK' if wake_up_result.rc | default(1) == 0 else 'SKIPPED' }}"

View File

@@ -1,6 +0,0 @@
mcp_servers:
mempalace:
command: "{{ mempalace_venv_path }}/bin/python"
args:
- -m
- mempalace.mcp_server

View File

@@ -1,21 +0,0 @@
wing: {{ mempalace_wing }}
palace: {{ mempalace_palace_path }}
rooms:
- name: sessions
description: Conversation history and durable agent transcripts
globs:
- "*.json"
- "*.jsonl"
- name: config
description: Hermes configuration and runtime settings
globs:
- "*.yaml"
- "*.yml"
- "*.toml"
- name: docs
description: Notes, markdown docs, and operating reports
globs:
- "*.md"
- "*.txt"
people: []
projects: []

View File

@@ -1,9 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
if command -v {{ mempalace_venv_path }}/bin/mempalace >/dev/null 2>&1; then
mkdir -p "{{ mempalace_wakeup_dir }}"
{{ mempalace_venv_path }}/bin/mempalace wake-up --config {{ mempalace_config_path }} > "{{ mempalace_wakeup_file }}"
export HERMES_MEMPALACE_WAKEUP_FILE="{{ mempalace_wakeup_file }}"
printf '[MemPalace] wake-up context refreshed: %s\n' "$HERMES_MEMPALACE_WAKEUP_FILE"
fi

View File

@@ -146,23 +146,6 @@ That bundle writes:
- `session-start-mempalace.sh`
- `issue-568-comment-template.md`
## Fleet Ansible deployment
Deploy MemPalace to Ezra (or the whole fleet) with the Ansible playbook:
```bash
ansible-playbook -i ansible/inventory/hosts.ini ansible/playbooks/deploy_mempalace.yml --limit ezra
```
This playbook:
1. Creates a dedicated venv and installs `mempalace==3.0.0`
2. Deploys `mempalace.yaml`, MCP config, and session-start hook
3. Mines the Hermes home and sessions directories
4. Runs a search smoke test
5. Generates the wake-up context file
Set `mempalace_run_mining=false` to skip mining on hosts where the corpus is already populated.
## Why this shape
- `wing: ezra_home` matches the issue's Ezra-specific integration target.

View File

@@ -0,0 +1,245 @@
#!/usr/bin/env python3
"""Fleet cost report generator.
Reads Timmy's sovereignty metrics database and estimates paid API spend by
agent/provider lane. Default output targets the local timmy-config reports
folder so the cost report can be filed from the sidecar repo.
"""
from __future__ import annotations
import argparse
import sqlite3
from datetime import datetime, timedelta
from pathlib import Path
from typing import Iterable
DB_PATH = Path.home() / ".timmy" / "metrics" / "model_metrics.db"
AGENT_LANES = (
{
"agent": "Timmy Cloud Lane",
"provider": "OpenRouter",
"patterns": ("openrouter/", "google/", "deepseek/", "x-ai/", "mistral/"),
"notes": "Cloud fallback and external reasoning routed through OpenRouter-compatible lanes.",
},
{
"agent": "Ezra",
"provider": "Anthropic",
"patterns": ("claude-", "anthropic/claude"),
"notes": "Archivist / long-form reasoning house on Claude-family models.",
},
{
"agent": "Bezalel",
"provider": "OpenAI",
"patterns": ("gpt-", "openai/", "codex"),
"notes": "Forge / implementation house on Codex/OpenAI-backed execution lanes.",
},
{
"agent": "Allegro",
"provider": "Kimi / Moonshot",
"patterns": ("kimi", "moonshot"),
"notes": "Tempo-and-dispatch house on Kimi / Moonshot direct API lanes.",
},
)
def default_report_path(report_date: str | None = None) -> Path:
if report_date is None:
report_date = datetime.now().strftime("%Y-%m-%d")
return Path.home() / "code" / "timmy-config" / "reports" / "production" / f"{report_date}-fleet-cost-report.md"
def match_lane(model: str) -> dict | None:
lowered = (model or "").lower()
for lane in AGENT_LANES:
if any(pattern in lowered for pattern in lane["patterns"]):
return lane
return None
def load_cost_rows(days: int = 30, db_path: Path = DB_PATH) -> list[tuple[str, int, int, int, float]]:
if not db_path.exists():
return []
cutoff = (datetime.now() - timedelta(days=days)).timestamp()
with sqlite3.connect(str(db_path)) as conn:
rows = conn.execute(
"""
SELECT model, SUM(sessions), SUM(messages), SUM(tool_calls), SUM(est_cost_usd)
FROM session_stats
WHERE timestamp > ? AND is_local = 0
GROUP BY model
ORDER BY SUM(est_cost_usd) DESC, model ASC
""",
(cutoff,),
).fetchall()
return [
(model, int(sessions or 0), int(messages or 0), int(tool_calls or 0), float(cost or 0.0))
for model, sessions, messages, tool_calls, cost in rows
]
def summarize_rows(rows: Iterable[tuple[str, int, int, int, float]], days: int = 30) -> dict:
rows = list(rows)
agents: dict[str, dict] = {}
providers_seen: set[str] = set()
inventory = [
{
"agent": lane["agent"],
"provider": lane["provider"],
"notes": lane["notes"],
}
for lane in AGENT_LANES
]
for lane in AGENT_LANES:
agents[lane["agent"]] = {
"provider": lane["provider"],
"models": [],
"sessions": 0,
"messages": 0,
"tool_calls": 0,
"monthly_cost_usd": 0.0,
"daily_cost_usd": 0.0,
"notes": lane["notes"],
}
unassigned = {
"provider": "Unassigned",
"models": [],
"sessions": 0,
"messages": 0,
"tool_calls": 0,
"monthly_cost_usd": 0.0,
"daily_cost_usd": 0.0,
"notes": "Observed paid-model spend not yet mapped to a named wizard house.",
}
for model, sessions, messages, tool_calls, monthly_cost in rows:
lane = match_lane(model)
if lane is None:
bucket = unassigned
else:
bucket = agents[lane["agent"]]
providers_seen.add(lane["provider"])
bucket["models"].append(
{
"model": model,
"sessions": sessions,
"messages": messages,
"tool_calls": tool_calls,
"monthly_cost_usd": round(monthly_cost, 4),
}
)
bucket["sessions"] += sessions
bucket["messages"] += messages
bucket["tool_calls"] += tool_calls
bucket["monthly_cost_usd"] += monthly_cost
for bucket in list(agents.values()) + [unassigned]:
bucket["monthly_cost_usd"] = round(bucket["monthly_cost_usd"], 4)
bucket["daily_cost_usd"] = round(bucket["monthly_cost_usd"] / max(days, 1), 4)
if unassigned["models"]:
agents["Unassigned"] = unassigned
providers_seen.add("Unassigned")
total_monthly = round(sum(item["monthly_cost_usd"] for item in agents.values()), 4)
total_daily = round(sum(item["daily_cost_usd"] for item in agents.values()), 4)
provider_order = sorted(providers_seen)
if "Unassigned" in provider_order:
provider_order = [p for p in provider_order if p != "Unassigned"] + ["Unassigned"]
return {
"days": days,
"providers": provider_order,
"inventory": inventory,
"agents": agents,
"total_monthly_cost_usd": total_monthly,
"total_daily_cost_usd": total_daily,
}
def render_markdown(summary: dict, report_date: str | None = None) -> str:
if report_date is None:
report_date = datetime.now().strftime("%Y-%m-%d")
lines = [
f"# Fleet Cost Report — {report_date}",
"",
f"Window: last {summary['days']} days of paid-model session stats from `~/.timmy/metrics/model_metrics.db`.",
"",
"## Paid API inventory",
"",
"| Agent | Provider | Notes |",
"| --- | --- | --- |",
]
for item in summary["inventory"]:
lines.append(f"| {item['agent']} | {item['provider']} | {item['notes']} |")
lines.extend(
[
"",
"## Estimated cost per agent per day",
"",
"| Agent | Provider | Daily cost | Monthly estimate | Sessions | Messages | Tool calls |",
"| --- | --- | ---: | ---: | ---: | ---: | ---: |",
]
)
for agent, data in summary["agents"].items():
lines.append(
f"| {agent} | {data['provider']} | ${data['daily_cost_usd']:.2f} | ${data['monthly_cost_usd']:.2f} | {data['sessions']} | {data['messages']} | {data['tool_calls']} |"
)
lines.extend(
[
"",
f"Total estimated daily paid spend: ${summary['total_daily_cost_usd']:.2f}",
f"Total estimated monthly paid spend: ${summary['total_monthly_cost_usd']:.2f}",
"",
"## Model evidence",
"",
]
)
for agent, data in summary["agents"].items():
lines.append(f"### {agent}")
if not data["models"]:
lines.append("- No paid-model sessions observed in the selected window.")
else:
for model in data["models"]:
lines.append(
f"- `{model['model']}` — {model['sessions']} sessions / {model['messages']} messages / {model['tool_calls']} tool calls / ${model['monthly_cost_usd']:.2f} est."
)
lines.append("")
lines.append("Generated by `python3 scripts/fleet_cost_report.py --days 30`. Default output path targets the local timmy-config report lane.")
lines.append("")
return "\n".join(lines)
def write_report(output_path: Path, summary: dict, report_date: str | None = None) -> Path:
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(render_markdown(summary, report_date=report_date), encoding="utf-8")
return output_path
def main() -> int:
parser = argparse.ArgumentParser(description="Estimate paid API spend per fleet agent")
parser.add_argument("--days", type=int, default=30, help="Lookback window in days")
parser.add_argument("--db-path", default=str(DB_PATH), help="Path to model_metrics.db")
parser.add_argument("--output", help="Optional markdown output path")
parser.add_argument("--date", help="Override report date (YYYY-MM-DD)")
args = parser.parse_args()
rows = load_cost_rows(days=args.days, db_path=Path(args.db_path).expanduser())
summary = summarize_rows(rows, days=args.days)
report_date = args.date or datetime.now().strftime("%Y-%m-%d")
output_path = Path(args.output).expanduser() if args.output else default_report_path(report_date)
write_report(output_path, summary, report_date=report_date)
print(output_path)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -0,0 +1,77 @@
from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path
import tempfile
import unittest
ROOT = Path(__file__).resolve().parent.parent
SCRIPT_PATH = ROOT / "scripts" / "fleet_cost_report.py"
def load_module():
spec = spec_from_file_location("fleet_cost_report", SCRIPT_PATH)
module = module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)
return module
class TestFleetCostReport(unittest.TestCase):
def test_default_output_targets_timmy_config_report_path(self):
module = load_module()
output_path = module.default_report_path("2026-04-22")
self.assertIn("timmy-config", str(output_path))
self.assertTrue(str(output_path).endswith("2026-04-22-fleet-cost-report.md"))
def test_summary_groups_paid_costs_by_agent_and_provider(self):
module = load_module()
rows = [
("claude-sonnet-4-6", 12, 120, 24, 6.0),
("gpt-5.4", 6, 60, 12, 3.0),
("openrouter/google/gemini-2.5-pro", 4, 40, 8, 2.0),
("kimi-k2", 2, 20, 4, 1.0),
]
summary = module.summarize_rows(rows, days=30)
self.assertEqual(summary["providers"], ["Anthropic", "Kimi / Moonshot", "OpenAI", "OpenRouter"])
self.assertAlmostEqual(summary["agents"]["Ezra"]["monthly_cost_usd"], 6.0)
self.assertAlmostEqual(summary["agents"]["Bezalel"]["monthly_cost_usd"], 3.0)
self.assertAlmostEqual(summary["agents"]["Timmy Cloud Lane"]["monthly_cost_usd"], 2.0)
self.assertAlmostEqual(summary["agents"]["Allegro"]["monthly_cost_usd"], 1.0)
self.assertAlmostEqual(summary["agents"]["Ezra"]["daily_cost_usd"], 0.2)
def test_report_render_mentions_inventory_and_agent_costs(self):
module = load_module()
rows = [
("claude-sonnet-4-6", 12, 120, 24, 6.0),
("gpt-5.4", 6, 60, 12, 3.0),
("openrouter/google/gemini-2.5-pro", 4, 40, 8, 2.0),
]
summary = module.summarize_rows(rows, days=30)
report = module.render_markdown(summary, report_date="2026-04-22")
self.assertIn("# Fleet Cost Report — 2026-04-22", report)
self.assertIn("## Paid API inventory", report)
self.assertIn("Anthropic", report)
self.assertIn("OpenRouter", report)
self.assertIn("OpenAI", report)
self.assertIn("## Estimated cost per agent per day", report)
self.assertIn("Timmy Cloud Lane", report)
self.assertIn("Ezra", report)
self.assertIn("Bezalel", report)
def test_write_report_creates_markdown_file(self):
module = load_module()
rows = [("claude-sonnet-4-6", 1, 10, 2, 0.5)]
summary = module.summarize_rows(rows, days=30)
with tempfile.TemporaryDirectory() as tmpdir:
dest = Path(tmpdir) / "fleet-cost.md"
module.write_report(dest, summary, report_date="2026-04-22")
self.assertTrue(dest.exists())
text = dest.read_text()
self.assertIn("Fleet Cost Report", text)
self.assertIn("Ezra", text)
if __name__ == "__main__":
unittest.main()

View File

@@ -1,92 +0,0 @@
from pathlib import Path
import unittest
ROOT = Path(__file__).resolve().parent.parent
ROLE_PATH = ROOT / "ansible" / "roles" / "mempalace"
PLAYBOOK_PATH = ROOT / "ansible" / "playbooks" / "deploy_mempalace.yml"
class TestMempalaceAnsibleRole(unittest.TestCase):
def test_role_directory_structure_exists(self):
self.assertTrue(ROLE_PATH.exists(), "mempalace role directory missing")
for subdir in ["tasks", "templates", "defaults", "meta"]:
self.assertTrue(
(ROLE_PATH / subdir).exists(),
f"mempalace role subdir missing: {subdir}",
)
def test_role_defaults_contains_required_variables(self):
defaults_path = ROLE_PATH / "defaults" / "main.yml"
self.assertTrue(defaults_path.exists())
text = defaults_path.read_text(encoding="utf-8")
required_vars = [
"mempalace_package_spec",
"mempalace_hermes_home",
"mempalace_sessions_dir",
"mempalace_palace_path",
"mempalace_wing",
"mempalace_wakeup_dir",
"mempalace_wakeup_file",
"mempalace_venv_path",
"mempalace_config_path",
"mempalace_mcp_config_path",
"mempalace_session_hook_path",
"mempalace_run_mining",
"mempalace_run_search_test",
"mempalace_run_wake_up",
]
for var in required_vars:
self.assertIn(var, text, f"missing default var: {var}")
def test_role_tasks_contain_required_steps(self):
tasks_path = ROLE_PATH / "tasks" / "main.yml"
self.assertTrue(tasks_path.exists())
text = tasks_path.read_text(encoding="utf-8")
required_steps = [
"Create mempalace virtual environment",
"Install mempalace package",
"Deploy mempalace.yaml configuration",
"Deploy Hermes MCP mempalace config",
"Deploy session-start wake-up hook",
"Mine Hermes home directory",
"Mine session history",
"Run search test",
"Generate wake-up context",
]
for step in required_steps:
self.assertIn(step, text, f"missing task: {step}")
def test_role_templates_are_valid(self):
yaml_template = ROLE_PATH / "templates" / "mempalace.yaml.j2"
mcp_template = ROLE_PATH / "templates" / "hermes-mcp-mempalace.yaml.j2"
hook_template = ROLE_PATH / "templates" / "session-start-mempalace.sh.j2"
self.assertTrue(yaml_template.exists())
self.assertTrue(mcp_template.exists())
self.assertTrue(hook_template.exists())
yaml_text = yaml_template.read_text(encoding="utf-8")
self.assertIn("wing: {{ mempalace_wing }}", yaml_text)
self.assertIn("palace: {{ mempalace_palace_path }}", yaml_text)
self.assertIn("rooms:", yaml_text)
mcp_text = mcp_template.read_text(encoding="utf-8")
self.assertIn("mcp_servers:", mcp_text)
self.assertIn("mempalace:", mcp_text)
self.assertIn("mempalace.mcp_server", mcp_text)
hook_text = hook_template.read_text(encoding="utf-8")
self.assertIn("mempalace wake-up", hook_text)
self.assertIn("HERMES_MEMPALACE_WAKEUP_FILE", hook_text)
def test_playbook_exists_and_targets_fleet(self):
self.assertTrue(PLAYBOOK_PATH.exists(), "deploy_mempalace.yml playbook missing")
text = PLAYBOOK_PATH.read_text(encoding="utf-8")
self.assertIn("hosts: fleet", text)
self.assertIn("../roles/mempalace", text)
self.assertIn("mempalace_venv_path", text)
if __name__ == "__main__":
unittest.main()

View File

@@ -85,8 +85,6 @@ class TestMempalaceEzraIntegration(unittest.TestCase):
"mcp_servers:",
"HERMES_MEMPALACE_WAKEUP_FILE",
"Metrics reply for #568",
"Fleet Ansible deployment",
"ansible-playbook",
]
for snippet in required:
self.assertIn(snippet, text)