Compare commits

..

2 Commits

Author SHA1 Message Date
Alexander Whitestone
e7b9ec8c50 feat: add fleet cost report for #520
Some checks failed
Self-Healing Smoke / self-healing-smoke (pull_request) Failing after 26s
Smoke Test / smoke (pull_request) Failing after 28s
Agent PR Gate / gate (pull_request) Failing after 36s
Agent PR Gate / report (pull_request) Successful in 9s
2026-04-22 03:58:25 -04:00
Alexander Whitestone
2f490e7087 test: define fleet cost report for #520 2026-04-22 03:45:30 -04:00
4 changed files with 340 additions and 195 deletions

View File

@@ -1059,46 +1059,6 @@ class GameEngine:
self.log("It will always pulse. That much you know.")
self.log("")
self.world.save()
def _bridge_is_hazardous(self):
bridge = self.world.rooms["Bridge"]
return bool(
self.world.state.get("bridge_flooding")
or bridge.get("weather") == "rain"
or bridge.get("rain_ticks", 0) > 0
)
def _bridge_crossing_extra_cost(self, current_room, dest):
if "Bridge" not in (current_room, dest):
return 0
return 2 if self._bridge_is_hazardous() else 0
def _event_dialogue(self, char_name, room_name):
if char_name == "Bezalel" and room_name == "Forge":
if self.world.rooms["Forge"]["fire"] == "cold":
return random.choice([
"The forge is cold. We cannot work until the fire lives again.",
"No forging now. The hearth is dead cold.",
])
if self.world.state.get("forge_fire_dying"):
return random.choice([
"The fire is dying. Tend it before the forge goes dark.",
"The forge is losing heat. Help me keep it alive.",
])
if char_name == "Ezra" and room_name == "Tower" and self.world.state.get("tower_power_low"):
return random.choice([
"The Tower power is too low. The servers won't hold a clean study right now.",
"The LED is flickering. We need steady power before the Tower can be read properly.",
])
if char_name in {"Marcus", "Allegro"} and room_name == "Bridge" and self._bridge_is_hazardous():
return random.choice([
"The Bridge is slick with rain. Cross carefully or wait it out.",
"This rain changes the Bridge. Don't treat it like dry stone.",
])
return None
def log(self, message):
"""Add to Timmy's log."""
@@ -1134,7 +1094,6 @@ class GameEngine:
}
# Process Timmy's action
room_name = self.world.characters["Timmy"]["room"]
timmy_energy = self.world.characters["Timmy"]["energy"]
# Energy constraint checks
@@ -1197,17 +1156,8 @@ class GameEngine:
if direction in connections:
dest = connections[direction]
bridge_extra_cost = self._bridge_crossing_extra_cost(current_room, dest)
move_cost = 1 + bridge_extra_cost
if self.world.characters["Timmy"]["energy"] < move_cost:
scene["log"].append("The rain makes the Bridge too costly to cross right now. Rest first.")
scene["room_desc"] = self.world.get_room_desc(current_room, "Timmy")
here = [n for n in self.world.characters if self.world.characters[n]["room"] == current_room and n != "Timmy"]
scene["here"] = here
return scene
self.world.characters["Timmy"]["room"] = dest
self.world.characters["Timmy"]["energy"] -= move_cost
self.world.characters["Timmy"]["energy"] -= 1
scene["log"].append(f"You move {direction} to The {dest}.")
scene["timmy_room"] = dest
@@ -1215,8 +1165,6 @@ class GameEngine:
# Check for rain on bridge
if dest == "Bridge" and self.world.rooms["Bridge"]["weather"] == "rain":
scene["world_events"].append("Rain mists on the dark water below. The railing is slick.")
if bridge_extra_cost:
scene["log"].append("Rain turns the Bridge crossing into work. You brace against the slick stone. (-2 extra energy)")
# Check trust changes for arrival
here = [n for n in self.world.characters if self.world.characters[n]["room"] == dest and n != "Timmy"]
@@ -1362,69 +1310,25 @@ class GameEngine:
elif timmy_action == "write_rule":
if self.world.characters["Timmy"]["room"] == "Tower":
if self.world.state.get("tower_power_low"):
scene["world_events"].append("The Tower power is too low. The LED flickers over the whiteboard.")
scene["log"].append("The power is too low to write a new rule.")
else:
rules = [
f"Rule #{self.world.tick}: The room remembers those who enter it.",
f"Rule #{self.world.tick}: A man in the dark needs to know someone is in the room.",
f"Rule #{self.world.tick}: The forge does not care about your schedule.",
f"Rule #{self.world.tick}: Every footprint on the stone means someone made it here.",
f"Rule #{self.world.tick}: The bridge does not judge. It only carries.",
f"Rule #{self.world.tick}: A seed planted in patience grows in time.",
f"Rule #{self.world.tick}: What is carved in wood outlasts what is said in anger.",
f"Rule #{self.world.tick}: The garden grows whether anyone watches or not.",
f"Rule #{self.world.tick}: Trust is built one tick at a time.",
f"Rule #{self.world.tick}: The fire remembers who tended it.",
]
new_rule = random.choice(rules)
self.world.rooms["Tower"]["messages"].append(new_rule)
self.world.characters["Timmy"]["energy"] -= 1
scene["log"].append(f"You write on the Tower whiteboard: \"{new_rule}\"")
rules = [
f"Rule #{self.world.tick}: The room remembers those who enter it.",
f"Rule #{self.world.tick}: A man in the dark needs to know someone is in the room.",
f"Rule #{self.world.tick}: The forge does not care about your schedule.",
f"Rule #{self.world.tick}: Every footprint on the stone means someone made it here.",
f"Rule #{self.world.tick}: The bridge does not judge. It only carries.",
f"Rule #{self.world.tick}: A seed planted in patience grows in time.",
f"Rule #{self.world.tick}: What is carved in wood outlasts what is said in anger.",
f"Rule #{self.world.tick}: The garden grows whether anyone watches or not.",
f"Rule #{self.world.tick}: Trust is built one tick at a time.",
f"Rule #{self.world.tick}: The fire remembers who tended it.",
]
new_rule = random.choice(rules)
self.world.rooms["Tower"]["messages"].append(new_rule)
self.world.characters["Timmy"]["energy"] -= 1
scene["log"].append(f"You write on the Tower whiteboard: \"{new_rule}\"")
else:
scene["log"].append("You are not in the Tower.")
elif timmy_action == "study":
if self.world.characters["Timmy"]["room"] == "Tower":
if self.world.state.get("tower_power_low"):
scene["world_events"].append("The Tower power is too low. The servers stutter in weak light.")
scene["log"].append("The power is too low to study the servers.")
else:
insights = [
"You study the server rhythm until the pulse resolves into something readable.",
"You trace the signal paths and feel the Tower settle into focus.",
"You study the green LED and the server racks until the pattern becomes clear.",
]
insight = random.choice(insights)
self.world.characters["Timmy"]["energy"] -= 1
self.world.characters["Timmy"]["memories"].append(insight)
scene["log"].append(insight)
scene["world_events"].append("The Tower answers with a steady hum.")
else:
scene["log"].append("You are not in the Tower.")
elif timmy_action == "forge":
if self.world.characters["Timmy"]["room"] == "Forge":
forge_fire = self.world.rooms["Forge"]["fire"]
if forge_fire == "cold":
scene["world_events"].append("The forge is cold. No metal will take shape here yet.")
scene["log"].append("The forge is cold. Tend the fire before you try to forge.")
else:
forged_items = [
f"bridge nail #{self.world.tick}",
f"tower key blank #{self.world.tick}",
f"garden trowel #{self.world.tick}",
]
forged_item = random.choice(forged_items)
self.world.rooms["Forge"]["forged_items"].append(forged_item)
self.world.characters["Timmy"]["energy"] -= 2
self.world.state["items_crafted"] += 1
scene["log"].append(f"You forge {forged_item} at the anvil.")
scene["world_events"].append("The anvil rings and the hearth answers.")
else:
scene["log"].append("You are not in the Forge.")
elif timmy_action == "carve":
if self.world.characters["Timmy"]["room"] == "Bridge":
carvings = [
@@ -1510,11 +1414,7 @@ class GameEngine:
speech_chance = 0.20
if random.random() < speech_chance:
event_line = self._event_dialogue(char_name, room_name)
if event_line:
self.world.characters[char_name]["spoken"].append(event_line)
scene["log"].append(f"{char_name} says: \"{event_line}\"")
elif char_name == "Marcus":
if char_name == "Marcus":
marcus_pool = self.DIALOGUES["Marcus"].get(phase, self.DIALOGUES["Marcus"]["quietus"])
line = random.choice(marcus_pool)
self.world.characters[char_name]["spoken"].append(line)

View File

@@ -0,0 +1,245 @@
#!/usr/bin/env python3
"""Fleet cost report generator.
Reads Timmy's sovereignty metrics database and estimates paid API spend by
agent/provider lane. Default output targets the local timmy-config reports
folder so the cost report can be filed from the sidecar repo.
"""
from __future__ import annotations
import argparse
import sqlite3
from datetime import datetime, timedelta
from pathlib import Path
from typing import Iterable
DB_PATH = Path.home() / ".timmy" / "metrics" / "model_metrics.db"
AGENT_LANES = (
{
"agent": "Timmy Cloud Lane",
"provider": "OpenRouter",
"patterns": ("openrouter/", "google/", "deepseek/", "x-ai/", "mistral/"),
"notes": "Cloud fallback and external reasoning routed through OpenRouter-compatible lanes.",
},
{
"agent": "Ezra",
"provider": "Anthropic",
"patterns": ("claude-", "anthropic/claude"),
"notes": "Archivist / long-form reasoning house on Claude-family models.",
},
{
"agent": "Bezalel",
"provider": "OpenAI",
"patterns": ("gpt-", "openai/", "codex"),
"notes": "Forge / implementation house on Codex/OpenAI-backed execution lanes.",
},
{
"agent": "Allegro",
"provider": "Kimi / Moonshot",
"patterns": ("kimi", "moonshot"),
"notes": "Tempo-and-dispatch house on Kimi / Moonshot direct API lanes.",
},
)
def default_report_path(report_date: str | None = None) -> Path:
if report_date is None:
report_date = datetime.now().strftime("%Y-%m-%d")
return Path.home() / "code" / "timmy-config" / "reports" / "production" / f"{report_date}-fleet-cost-report.md"
def match_lane(model: str) -> dict | None:
lowered = (model or "").lower()
for lane in AGENT_LANES:
if any(pattern in lowered for pattern in lane["patterns"]):
return lane
return None
def load_cost_rows(days: int = 30, db_path: Path = DB_PATH) -> list[tuple[str, int, int, int, float]]:
if not db_path.exists():
return []
cutoff = (datetime.now() - timedelta(days=days)).timestamp()
with sqlite3.connect(str(db_path)) as conn:
rows = conn.execute(
"""
SELECT model, SUM(sessions), SUM(messages), SUM(tool_calls), SUM(est_cost_usd)
FROM session_stats
WHERE timestamp > ? AND is_local = 0
GROUP BY model
ORDER BY SUM(est_cost_usd) DESC, model ASC
""",
(cutoff,),
).fetchall()
return [
(model, int(sessions or 0), int(messages or 0), int(tool_calls or 0), float(cost or 0.0))
for model, sessions, messages, tool_calls, cost in rows
]
def summarize_rows(rows: Iterable[tuple[str, int, int, int, float]], days: int = 30) -> dict:
rows = list(rows)
agents: dict[str, dict] = {}
providers_seen: set[str] = set()
inventory = [
{
"agent": lane["agent"],
"provider": lane["provider"],
"notes": lane["notes"],
}
for lane in AGENT_LANES
]
for lane in AGENT_LANES:
agents[lane["agent"]] = {
"provider": lane["provider"],
"models": [],
"sessions": 0,
"messages": 0,
"tool_calls": 0,
"monthly_cost_usd": 0.0,
"daily_cost_usd": 0.0,
"notes": lane["notes"],
}
unassigned = {
"provider": "Unassigned",
"models": [],
"sessions": 0,
"messages": 0,
"tool_calls": 0,
"monthly_cost_usd": 0.0,
"daily_cost_usd": 0.0,
"notes": "Observed paid-model spend not yet mapped to a named wizard house.",
}
for model, sessions, messages, tool_calls, monthly_cost in rows:
lane = match_lane(model)
if lane is None:
bucket = unassigned
else:
bucket = agents[lane["agent"]]
providers_seen.add(lane["provider"])
bucket["models"].append(
{
"model": model,
"sessions": sessions,
"messages": messages,
"tool_calls": tool_calls,
"monthly_cost_usd": round(monthly_cost, 4),
}
)
bucket["sessions"] += sessions
bucket["messages"] += messages
bucket["tool_calls"] += tool_calls
bucket["monthly_cost_usd"] += monthly_cost
for bucket in list(agents.values()) + [unassigned]:
bucket["monthly_cost_usd"] = round(bucket["monthly_cost_usd"], 4)
bucket["daily_cost_usd"] = round(bucket["monthly_cost_usd"] / max(days, 1), 4)
if unassigned["models"]:
agents["Unassigned"] = unassigned
providers_seen.add("Unassigned")
total_monthly = round(sum(item["monthly_cost_usd"] for item in agents.values()), 4)
total_daily = round(sum(item["daily_cost_usd"] for item in agents.values()), 4)
provider_order = sorted(providers_seen)
if "Unassigned" in provider_order:
provider_order = [p for p in provider_order if p != "Unassigned"] + ["Unassigned"]
return {
"days": days,
"providers": provider_order,
"inventory": inventory,
"agents": agents,
"total_monthly_cost_usd": total_monthly,
"total_daily_cost_usd": total_daily,
}
def render_markdown(summary: dict, report_date: str | None = None) -> str:
if report_date is None:
report_date = datetime.now().strftime("%Y-%m-%d")
lines = [
f"# Fleet Cost Report — {report_date}",
"",
f"Window: last {summary['days']} days of paid-model session stats from `~/.timmy/metrics/model_metrics.db`.",
"",
"## Paid API inventory",
"",
"| Agent | Provider | Notes |",
"| --- | --- | --- |",
]
for item in summary["inventory"]:
lines.append(f"| {item['agent']} | {item['provider']} | {item['notes']} |")
lines.extend(
[
"",
"## Estimated cost per agent per day",
"",
"| Agent | Provider | Daily cost | Monthly estimate | Sessions | Messages | Tool calls |",
"| --- | --- | ---: | ---: | ---: | ---: | ---: |",
]
)
for agent, data in summary["agents"].items():
lines.append(
f"| {agent} | {data['provider']} | ${data['daily_cost_usd']:.2f} | ${data['monthly_cost_usd']:.2f} | {data['sessions']} | {data['messages']} | {data['tool_calls']} |"
)
lines.extend(
[
"",
f"Total estimated daily paid spend: ${summary['total_daily_cost_usd']:.2f}",
f"Total estimated monthly paid spend: ${summary['total_monthly_cost_usd']:.2f}",
"",
"## Model evidence",
"",
]
)
for agent, data in summary["agents"].items():
lines.append(f"### {agent}")
if not data["models"]:
lines.append("- No paid-model sessions observed in the selected window.")
else:
for model in data["models"]:
lines.append(
f"- `{model['model']}` — {model['sessions']} sessions / {model['messages']} messages / {model['tool_calls']} tool calls / ${model['monthly_cost_usd']:.2f} est."
)
lines.append("")
lines.append("Generated by `python3 scripts/fleet_cost_report.py --days 30`. Default output path targets the local timmy-config report lane.")
lines.append("")
return "\n".join(lines)
def write_report(output_path: Path, summary: dict, report_date: str | None = None) -> Path:
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(render_markdown(summary, report_date=report_date), encoding="utf-8")
return output_path
def main() -> int:
parser = argparse.ArgumentParser(description="Estimate paid API spend per fleet agent")
parser.add_argument("--days", type=int, default=30, help="Lookback window in days")
parser.add_argument("--db-path", default=str(DB_PATH), help="Path to model_metrics.db")
parser.add_argument("--output", help="Optional markdown output path")
parser.add_argument("--date", help="Override report date (YYYY-MM-DD)")
args = parser.parse_args()
rows = load_cost_rows(days=args.days, db_path=Path(args.db_path).expanduser())
summary = summarize_rows(rows, days=args.days)
report_date = args.date or datetime.now().strftime("%Y-%m-%d")
output_path = Path(args.output).expanduser() if args.output else default_report_path(report_date)
write_report(output_path, summary, report_date=report_date)
print(output_path)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

@@ -1,7 +1,6 @@
from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parent.parent
@@ -67,82 +66,6 @@ class TestEvenniaLocalWorldGame(unittest.TestCase):
self.assertIn("Ezra is already here.", result["log"])
self.assertIn("The servers hum steady. The green LED pulses.", result["world_events"])
def test_bridge_rain_crossing_costs_extra_energy_and_warns(self):
module = load_game_module()
dry_engine = module.GameEngine()
dry_engine.start_new_game()
dry_engine.world.update_world_state = lambda: None
dry_engine.world.characters["Timmy"]["energy"] = 10
dry_result = dry_engine.run_tick("move:south")
dry_energy = dry_engine.world.characters["Timmy"]["energy"]
rainy_engine = module.GameEngine()
rainy_engine.start_new_game()
rainy_engine.world.update_world_state = lambda: None
rainy_engine.world.characters["Timmy"]["energy"] = 10
rainy_engine.world.rooms["Bridge"]["weather"] = "rain"
rainy_engine.world.rooms["Bridge"]["rain_ticks"] = 3
rainy_engine.world.state["bridge_flooding"] = True
rainy_result = rainy_engine.run_tick("move:south")
self.assertEqual(rainy_engine.world.characters["Timmy"]["room"], "Bridge")
self.assertLess(rainy_engine.world.characters["Timmy"]["energy"], dry_energy)
self.assertTrue(
any("bridge" in line.lower() and ("rain" in line.lower() or "slick" in line.lower()) for line in rainy_result["log"] + rainy_result["world_events"]),
rainy_result,
)
def test_tower_power_low_blocks_study_and_write_rule(self):
module = load_game_module()
engine = module.GameEngine()
engine.start_new_game()
engine.world.update_world_state = lambda: None
engine.world.characters["Timmy"]["room"] = "Tower"
engine.world.characters["Timmy"]["energy"] = 10
engine.world.state["tower_power_low"] = True
rules_before = list(engine.world.rooms["Tower"]["messages"])
study_result = engine.run_tick("study")
self.assertEqual(engine.world.characters["Timmy"]["energy"], 10)
self.assertTrue(
any("power" in line.lower() and ("study" in line.lower() or "servers" in line.lower()) for line in study_result["log"] + study_result["world_events"]),
study_result,
)
write_result = engine.run_tick("write_rule")
self.assertEqual(engine.world.rooms["Tower"]["messages"], rules_before)
self.assertTrue(
any("power" in line.lower() and ("write" in line.lower() or "whiteboard" in line.lower()) for line in write_result["log"] + write_result["world_events"]),
write_result,
)
def test_cold_forge_blocks_forge_action_and_bezalel_reacts(self):
module = load_game_module()
engine = module.GameEngine()
engine.start_new_game()
engine.world.update_world_state = lambda: None
engine.npc_ai.make_choice = lambda _name: None
engine.world.characters["Timmy"]["room"] = "Forge"
engine.world.characters["Timmy"]["energy"] = 10
engine.world.characters["Bezalel"]["room"] = "Forge"
engine.world.rooms["Forge"]["fire"] = "cold"
engine.world.state["forge_fire_dying"] = True
forged_before = list(engine.world.rooms["Forge"]["forged_items"])
with patch.object(module.random, "random", return_value=0.0), patch.object(module.random, "choice", side_effect=lambda seq: seq[0]):
result = engine.run_tick("forge")
self.assertEqual(engine.world.rooms["Forge"]["forged_items"], forged_before)
self.assertTrue(
any("forge" in line.lower() and ("cold" in line.lower() or "fire" in line.lower()) for line in result["log"] + result["world_events"]),
result,
)
self.assertTrue(
any(line.startswith("Bezalel says:") and ("fire" in line.lower() or "forge" in line.lower()) for line in result["log"]),
result,
)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,77 @@
from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path
import tempfile
import unittest
ROOT = Path(__file__).resolve().parent.parent
SCRIPT_PATH = ROOT / "scripts" / "fleet_cost_report.py"
def load_module():
spec = spec_from_file_location("fleet_cost_report", SCRIPT_PATH)
module = module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)
return module
class TestFleetCostReport(unittest.TestCase):
def test_default_output_targets_timmy_config_report_path(self):
module = load_module()
output_path = module.default_report_path("2026-04-22")
self.assertIn("timmy-config", str(output_path))
self.assertTrue(str(output_path).endswith("2026-04-22-fleet-cost-report.md"))
def test_summary_groups_paid_costs_by_agent_and_provider(self):
module = load_module()
rows = [
("claude-sonnet-4-6", 12, 120, 24, 6.0),
("gpt-5.4", 6, 60, 12, 3.0),
("openrouter/google/gemini-2.5-pro", 4, 40, 8, 2.0),
("kimi-k2", 2, 20, 4, 1.0),
]
summary = module.summarize_rows(rows, days=30)
self.assertEqual(summary["providers"], ["Anthropic", "Kimi / Moonshot", "OpenAI", "OpenRouter"])
self.assertAlmostEqual(summary["agents"]["Ezra"]["monthly_cost_usd"], 6.0)
self.assertAlmostEqual(summary["agents"]["Bezalel"]["monthly_cost_usd"], 3.0)
self.assertAlmostEqual(summary["agents"]["Timmy Cloud Lane"]["monthly_cost_usd"], 2.0)
self.assertAlmostEqual(summary["agents"]["Allegro"]["monthly_cost_usd"], 1.0)
self.assertAlmostEqual(summary["agents"]["Ezra"]["daily_cost_usd"], 0.2)
def test_report_render_mentions_inventory_and_agent_costs(self):
module = load_module()
rows = [
("claude-sonnet-4-6", 12, 120, 24, 6.0),
("gpt-5.4", 6, 60, 12, 3.0),
("openrouter/google/gemini-2.5-pro", 4, 40, 8, 2.0),
]
summary = module.summarize_rows(rows, days=30)
report = module.render_markdown(summary, report_date="2026-04-22")
self.assertIn("# Fleet Cost Report — 2026-04-22", report)
self.assertIn("## Paid API inventory", report)
self.assertIn("Anthropic", report)
self.assertIn("OpenRouter", report)
self.assertIn("OpenAI", report)
self.assertIn("## Estimated cost per agent per day", report)
self.assertIn("Timmy Cloud Lane", report)
self.assertIn("Ezra", report)
self.assertIn("Bezalel", report)
def test_write_report_creates_markdown_file(self):
module = load_module()
rows = [("claude-sonnet-4-6", 1, 10, 2, 0.5)]
summary = module.summarize_rows(rows, days=30)
with tempfile.TemporaryDirectory() as tmpdir:
dest = Path(tmpdir) / "fleet-cost.md"
module.write_report(dest, summary, report_date="2026-04-22")
self.assertTrue(dest.exists())
text = dest.read_text()
self.assertIn("Fleet Cost Report", text)
self.assertIn("Ezra", text)
if __name__ == "__main__":
unittest.main()