perplexity/Timmy-time-dashboard
Archived
1
0
Fork 0
forked from Rockachopa/Timmy-time-dashboard
Code Issues Pull Requests 1 Activity

[claude] Add agent performance regression benchmark suite (#1015) (#1053)

Browse Source
This commit is contained in:
Claude (Opus 4.6)
2026-03-22 23:55:27 +00:00
parent c0f6ca9fc2
commit 45bde4df58
7 changed files with 1045 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/infrastructure/world/benchmark/__init__.py Normal file
View File

@@ -0,0 +1,17 @@
"""Performance regression suite for Morrowind agent scenarios.
Provides standardised benchmark scenarios, a runner that executes them
through the heartbeat loop with a mock (or live) world adapter, and
metrics collection for CI-integrated regression detection.
"""
from infrastructure.world.benchmark.metrics import BenchmarkMetrics
from infrastructure.world.benchmark.runner import BenchmarkRunner
from infrastructure.world.benchmark.scenarios import BenchmarkScenario, load_scenarios
__all__ = [
"BenchmarkMetrics",
"BenchmarkRunner",
"BenchmarkScenario",
"load_scenarios",
]

195
src/infrastructure/world/benchmark/metrics.py Normal file
View File

@@ -0,0 +1,195 @@
"""Benchmark metrics collection and persistence.
Tracks per-scenario results: cycles used, wall-clock time, success,
LLM call count, and estimated metabolic cost. Results are persisted
as JSONL for trend analysis and CI regression gates.
"""
from __future__ import annotations
import json
import logging
from dataclasses import asdict, dataclass, field
from pathlib import Path
logger = logging.getLogger(__name__)
@dataclass
class ScenarioResult:
"""Outcome of running a single benchmark scenario.
Attributes:
scenario_name: Human-readable scenario name.
success: Whether the goal predicate was satisfied.
cycles_used: Number of heartbeat cycles executed.
max_cycles: The scenario's cycle budget.
wall_time_ms: Total wall-clock time in milliseconds.
llm_calls: Number of LLM inference calls made.
metabolic_cost: Estimated resource cost (arbitrary unit, ≈ tokens).
error: Error message if the run crashed.
tags: Scenario tags (copied for filtering).
"""
scenario_name: str
success: bool = False
cycles_used: int = 0
max_cycles: int = 0
wall_time_ms: int = 0
llm_calls: int = 0
metabolic_cost: float = 0.0
error: str | None = None
tags: list[str] = field(default_factory=list)
@dataclass
class BenchmarkMetrics:
"""Aggregated metrics across all scenarios in a benchmark run.
Attributes:
results: Per-scenario results.
total_time_ms: Total wall-clock time for the full suite.
timestamp: ISO-8601 timestamp of the run.
commit_sha: Git commit SHA (if available).
"""
results: list[ScenarioResult] = field(default_factory=list)
total_time_ms: int = 0
timestamp: str = ""
commit_sha: str = ""
# -- derived properties ------------------------------------------------
@property
def pass_count(self) -> int:
return sum(1 for r in self.results if r.success)
@property
def fail_count(self) -> int:
return sum(1 for r in self.results if not r.success)
@property
def success_rate(self) -> float:
if not self.results:
return 0.0
return self.pass_count / len(self.results)
@property
def total_llm_calls(self) -> int:
return sum(r.llm_calls for r in self.results)
@property
def total_metabolic_cost(self) -> float:
return sum(r.metabolic_cost for r in self.results)
# -- persistence -------------------------------------------------------
def save(self, path: Path) -> None:
"""Append this run's results to a JSONL file at *path*."""
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
record = {
"timestamp": self.timestamp,
"commit_sha": self.commit_sha,
"total_time_ms": self.total_time_ms,
"success_rate": round(self.success_rate, 4),
"total_llm_calls": self.total_llm_calls,
"total_metabolic_cost": round(self.total_metabolic_cost, 2),
"scenarios": [asdict(r) for r in self.results],
}
with path.open("a") as f:
f.write(json.dumps(record) + "\n")
logger.info("Benchmark results saved to %s", path)
# -- summary -----------------------------------------------------------
def summary(self) -> str:
"""Return a human-readable summary of the benchmark run."""
lines = [
"=== Benchmark Summary ===",
f"Scenarios: {len(self.results)} "
f"Passed: {self.pass_count} "
f"Failed: {self.fail_count} "
f"Success rate: {self.success_rate:.0%}",
f"Total time: {self.total_time_ms} ms "
f"LLM calls: {self.total_llm_calls} "
f"Metabolic cost: {self.total_metabolic_cost:.1f}",
]
if self.commit_sha:
lines.append(f"Commit: {self.commit_sha}")
lines.append("")
for r in self.results:
status = "PASS" if r.success else "FAIL"
lines.append(
f" [{status}] {r.scenario_name} — "
f"{r.cycles_used}/{r.max_cycles} cycles, "
f"{r.wall_time_ms} ms, "
f"{r.llm_calls} LLM calls"
)
if r.error:
lines.append(f" Error: {r.error}")
return "\n".join(lines)
def load_history(path: Path) -> list[dict]:
"""Load benchmark history from a JSONL file.
Returns:
List of run records, most recent first.
"""
path = Path(path)
if not path.exists():
return []
records: list[dict] = []
for line in path.read_text().strip().splitlines():
try:
records.append(json.loads(line))
except json.JSONDecodeError:
continue
return list(reversed(records))
def compare_runs(
current: BenchmarkMetrics,
baseline: BenchmarkMetrics,
) -> str:
"""Compare two benchmark runs and report regressions.
Returns:
Human-readable comparison report.
"""
lines = ["=== Regression Report ==="]
# Overall
rate_delta = current.success_rate - baseline.success_rate
lines.append(
f"Success rate: {baseline.success_rate:.0%} -> {current.success_rate:.0%} "
f"({rate_delta:+.0%})"
)
cost_delta = current.total_metabolic_cost - baseline.total_metabolic_cost
if baseline.total_metabolic_cost > 0:
cost_pct = (cost_delta / baseline.total_metabolic_cost) * 100
lines.append(
f"Metabolic cost: {baseline.total_metabolic_cost:.1f} -> "
f"{current.total_metabolic_cost:.1f} ({cost_pct:+.1f}%)"
)
# Per-scenario
baseline_map = {r.scenario_name: r for r in baseline.results}
for r in current.results:
b = baseline_map.get(r.scenario_name)
if b is None:
lines.append(f" [NEW] {r.scenario_name}")
continue
if b.success and not r.success:
lines.append(f" [REGRESSION] {r.scenario_name} — was PASS, now FAIL")
elif not b.success and r.success:
lines.append(f" [IMPROVEMENT] {r.scenario_name} — was FAIL, now PASS")
elif r.cycles_used > b.cycles_used * 1.5:
lines.append(
f" [SLOWER] {r.scenario_name} — "
f"{b.cycles_used} -> {r.cycles_used} cycles (+{r.cycles_used - b.cycles_used})"
)
return "\n".join(lines)

167
src/infrastructure/world/benchmark/runner.py Normal file
View File

@@ -0,0 +1,167 @@
"""Benchmark runner — executes scenarios through the heartbeat loop.
Wires each ``BenchmarkScenario`` into a ``MockWorldAdapter`` (or a
supplied adapter), runs the heartbeat for up to ``max_cycles``, and
collects ``BenchmarkMetrics``.
"""
from __future__ import annotations
import logging
import subprocess
import time
from datetime import UTC, datetime
from infrastructure.world.adapters.mock import MockWorldAdapter
from infrastructure.world.benchmark.metrics import BenchmarkMetrics, ScenarioResult
from infrastructure.world.benchmark.scenarios import BenchmarkScenario
from infrastructure.world.interface import WorldInterface
from loop.heartbeat import Heartbeat
logger = logging.getLogger(__name__)
# Rough estimate: each heartbeat cycle costs ~1 unit of metabolic cost
# (gather + reason + act phases each touch the LLM router once).
_COST_PER_CYCLE = 3.0 # three phases per cycle
class BenchmarkRunner:
"""Run benchmark scenarios and collect metrics.
Parameters
----------
adapter_factory:
Optional callable that returns a ``WorldInterface`` for a given
scenario. Defaults to building a ``MockWorldAdapter`` from the
scenario's start state.
heartbeat_interval:
Seconds between heartbeat ticks (0 for immediate).
"""
def __init__(
self,
*,
adapter_factory=None,
heartbeat_interval: float = 0.0,
) -> None:
self._adapter_factory = adapter_factory or self._default_adapter
self._interval = heartbeat_interval
# -- public API --------------------------------------------------------
async def run(
self,
scenarios: list[BenchmarkScenario],
) -> BenchmarkMetrics:
"""Execute all *scenarios* and return aggregated metrics."""
metrics = BenchmarkMetrics(
timestamp=datetime.now(UTC).isoformat(),
commit_sha=self._git_sha(),
)
suite_start = time.monotonic()
for scenario in scenarios:
logger.info("Benchmark: starting '%s'", scenario.name)
result = await self._run_scenario(scenario)
metrics.results.append(result)
status = "PASS" if result.success else "FAIL"
logger.info(
"Benchmark: '%s' %s (%d/%d cycles, %d ms)",
scenario.name,
status,
result.cycles_used,
result.max_cycles,
result.wall_time_ms,
)
metrics.total_time_ms = int((time.monotonic() - suite_start) * 1000)
return metrics
# -- internal ----------------------------------------------------------
async def _run_scenario(self, scenario: BenchmarkScenario) -> ScenarioResult:
"""Run a single scenario through the heartbeat loop."""
result = ScenarioResult(
scenario_name=scenario.name,
max_cycles=scenario.max_cycles,
tags=list(scenario.tags),
)
adapter = self._adapter_factory(scenario)
adapter.connect()
hb = Heartbeat(world=adapter, interval=self._interval)
actions: list[dict] = []
start = time.monotonic()
try:
for cycle in range(1, scenario.max_cycles + 1):
record = await hb.run_once()
result.cycles_used = cycle
# Track LLM calls (each cycle has 3 phases that may call LLM)
result.llm_calls += 3
# Accumulate actions for goal predicate
if record.action_taken and record.action_taken != "idle":
actions.append(
{
"action": record.action_taken,
"target": record.observation.get("location", ""),
"status": record.action_status,
}
)
# Update adapter location if scenario simulates movement
current_location = self._get_current_location(adapter)
# Check goal predicate
if scenario.goal_predicate is not None:
if scenario.goal_predicate(actions, current_location):
result.success = True
break
elif cycle == scenario.max_cycles:
# No predicate — success if we survived all cycles
result.success = True
except Exception as exc:
logger.warning("Benchmark scenario '%s' crashed: %s", scenario.name, exc)
result.error = str(exc)
finally:
adapter.disconnect()
result.wall_time_ms = int((time.monotonic() - start) * 1000)
result.metabolic_cost = result.cycles_used * _COST_PER_CYCLE
return result
@staticmethod
def _default_adapter(scenario: BenchmarkScenario) -> WorldInterface:
"""Build a MockWorldAdapter from a scenario's starting state."""
return MockWorldAdapter(
location=scenario.start_location,
entities=list(scenario.entities),
events=list(scenario.events),
)
@staticmethod
def _get_current_location(adapter: WorldInterface) -> str:
"""Read the current location from the adapter."""
try:
perception = adapter.observe()
return perception.location
except Exception:
return ""
@staticmethod
def _git_sha() -> str:
"""Best-effort: return the current git commit SHA."""
try:
result = subprocess.run(
["git", "rev-parse", "--short", "HEAD"],
capture_output=True,
text=True,
timeout=5,
)
return result.stdout.strip() if result.returncode == 0 else ""
except (OSError, subprocess.TimeoutExpired):
return ""

160
src/infrastructure/world/benchmark/scenarios.py Normal file
View File

@@ -0,0 +1,160 @@
"""Benchmark scenario definitions for Morrowind agent regression testing.
Each scenario specifies a starting location, goal conditions, world state
(entities, events), and maximum cycles allowed. The runner feeds these
into the heartbeat loop and checks completion against the goal predicate.
"""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass, field
@dataclass(frozen=True)
class BenchmarkScenario:
"""A reproducible agent task used to detect performance regressions.
Attributes:
name: Human-readable scenario name.
description: What the scenario tests.
start_location: Where the agent begins.
goal_location: Target location (if navigation scenario).
entities: NPCs / objects present in the world.
events: Game events injected each cycle.
max_cycles: Hard cap on heartbeat cycles before failure.
goal_predicate: Optional callable ``(actions, location) -> bool``
evaluated after each cycle to check early success.
tags: Freeform tags for filtering (e.g. "navigation", "quest").
"""
name: str
description: str
start_location: str
goal_location: str = ""
entities: list[str] = field(default_factory=list)
events: list[str] = field(default_factory=list)
max_cycles: int = 50
goal_predicate: Callable | None = None
tags: list[str] = field(default_factory=list)
# ---------------------------------------------------------------------------
# Goal predicates
# ---------------------------------------------------------------------------
def _reached_location(target: str) -> Callable:
"""Return a predicate that checks whether the agent reached *target*."""
def predicate(actions: list[dict], current_location: str) -> bool:
return current_location.lower() == target.lower()
return predicate
def _interacted_with(npc: str) -> Callable:
"""Return a predicate that checks for a speak/interact action with *npc*."""
def predicate(actions: list[dict], current_location: str) -> bool:
for act in actions:
if act.get("action") in ("speak", "interact", "talk"):
if act.get("target", "").lower() == npc.lower():
return True
return False
return predicate
# ---------------------------------------------------------------------------
# Built-in scenarios
# ---------------------------------------------------------------------------
BUILTIN_SCENARIOS: list[BenchmarkScenario] = [
BenchmarkScenario(
name="Walk Seyda Neen to Balmora",
description=(
"Navigate from the starting village to Balmora via the road. "
"Tests basic navigation and pathfinding."
),
start_location="Seyda Neen",
goal_location="Balmora",
entities=["Silt Strider", "Road Sign", "Mudcrab"],
events=["player_spawned"],
max_cycles=30,
goal_predicate=_reached_location("Balmora"),
tags=["navigation", "basic"],
),
BenchmarkScenario(
name="Fargoth's Ring",
description=(
"Complete the Fargoth quest: find Fargoth, receive the ring, "
"and return it. Tests NPC interaction and quest logic."
),
start_location="Seyda Neen",
goal_location="Seyda Neen",
entities=["Fargoth", "Arrille", "Guard"],
events=["quest_available:fargoth_ring"],
max_cycles=40,
goal_predicate=_interacted_with("Fargoth"),
tags=["quest", "npc_interaction"],
),
BenchmarkScenario(
name="Balmora Guild Navigation",
description=(
"Walk from Balmora South Wall Corner Club to the Fighters Guild. "
"Tests intra-city navigation with multiple NPCs present."
),
start_location="Balmora, South Wall Corner Club",
goal_location="Balmora, Fighters Guild",
entities=["Guard", "Merchant", "Caius Cosades"],
events=["player_entered"],
max_cycles=20,
goal_predicate=_reached_location("Balmora, Fighters Guild"),
tags=["navigation", "city"],
),
BenchmarkScenario(
name="Combat Encounter — Mudcrab",
description=(
"Engage and defeat a single Mudcrab on the road between "
"Seyda Neen and Balmora. Tests combat action selection."
),
start_location="Bitter Coast Road",
goal_location="Bitter Coast Road",
entities=["Mudcrab"],
events=["hostile_entity_nearby"],
max_cycles=15,
goal_predicate=None, # Success = survived max_cycles without crash
tags=["combat", "basic"],
),
BenchmarkScenario(
name="Passive Observation — Balmora Market",
description=(
"Observe the Balmora market for 10 cycles without acting. "
"Tests that the agent can reason without unnecessary actions."
),
start_location="Balmora, Market Square",
goal_location="",
entities=["Merchant", "Guard", "Pilgrim", "Trader"],
events=["market_day"],
max_cycles=10,
tags=["observation", "passive"],
),
]
def load_scenarios(
tags: list[str] | None = None,
) -> list[BenchmarkScenario]:
"""Return built-in scenarios, optionally filtered by tags.
Args:
tags: If provided, only return scenarios whose tags overlap.
Returns:
List of matching ``BenchmarkScenario`` instances.
"""
if tags is None:
return list(BUILTIN_SCENARIOS)
tag_set = set(tags)
return [s for s in BUILTIN_SCENARIOS if tag_set & set(s.tags)]