forked from Rockachopa/Timmy-time-dashboard
Compare commits
1 Commits
kimi/issue
...
kimi/issue
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
919a011cae |
98
config/stress_modes.yaml
Normal file
98
config/stress_modes.yaml
Normal file
@@ -0,0 +1,98 @@
|
||||
# ── System Stress Modes Configuration ────────────────────────────────────────
|
||||
#
|
||||
# This configuration defines how token rewards adapt based on system stress.
|
||||
# When the system detects elevated stress (flaky tests, growing backlog,
|
||||
# CI failures), quest rewards are adjusted to incentivize agents to focus
|
||||
# on the most critical areas.
|
||||
#
|
||||
# ── How It Works ─────────────────────────────────────────────────────────────
|
||||
#
|
||||
# 1. SIGNALS: System metrics are monitored continuously
|
||||
# 2. SCORE: Weighted contributions from triggered signals create a stress score
|
||||
# 3. MODE: Score determines the stress mode (calm, elevated, high)
|
||||
# 4. MULTIPLIERS: Token rewards are multiplied based on the current mode
|
||||
#
|
||||
# ── Stress Thresholds ────────────────────────────────────────────────────────
|
||||
|
||||
thresholds:
|
||||
# Minimum score to enter elevated mode (0.0 - 1.0)
|
||||
elevated_min: 0.3
|
||||
|
||||
# Minimum score to enter high stress mode (0.0 - 1.0)
|
||||
high_min: 0.6
|
||||
|
||||
# ── Stress Signals ───────────────────────────────────────────────────────────
|
||||
#
|
||||
# Each signal has:
|
||||
# - threshold: Value at which signal is considered "triggered"
|
||||
# - weight: Contribution to overall stress score (should sum to ~1.0)
|
||||
|
||||
signals:
|
||||
flaky_test_rate:
|
||||
threshold: 0.15 # 15% of tests showing flakiness
|
||||
weight: 0.30
|
||||
description: "Percentage of test runs that are flaky"
|
||||
|
||||
p1_backlog_growth:
|
||||
threshold: 5 # 5 new P1 issues in lookback period
|
||||
weight: 0.25
|
||||
description: "Net growth in P1 priority issues over 7 days"
|
||||
|
||||
ci_failure_rate:
|
||||
threshold: 0.20 # 20% of CI runs failing
|
||||
weight: 0.25
|
||||
description: "Percentage of CI runs failing in lookback period"
|
||||
|
||||
open_bug_count:
|
||||
threshold: 20 # 20 open bugs
|
||||
weight: 0.20
|
||||
description: "Total open issues labeled as 'bug'"
|
||||
|
||||
# ── Token Multipliers ────────────────────────────────────────────────────────
|
||||
#
|
||||
# Multipliers are applied to quest rewards based on current stress mode.
|
||||
# Values > 1.0 increase rewards, < 1.0 decrease rewards.
|
||||
#
|
||||
# Quest types:
|
||||
# - test_improve: Test coverage/quality improvements
|
||||
# - docs_update: Documentation updates
|
||||
# - issue_count: Closing specific issue types
|
||||
# - issue_reduce: Reducing overall issue backlog
|
||||
# - daily_run: Daily Run session completion
|
||||
# - custom: Special/manual quests
|
||||
# - exploration: Exploratory work
|
||||
# - refactor: Code refactoring
|
||||
|
||||
multipliers:
|
||||
calm:
|
||||
# Calm periods: incentivize maintenance and exploration
|
||||
test_improve: 1.0
|
||||
docs_update: 1.2
|
||||
issue_count: 1.0
|
||||
issue_reduce: 1.0
|
||||
daily_run: 1.0
|
||||
custom: 1.0
|
||||
exploration: 1.3
|
||||
refactor: 1.2
|
||||
|
||||
elevated:
|
||||
# Elevated stress: start emphasizing stability
|
||||
test_improve: 1.2
|
||||
docs_update: 1.0
|
||||
issue_count: 1.1
|
||||
issue_reduce: 1.1
|
||||
daily_run: 1.0
|
||||
custom: 1.0
|
||||
exploration: 1.0
|
||||
refactor: 0.9 # Discourage risky changes
|
||||
|
||||
high:
|
||||
# High stress: crisis mode, focus on stabilization
|
||||
test_improve: 1.5 # Strongly incentivize testing
|
||||
docs_update: 0.8 # Deprioritize docs
|
||||
issue_count: 1.3 # Reward closing issues
|
||||
issue_reduce: 1.4 # Strongly reward reducing backlog
|
||||
daily_run: 1.1
|
||||
custom: 1.0
|
||||
exploration: 0.7 # Discourage exploration
|
||||
refactor: 0.6 # Discourage refactors during crisis
|
||||
@@ -44,7 +44,6 @@ from dashboard.routes.mobile import router as mobile_router
|
||||
from dashboard.routes.models import api_router as models_api_router
|
||||
from dashboard.routes.models import router as models_router
|
||||
from dashboard.routes.quests import router as quests_router
|
||||
from dashboard.routes.scorecards import router as scorecards_router
|
||||
from dashboard.routes.spark import router as spark_router
|
||||
from dashboard.routes.system import router as system_router
|
||||
from dashboard.routes.tasks import router as tasks_router
|
||||
@@ -630,7 +629,6 @@ app.include_router(matrix_router)
|
||||
app.include_router(tower_router)
|
||||
app.include_router(daily_run_router)
|
||||
app.include_router(quests_router)
|
||||
app.include_router(scorecards_router)
|
||||
|
||||
|
||||
@app.websocket("/ws")
|
||||
|
||||
@@ -187,6 +187,76 @@ async def reload_quest_config_api() -> JSONResponse:
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stress Mode Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@router.get("/api/stress")
|
||||
async def get_stress_status_api() -> JSONResponse:
|
||||
"""Get current stress mode status and multipliers.
|
||||
|
||||
Returns:
|
||||
Current stress mode, score, active signals, and multipliers
|
||||
"""
|
||||
try:
|
||||
from timmy.stress_detector import (
|
||||
detect_stress_mode,
|
||||
get_stress_summary,
|
||||
)
|
||||
|
||||
snapshot = detect_stress_mode()
|
||||
summary = get_stress_summary()
|
||||
|
||||
return JSONResponse(
|
||||
{
|
||||
"status": "ok",
|
||||
"stress": summary,
|
||||
"raw": snapshot.to_dict(),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to get stress status: %s", exc)
|
||||
return JSONResponse(
|
||||
{
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
},
|
||||
status_code=500,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/stress/refresh")
|
||||
async def refresh_stress_detection_api() -> JSONResponse:
|
||||
"""Force a fresh stress detection check.
|
||||
|
||||
Normally stress is cached for 60 seconds. This endpoint
|
||||
bypasses the cache for immediate results.
|
||||
"""
|
||||
try:
|
||||
from timmy.stress_detector import detect_stress_mode, get_stress_summary
|
||||
|
||||
snapshot = detect_stress_mode(force_refresh=True)
|
||||
summary = get_stress_summary()
|
||||
|
||||
return JSONResponse(
|
||||
{
|
||||
"status": "ok",
|
||||
"stress": summary,
|
||||
"raw": snapshot.to_dict(),
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to refresh stress detection: %s", exc)
|
||||
return JSONResponse(
|
||||
{
|
||||
"status": "error",
|
||||
"error": str(exc),
|
||||
},
|
||||
status_code=500,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dashboard UI Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -1,353 +0,0 @@
|
||||
"""Agent scorecard routes — API endpoints for generating and viewing scorecards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
from dashboard.services.scorecard_service import (
|
||||
PeriodType,
|
||||
generate_all_scorecards,
|
||||
generate_scorecard,
|
||||
get_tracked_agents,
|
||||
)
|
||||
from dashboard.templating import templates
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/scorecards", tags=["scorecards"])
|
||||
|
||||
|
||||
def _format_period_label(period_type: PeriodType) -> str:
|
||||
"""Format a period type for display."""
|
||||
return "Daily" if period_type == PeriodType.daily else "Weekly"
|
||||
|
||||
|
||||
@router.get("/api/agents")
|
||||
async def list_tracked_agents() -> dict[str, list[str]]:
|
||||
"""Return the list of tracked agent IDs.
|
||||
|
||||
Returns:
|
||||
Dict with "agents" key containing list of agent IDs
|
||||
"""
|
||||
return {"agents": get_tracked_agents()}
|
||||
|
||||
|
||||
@router.get("/api/{agent_id}")
|
||||
async def get_agent_scorecard(
|
||||
agent_id: str,
|
||||
period: str = Query(default="daily", description="Period type: 'daily' or 'weekly'"),
|
||||
) -> JSONResponse:
|
||||
"""Generate a scorecard for a specific agent.
|
||||
|
||||
Args:
|
||||
agent_id: The agent ID (e.g., 'kimi', 'claude')
|
||||
period: 'daily' or 'weekly' (default: daily)
|
||||
|
||||
Returns:
|
||||
JSON response with scorecard data
|
||||
"""
|
||||
try:
|
||||
period_type = PeriodType(period.lower())
|
||||
except ValueError:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": f"Invalid period '{period}'. Use 'daily' or 'weekly'."},
|
||||
)
|
||||
|
||||
try:
|
||||
scorecard = generate_scorecard(agent_id, period_type)
|
||||
|
||||
if scorecard is None:
|
||||
return JSONResponse(
|
||||
status_code=404,
|
||||
content={"error": f"No scorecard found for agent '{agent_id}'"},
|
||||
)
|
||||
|
||||
return JSONResponse(content=scorecard.to_dict())
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Failed to generate scorecard for %s: %s", agent_id, exc)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": f"Failed to generate scorecard: {str(exc)}"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api")
|
||||
async def get_all_scorecards(
|
||||
period: str = Query(default="daily", description="Period type: 'daily' or 'weekly'"),
|
||||
) -> JSONResponse:
|
||||
"""Generate scorecards for all tracked agents.
|
||||
|
||||
Args:
|
||||
period: 'daily' or 'weekly' (default: daily)
|
||||
|
||||
Returns:
|
||||
JSON response with list of scorecard data
|
||||
"""
|
||||
try:
|
||||
period_type = PeriodType(period.lower())
|
||||
except ValueError:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": f"Invalid period '{period}'. Use 'daily' or 'weekly'."},
|
||||
)
|
||||
|
||||
try:
|
||||
scorecards = generate_all_scorecards(period_type)
|
||||
return JSONResponse(
|
||||
content={
|
||||
"period": period_type.value,
|
||||
"scorecards": [s.to_dict() for s in scorecards],
|
||||
"count": len(scorecards),
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Failed to generate scorecards: %s", exc)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": f"Failed to generate scorecards: {str(exc)}"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def scorecards_page(request: Request) -> HTMLResponse:
|
||||
"""Render the scorecards dashboard page.
|
||||
|
||||
Returns:
|
||||
HTML page with scorecard interface
|
||||
"""
|
||||
agents = get_tracked_agents()
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"scorecards.html",
|
||||
{
|
||||
"agents": agents,
|
||||
"periods": ["daily", "weekly"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/panel/{agent_id}", response_class=HTMLResponse)
|
||||
async def agent_scorecard_panel(
|
||||
request: Request,
|
||||
agent_id: str,
|
||||
period: str = Query(default="daily"),
|
||||
) -> HTMLResponse:
|
||||
"""Render an individual agent scorecard panel (for HTMX).
|
||||
|
||||
Args:
|
||||
request: The request object
|
||||
agent_id: The agent ID
|
||||
period: 'daily' or 'weekly'
|
||||
|
||||
Returns:
|
||||
HTML panel with scorecard content
|
||||
"""
|
||||
try:
|
||||
period_type = PeriodType(period.lower())
|
||||
except ValueError:
|
||||
period_type = PeriodType.daily
|
||||
|
||||
try:
|
||||
scorecard = generate_scorecard(agent_id, period_type)
|
||||
|
||||
if scorecard is None:
|
||||
return HTMLResponse(
|
||||
content=f"""
|
||||
<div class="card mc-panel">
|
||||
<h5 class="card-title">{agent_id.title()}</h5>
|
||||
<p class="text-muted">No activity recorded for this period.</p>
|
||||
</div>
|
||||
""",
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
data = scorecard.to_dict()
|
||||
|
||||
# Build patterns HTML
|
||||
patterns_html = ""
|
||||
if data["patterns"]:
|
||||
patterns_list = "".join([f"<li>{p}</li>" for p in data["patterns"]])
|
||||
patterns_html = f"""
|
||||
<div class="mt-3">
|
||||
<h6>Patterns</h6>
|
||||
<ul class="list-unstyled text-info">
|
||||
{patterns_list}
|
||||
</ul>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# Build bullets HTML
|
||||
bullets_html = "".join([f"<li>{b}</li>" for b in data["narrative_bullets"]])
|
||||
|
||||
# Build metrics summary
|
||||
metrics = data["metrics"]
|
||||
|
||||
html_content = f"""
|
||||
<div class="card mc-panel">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<h5 class="card-title mb-0">{agent_id.title()}</h5>
|
||||
<span class="badge bg-secondary">{_format_period_label(period_type)}</span>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<ul class="list-unstyled mb-3">
|
||||
{bullets_html}
|
||||
</ul>
|
||||
|
||||
<div class="row text-center small">
|
||||
<div class="col">
|
||||
<div class="text-muted">PRs</div>
|
||||
<div class="fw-bold">{metrics["prs_opened"]}/{metrics["prs_merged"]}</div>
|
||||
<div class="text-muted" style="font-size: 0.75rem;">
|
||||
{int(metrics["pr_merge_rate"] * 100)}% merged
|
||||
</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Issues</div>
|
||||
<div class="fw-bold">{metrics["issues_touched"]}</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Tests</div>
|
||||
<div class="fw-bold">{metrics["tests_affected"]}</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Tokens</div>
|
||||
<div class="fw-bold {"text-success" if metrics["token_net"] >= 0 else "text-danger"}">
|
||||
{"+" if metrics["token_net"] > 0 else ""}{metrics["token_net"]}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{patterns_html}
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
return HTMLResponse(content=html_content)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Failed to render scorecard panel for %s: %s", agent_id, exc)
|
||||
return HTMLResponse(
|
||||
content=f"""
|
||||
<div class="card mc-panel border-danger">
|
||||
<h5 class="card-title">{agent_id.title()}</h5>
|
||||
<p class="text-danger">Error loading scorecard: {str(exc)}</p>
|
||||
</div>
|
||||
""",
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/all/panels", response_class=HTMLResponse)
|
||||
async def all_scorecard_panels(
|
||||
request: Request,
|
||||
period: str = Query(default="daily"),
|
||||
) -> HTMLResponse:
|
||||
"""Render all agent scorecard panels (for HTMX).
|
||||
|
||||
Args:
|
||||
request: The request object
|
||||
period: 'daily' or 'weekly'
|
||||
|
||||
Returns:
|
||||
HTML with all scorecard panels
|
||||
"""
|
||||
try:
|
||||
period_type = PeriodType(period.lower())
|
||||
except ValueError:
|
||||
period_type = PeriodType.daily
|
||||
|
||||
try:
|
||||
scorecards = generate_all_scorecards(period_type)
|
||||
|
||||
panels: list[str] = []
|
||||
for scorecard in scorecards:
|
||||
data = scorecard.to_dict()
|
||||
|
||||
# Build patterns HTML
|
||||
patterns_html = ""
|
||||
if data["patterns"]:
|
||||
patterns_list = "".join([f"<li>{p}</li>" for p in data["patterns"]])
|
||||
patterns_html = f"""
|
||||
<div class="mt-3">
|
||||
<h6>Patterns</h6>
|
||||
<ul class="list-unstyled text-info">
|
||||
{patterns_list}
|
||||
</ul>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# Build bullets HTML
|
||||
bullets_html = "".join([f"<li>{b}</li>" for b in data["narrative_bullets"]])
|
||||
metrics = data["metrics"]
|
||||
|
||||
panel_html = f"""
|
||||
<div class="col-md-6 col-lg-4 mb-3">
|
||||
<div class="card mc-panel">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<h5 class="card-title mb-0">{scorecard.agent_id.title()}</h5>
|
||||
<span class="badge bg-secondary">{_format_period_label(period_type)}</span>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<ul class="list-unstyled mb-3">
|
||||
{bullets_html}
|
||||
</ul>
|
||||
|
||||
<div class="row text-center small">
|
||||
<div class="col">
|
||||
<div class="text-muted">PRs</div>
|
||||
<div class="fw-bold">{metrics["prs_opened"]}/{metrics["prs_merged"]}</div>
|
||||
<div class="text-muted" style="font-size: 0.75rem;">
|
||||
{int(metrics["pr_merge_rate"] * 100)}% merged
|
||||
</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Issues</div>
|
||||
<div class="fw-bold">{metrics["issues_touched"]}</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Tests</div>
|
||||
<div class="fw-bold">{metrics["tests_affected"]}</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Tokens</div>
|
||||
<div class="fw-bold {"text-success" if metrics["token_net"] >= 0 else "text-danger"}">
|
||||
{"+" if metrics["token_net"] > 0 else ""}{metrics["token_net"]}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{patterns_html}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
panels.append(panel_html)
|
||||
|
||||
html_content = f"""
|
||||
<div class="row">
|
||||
{"".join(panels)}
|
||||
</div>
|
||||
<div class="text-muted small mt-2">
|
||||
Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S UTC")}
|
||||
</div>
|
||||
"""
|
||||
|
||||
return HTMLResponse(content=html_content)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Failed to render all scorecard panels: %s", exc)
|
||||
return HTMLResponse(
|
||||
content=f"""
|
||||
<div class="alert alert-danger">
|
||||
Error loading scorecards: {str(exc)}
|
||||
</div>
|
||||
""",
|
||||
status_code=200,
|
||||
)
|
||||
@@ -1,17 +0,0 @@
|
||||
"""Dashboard services for business logic."""
|
||||
|
||||
from dashboard.services.scorecard_service import (
|
||||
PeriodType,
|
||||
ScorecardSummary,
|
||||
generate_all_scorecards,
|
||||
generate_scorecard,
|
||||
get_tracked_agents,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"PeriodType",
|
||||
"ScorecardSummary",
|
||||
"generate_all_scorecards",
|
||||
"generate_scorecard",
|
||||
"get_tracked_agents",
|
||||
]
|
||||
@@ -1,515 +0,0 @@
|
||||
"""Agent scorecard service — track and summarize agent performance.
|
||||
|
||||
Generates daily/weekly scorecards showing:
|
||||
- Issues touched, PRs opened/merged
|
||||
- Tests affected, tokens earned/spent
|
||||
- Pattern highlights (merge rate, activity quality)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
from infrastructure.events.bus import Event, get_event_bus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Bot/agent usernames to track
|
||||
TRACKED_AGENTS = frozenset({"hermes", "kimi", "manus", "claude", "gemini"})
|
||||
|
||||
|
||||
class PeriodType(StrEnum):
|
||||
daily = "daily"
|
||||
weekly = "weekly"
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentMetrics:
|
||||
"""Raw metrics collected for an agent over a period."""
|
||||
|
||||
agent_id: str
|
||||
issues_touched: set[int] = field(default_factory=set)
|
||||
prs_opened: set[int] = field(default_factory=set)
|
||||
prs_merged: set[int] = field(default_factory=set)
|
||||
tests_affected: set[str] = field(default_factory=set)
|
||||
tokens_earned: int = 0
|
||||
tokens_spent: int = 0
|
||||
commits: int = 0
|
||||
comments: int = 0
|
||||
|
||||
@property
|
||||
def pr_merge_rate(self) -> float:
|
||||
"""Calculate PR merge rate (0.0 - 1.0)."""
|
||||
opened = len(self.prs_opened)
|
||||
if opened == 0:
|
||||
return 0.0
|
||||
return len(self.prs_merged) / opened
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScorecardSummary:
|
||||
"""A generated scorecard with narrative summary."""
|
||||
|
||||
agent_id: str
|
||||
period_type: PeriodType
|
||||
period_start: datetime
|
||||
period_end: datetime
|
||||
metrics: AgentMetrics
|
||||
narrative_bullets: list[str] = field(default_factory=list)
|
||||
patterns: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Convert scorecard to dictionary for JSON serialization."""
|
||||
return {
|
||||
"agent_id": self.agent_id,
|
||||
"period_type": self.period_type.value,
|
||||
"period_start": self.period_start.isoformat(),
|
||||
"period_end": self.period_end.isoformat(),
|
||||
"metrics": {
|
||||
"issues_touched": len(self.metrics.issues_touched),
|
||||
"prs_opened": len(self.metrics.prs_opened),
|
||||
"prs_merged": len(self.metrics.prs_merged),
|
||||
"pr_merge_rate": round(self.metrics.pr_merge_rate, 2),
|
||||
"tests_affected": len(self.tests_affected),
|
||||
"commits": self.metrics.commits,
|
||||
"comments": self.metrics.comments,
|
||||
"tokens_earned": self.metrics.tokens_earned,
|
||||
"tokens_spent": self.metrics.tokens_spent,
|
||||
"token_net": self.metrics.tokens_earned - self.metrics.tokens_spent,
|
||||
},
|
||||
"narrative_bullets": self.narrative_bullets,
|
||||
"patterns": self.patterns,
|
||||
}
|
||||
|
||||
@property
|
||||
def tests_affected(self) -> set[str]:
|
||||
"""Alias for metrics.tests_affected."""
|
||||
return self.metrics.tests_affected
|
||||
|
||||
|
||||
def _get_period_bounds(
|
||||
period_type: PeriodType, reference_date: datetime | None = None
|
||||
) -> tuple[datetime, datetime]:
|
||||
"""Calculate start and end timestamps for a period.
|
||||
|
||||
Args:
|
||||
period_type: daily or weekly
|
||||
reference_date: The date to calculate from (defaults to now)
|
||||
|
||||
Returns:
|
||||
Tuple of (period_start, period_end) in UTC
|
||||
"""
|
||||
if reference_date is None:
|
||||
reference_date = datetime.now(UTC)
|
||||
|
||||
# Normalize to start of day
|
||||
end = reference_date.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
if period_type == PeriodType.daily:
|
||||
start = end - timedelta(days=1)
|
||||
else: # weekly
|
||||
start = end - timedelta(days=7)
|
||||
|
||||
return start, end
|
||||
|
||||
|
||||
def _collect_events_for_period(
|
||||
start: datetime, end: datetime, agent_id: str | None = None
|
||||
) -> list[Event]:
|
||||
"""Collect events from the event bus for a time period.
|
||||
|
||||
Args:
|
||||
start: Period start time
|
||||
end: Period end time
|
||||
agent_id: Optional agent filter
|
||||
|
||||
Returns:
|
||||
List of matching events
|
||||
"""
|
||||
bus = get_event_bus()
|
||||
events: list[Event] = []
|
||||
|
||||
# Query persisted events for relevant types
|
||||
event_types = [
|
||||
"gitea.push",
|
||||
"gitea.issue.opened",
|
||||
"gitea.issue.comment",
|
||||
"gitea.pull_request",
|
||||
"agent.task.completed",
|
||||
"test.execution",
|
||||
]
|
||||
|
||||
for event_type in event_types:
|
||||
try:
|
||||
type_events = bus.replay(
|
||||
event_type=event_type,
|
||||
source=agent_id,
|
||||
limit=1000,
|
||||
)
|
||||
events.extend(type_events)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to replay events for %s: %s", event_type, exc)
|
||||
|
||||
# Filter by timestamp
|
||||
filtered = []
|
||||
for event in events:
|
||||
try:
|
||||
event_time = datetime.fromisoformat(event.timestamp.replace("Z", "+00:00"))
|
||||
if start <= event_time < end:
|
||||
filtered.append(event)
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
def _extract_actor_from_event(event: Event) -> str:
|
||||
"""Extract the actor/agent from an event."""
|
||||
# Try data fields first
|
||||
if "actor" in event.data:
|
||||
return event.data["actor"]
|
||||
if "agent_id" in event.data:
|
||||
return event.data["agent_id"]
|
||||
# Fall back to source
|
||||
return event.source
|
||||
|
||||
|
||||
def _is_tracked_agent(actor: str) -> bool:
|
||||
"""Check if an actor is a tracked agent."""
|
||||
return actor.lower() in TRACKED_AGENTS
|
||||
|
||||
|
||||
def _aggregate_metrics(events: list[Event]) -> dict[str, AgentMetrics]:
|
||||
"""Aggregate metrics from events grouped by agent.
|
||||
|
||||
Args:
|
||||
events: List of events to process
|
||||
|
||||
Returns:
|
||||
Dict mapping agent_id -> AgentMetrics
|
||||
"""
|
||||
metrics_by_agent: dict[str, AgentMetrics] = {}
|
||||
|
||||
for event in events:
|
||||
actor = _extract_actor_from_event(event)
|
||||
|
||||
# Skip non-agent events unless they explicitly have an agent_id
|
||||
if not _is_tracked_agent(actor) and "agent_id" not in event.data:
|
||||
continue
|
||||
|
||||
if actor not in metrics_by_agent:
|
||||
metrics_by_agent[actor] = AgentMetrics(agent_id=actor)
|
||||
|
||||
metrics = metrics_by_agent[actor]
|
||||
|
||||
# Process based on event type
|
||||
event_type = event.type
|
||||
|
||||
if event_type == "gitea.push":
|
||||
metrics.commits += event.data.get("num_commits", 1)
|
||||
|
||||
elif event_type == "gitea.issue.opened":
|
||||
issue_num = event.data.get("issue_number", 0)
|
||||
if issue_num:
|
||||
metrics.issues_touched.add(issue_num)
|
||||
|
||||
elif event_type == "gitea.issue.comment":
|
||||
metrics.comments += 1
|
||||
issue_num = event.data.get("issue_number", 0)
|
||||
if issue_num:
|
||||
metrics.issues_touched.add(issue_num)
|
||||
|
||||
elif event_type == "gitea.pull_request":
|
||||
pr_num = event.data.get("pr_number", 0)
|
||||
action = event.data.get("action", "")
|
||||
merged = event.data.get("merged", False)
|
||||
|
||||
if pr_num:
|
||||
if action == "opened":
|
||||
metrics.prs_opened.add(pr_num)
|
||||
elif action == "closed" and merged:
|
||||
metrics.prs_merged.add(pr_num)
|
||||
# Also count as touched issue for tracking
|
||||
metrics.issues_touched.add(pr_num)
|
||||
|
||||
elif event_type == "agent.task.completed":
|
||||
# Extract test files from task data
|
||||
affected = event.data.get("tests_affected", [])
|
||||
for test in affected:
|
||||
metrics.tests_affected.add(test)
|
||||
|
||||
# Token rewards from task completion
|
||||
reward = event.data.get("token_reward", 0)
|
||||
if reward:
|
||||
metrics.tokens_earned += reward
|
||||
|
||||
elif event_type == "test.execution":
|
||||
# Track test files that were executed
|
||||
test_files = event.data.get("test_files", [])
|
||||
for test in test_files:
|
||||
metrics.tests_affected.add(test)
|
||||
|
||||
return metrics_by_agent
|
||||
|
||||
|
||||
def _query_token_transactions(agent_id: str, start: datetime, end: datetime) -> tuple[int, int]:
|
||||
"""Query the lightning ledger for token transactions.
|
||||
|
||||
Args:
|
||||
agent_id: The agent to query for
|
||||
start: Period start
|
||||
end: Period end
|
||||
|
||||
Returns:
|
||||
Tuple of (tokens_earned, tokens_spent)
|
||||
"""
|
||||
try:
|
||||
from lightning.ledger import get_transactions
|
||||
|
||||
transactions = get_transactions(limit=1000)
|
||||
|
||||
earned = 0
|
||||
spent = 0
|
||||
|
||||
for tx in transactions:
|
||||
# Filter by agent if specified
|
||||
if tx.agent_id and tx.agent_id != agent_id:
|
||||
continue
|
||||
|
||||
# Filter by timestamp
|
||||
try:
|
||||
tx_time = datetime.fromisoformat(tx.created_at.replace("Z", "+00:00"))
|
||||
if not (start <= tx_time < end):
|
||||
continue
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
|
||||
if tx.tx_type.value == "incoming":
|
||||
earned += tx.amount_sats
|
||||
else:
|
||||
spent += tx.amount_sats
|
||||
|
||||
return earned, spent
|
||||
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to query token transactions: %s", exc)
|
||||
return 0, 0
|
||||
|
||||
|
||||
def _generate_narrative_bullets(metrics: AgentMetrics, period_type: PeriodType) -> list[str]:
|
||||
"""Generate narrative summary bullets for a scorecard.
|
||||
|
||||
Args:
|
||||
metrics: The agent's metrics
|
||||
period_type: daily or weekly
|
||||
|
||||
Returns:
|
||||
List of narrative bullet points
|
||||
"""
|
||||
bullets: list[str] = []
|
||||
period_label = "day" if period_type == PeriodType.daily else "week"
|
||||
|
||||
# Activity summary
|
||||
activities = []
|
||||
if metrics.commits:
|
||||
activities.append(f"{metrics.commits} commit{'s' if metrics.commits != 1 else ''}")
|
||||
if len(metrics.prs_opened):
|
||||
activities.append(
|
||||
f"{len(metrics.prs_opened)} PR{'s' if len(metrics.prs_opened) != 1 else ''} opened"
|
||||
)
|
||||
if len(metrics.prs_merged):
|
||||
activities.append(
|
||||
f"{len(metrics.prs_merged)} PR{'s' if len(metrics.prs_merged) != 1 else ''} merged"
|
||||
)
|
||||
if len(metrics.issues_touched):
|
||||
activities.append(
|
||||
f"{len(metrics.issues_touched)} issue{'s' if len(metrics.issues_touched) != 1 else ''} touched"
|
||||
)
|
||||
if metrics.comments:
|
||||
activities.append(f"{metrics.comments} comment{'s' if metrics.comments != 1 else ''}")
|
||||
|
||||
if activities:
|
||||
bullets.append(f"Active across {', '.join(activities)} this {period_label}.")
|
||||
|
||||
# Test activity
|
||||
if len(metrics.tests_affected):
|
||||
bullets.append(
|
||||
f"Affected {len(metrics.tests_affected)} test file{'s' if len(metrics.tests_affected) != 1 else ''}."
|
||||
)
|
||||
|
||||
# Token summary
|
||||
net_tokens = metrics.tokens_earned - metrics.tokens_spent
|
||||
if metrics.tokens_earned or metrics.tokens_spent:
|
||||
if net_tokens > 0:
|
||||
bullets.append(
|
||||
f"Net earned {net_tokens} tokens ({metrics.tokens_earned} earned, {metrics.tokens_spent} spent)."
|
||||
)
|
||||
elif net_tokens < 0:
|
||||
bullets.append(
|
||||
f"Net spent {abs(net_tokens)} tokens ({metrics.tokens_earned} earned, {metrics.tokens_spent} spent)."
|
||||
)
|
||||
else:
|
||||
bullets.append(
|
||||
f"Balanced token flow ({metrics.tokens_earned} earned, {metrics.tokens_spent} spent)."
|
||||
)
|
||||
|
||||
# Handle empty case
|
||||
if not bullets:
|
||||
bullets.append(f"No recorded activity this {period_label}.")
|
||||
|
||||
return bullets
|
||||
|
||||
|
||||
def _detect_patterns(metrics: AgentMetrics) -> list[str]:
|
||||
"""Detect interesting patterns in agent behavior.
|
||||
|
||||
Args:
|
||||
metrics: The agent's metrics
|
||||
|
||||
Returns:
|
||||
List of pattern descriptions
|
||||
"""
|
||||
patterns: list[str] = []
|
||||
|
||||
pr_opened = len(metrics.prs_opened)
|
||||
merge_rate = metrics.pr_merge_rate
|
||||
|
||||
# Merge rate patterns
|
||||
if pr_opened >= 3:
|
||||
if merge_rate >= 0.8:
|
||||
patterns.append("High merge rate with few failures — code quality focus.")
|
||||
elif merge_rate <= 0.3:
|
||||
patterns.append("Lots of noisy PRs, low merge rate — may need review support.")
|
||||
|
||||
# Activity patterns
|
||||
if metrics.commits > 10 and pr_opened == 0:
|
||||
patterns.append("High commit volume without PRs — working directly on main?")
|
||||
|
||||
if len(metrics.issues_touched) > 5 and metrics.comments == 0:
|
||||
patterns.append("Touching many issues but low comment volume — silent worker.")
|
||||
|
||||
if metrics.comments > len(metrics.issues_touched) * 2:
|
||||
patterns.append("Highly communicative — lots of discussion relative to work items.")
|
||||
|
||||
# Token patterns
|
||||
net_tokens = metrics.tokens_earned - metrics.tokens_spent
|
||||
if net_tokens > 100:
|
||||
patterns.append("Strong token accumulation — high value delivery.")
|
||||
elif net_tokens < -50:
|
||||
patterns.append("High token spend — may be in experimentation phase.")
|
||||
|
||||
return patterns
|
||||
|
||||
|
||||
def generate_scorecard(
|
||||
agent_id: str,
|
||||
period_type: PeriodType = PeriodType.daily,
|
||||
reference_date: datetime | None = None,
|
||||
) -> ScorecardSummary | None:
|
||||
"""Generate a scorecard for a single agent.
|
||||
|
||||
Args:
|
||||
agent_id: The agent to generate scorecard for
|
||||
period_type: daily or weekly
|
||||
reference_date: The date to calculate from (defaults to now)
|
||||
|
||||
Returns:
|
||||
ScorecardSummary or None if agent has no activity
|
||||
"""
|
||||
start, end = _get_period_bounds(period_type, reference_date)
|
||||
|
||||
# Collect events
|
||||
events = _collect_events_for_period(start, end, agent_id)
|
||||
|
||||
# Aggregate metrics
|
||||
all_metrics = _aggregate_metrics(events)
|
||||
|
||||
# Get metrics for this specific agent
|
||||
if agent_id not in all_metrics:
|
||||
# Create empty metrics - still generate a scorecard
|
||||
metrics = AgentMetrics(agent_id=agent_id)
|
||||
else:
|
||||
metrics = all_metrics[agent_id]
|
||||
|
||||
# Augment with token data from ledger
|
||||
tokens_earned, tokens_spent = _query_token_transactions(agent_id, start, end)
|
||||
metrics.tokens_earned = max(metrics.tokens_earned, tokens_earned)
|
||||
metrics.tokens_spent = max(metrics.tokens_spent, tokens_spent)
|
||||
|
||||
# Generate narrative and patterns
|
||||
narrative = _generate_narrative_bullets(metrics, period_type)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
return ScorecardSummary(
|
||||
agent_id=agent_id,
|
||||
period_type=period_type,
|
||||
period_start=start,
|
||||
period_end=end,
|
||||
metrics=metrics,
|
||||
narrative_bullets=narrative,
|
||||
patterns=patterns,
|
||||
)
|
||||
|
||||
|
||||
def generate_all_scorecards(
|
||||
period_type: PeriodType = PeriodType.daily,
|
||||
reference_date: datetime | None = None,
|
||||
) -> list[ScorecardSummary]:
|
||||
"""Generate scorecards for all tracked agents.
|
||||
|
||||
Args:
|
||||
period_type: daily or weekly
|
||||
reference_date: The date to calculate from (defaults to now)
|
||||
|
||||
Returns:
|
||||
List of ScorecardSummary for all agents with activity
|
||||
"""
|
||||
start, end = _get_period_bounds(period_type, reference_date)
|
||||
|
||||
# Collect all events
|
||||
events = _collect_events_for_period(start, end)
|
||||
|
||||
# Aggregate metrics for all agents
|
||||
all_metrics = _aggregate_metrics(events)
|
||||
|
||||
# Include tracked agents even if no activity
|
||||
for agent_id in TRACKED_AGENTS:
|
||||
if agent_id not in all_metrics:
|
||||
all_metrics[agent_id] = AgentMetrics(agent_id=agent_id)
|
||||
|
||||
# Generate scorecards
|
||||
scorecards: list[ScorecardSummary] = []
|
||||
|
||||
for agent_id, metrics in all_metrics.items():
|
||||
# Augment with token data
|
||||
tokens_earned, tokens_spent = _query_token_transactions(agent_id, start, end)
|
||||
metrics.tokens_earned = max(metrics.tokens_earned, tokens_earned)
|
||||
metrics.tokens_spent = max(metrics.tokens_spent, tokens_spent)
|
||||
|
||||
narrative = _generate_narrative_bullets(metrics, period_type)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
scorecard = ScorecardSummary(
|
||||
agent_id=agent_id,
|
||||
period_type=period_type,
|
||||
period_start=start,
|
||||
period_end=end,
|
||||
metrics=metrics,
|
||||
narrative_bullets=narrative,
|
||||
patterns=patterns,
|
||||
)
|
||||
scorecards.append(scorecard)
|
||||
|
||||
# Sort by agent_id for consistent ordering
|
||||
scorecards.sort(key=lambda s: s.agent_id)
|
||||
|
||||
return scorecards
|
||||
|
||||
|
||||
def get_tracked_agents() -> list[str]:
|
||||
"""Return the list of tracked agent IDs."""
|
||||
return sorted(TRACKED_AGENTS)
|
||||
@@ -51,7 +51,6 @@
|
||||
<a href="/thinking" class="mc-test-link mc-link-thinking">THINKING</a>
|
||||
<a href="/swarm/mission-control" class="mc-test-link">MISSION CTRL</a>
|
||||
<a href="/swarm/live" class="mc-test-link">SWARM</a>
|
||||
<a href="/scorecards" class="mc-test-link">SCORECARDS</a>
|
||||
<a href="/bugs" class="mc-test-link mc-link-bugs">BUGS</a>
|
||||
</div>
|
||||
</div>
|
||||
@@ -124,7 +123,6 @@
|
||||
<a href="/thinking" class="mc-mobile-link">THINKING</a>
|
||||
<a href="/swarm/mission-control" class="mc-mobile-link">MISSION CONTROL</a>
|
||||
<a href="/swarm/live" class="mc-mobile-link">SWARM</a>
|
||||
<a href="/scorecards" class="mc-mobile-link">SCORECARDS</a>
|
||||
<a href="/bugs" class="mc-mobile-link">BUGS</a>
|
||||
<div class="mc-mobile-section-label">INTELLIGENCE</div>
|
||||
<a href="/spark/ui" class="mc-mobile-link">SPARK</a>
|
||||
|
||||
@@ -1,113 +0,0 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Agent Scorecards - Timmy Time{% endblock %}
|
||||
|
||||
{% block extra_styles %}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="container-fluid py-4">
|
||||
<!-- Header -->
|
||||
<div class="d-flex justify-content-between align-items-center mb-4">
|
||||
<div>
|
||||
<h1 class="h3 mb-0">AGENT SCORECARDS</h1>
|
||||
<p class="text-muted small mb-0">Track agent performance across issues, PRs, tests, and tokens</p>
|
||||
</div>
|
||||
<div class="d-flex gap-2">
|
||||
<select id="period-select" class="form-select form-select-sm" style="width: auto;">
|
||||
<option value="daily" selected>Daily</option>
|
||||
<option value="weekly">Weekly</option>
|
||||
</select>
|
||||
<button class="btn btn-sm btn-primary" onclick="refreshScorecards()">
|
||||
<span>Refresh</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Scorecards Grid -->
|
||||
<div id="scorecards-container"
|
||||
hx-get="/scorecards/all/panels?period=daily"
|
||||
hx-trigger="load"
|
||||
hx-swap="innerHTML">
|
||||
<div class="text-center py-5">
|
||||
<div class="spinner-border text-secondary" role="status">
|
||||
<span class="visually-hidden">Loading...</span>
|
||||
</div>
|
||||
<p class="text-muted mt-2">Loading scorecards...</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- API Reference -->
|
||||
<div class="mt-5 pt-4 border-top">
|
||||
<h5 class="text-muted">API Reference</h5>
|
||||
<div class="row g-3">
|
||||
<div class="col-md-6">
|
||||
<div class="card mc-panel">
|
||||
<div class="card-body">
|
||||
<h6 class="card-title">List Tracked Agents</h6>
|
||||
<code>GET /scorecards/api/agents</code>
|
||||
<p class="small text-muted mt-2">Returns all tracked agent IDs</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="card mc-panel">
|
||||
<div class="card-body">
|
||||
<h6 class="card-title">Get All Scorecards</h6>
|
||||
<code>GET /scorecards/api?period=daily|weekly</code>
|
||||
<p class="small text-muted mt-2">Returns scorecards for all agents</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="card mc-panel">
|
||||
<div class="card-body">
|
||||
<h6 class="card-title">Get Agent Scorecard</h6>
|
||||
<code>GET /scorecards/api/{agent_id}?period=daily|weekly</code>
|
||||
<p class="small text-muted mt-2">Returns scorecard for a specific agent</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="card mc-panel">
|
||||
<div class="card-body">
|
||||
<h6 class="card-title">HTML Panel (HTMX)</h6>
|
||||
<code>GET /scorecards/panel/{agent_id}?period=daily|weekly</code>
|
||||
<p class="small text-muted mt-2">Returns HTML panel for embedding</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Period selector change handler
|
||||
document.getElementById('period-select').addEventListener('change', function() {
|
||||
refreshScorecards();
|
||||
});
|
||||
|
||||
function refreshScorecards() {
|
||||
var period = document.getElementById('period-select').value;
|
||||
var container = document.getElementById('scorecards-container');
|
||||
|
||||
// Show loading state
|
||||
container.innerHTML = `
|
||||
<div class="text-center py-5">
|
||||
<div class="spinner-border text-secondary" role="status">
|
||||
<span class="visually-hidden">Loading...</span>
|
||||
</div>
|
||||
<p class="text-muted mt-2">Loading scorecards...</p>
|
||||
</div>
|
||||
`;
|
||||
|
||||
// Trigger HTMX request
|
||||
htmx.ajax('GET', '/scorecards/all/panels?period=' + period, {
|
||||
target: '#scorecards-container',
|
||||
swap: 'innerHTML'
|
||||
});
|
||||
}
|
||||
|
||||
// Auto-refresh every 5 minutes
|
||||
setInterval(refreshScorecards, 300000);
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -269,6 +269,22 @@ def _is_on_cooldown(progress: QuestProgress, quest: QuestDefinition) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _apply_stress_multiplier(base_reward: int, quest_type: QuestType) -> tuple[int, float]:
|
||||
"""Apply stress-based multiplier to quest reward.
|
||||
|
||||
Returns:
|
||||
Tuple of (adjusted_reward, multiplier_used)
|
||||
"""
|
||||
try:
|
||||
from timmy.stress_detector import apply_multiplier
|
||||
|
||||
multiplier = apply_multiplier(base_reward, quest_type.value)
|
||||
return multiplier, multiplier / max(base_reward, 1)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to apply stress multiplier: %s", exc)
|
||||
return base_reward, 1.0
|
||||
|
||||
|
||||
def claim_quest_reward(quest_id: str, agent_id: str) -> dict[str, Any] | None:
|
||||
"""Claim the token reward for a completed quest.
|
||||
|
||||
@@ -292,13 +308,18 @@ def claim_quest_reward(quest_id: str, agent_id: str) -> dict[str, Any] | None:
|
||||
return None
|
||||
|
||||
try:
|
||||
# Apply stress-based multiplier
|
||||
adjusted_reward, multiplier = _apply_stress_multiplier(
|
||||
quest.reward_tokens, quest.quest_type
|
||||
)
|
||||
|
||||
# Award tokens via ledger
|
||||
from lightning.ledger import create_invoice_entry, mark_settled
|
||||
|
||||
# Create a mock invoice for the reward
|
||||
invoice_entry = create_invoice_entry(
|
||||
payment_hash=f"quest_{quest_id}_{agent_id}_{int(time.time())}",
|
||||
amount_sats=quest.reward_tokens,
|
||||
amount_sats=adjusted_reward,
|
||||
memo=f"Quest reward: {quest.name}",
|
||||
source="quest_reward",
|
||||
agent_id=agent_id,
|
||||
@@ -320,12 +341,21 @@ def claim_quest_reward(quest_id: str, agent_id: str) -> dict[str, Any] | None:
|
||||
progress.completed_at = ""
|
||||
progress.claimed_at = ""
|
||||
|
||||
notification = quest.notification_message.format(tokens=quest.reward_tokens)
|
||||
# Build notification with multiplier info
|
||||
notification = quest.notification_message.format(tokens=adjusted_reward)
|
||||
if multiplier != 1.0:
|
||||
pct = int((multiplier - 1.0) * 100)
|
||||
if pct > 0:
|
||||
notification += f" (+{pct}% stress bonus)"
|
||||
else:
|
||||
notification += f" ({pct}% stress adjustment)"
|
||||
|
||||
return {
|
||||
"quest_id": quest_id,
|
||||
"agent_id": agent_id,
|
||||
"tokens_awarded": quest.reward_tokens,
|
||||
"tokens_awarded": adjusted_reward,
|
||||
"base_reward": quest.reward_tokens,
|
||||
"multiplier": round(multiplier, 2),
|
||||
"notification": notification,
|
||||
"completion_count": progress.completion_count,
|
||||
}
|
||||
@@ -467,6 +497,14 @@ def get_agent_quests_status(agent_id: str) -> dict[str, Any]:
|
||||
total_rewards = 0
|
||||
completed_count = 0
|
||||
|
||||
# Get current stress mode for adjusted rewards display
|
||||
try:
|
||||
from timmy.stress_detector import get_current_stress_mode, get_multiplier
|
||||
|
||||
current_mode = get_current_stress_mode()
|
||||
except Exception:
|
||||
current_mode = None
|
||||
|
||||
for quest_id, quest in definitions.items():
|
||||
progress = get_quest_progress(quest_id, agent_id)
|
||||
if not progress:
|
||||
@@ -474,11 +512,23 @@ def get_agent_quests_status(agent_id: str) -> dict[str, Any]:
|
||||
|
||||
is_on_cooldown = _is_on_cooldown(progress, quest) if quest.repeatable else False
|
||||
|
||||
# Calculate adjusted reward with stress multiplier
|
||||
adjusted_reward = quest.reward_tokens
|
||||
multiplier = 1.0
|
||||
if current_mode:
|
||||
try:
|
||||
multiplier = get_multiplier(quest.quest_type.value, current_mode)
|
||||
adjusted_reward = int(quest.reward_tokens * multiplier)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
quest_info = {
|
||||
"quest_id": quest_id,
|
||||
"name": quest.name,
|
||||
"description": quest.description,
|
||||
"reward_tokens": quest.reward_tokens,
|
||||
"adjusted_reward": adjusted_reward,
|
||||
"multiplier": round(multiplier, 2),
|
||||
"type": quest.quest_type.value,
|
||||
"enabled": quest.enabled,
|
||||
"repeatable": quest.repeatable,
|
||||
@@ -509,6 +559,7 @@ def get_agent_quests_status(agent_id: str) -> dict[str, Any]:
|
||||
"total_tokens_earned": total_rewards,
|
||||
"total_quests_completed": completed_count,
|
||||
"active_quests_count": len([q for q in quests_status if q["enabled"]]),
|
||||
"stress_mode": current_mode.value if current_mode else None,
|
||||
}
|
||||
|
||||
|
||||
|
||||
565
src/timmy/stress_detector.py
Normal file
565
src/timmy/stress_detector.py
Normal file
@@ -0,0 +1,565 @@
|
||||
"""System stress detection for adaptive token rewards.
|
||||
|
||||
Monitors system signals like flakiness, backlog growth, and CI failures
|
||||
to determine the current stress mode. Token rewards are then adjusted
|
||||
based on the stress mode to incentivize agents to focus on critical areas.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Path to stress mode configuration
|
||||
STRESS_CONFIG_PATH = Path(settings.repo_root) / "config" / "stress_modes.yaml"
|
||||
|
||||
|
||||
class StressMode(StrEnum):
|
||||
"""System stress modes.
|
||||
|
||||
- CALM: Normal operations, incentivize exploration and refactoring
|
||||
- ELEVATED: Some stress signals detected, balance incentives
|
||||
- HIGH: Critical stress, strongly incentivize bug fixes and stabilization
|
||||
"""
|
||||
|
||||
CALM = "calm"
|
||||
ELEVATED = "elevated"
|
||||
HIGH = "high"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StressSignal:
|
||||
"""A single stress signal reading."""
|
||||
|
||||
name: str
|
||||
value: float
|
||||
threshold: float
|
||||
weight: float
|
||||
timestamp: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
|
||||
|
||||
@property
|
||||
def is_triggered(self) -> bool:
|
||||
"""Whether this signal exceeds its threshold."""
|
||||
return self.value >= self.threshold
|
||||
|
||||
@property
|
||||
def contribution(self) -> float:
|
||||
"""Calculate this signal's contribution to stress score."""
|
||||
if not self.is_triggered:
|
||||
return 0.0
|
||||
# Contribution is weighted ratio of value to threshold
|
||||
return min(1.0, (self.value / max(self.threshold, 1.0))) * self.weight
|
||||
|
||||
|
||||
@dataclass
|
||||
class StressSnapshot:
|
||||
"""Complete stress assessment at a point in time."""
|
||||
|
||||
mode: StressMode
|
||||
score: float
|
||||
signals: list[StressSignal]
|
||||
multipliers: dict[str, float]
|
||||
timestamp: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Convert to dictionary for serialization."""
|
||||
return {
|
||||
"mode": self.mode.value,
|
||||
"score": round(self.score, 3),
|
||||
"signals": [
|
||||
{
|
||||
"name": s.name,
|
||||
"value": s.value,
|
||||
"threshold": s.threshold,
|
||||
"triggered": s.is_triggered,
|
||||
"contribution": round(s.contribution, 3),
|
||||
}
|
||||
for s in self.signals
|
||||
],
|
||||
"multipliers": self.multipliers,
|
||||
"timestamp": self.timestamp,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class StressThresholds:
|
||||
"""Thresholds for entering/exiting stress modes."""
|
||||
|
||||
elevated_min: float = 0.3
|
||||
high_min: float = 0.6
|
||||
|
||||
def get_mode_for_score(self, score: float) -> StressMode:
|
||||
"""Determine stress mode based on score."""
|
||||
if score >= self.high_min:
|
||||
return StressMode.HIGH
|
||||
elif score >= self.elevated_min:
|
||||
return StressMode.ELEVATED
|
||||
return StressMode.CALM
|
||||
|
||||
|
||||
# In-memory storage for stress state
|
||||
_current_snapshot: StressSnapshot | None = None
|
||||
_last_check_time: datetime | None = None
|
||||
_config_cache: dict[str, Any] | None = None
|
||||
_config_mtime: float = 0.0
|
||||
|
||||
|
||||
def _load_stress_config() -> dict[str, Any]:
|
||||
"""Load stress mode configuration from YAML.
|
||||
|
||||
Returns:
|
||||
Configuration dictionary with default fallbacks
|
||||
"""
|
||||
global _config_cache, _config_mtime
|
||||
|
||||
# Check if config file has been modified
|
||||
if STRESS_CONFIG_PATH.exists():
|
||||
mtime = STRESS_CONFIG_PATH.stat().st_mtime
|
||||
if mtime != _config_mtime or _config_cache is None:
|
||||
try:
|
||||
raw = STRESS_CONFIG_PATH.read_text()
|
||||
_config_cache = yaml.safe_load(raw) or {}
|
||||
_config_mtime = mtime
|
||||
logger.debug("Loaded stress config from %s", STRESS_CONFIG_PATH)
|
||||
except (OSError, yaml.YAMLError) as exc:
|
||||
logger.warning("Failed to load stress config: %s", exc)
|
||||
_config_cache = {}
|
||||
|
||||
if _config_cache is None:
|
||||
_config_cache = {}
|
||||
|
||||
return _config_cache
|
||||
|
||||
|
||||
def get_default_config() -> dict[str, Any]:
|
||||
"""Get default stress configuration."""
|
||||
return {
|
||||
"thresholds": {
|
||||
"elevated_min": 0.3,
|
||||
"high_min": 0.6,
|
||||
},
|
||||
"signals": {
|
||||
"flaky_test_rate": {
|
||||
"threshold": 0.15, # 15% flaky test rate
|
||||
"weight": 0.3,
|
||||
"description": "Percentage of tests that are flaky",
|
||||
},
|
||||
"p1_backlog_growth": {
|
||||
"threshold": 5, # 5 new P1 issues
|
||||
"weight": 0.25,
|
||||
"description": "Net growth in P1 priority issues",
|
||||
},
|
||||
"ci_failure_rate": {
|
||||
"threshold": 0.2, # 20% CI failure rate
|
||||
"weight": 0.25,
|
||||
"description": "Percentage of CI runs failing",
|
||||
},
|
||||
"open_bug_count": {
|
||||
"threshold": 20, # 20 open bugs
|
||||
"weight": 0.2,
|
||||
"description": "Total open issues labeled as bugs",
|
||||
},
|
||||
},
|
||||
"multipliers": {
|
||||
StressMode.CALM.value: {
|
||||
"test_improve": 1.0,
|
||||
"docs_update": 1.2, # Calm periods good for docs
|
||||
"issue_count": 1.0,
|
||||
"issue_reduce": 1.0,
|
||||
"daily_run": 1.0,
|
||||
"custom": 1.0,
|
||||
"exploration": 1.3, # Encourage exploration
|
||||
"refactor": 1.2, # Encourage refactoring
|
||||
},
|
||||
StressMode.ELEVATED.value: {
|
||||
"test_improve": 1.2, # Start emphasizing tests
|
||||
"docs_update": 1.0,
|
||||
"issue_count": 1.1,
|
||||
"issue_reduce": 1.1,
|
||||
"daily_run": 1.0,
|
||||
"custom": 1.0,
|
||||
"exploration": 1.0,
|
||||
"refactor": 0.9, # Discourage risky refactors
|
||||
},
|
||||
StressMode.HIGH.value: {
|
||||
"test_improve": 1.5, # Strongly incentivize testing
|
||||
"docs_update": 0.8, # Deprioritize docs
|
||||
"issue_count": 1.3, # Reward closing issues
|
||||
"issue_reduce": 1.4, # Strongly reward reducing backlog
|
||||
"daily_run": 1.1,
|
||||
"custom": 1.0,
|
||||
"exploration": 0.7, # Discourage exploration
|
||||
"refactor": 0.6, # Discourage refactors during crisis
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _get_config_value(key_path: str, default: Any = None) -> Any:
|
||||
"""Get a value from config using dot notation path."""
|
||||
config = _load_stress_config()
|
||||
keys = key_path.split(".")
|
||||
value = config
|
||||
for key in keys:
|
||||
if isinstance(value, dict):
|
||||
value = value.get(key)
|
||||
else:
|
||||
return default
|
||||
return value if value is not None else default
|
||||
|
||||
|
||||
def _calculate_flaky_test_rate() -> float:
|
||||
"""Calculate current flaky test rate from available data."""
|
||||
try:
|
||||
# Try to load from daily run metrics or test results
|
||||
test_results_path = Path(settings.repo_root) / ".loop" / "test_results.jsonl"
|
||||
if not test_results_path.exists():
|
||||
return 0.0
|
||||
|
||||
# Count recent test runs and flaky results
|
||||
now = datetime.now(UTC)
|
||||
cutoff = now - timedelta(days=7)
|
||||
|
||||
total_runs = 0
|
||||
flaky_runs = 0
|
||||
|
||||
if test_results_path.exists():
|
||||
for line in test_results_path.read_text().strip().splitlines():
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
ts_str = entry.get("timestamp", "")
|
||||
if not ts_str:
|
||||
continue
|
||||
ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
|
||||
if ts >= cutoff:
|
||||
total_runs += 1
|
||||
if entry.get("is_flaky", False):
|
||||
flaky_runs += 1
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
|
||||
return flaky_runs / max(total_runs, 1)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to calculate flaky test rate: %s", exc)
|
||||
return 0.0
|
||||
|
||||
|
||||
def _calculate_p1_backlog_growth() -> float:
|
||||
"""Calculate P1 issue backlog growth."""
|
||||
try:
|
||||
from dashboard.routes.daily_run import GiteaClient, _load_config
|
||||
|
||||
config = _load_config()
|
||||
token = config.get("token")
|
||||
client = GiteaClient(config, token)
|
||||
|
||||
if not client.is_available():
|
||||
return 0.0
|
||||
|
||||
# Get current P1 issues
|
||||
now = datetime.now(UTC)
|
||||
cutoff_current = now - timedelta(days=7)
|
||||
cutoff_previous = now - timedelta(days=14)
|
||||
|
||||
issues = client.get_paginated("issues", {"state": "all", "labels": "P1", "limit": 100})
|
||||
|
||||
current_count = 0
|
||||
previous_count = 0
|
||||
|
||||
for issue in issues:
|
||||
created_at = issue.get("created_at", "")
|
||||
if not created_at:
|
||||
continue
|
||||
try:
|
||||
created = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
|
||||
if created >= cutoff_current:
|
||||
current_count += 1
|
||||
elif created >= cutoff_previous:
|
||||
previous_count += 1
|
||||
except (ValueError, TypeError):
|
||||
continue
|
||||
|
||||
# Return net growth (positive means growing backlog)
|
||||
return max(0, current_count - previous_count)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to calculate P1 backlog growth: %s", exc)
|
||||
return 0.0
|
||||
|
||||
|
||||
def _calculate_ci_failure_rate() -> float:
|
||||
"""Calculate CI failure rate from recent runs."""
|
||||
try:
|
||||
# Try to get CI metrics from Gitea or local files
|
||||
ci_results_path = Path(settings.repo_root) / ".loop" / "ci_results.jsonl"
|
||||
if not ci_results_path.exists():
|
||||
return 0.0
|
||||
|
||||
now = datetime.now(UTC)
|
||||
cutoff = now - timedelta(days=7)
|
||||
|
||||
total_runs = 0
|
||||
failed_runs = 0
|
||||
|
||||
for line in ci_results_path.read_text().strip().splitlines():
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
ts_str = entry.get("timestamp", "")
|
||||
if not ts_str:
|
||||
continue
|
||||
ts = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
|
||||
if ts >= cutoff:
|
||||
total_runs += 1
|
||||
if entry.get("status") != "success":
|
||||
failed_runs += 1
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
|
||||
return failed_runs / max(total_runs, 1)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to calculate CI failure rate: %s", exc)
|
||||
return 0.0
|
||||
|
||||
|
||||
def _calculate_open_bug_count() -> float:
|
||||
"""Calculate current open bug count."""
|
||||
try:
|
||||
from dashboard.routes.daily_run import GiteaClient, _load_config
|
||||
|
||||
config = _load_config()
|
||||
token = config.get("token")
|
||||
client = GiteaClient(config, token)
|
||||
|
||||
if not client.is_available():
|
||||
return 0.0
|
||||
|
||||
issues = client.get_paginated("issues", {"state": "open", "labels": "bug", "limit": 100})
|
||||
|
||||
return float(len(issues))
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to calculate open bug count: %s", exc)
|
||||
return 0.0
|
||||
|
||||
|
||||
def _collect_stress_signals() -> list[StressSignal]:
|
||||
"""Collect all stress signals from the system."""
|
||||
config = _load_stress_config()
|
||||
default_config = get_default_config()
|
||||
signals_config = config.get("signals", default_config["signals"])
|
||||
|
||||
signals = []
|
||||
|
||||
# Define signal collectors
|
||||
collectors = {
|
||||
"flaky_test_rate": _calculate_flaky_test_rate,
|
||||
"p1_backlog_growth": _calculate_p1_backlog_growth,
|
||||
"ci_failure_rate": _calculate_ci_failure_rate,
|
||||
"open_bug_count": _calculate_open_bug_count,
|
||||
}
|
||||
|
||||
for signal_name, collector in collectors.items():
|
||||
signal_cfg = signals_config.get(signal_name, {})
|
||||
default_cfg = default_config["signals"].get(signal_name, {})
|
||||
|
||||
try:
|
||||
value = collector()
|
||||
threshold = signal_cfg.get("threshold", default_cfg.get("threshold", 1.0))
|
||||
weight = signal_cfg.get("weight", default_cfg.get("weight", 0.25))
|
||||
|
||||
signals.append(
|
||||
StressSignal(
|
||||
name=signal_name,
|
||||
value=value,
|
||||
threshold=threshold,
|
||||
weight=weight,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to collect signal %s: %s", signal_name, exc)
|
||||
|
||||
return signals
|
||||
|
||||
|
||||
def _calculate_stress_score(signals: list[StressSignal]) -> float:
|
||||
"""Calculate overall stress score from signals.
|
||||
|
||||
Score is weighted sum of triggered signal contributions,
|
||||
normalized to 0-1 range.
|
||||
"""
|
||||
if not signals:
|
||||
return 0.0
|
||||
|
||||
total_weight = sum(s.weight for s in signals)
|
||||
if total_weight == 0:
|
||||
return 0.0
|
||||
|
||||
triggered_contribution = sum(s.contribution for s in signals)
|
||||
return min(1.0, triggered_contribution / total_weight)
|
||||
|
||||
|
||||
def _get_multipliers_for_mode(mode: StressMode) -> dict[str, float]:
|
||||
"""Get token multipliers for a specific stress mode."""
|
||||
config = _load_stress_config()
|
||||
default_config = get_default_config()
|
||||
|
||||
multipliers = config.get("multipliers", default_config["multipliers"])
|
||||
mode_multipliers = multipliers.get(mode.value, {})
|
||||
default_mode_multipliers = default_config["multipliers"].get(mode.value, {})
|
||||
|
||||
# Merge with defaults
|
||||
result = default_mode_multipliers.copy()
|
||||
result.update(mode_multipliers)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def detect_stress_mode(
|
||||
force_refresh: bool = False,
|
||||
min_check_interval_seconds: int = 60,
|
||||
) -> StressSnapshot:
|
||||
"""Detect current system stress mode.
|
||||
|
||||
Args:
|
||||
force_refresh: Force a new check even if recently checked
|
||||
min_check_interval_seconds: Minimum seconds between checks
|
||||
|
||||
Returns:
|
||||
StressSnapshot with mode, score, signals, and multipliers
|
||||
"""
|
||||
global _current_snapshot, _last_check_time
|
||||
|
||||
now = datetime.now(UTC)
|
||||
|
||||
# Return cached snapshot if recent and not forced
|
||||
if not force_refresh and _current_snapshot is not None and _last_check_time is not None:
|
||||
elapsed = (now - _last_check_time).total_seconds()
|
||||
if elapsed < min_check_interval_seconds:
|
||||
return _current_snapshot
|
||||
|
||||
# Collect signals and calculate stress
|
||||
signals = _collect_stress_signals()
|
||||
score = _calculate_stress_score(signals)
|
||||
|
||||
# Determine mode from score
|
||||
config = _load_stress_config()
|
||||
default_config = get_default_config()
|
||||
thresholds_cfg = config.get("thresholds", default_config["thresholds"])
|
||||
thresholds = StressThresholds(
|
||||
elevated_min=thresholds_cfg.get("elevated_min", 0.3),
|
||||
high_min=thresholds_cfg.get("high_min", 0.6),
|
||||
)
|
||||
mode = thresholds.get_mode_for_score(score)
|
||||
|
||||
# Get multipliers for this mode
|
||||
multipliers = _get_multipliers_for_mode(mode)
|
||||
|
||||
# Create snapshot
|
||||
snapshot = StressSnapshot(
|
||||
mode=mode,
|
||||
score=score,
|
||||
signals=signals,
|
||||
multipliers=multipliers,
|
||||
timestamp=now.isoformat(),
|
||||
)
|
||||
|
||||
# Cache result
|
||||
_current_snapshot = snapshot
|
||||
_last_check_time = now
|
||||
|
||||
# Log mode changes
|
||||
if _current_snapshot is not None and _current_snapshot.mode != mode:
|
||||
logger.info(
|
||||
"Stress mode changed: %s -> %s (score: %.2f)",
|
||||
_current_snapshot.mode.value if _current_snapshot else "none",
|
||||
mode.value,
|
||||
score,
|
||||
)
|
||||
|
||||
return snapshot
|
||||
|
||||
|
||||
def get_current_stress_mode() -> StressMode:
|
||||
"""Get current stress mode (uses cached or fresh detection)."""
|
||||
snapshot = detect_stress_mode()
|
||||
return snapshot.mode
|
||||
|
||||
|
||||
def get_multiplier(quest_type: str, mode: StressMode | None = None) -> float:
|
||||
"""Get token multiplier for a quest type.
|
||||
|
||||
Args:
|
||||
quest_type: Type of quest (test_improve, issue_count, etc.)
|
||||
mode: Specific mode to get multiplier for, or None for current
|
||||
|
||||
Returns:
|
||||
Multiplier value (1.0 = normal, 1.5 = 50% bonus, etc.)
|
||||
"""
|
||||
if mode is None:
|
||||
mode = get_current_stress_mode()
|
||||
|
||||
multipliers = _get_multipliers_for_mode(mode)
|
||||
return multipliers.get(quest_type, 1.0)
|
||||
|
||||
|
||||
def apply_multiplier(base_reward: int, quest_type: str) -> int:
|
||||
"""Apply stress-based multiplier to a base reward.
|
||||
|
||||
Args:
|
||||
base_reward: Base token reward amount
|
||||
quest_type: Type of quest for multiplier lookup
|
||||
|
||||
Returns:
|
||||
Adjusted reward amount (always >= 1)
|
||||
"""
|
||||
multiplier = get_multiplier(quest_type)
|
||||
adjusted = int(base_reward * multiplier)
|
||||
return max(1, adjusted)
|
||||
|
||||
|
||||
def get_stress_summary() -> dict[str, Any]:
|
||||
"""Get a human-readable summary of current stress state."""
|
||||
snapshot = detect_stress_mode()
|
||||
|
||||
# Generate explanation
|
||||
explanations = {
|
||||
StressMode.CALM: "System is calm. Good time for exploration and refactoring.",
|
||||
StressMode.ELEVATED: "Elevated stress detected. Focus on stability and tests.",
|
||||
StressMode.HIGH: "HIGH STRESS MODE. Prioritize bug fixes and test hardening.",
|
||||
}
|
||||
|
||||
triggered_signals = [s for s in snapshot.signals if s.is_triggered]
|
||||
|
||||
return {
|
||||
"mode": snapshot.mode.value,
|
||||
"score": round(snapshot.score, 3),
|
||||
"explanation": explanations.get(snapshot.mode, "Unknown mode"),
|
||||
"active_signals": [
|
||||
{
|
||||
"name": s.name,
|
||||
"value": round(s.value, 3),
|
||||
"threshold": s.threshold,
|
||||
}
|
||||
for s in triggered_signals
|
||||
],
|
||||
"current_multipliers": snapshot.multipliers,
|
||||
"last_updated": snapshot.timestamp,
|
||||
}
|
||||
|
||||
|
||||
def reset_stress_state() -> None:
|
||||
"""Reset stress state cache (useful for testing)."""
|
||||
global _current_snapshot, _last_check_time, _config_cache, _config_mtime
|
||||
_current_snapshot = None
|
||||
_last_check_time = None
|
||||
_config_cache = None
|
||||
_config_mtime = 0.0
|
||||
@@ -1,680 +0,0 @@
|
||||
"""Tests for agent scorecard functionality."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from dashboard.services.scorecard_service import (
|
||||
AgentMetrics,
|
||||
PeriodType,
|
||||
ScorecardSummary,
|
||||
_aggregate_metrics,
|
||||
_detect_patterns,
|
||||
_extract_actor_from_event,
|
||||
_generate_narrative_bullets,
|
||||
_get_period_bounds,
|
||||
_is_tracked_agent,
|
||||
_query_token_transactions,
|
||||
generate_all_scorecards,
|
||||
generate_scorecard,
|
||||
get_tracked_agents,
|
||||
)
|
||||
from infrastructure.events.bus import Event
|
||||
|
||||
|
||||
class TestPeriodBounds:
|
||||
"""Test period boundary calculations."""
|
||||
|
||||
def test_daily_period_bounds(self):
|
||||
"""Test daily period returns correct 24-hour window."""
|
||||
reference = datetime(2026, 3, 21, 12, 30, 45, tzinfo=UTC)
|
||||
start, end = _get_period_bounds(PeriodType.daily, reference)
|
||||
|
||||
assert end == datetime(2026, 3, 21, 0, 0, 0, tzinfo=UTC)
|
||||
assert start == datetime(2026, 3, 20, 0, 0, 0, tzinfo=UTC)
|
||||
assert (end - start) == timedelta(days=1)
|
||||
|
||||
def test_weekly_period_bounds(self):
|
||||
"""Test weekly period returns correct 7-day window."""
|
||||
reference = datetime(2026, 3, 21, 12, 30, 45, tzinfo=UTC)
|
||||
start, end = _get_period_bounds(PeriodType.weekly, reference)
|
||||
|
||||
assert end == datetime(2026, 3, 21, 0, 0, 0, tzinfo=UTC)
|
||||
assert start == datetime(2026, 3, 14, 0, 0, 0, tzinfo=UTC)
|
||||
assert (end - start) == timedelta(days=7)
|
||||
|
||||
def test_default_reference_date(self):
|
||||
"""Test default reference date uses current time."""
|
||||
start, end = _get_period_bounds(PeriodType.daily)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
# End should be start of current day (midnight)
|
||||
expected_end = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
assert end == expected_end
|
||||
# Start should be 24 hours before end
|
||||
assert (end - start) == timedelta(days=1)
|
||||
|
||||
|
||||
class TestTrackedAgents:
|
||||
"""Test agent tracking functions."""
|
||||
|
||||
def test_get_tracked_agents(self):
|
||||
"""Test get_tracked_agents returns sorted list."""
|
||||
agents = get_tracked_agents()
|
||||
assert isinstance(agents, list)
|
||||
assert "kimi" in agents
|
||||
assert "claude" in agents
|
||||
assert "gemini" in agents
|
||||
assert "hermes" in agents
|
||||
assert "manus" in agents
|
||||
assert agents == sorted(agents)
|
||||
|
||||
def test_is_tracked_agent_true(self):
|
||||
"""Test _is_tracked_agent returns True for tracked agents."""
|
||||
assert _is_tracked_agent("kimi") is True
|
||||
assert _is_tracked_agent("KIMI") is True # case insensitive
|
||||
assert _is_tracked_agent("claude") is True
|
||||
assert _is_tracked_agent("hermes") is True
|
||||
|
||||
def test_is_tracked_agent_false(self):
|
||||
"""Test _is_tracked_agent returns False for untracked agents."""
|
||||
assert _is_tracked_agent("unknown") is False
|
||||
assert _is_tracked_agent("rockachopa") is False
|
||||
assert _is_tracked_agent("") is False
|
||||
|
||||
|
||||
class TestExtractActor:
|
||||
"""Test actor extraction from events."""
|
||||
|
||||
def test_extract_from_actor_field(self):
|
||||
"""Test extraction from data.actor field."""
|
||||
event = Event(type="test", source="system", data={"actor": "kimi"})
|
||||
assert _extract_actor_from_event(event) == "kimi"
|
||||
|
||||
def test_extract_from_agent_id_field(self):
|
||||
"""Test extraction from data.agent_id field."""
|
||||
event = Event(type="test", source="system", data={"agent_id": "claude"})
|
||||
assert _extract_actor_from_event(event) == "claude"
|
||||
|
||||
def test_extract_from_source_fallback(self):
|
||||
"""Test fallback to event.source."""
|
||||
event = Event(type="test", source="gemini", data={})
|
||||
assert _extract_actor_from_event(event) == "gemini"
|
||||
|
||||
def test_actor_priority_over_agent_id(self):
|
||||
"""Test actor field takes priority over agent_id."""
|
||||
event = Event(type="test", source="system", data={"actor": "kimi", "agent_id": "claude"})
|
||||
assert _extract_actor_from_event(event) == "kimi"
|
||||
|
||||
|
||||
class TestAggregateMetrics:
|
||||
"""Test metrics aggregation from events."""
|
||||
|
||||
def test_empty_events(self):
|
||||
"""Test aggregation with no events returns empty dict."""
|
||||
result = _aggregate_metrics([])
|
||||
assert result == {}
|
||||
|
||||
def test_push_event_aggregation(self):
|
||||
"""Test push events aggregate commits correctly."""
|
||||
events = [
|
||||
Event(type="gitea.push", source="gitea", data={"actor": "kimi", "num_commits": 3}),
|
||||
Event(type="gitea.push", source="gitea", data={"actor": "kimi", "num_commits": 2}),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "kimi" in result
|
||||
assert result["kimi"].commits == 5
|
||||
|
||||
def test_issue_opened_aggregation(self):
|
||||
"""Test issue opened events aggregate correctly."""
|
||||
events = [
|
||||
Event(
|
||||
type="gitea.issue.opened",
|
||||
source="gitea",
|
||||
data={"actor": "claude", "issue_number": 100},
|
||||
),
|
||||
Event(
|
||||
type="gitea.issue.opened",
|
||||
source="gitea",
|
||||
data={"actor": "claude", "issue_number": 101},
|
||||
),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "claude" in result
|
||||
assert len(result["claude"].issues_touched) == 2
|
||||
assert 100 in result["claude"].issues_touched
|
||||
assert 101 in result["claude"].issues_touched
|
||||
|
||||
def test_comment_aggregation(self):
|
||||
"""Test comment events aggregate correctly."""
|
||||
events = [
|
||||
Event(
|
||||
type="gitea.issue.comment",
|
||||
source="gitea",
|
||||
data={"actor": "gemini", "issue_number": 100},
|
||||
),
|
||||
Event(
|
||||
type="gitea.issue.comment",
|
||||
source="gitea",
|
||||
data={"actor": "gemini", "issue_number": 101},
|
||||
),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "gemini" in result
|
||||
assert result["gemini"].comments == 2
|
||||
assert len(result["gemini"].issues_touched) == 2 # Comments touch issues too
|
||||
|
||||
def test_pr_events_aggregation(self):
|
||||
"""Test PR open and merge events aggregate correctly."""
|
||||
events = [
|
||||
Event(
|
||||
type="gitea.pull_request",
|
||||
source="gitea",
|
||||
data={"actor": "kimi", "pr_number": 50, "action": "opened"},
|
||||
),
|
||||
Event(
|
||||
type="gitea.pull_request",
|
||||
source="gitea",
|
||||
data={"actor": "kimi", "pr_number": 50, "action": "closed", "merged": True},
|
||||
),
|
||||
Event(
|
||||
type="gitea.pull_request",
|
||||
source="gitea",
|
||||
data={"actor": "kimi", "pr_number": 51, "action": "opened"},
|
||||
),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "kimi" in result
|
||||
assert len(result["kimi"].prs_opened) == 2
|
||||
assert len(result["kimi"].prs_merged) == 1
|
||||
assert 50 in result["kimi"].prs_merged
|
||||
|
||||
def test_untracked_agent_filtered(self):
|
||||
"""Test events from untracked agents are filtered out."""
|
||||
events = [
|
||||
Event(
|
||||
type="gitea.push", source="gitea", data={"actor": "rockachopa", "num_commits": 5}
|
||||
),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "rockachopa" not in result
|
||||
|
||||
def test_task_completion_aggregation(self):
|
||||
"""Test task completion events aggregate test files."""
|
||||
events = [
|
||||
Event(
|
||||
type="agent.task.completed",
|
||||
source="gitea",
|
||||
data={
|
||||
"agent_id": "kimi",
|
||||
"tests_affected": ["test_foo.py", "test_bar.py"],
|
||||
"token_reward": 10,
|
||||
},
|
||||
),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "kimi" in result
|
||||
assert len(result["kimi"].tests_affected) == 2
|
||||
assert "test_foo.py" in result["kimi"].tests_affected
|
||||
assert result["kimi"].tokens_earned == 10
|
||||
|
||||
|
||||
class TestAgentMetrics:
|
||||
"""Test AgentMetrics class."""
|
||||
|
||||
def test_merge_rate_zero_prs(self):
|
||||
"""Test merge rate is 0 when no PRs opened."""
|
||||
metrics = AgentMetrics(agent_id="kimi")
|
||||
assert metrics.pr_merge_rate == 0.0
|
||||
|
||||
def test_merge_rate_perfect(self):
|
||||
"""Test 100% merge rate calculation."""
|
||||
metrics = AgentMetrics(agent_id="kimi", prs_opened={1, 2, 3}, prs_merged={1, 2, 3})
|
||||
assert metrics.pr_merge_rate == 1.0
|
||||
|
||||
def test_merge_rate_partial(self):
|
||||
"""Test partial merge rate calculation."""
|
||||
metrics = AgentMetrics(agent_id="kimi", prs_opened={1, 2, 3, 4}, prs_merged={1, 2})
|
||||
assert metrics.pr_merge_rate == 0.5
|
||||
|
||||
|
||||
class TestDetectPatterns:
|
||||
"""Test pattern detection logic."""
|
||||
|
||||
def test_high_merge_rate_pattern(self):
|
||||
"""Test detection of high merge rate pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
prs_opened={1, 2, 3, 4, 5},
|
||||
prs_merged={1, 2, 3, 4}, # 80% merge rate
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("High merge rate" in p for p in patterns)
|
||||
|
||||
def test_low_merge_rate_pattern(self):
|
||||
"""Test detection of low merge rate pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
prs_opened={1, 2, 3, 4, 5},
|
||||
prs_merged={1}, # 20% merge rate
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("low merge rate" in p for p in patterns)
|
||||
|
||||
def test_high_commits_no_prs_pattern(self):
|
||||
"""Test detection of direct-to-main commits pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
commits=15,
|
||||
prs_opened=set(),
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("High commit volume without PRs" in p for p in patterns)
|
||||
|
||||
def test_silent_worker_pattern(self):
|
||||
"""Test detection of silent worker pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
issues_touched={1, 2, 3, 4, 5, 6},
|
||||
comments=0,
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("silent worker" in p for p in patterns)
|
||||
|
||||
def test_communicative_pattern(self):
|
||||
"""Test detection of highly communicative pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
issues_touched={1, 2}, # 2 issues
|
||||
comments=10, # 5x comments per issue
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("Highly communicative" in p for p in patterns)
|
||||
|
||||
def test_token_accumulation_pattern(self):
|
||||
"""Test detection of token accumulation pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tokens_earned=150,
|
||||
tokens_spent=10,
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("Strong token accumulation" in p for p in patterns)
|
||||
|
||||
def test_token_spend_pattern(self):
|
||||
"""Test detection of high token spend pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tokens_earned=10,
|
||||
tokens_spent=100,
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("High token spend" in p for p in patterns)
|
||||
|
||||
|
||||
class TestGenerateNarrative:
|
||||
"""Test narrative bullet generation."""
|
||||
|
||||
def test_empty_metrics_narrative(self):
|
||||
"""Test narrative for empty metrics mentions no activity."""
|
||||
metrics = AgentMetrics(agent_id="kimi")
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
assert len(bullets) == 1
|
||||
assert "No recorded activity" in bullets[0]
|
||||
|
||||
def test_activity_summary_narrative(self):
|
||||
"""Test narrative includes activity summary."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
commits=5,
|
||||
prs_opened={1, 2},
|
||||
prs_merged={1},
|
||||
)
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
activity_bullet = next((b for b in bullets if "Active across" in b), None)
|
||||
assert activity_bullet is not None
|
||||
assert "5 commits" in activity_bullet
|
||||
assert "2 PRs opened" in activity_bullet
|
||||
assert "1 PR merged" in activity_bullet
|
||||
|
||||
def test_tests_affected_narrative(self):
|
||||
"""Test narrative includes tests affected."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tests_affected={"test_a.py", "test_b.py"},
|
||||
)
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
assert any("2 test files" in b for b in bullets)
|
||||
|
||||
def test_tokens_earned_narrative(self):
|
||||
"""Test narrative includes token earnings."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tokens_earned=100,
|
||||
tokens_spent=20,
|
||||
)
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
assert any("Net earned 80 tokens" in b for b in bullets)
|
||||
|
||||
def test_tokens_spent_narrative(self):
|
||||
"""Test narrative includes token spending."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tokens_earned=20,
|
||||
tokens_spent=100,
|
||||
)
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
assert any("Net spent 80 tokens" in b for b in bullets)
|
||||
|
||||
def test_balanced_tokens_narrative(self):
|
||||
"""Test narrative for balanced token flow."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tokens_earned=100,
|
||||
tokens_spent=100,
|
||||
)
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
assert any("Balanced token flow" in b for b in bullets)
|
||||
|
||||
|
||||
class TestScorecardSummary:
|
||||
"""Test ScorecardSummary dataclass."""
|
||||
|
||||
def test_to_dict_structure(self):
|
||||
"""Test to_dict returns expected structure."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
issues_touched={1, 2},
|
||||
prs_opened={10, 11},
|
||||
prs_merged={10},
|
||||
tokens_earned=100,
|
||||
tokens_spent=20,
|
||||
)
|
||||
summary = ScorecardSummary(
|
||||
agent_id="kimi",
|
||||
period_type=PeriodType.daily,
|
||||
period_start=datetime.now(UTC),
|
||||
period_end=datetime.now(UTC),
|
||||
metrics=metrics,
|
||||
narrative_bullets=["Test bullet"],
|
||||
patterns=["Test pattern"],
|
||||
)
|
||||
data = summary.to_dict()
|
||||
|
||||
assert data["agent_id"] == "kimi"
|
||||
assert data["period_type"] == "daily"
|
||||
assert "metrics" in data
|
||||
assert data["metrics"]["issues_touched"] == 2
|
||||
assert data["metrics"]["prs_opened"] == 2
|
||||
assert data["metrics"]["prs_merged"] == 1
|
||||
assert data["metrics"]["pr_merge_rate"] == 0.5
|
||||
assert data["metrics"]["tokens_earned"] == 100
|
||||
assert data["metrics"]["token_net"] == 80
|
||||
assert data["narrative_bullets"] == ["Test bullet"]
|
||||
assert data["patterns"] == ["Test pattern"]
|
||||
|
||||
|
||||
class TestQueryTokenTransactions:
|
||||
"""Test token transaction querying."""
|
||||
|
||||
def test_empty_ledger(self):
|
||||
"""Test empty ledger returns zero values."""
|
||||
with patch("lightning.ledger.get_transactions", return_value=[]):
|
||||
earned, spent = _query_token_transactions("kimi", datetime.now(UTC), datetime.now(UTC))
|
||||
assert earned == 0
|
||||
assert spent == 0
|
||||
|
||||
def test_ledger_with_transactions(self):
|
||||
"""Test ledger aggregation of transactions."""
|
||||
now = datetime.now(UTC)
|
||||
mock_tx = [
|
||||
MagicMock(
|
||||
agent_id="kimi",
|
||||
tx_type=MagicMock(value="incoming"),
|
||||
amount_sats=100,
|
||||
created_at=now.isoformat(),
|
||||
),
|
||||
MagicMock(
|
||||
agent_id="kimi",
|
||||
tx_type=MagicMock(value="outgoing"),
|
||||
amount_sats=30,
|
||||
created_at=now.isoformat(),
|
||||
),
|
||||
]
|
||||
with patch("lightning.ledger.get_transactions", return_value=mock_tx):
|
||||
earned, spent = _query_token_transactions(
|
||||
"kimi", now - timedelta(hours=1), now + timedelta(hours=1)
|
||||
)
|
||||
assert earned == 100
|
||||
assert spent == 30
|
||||
|
||||
def test_ledger_filters_by_agent(self):
|
||||
"""Test ledger filters transactions by agent_id."""
|
||||
now = datetime.now(UTC)
|
||||
mock_tx = [
|
||||
MagicMock(
|
||||
agent_id="claude",
|
||||
tx_type=MagicMock(value="incoming"),
|
||||
amount_sats=100,
|
||||
created_at=now.isoformat(),
|
||||
),
|
||||
]
|
||||
with patch("lightning.ledger.get_transactions", return_value=mock_tx):
|
||||
earned, spent = _query_token_transactions(
|
||||
"kimi", now - timedelta(hours=1), now + timedelta(hours=1)
|
||||
)
|
||||
assert earned == 0 # Transaction was for claude, not kimi
|
||||
|
||||
def test_ledger_filters_by_time(self):
|
||||
"""Test ledger filters transactions by time range."""
|
||||
now = datetime.now(UTC)
|
||||
old_time = now - timedelta(days=2)
|
||||
mock_tx = [
|
||||
MagicMock(
|
||||
agent_id="kimi",
|
||||
tx_type=MagicMock(value="incoming"),
|
||||
amount_sats=100,
|
||||
created_at=old_time.isoformat(),
|
||||
),
|
||||
]
|
||||
with patch("lightning.ledger.get_transactions", return_value=mock_tx):
|
||||
# Query for today only
|
||||
earned, spent = _query_token_transactions(
|
||||
"kimi", now - timedelta(hours=1), now + timedelta(hours=1)
|
||||
)
|
||||
assert earned == 0 # Transaction was 2 days ago
|
||||
|
||||
|
||||
class TestGenerateScorecard:
|
||||
"""Test scorecard generation."""
|
||||
|
||||
def test_generate_scorecard_no_activity(self):
|
||||
"""Test scorecard generation for agent with no activity."""
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._collect_events_for_period", return_value=[]
|
||||
):
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._query_token_transactions",
|
||||
return_value=(0, 0),
|
||||
):
|
||||
scorecard = generate_scorecard("kimi", PeriodType.daily)
|
||||
|
||||
assert scorecard is not None
|
||||
assert scorecard.agent_id == "kimi"
|
||||
assert scorecard.period_type == PeriodType.daily
|
||||
assert len(scorecard.narrative_bullets) == 1
|
||||
assert "No recorded activity" in scorecard.narrative_bullets[0]
|
||||
|
||||
def test_generate_scorecard_with_activity(self):
|
||||
"""Test scorecard generation includes activity."""
|
||||
events = [
|
||||
Event(type="gitea.push", source="gitea", data={"actor": "kimi", "num_commits": 5}),
|
||||
]
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._collect_events_for_period", return_value=events
|
||||
):
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._query_token_transactions",
|
||||
return_value=(100, 20),
|
||||
):
|
||||
scorecard = generate_scorecard("kimi", PeriodType.daily)
|
||||
|
||||
assert scorecard is not None
|
||||
assert scorecard.metrics.commits == 5
|
||||
assert scorecard.metrics.tokens_earned == 100
|
||||
assert scorecard.metrics.tokens_spent == 20
|
||||
|
||||
|
||||
class TestGenerateAllScorecards:
|
||||
"""Test generating scorecards for all agents."""
|
||||
|
||||
def test_generates_for_all_tracked_agents(self):
|
||||
"""Test all tracked agents get scorecards even with no activity."""
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._collect_events_for_period", return_value=[]
|
||||
):
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._query_token_transactions",
|
||||
return_value=(0, 0),
|
||||
):
|
||||
scorecards = generate_all_scorecards(PeriodType.daily)
|
||||
|
||||
agent_ids = {s.agent_id for s in scorecards}
|
||||
expected = {"kimi", "claude", "gemini", "hermes", "manus"}
|
||||
assert expected.issubset(agent_ids)
|
||||
|
||||
def test_scorecards_sorted(self):
|
||||
"""Test scorecards are sorted by agent_id."""
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._collect_events_for_period", return_value=[]
|
||||
):
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._query_token_transactions",
|
||||
return_value=(0, 0),
|
||||
):
|
||||
scorecards = generate_all_scorecards(PeriodType.daily)
|
||||
|
||||
agent_ids = [s.agent_id for s in scorecards]
|
||||
assert agent_ids == sorted(agent_ids)
|
||||
|
||||
|
||||
class TestScorecardRoutes:
|
||||
"""Test scorecard API routes."""
|
||||
|
||||
def test_list_agents_endpoint(self, client):
|
||||
"""Test GET /scorecards/api/agents returns tracked agents."""
|
||||
response = client.get("/scorecards/api/agents")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "agents" in data
|
||||
assert "kimi" in data["agents"]
|
||||
assert "claude" in data["agents"]
|
||||
|
||||
def test_get_scorecard_endpoint(self, client):
|
||||
"""Test GET /scorecards/api/{agent_id} returns scorecard."""
|
||||
with patch("dashboard.routes.scorecards.generate_scorecard") as mock_generate:
|
||||
mock_generate.return_value = ScorecardSummary(
|
||||
agent_id="kimi",
|
||||
period_type=PeriodType.daily,
|
||||
period_start=datetime.now(UTC),
|
||||
period_end=datetime.now(UTC),
|
||||
metrics=AgentMetrics(agent_id="kimi"),
|
||||
narrative_bullets=["Test bullet"],
|
||||
patterns=[],
|
||||
)
|
||||
response = client.get("/scorecards/api/kimi?period=daily")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["agent_id"] == "kimi"
|
||||
assert data["period_type"] == "daily"
|
||||
|
||||
def test_get_scorecard_invalid_period(self, client):
|
||||
"""Test GET with invalid period returns 400."""
|
||||
response = client.get("/scorecards/api/kimi?period=invalid")
|
||||
assert response.status_code == 400
|
||||
assert "error" in response.json()
|
||||
|
||||
def test_get_all_scorecards_endpoint(self, client):
|
||||
"""Test GET /scorecards/api returns all scorecards."""
|
||||
with patch("dashboard.routes.scorecards.generate_all_scorecards") as mock_generate:
|
||||
mock_generate.return_value = [
|
||||
ScorecardSummary(
|
||||
agent_id="kimi",
|
||||
period_type=PeriodType.daily,
|
||||
period_start=datetime.now(UTC),
|
||||
period_end=datetime.now(UTC),
|
||||
metrics=AgentMetrics(agent_id="kimi"),
|
||||
narrative_bullets=[],
|
||||
patterns=[],
|
||||
),
|
||||
]
|
||||
response = client.get("/scorecards/api?period=daily")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["period"] == "daily"
|
||||
assert "scorecards" in data
|
||||
assert len(data["scorecards"]) == 1
|
||||
|
||||
def test_scorecards_page_renders(self, client):
|
||||
"""Test GET /scorecards returns HTML page."""
|
||||
response = client.get("/scorecards")
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers.get("content-type", "")
|
||||
assert "AGENT SCORECARDS" in response.text
|
||||
|
||||
def test_scorecard_panel_renders(self, client):
|
||||
"""Test GET /scorecards/panel/{agent_id} returns HTML."""
|
||||
with patch("dashboard.routes.scorecards.generate_scorecard") as mock_generate:
|
||||
mock_generate.return_value = ScorecardSummary(
|
||||
agent_id="kimi",
|
||||
period_type=PeriodType.daily,
|
||||
period_start=datetime.now(UTC),
|
||||
period_end=datetime.now(UTC),
|
||||
metrics=AgentMetrics(agent_id="kimi", commits=5),
|
||||
narrative_bullets=["Active across 5 commits this day."],
|
||||
patterns=["High activity"],
|
||||
)
|
||||
response = client.get("/scorecards/panel/kimi?period=daily")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers.get("content-type", "")
|
||||
assert "Kimi" in response.text
|
||||
|
||||
def test_all_panels_renders(self, client):
|
||||
"""Test GET /scorecards/all/panels returns HTML with all panels."""
|
||||
with patch("dashboard.routes.scorecards.generate_all_scorecards") as mock_generate:
|
||||
mock_generate.return_value = [
|
||||
ScorecardSummary(
|
||||
agent_id="kimi",
|
||||
period_type=PeriodType.daily,
|
||||
period_start=datetime.now(UTC),
|
||||
period_end=datetime.now(UTC),
|
||||
metrics=AgentMetrics(agent_id="kimi"),
|
||||
narrative_bullets=[],
|
||||
patterns=[],
|
||||
),
|
||||
]
|
||||
response = client.get("/scorecards/all/panels?period=daily")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers.get("content-type", "")
|
||||
294
tests/unit/test_stress_detector.py
Normal file
294
tests/unit/test_stress_detector.py
Normal file
@@ -0,0 +1,294 @@
|
||||
"""Unit tests for the stress detector module.
|
||||
|
||||
Tests stress signal calculation, mode detection, multipliers,
|
||||
and integration with the quest system.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from timmy.stress_detector import (
|
||||
StressMode,
|
||||
StressSignal,
|
||||
StressSnapshot,
|
||||
StressThresholds,
|
||||
_calculate_stress_score,
|
||||
_get_multipliers_for_mode,
|
||||
apply_multiplier,
|
||||
get_default_config,
|
||||
reset_stress_state,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_stress_state():
|
||||
"""Reset stress state between tests."""
|
||||
reset_stress_state()
|
||||
yield
|
||||
reset_stress_state()
|
||||
|
||||
|
||||
# ── Stress Mode Tests ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStressMode:
|
||||
def test_stress_mode_values(self):
|
||||
"""StressMode enum has expected values."""
|
||||
assert StressMode.CALM.value == "calm"
|
||||
assert StressMode.ELEVATED.value == "elevated"
|
||||
assert StressMode.HIGH.value == "high"
|
||||
|
||||
|
||||
# ── Stress Signal Tests ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStressSignal:
|
||||
def test_signal_not_triggered(self):
|
||||
"""Signal with value below threshold is not triggered."""
|
||||
signal = StressSignal(
|
||||
name="test_signal",
|
||||
value=5.0,
|
||||
threshold=10.0,
|
||||
weight=0.5,
|
||||
)
|
||||
assert not signal.is_triggered
|
||||
assert signal.contribution == 0.0
|
||||
|
||||
def test_signal_triggered(self):
|
||||
"""Signal with value at threshold is triggered."""
|
||||
signal = StressSignal(
|
||||
name="test_signal",
|
||||
value=10.0,
|
||||
threshold=10.0,
|
||||
weight=0.5,
|
||||
)
|
||||
assert signal.is_triggered
|
||||
assert signal.contribution == 0.5 # weight * min(1, value/threshold)
|
||||
|
||||
def test_signal_contribution_capped(self):
|
||||
"""Signal contribution is capped at weight when value >> threshold."""
|
||||
signal = StressSignal(
|
||||
name="test_signal",
|
||||
value=100.0,
|
||||
threshold=10.0,
|
||||
weight=0.5,
|
||||
)
|
||||
assert signal.is_triggered
|
||||
assert signal.contribution == 0.5 # Capped at weight
|
||||
|
||||
def test_signal_partial_contribution(self):
|
||||
"""Signal contribution scales with value/threshold ratio."""
|
||||
signal = StressSignal(
|
||||
name="test_signal",
|
||||
value=15.0,
|
||||
threshold=10.0,
|
||||
weight=0.5,
|
||||
)
|
||||
assert signal.is_triggered
|
||||
# contribution = min(1, 15/10) * 0.5 = 0.5 (capped)
|
||||
assert signal.contribution == 0.5
|
||||
|
||||
|
||||
# ── Stress Thresholds Tests ────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStressThresholds:
|
||||
def test_calm_mode(self):
|
||||
"""Score below elevated_min returns CALM mode."""
|
||||
thresholds = StressThresholds(elevated_min=0.3, high_min=0.6)
|
||||
assert thresholds.get_mode_for_score(0.0) == StressMode.CALM
|
||||
assert thresholds.get_mode_for_score(0.1) == StressMode.CALM
|
||||
assert thresholds.get_mode_for_score(0.29) == StressMode.CALM
|
||||
|
||||
def test_elevated_mode(self):
|
||||
"""Score between elevated_min and high_min returns ELEVATED mode."""
|
||||
thresholds = StressThresholds(elevated_min=0.3, high_min=0.6)
|
||||
assert thresholds.get_mode_for_score(0.3) == StressMode.ELEVATED
|
||||
assert thresholds.get_mode_for_score(0.5) == StressMode.ELEVATED
|
||||
assert thresholds.get_mode_for_score(0.59) == StressMode.ELEVATED
|
||||
|
||||
def test_high_mode(self):
|
||||
"""Score at or above high_min returns HIGH mode."""
|
||||
thresholds = StressThresholds(elevated_min=0.3, high_min=0.6)
|
||||
assert thresholds.get_mode_for_score(0.6) == StressMode.HIGH
|
||||
assert thresholds.get_mode_for_score(0.8) == StressMode.HIGH
|
||||
assert thresholds.get_mode_for_score(1.0) == StressMode.HIGH
|
||||
|
||||
|
||||
# ── Stress Score Calculation Tests ─────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStressScoreCalculation:
|
||||
def test_empty_signals(self):
|
||||
"""Empty signal list returns zero stress score."""
|
||||
score = _calculate_stress_score([])
|
||||
assert score == 0.0
|
||||
|
||||
def test_no_triggered_signals(self):
|
||||
"""No triggered signals means zero stress score."""
|
||||
signals = [
|
||||
StressSignal(name="s1", value=1.0, threshold=10.0, weight=0.5),
|
||||
StressSignal(name="s2", value=2.0, threshold=10.0, weight=0.5),
|
||||
]
|
||||
score = _calculate_stress_score(signals)
|
||||
assert score == 0.0
|
||||
|
||||
def test_single_triggered_signal(self):
|
||||
"""Single triggered signal contributes its weight."""
|
||||
signals = [
|
||||
StressSignal(name="s1", value=10.0, threshold=10.0, weight=0.5),
|
||||
]
|
||||
score = _calculate_stress_score(signals)
|
||||
# contribution = 0.5, total_weight = 0.5, score = 0.5/0.5 = 1.0
|
||||
assert score == 1.0
|
||||
|
||||
def test_mixed_signals(self):
|
||||
"""Mix of triggered and non-triggered signals."""
|
||||
signals = [
|
||||
StressSignal(name="s1", value=10.0, threshold=10.0, weight=0.3),
|
||||
StressSignal(name="s2", value=1.0, threshold=10.0, weight=0.3),
|
||||
StressSignal(name="s3", value=10.0, threshold=10.0, weight=0.4),
|
||||
]
|
||||
score = _calculate_stress_score(signals)
|
||||
# triggered contributions: 0.3 + 0.4 = 0.7
|
||||
# total_weight: 0.3 + 0.3 + 0.4 = 1.0
|
||||
# score = 0.7 / 1.0 = 0.7
|
||||
assert score == 0.7
|
||||
|
||||
def test_score_capped_at_one(self):
|
||||
"""Stress score is capped at 1.0."""
|
||||
signals = [
|
||||
StressSignal(name="s1", value=100.0, threshold=10.0, weight=1.0),
|
||||
StressSignal(name="s2", value=100.0, threshold=10.0, weight=1.0),
|
||||
]
|
||||
score = _calculate_stress_score(signals)
|
||||
assert score == 1.0 # Capped
|
||||
|
||||
|
||||
# ── Multiplier Tests ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestMultipliers:
|
||||
def test_default_config_structure(self):
|
||||
"""Default config has expected structure."""
|
||||
config = get_default_config()
|
||||
assert "thresholds" in config
|
||||
assert "signals" in config
|
||||
assert "multipliers" in config
|
||||
|
||||
def test_calm_mode_multipliers(self):
|
||||
"""Calm mode has expected multipliers."""
|
||||
multipliers = _get_multipliers_for_mode(StressMode.CALM)
|
||||
assert multipliers["test_improve"] == 1.0
|
||||
assert multipliers["docs_update"] == 1.2
|
||||
assert multipliers["exploration"] == 1.3
|
||||
assert multipliers["refactor"] == 1.2
|
||||
|
||||
def test_elevated_mode_multipliers(self):
|
||||
"""Elevated mode has expected multipliers."""
|
||||
multipliers = _get_multipliers_for_mode(StressMode.ELEVATED)
|
||||
assert multipliers["test_improve"] == 1.2
|
||||
assert multipliers["issue_reduce"] == 1.1
|
||||
assert multipliers["refactor"] == 0.9
|
||||
|
||||
def test_high_mode_multipliers(self):
|
||||
"""High stress mode has expected multipliers."""
|
||||
multipliers = _get_multipliers_for_mode(StressMode.HIGH)
|
||||
assert multipliers["test_improve"] == 1.5
|
||||
assert multipliers["issue_reduce"] == 1.4
|
||||
assert multipliers["exploration"] == 0.7
|
||||
assert multipliers["refactor"] == 0.6
|
||||
|
||||
def test_multiplier_fallback_for_unknown_type(self):
|
||||
"""Unknown quest types return default multiplier of 1.0."""
|
||||
multipliers = _get_multipliers_for_mode(StressMode.CALM)
|
||||
assert multipliers.get("unknown_type", 1.0) == 1.0
|
||||
|
||||
|
||||
# ── Apply Multiplier Tests ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestApplyMultiplier:
|
||||
def test_apply_multiplier_calm(self):
|
||||
"""Multiplier applies correctly in calm mode."""
|
||||
# This test uses get_multiplier which reads from current stress mode
|
||||
# Since we can't easily mock the stress mode, we test the apply_multiplier logic
|
||||
base = 100
|
||||
# In calm mode with test_improve = 1.0
|
||||
result = apply_multiplier(base, "unknown_type")
|
||||
assert result >= 1 # At least 1 token
|
||||
|
||||
def test_apply_multiplier_minimum_one(self):
|
||||
"""Applied reward is at least 1 token."""
|
||||
# Even with very low multiplier, result should be >= 1
|
||||
result = apply_multiplier(1, "any_type")
|
||||
assert result >= 1
|
||||
|
||||
|
||||
# ── Stress Snapshot Tests ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStressSnapshot:
|
||||
def test_snapshot_to_dict(self):
|
||||
"""Snapshot can be converted to dictionary."""
|
||||
signals = [
|
||||
StressSignal(name="test", value=10.0, threshold=5.0, weight=0.5),
|
||||
]
|
||||
snapshot = StressSnapshot(
|
||||
mode=StressMode.ELEVATED,
|
||||
score=0.5,
|
||||
signals=signals,
|
||||
multipliers={"test_improve": 1.2},
|
||||
)
|
||||
|
||||
data = snapshot.to_dict()
|
||||
assert data["mode"] == "elevated"
|
||||
assert data["score"] == 0.5
|
||||
assert len(data["signals"]) == 1
|
||||
assert data["multipliers"]["test_improve"] == 1.2
|
||||
|
||||
|
||||
# ── Integration Tests ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestStressDetectorIntegration:
|
||||
def test_reset_stress_state(self):
|
||||
"""Reset clears internal state."""
|
||||
# Just verify reset doesn't error
|
||||
reset_stress_state()
|
||||
|
||||
def test_default_config_contains_all_signals(self):
|
||||
"""Default config defines all expected signals."""
|
||||
config = get_default_config()
|
||||
signals = config["signals"]
|
||||
|
||||
expected_signals = [
|
||||
"flaky_test_rate",
|
||||
"p1_backlog_growth",
|
||||
"ci_failure_rate",
|
||||
"open_bug_count",
|
||||
]
|
||||
|
||||
for signal in expected_signals:
|
||||
assert signal in signals
|
||||
assert "threshold" in signals[signal]
|
||||
assert "weight" in signals[signal]
|
||||
|
||||
def test_default_config_contains_all_modes(self):
|
||||
"""Default config defines all stress modes."""
|
||||
config = get_default_config()
|
||||
multipliers = config["multipliers"]
|
||||
|
||||
assert "calm" in multipliers
|
||||
assert "elevated" in multipliers
|
||||
assert "high" in multipliers
|
||||
|
||||
def test_multiplier_weights_sum_approximately_one(self):
|
||||
"""Signal weights should approximately sum to 1.0."""
|
||||
config = get_default_config()
|
||||
signals = config["signals"]
|
||||
|
||||
total_weight = sum(s["weight"] for s in signals.values())
|
||||
# Allow some flexibility but should be close to 1.0
|
||||
assert 0.9 <= total_weight <= 1.1
|
||||
Reference in New Issue
Block a user