forked from Rockachopa/Timmy-time-dashboard
273 lines
10 KiB
Python
273 lines
10 KiB
Python
"""Unit tests for the sovereignty metrics emitter and store.
|
|
|
|
Refs: #954
|
|
"""
|
|
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
import pytest
|
|
|
|
from timmy.sovereignty.metrics import (
|
|
ALL_EVENT_TYPES,
|
|
SovereigntyMetricsStore,
|
|
emit_sovereignty_event,
|
|
get_cost_per_hour,
|
|
get_skills_crystallized,
|
|
get_sovereignty_pct,
|
|
record,
|
|
)
|
|
|
|
pytestmark = pytest.mark.unit
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path):
|
|
"""A fresh SovereigntyMetricsStore backed by a temp database."""
|
|
return SovereigntyMetricsStore(db_path=tmp_path / "test_sov.db")
|
|
|
|
|
|
# ── ALL_EVENT_TYPES ───────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestEventTypes:
|
|
def test_all_expected_event_types_present(self):
|
|
expected = {
|
|
"perception_cache_hit",
|
|
"perception_vlm_call",
|
|
"decision_rule_hit",
|
|
"decision_llm_call",
|
|
"narration_template",
|
|
"narration_llm",
|
|
"skill_crystallized",
|
|
"api_call",
|
|
"api_cost",
|
|
}
|
|
assert ALL_EVENT_TYPES == expected
|
|
|
|
|
|
# ── Record & retrieval ────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestRecord:
|
|
def test_record_inserts_event(self, store):
|
|
store.record("perception_cache_hit")
|
|
pct = store.get_sovereignty_pct("perception")
|
|
assert pct == 100.0
|
|
|
|
def test_record_with_metadata(self, store):
|
|
store.record("api_cost", metadata={"usd": 0.05})
|
|
cost = store.get_cost_per_hour()
|
|
assert cost > 0.0
|
|
|
|
def test_record_with_session_id(self, store):
|
|
store.record("skill_crystallized", session_id="sess-1")
|
|
assert store.get_skills_crystallized("sess-1") == 1
|
|
|
|
def test_record_unknown_type_does_not_raise(self, store):
|
|
"""Unknown event types are silently stored (no crash)."""
|
|
store.record("totally_unknown_event") # should not raise
|
|
|
|
|
|
# ── Sessions ──────────────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestSessions:
|
|
def test_start_session_returns_id(self, store):
|
|
sid = store.start_session(game="Bannerlord")
|
|
assert isinstance(sid, str)
|
|
assert len(sid) > 0
|
|
|
|
def test_start_session_accepts_custom_id(self, store):
|
|
sid = store.start_session(game="Bannerlord", session_id="my-session")
|
|
assert sid == "my-session"
|
|
|
|
def test_end_session_does_not_raise(self, store):
|
|
sid = store.start_session()
|
|
store.end_session(sid) # should not raise
|
|
|
|
def test_start_session_idempotent(self, store):
|
|
"""Starting a session with the same ID twice is a no-op."""
|
|
sid = store.start_session(session_id="dup")
|
|
sid2 = store.start_session(session_id="dup")
|
|
assert sid == sid2
|
|
|
|
|
|
# ── Sovereignty percentage ────────────────────────────────────────────────────
|
|
|
|
|
|
class TestGetSovereigntyPct:
|
|
def test_perception_all_cache_hits(self, store):
|
|
for _ in range(5):
|
|
store.record("perception_cache_hit")
|
|
assert store.get_sovereignty_pct("perception") == 100.0
|
|
|
|
def test_perception_mixed(self, store):
|
|
store.record("perception_cache_hit")
|
|
store.record("perception_vlm_call")
|
|
assert store.get_sovereignty_pct("perception") == 50.0
|
|
|
|
def test_decision_all_sovereign(self, store):
|
|
for _ in range(3):
|
|
store.record("decision_rule_hit")
|
|
assert store.get_sovereignty_pct("decision") == 100.0
|
|
|
|
def test_narration_all_sovereign(self, store):
|
|
store.record("narration_template")
|
|
store.record("narration_template")
|
|
assert store.get_sovereignty_pct("narration") == 100.0
|
|
|
|
def test_narration_all_llm(self, store):
|
|
store.record("narration_llm")
|
|
assert store.get_sovereignty_pct("narration") == 0.0
|
|
|
|
def test_no_events_returns_zero(self, store):
|
|
assert store.get_sovereignty_pct("perception") == 0.0
|
|
|
|
def test_unknown_layer_returns_zero(self, store):
|
|
assert store.get_sovereignty_pct("nonexistent_layer") == 0.0
|
|
|
|
def test_time_window_filters_old_events(self, store, tmp_path):
|
|
"""Events outside the time window are excluded."""
|
|
# Insert an event with a very old timestamp directly
|
|
import sqlite3
|
|
from contextlib import closing
|
|
|
|
with closing(sqlite3.connect(str(store._db_path))) as conn:
|
|
conn.execute(
|
|
"INSERT INTO events (timestamp, event_type, session_id, metadata_json) VALUES (?, ?, ?, ?)",
|
|
("2000-01-01T00:00:00+00:00", "perception_cache_hit", "", "{}"),
|
|
)
|
|
conn.commit()
|
|
|
|
# With a 60-second window, the old event should be excluded
|
|
pct = store.get_sovereignty_pct("perception", time_window=60)
|
|
assert pct == 0.0
|
|
|
|
def test_time_window_includes_recent_events(self, store):
|
|
store.record("decision_rule_hit")
|
|
pct = store.get_sovereignty_pct("decision", time_window=60)
|
|
assert pct == 100.0
|
|
|
|
|
|
# ── Cost per hour ─────────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestGetCostPerHour:
|
|
def test_no_events_returns_zero(self, store):
|
|
assert store.get_cost_per_hour() == 0.0
|
|
|
|
def test_single_cost_event(self, store):
|
|
# Record a cost of $1.00 within the last hour window
|
|
store.record("api_cost", metadata={"usd": 1.00})
|
|
cost = store.get_cost_per_hour(time_window=3600)
|
|
assert cost == pytest.approx(1.00, rel=1e-3)
|
|
|
|
def test_multiple_cost_events(self, store):
|
|
store.record("api_cost", metadata={"usd": 0.25})
|
|
store.record("api_cost", metadata={"usd": 0.75})
|
|
cost = store.get_cost_per_hour(time_window=3600)
|
|
assert cost == pytest.approx(1.00, rel=1e-3)
|
|
|
|
def test_missing_usd_field_is_zero(self, store):
|
|
store.record("api_cost", metadata={"model": "gpt-4"})
|
|
assert store.get_cost_per_hour() == 0.0
|
|
|
|
def test_cost_extrapolated_for_short_window(self, store):
|
|
"""Cost recorded in a 1800s window is doubled to get per-hour rate."""
|
|
store.record("api_cost", metadata={"usd": 0.5})
|
|
cost = store.get_cost_per_hour(time_window=1800)
|
|
assert cost == pytest.approx(1.0, rel=1e-3)
|
|
|
|
|
|
# ── Skills crystallised ───────────────────────────────────────────────────────
|
|
|
|
|
|
class TestGetSkillsCrystallized:
|
|
def test_no_skills_returns_zero(self, store):
|
|
assert store.get_skills_crystallized() == 0
|
|
|
|
def test_counts_all_sessions(self, store):
|
|
store.record("skill_crystallized", session_id="a")
|
|
store.record("skill_crystallized", session_id="b")
|
|
assert store.get_skills_crystallized() == 2
|
|
|
|
def test_filters_by_session(self, store):
|
|
store.record("skill_crystallized", session_id="sess-1")
|
|
store.record("skill_crystallized", session_id="sess-2")
|
|
assert store.get_skills_crystallized("sess-1") == 1
|
|
|
|
def test_session_with_no_skills(self, store):
|
|
store.record("skill_crystallized", session_id="sess-1")
|
|
assert store.get_skills_crystallized("sess-999") == 0
|
|
|
|
|
|
# ── Snapshot ──────────────────────────────────────────────────────────────────
|
|
|
|
|
|
class TestGetSnapshot:
|
|
def test_snapshot_structure(self, store):
|
|
snap = store.get_snapshot()
|
|
assert "sovereignty" in snap
|
|
assert "cost_per_hour" in snap
|
|
assert "skills_crystallized" in snap
|
|
|
|
def test_snapshot_sovereignty_has_all_layers(self, store):
|
|
snap = store.get_snapshot()
|
|
assert set(snap["sovereignty"].keys()) == {"perception", "decision", "narration"}
|
|
|
|
def test_snapshot_reflects_events(self, store):
|
|
store.record("perception_cache_hit")
|
|
store.record("skill_crystallized")
|
|
snap = store.get_snapshot()
|
|
assert snap["sovereignty"]["perception"] == 100.0
|
|
assert snap["skills_crystallized"] == 1
|
|
|
|
|
|
# ── Module-level convenience functions ───────────────────────────────────────
|
|
|
|
|
|
class TestModuleLevelFunctions:
|
|
def test_record_and_get_sovereignty_pct(self, tmp_path):
|
|
with (
|
|
patch("timmy.sovereignty.metrics._store", None),
|
|
patch("timmy.sovereignty.metrics.DB_PATH", tmp_path / "fn_test.db"),
|
|
):
|
|
record("decision_rule_hit")
|
|
pct = get_sovereignty_pct("decision")
|
|
assert pct == 100.0
|
|
|
|
def test_get_cost_per_hour_module_fn(self, tmp_path):
|
|
with (
|
|
patch("timmy.sovereignty.metrics._store", None),
|
|
patch("timmy.sovereignty.metrics.DB_PATH", tmp_path / "fn_test2.db"),
|
|
):
|
|
record("api_cost", {"usd": 0.5})
|
|
cost = get_cost_per_hour()
|
|
assert cost > 0.0
|
|
|
|
def test_get_skills_crystallized_module_fn(self, tmp_path):
|
|
with (
|
|
patch("timmy.sovereignty.metrics._store", None),
|
|
patch("timmy.sovereignty.metrics.DB_PATH", tmp_path / "fn_test3.db"),
|
|
):
|
|
record("skill_crystallized")
|
|
count = get_skills_crystallized()
|
|
assert count == 1
|
|
|
|
|
|
# ── emit_sovereignty_event ────────────────────────────────────────────────────
|
|
|
|
|
|
class TestEmitSovereigntyEvent:
|
|
@pytest.mark.asyncio
|
|
async def test_emit_records_and_publishes(self, tmp_path):
|
|
with (
|
|
patch("timmy.sovereignty.metrics._store", None),
|
|
patch("timmy.sovereignty.metrics.DB_PATH", tmp_path / "emit_test.db"),
|
|
patch("infrastructure.events.bus.emit", new_callable=AsyncMock) as mock_emit,
|
|
):
|
|
await emit_sovereignty_event("perception_cache_hit", {"frame": 42}, session_id="s1")
|
|
mock_emit.assert_called_once()
|
|
args = mock_emit.call_args[0]
|
|
assert args[0] == "sovereignty.event.perception_cache_hit"
|