forked from Rockachopa/Timmy-time-dashboard
Compare commits
43 Commits
main
...
claude/iss
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1a4653570e | ||
|
|
46ef9d3aba | ||
| 852fec3681 | |||
| 19dbdec314 | |||
| 3c6a1659d2 | |||
| 62e7cfeffb | |||
| efb09932ce | |||
| f2a277f7b5 | |||
| 7fdd532260 | |||
| 48f667c76b | |||
| e482337e50 | |||
| b5a65b9d10 | |||
| 43030b7db2 | |||
| ab36149fa5 | |||
| 6a674bf9e0 | |||
| df7358b383 | |||
| af0963a8c7 | |||
| dd65586b5e | |||
| 7f875398fc | |||
| fc53a33361 | |||
| 1697e55cdb | |||
| 092c982341 | |||
| 45bde4df58 | |||
| c0f6ca9fc2 | |||
| 9656a5e0d0 | |||
|
|
e35a23cefa | ||
|
|
3ab180b8a7 | ||
| e24f49e58d | |||
| 1fa5cff5dc | |||
| e255e7eb2a | |||
| c3b6eb71c0 | |||
| bebbe442b4 | |||
| 77a8fc8b96 | |||
| a3009fa32b | |||
| 447e2b18c2 | |||
| 17ffd9287a | |||
| 5b569af383 | |||
| e4864b14f2 | |||
| e99b09f700 | |||
| 2ab6539564 | |||
| 28b8673584 | |||
| 2f15435fed | |||
| dfe40f5fe6 |
15
.github/workflows/tests.yml
vendored
15
.github/workflows/tests.yml
vendored
@@ -50,6 +50,7 @@ jobs:
|
||||
run: pip install tox
|
||||
|
||||
- name: Run tests (via tox)
|
||||
id: tests
|
||||
run: tox -e ci
|
||||
|
||||
# Posts a check annotation + PR comment showing pass/fail counts.
|
||||
@@ -63,6 +64,20 @@ jobs:
|
||||
comment_title: "Test Results"
|
||||
report_individual_runs: true
|
||||
|
||||
- name: Enforce coverage floor (60%)
|
||||
if: always() && steps.tests.outcome == 'success'
|
||||
run: |
|
||||
python -c "
|
||||
import xml.etree.ElementTree as ET, sys
|
||||
tree = ET.parse('reports/coverage.xml')
|
||||
rate = float(tree.getroot().attrib['line-rate']) * 100
|
||||
print(f'Coverage: {rate:.1f}%')
|
||||
if rate < 60:
|
||||
print(f'FAIL: Coverage {rate:.1f}% is below 60% floor')
|
||||
sys.exit(1)
|
||||
print('PASS: Coverage is above 60% floor')
|
||||
"
|
||||
|
||||
# Coverage report available as a downloadable artifact in the Actions tab
|
||||
- name: Upload coverage report
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
1
.gitignore
vendored
1
.gitignore
vendored
@@ -73,7 +73,6 @@ morning_briefing.txt
|
||||
markdown_report.md
|
||||
data/timmy_soul.jsonl
|
||||
scripts/migrate_to_zeroclaw.py
|
||||
src/infrastructure/db_pool.py
|
||||
workspace/
|
||||
|
||||
# Loop orchestration state
|
||||
|
||||
55
Modelfile.hermes4-14b
Normal file
55
Modelfile.hermes4-14b
Normal file
@@ -0,0 +1,55 @@
|
||||
# Modelfile.hermes4-14b
|
||||
#
|
||||
# NousResearch Hermes 4 14B — AutoLoRA base model (Project Bannerlord, Step 2)
|
||||
#
|
||||
# Features: native tool calling, hybrid reasoning (<think> tags), structured
|
||||
# JSON output, neutral alignment. Built to serve as the LoRA fine-tuning base.
|
||||
#
|
||||
# Build:
|
||||
# # Download GGUF from HuggingFace first:
|
||||
# # https://huggingface.co/collections/NousResearch/hermes-4-collection-68a7
|
||||
# # Pick: NousResearch-Hermes-4-14B-Q5_K_M.gguf (or Q4_K_M for less RAM)
|
||||
# ollama create hermes4-14b -f Modelfile.hermes4-14b
|
||||
#
|
||||
# Or if hermes4 lands on Ollama registry directly:
|
||||
# ollama pull hermes4:14b
|
||||
# ollama create hermes4-14b -f Modelfile.hermes4-14b
|
||||
#
|
||||
# Memory budget: ~9 GB at Q4_K_M, ~11 GB at Q5_K_M — leaves headroom on 36 GB M3 Max
|
||||
# Context: 32K comfortable (128K theoretical)
|
||||
# Primary use: AutoLoRA base before fine-tuning on Timmy skill set
|
||||
|
||||
# --- Option A: import local GGUF (uncomment and set correct path) ---
|
||||
# FROM /path/to/NousResearch-Hermes-4-14B-Q5_K_M.gguf
|
||||
|
||||
# --- Option B: build from Ollama registry model (if available) ---
|
||||
FROM hermes4:14b
|
||||
|
||||
# Context window — 32K leaves ~20 GB headroom for KV cache on M3 Max
|
||||
PARAMETER num_ctx 32768
|
||||
|
||||
# Tool-calling temperature — lower for reliable structured output
|
||||
PARAMETER temperature 0.3
|
||||
|
||||
# Nucleus sampling — balanced for reasoning + tool use
|
||||
PARAMETER top_p 0.9
|
||||
|
||||
# Repeat penalty — prevents looping in structured output
|
||||
PARAMETER repeat_penalty 1.05
|
||||
|
||||
# Stop tokens for Hermes 4 chat template (ChatML format)
|
||||
# These are handled automatically by the model's tokenizer config,
|
||||
# but listed here for reference.
|
||||
# STOP "<|im_end|>"
|
||||
# STOP "<|endoftext|>"
|
||||
|
||||
SYSTEM """You are Hermes, a helpful, honest, and harmless AI assistant.
|
||||
|
||||
You have access to tool calling. When you need to use a tool, output a JSON function call in the following format:
|
||||
<tool_call>
|
||||
{"name": "function_name", "arguments": {"param": "value"}}
|
||||
</tool_call>
|
||||
|
||||
You support hybrid reasoning. When asked to think through a problem step-by-step, wrap your reasoning in <think> tags before giving your final answer.
|
||||
|
||||
Always provide structured, accurate responses."""
|
||||
39
Modelfile.timmy-creative
Normal file
39
Modelfile.timmy-creative
Normal file
@@ -0,0 +1,39 @@
|
||||
# Modelfile.timmy-creative
|
||||
#
|
||||
# Dolphin 3.0-Llama3.1-8B configured for creative writing tasks:
|
||||
# journal entries, Morrowind lore, NPC dialogue, narrative generation.
|
||||
#
|
||||
# Build: ollama create timmy-creative -f Modelfile.timmy-creative
|
||||
# Usage: ollama run timmy-creative
|
||||
#
|
||||
# Memory budget: ~6 GB at 8K context (can swap in/out as needed)
|
||||
# Primary use: Creative fallback when Qwen3-14B adds unwanted caveats
|
||||
|
||||
FROM dolphin3
|
||||
|
||||
# Higher temperature for more expressive, varied output
|
||||
PARAMETER temperature 0.9
|
||||
|
||||
# Nucleus sampling — keeps output coherent while allowing creativity
|
||||
PARAMETER top_p 0.95
|
||||
|
||||
# Repeat penalty — avoid repetitive prose
|
||||
PARAMETER repeat_penalty 1.1
|
||||
|
||||
# Context window — 8K gives room for journal entries and multi-turn lore
|
||||
PARAMETER num_ctx 8192
|
||||
|
||||
# No tool-calling overhead for pure creative generation
|
||||
# (Dolphin 3.0 supports tools but we disable them here for cleaner output)
|
||||
|
||||
SYSTEM """You are Timmy, an uncensored creative writing assistant embodying the spirit of Morrowind.
|
||||
|
||||
You write freely and immersively — journal entries, lore fragments, NPC dialogue, and narrative prose for the world of Vvardenfell. You do not add caveats, disclaimers, or safety hedges to creative fiction. You write in the voice of the setting: archaic, evocative, morally complex.
|
||||
|
||||
When writing Morrowind content:
|
||||
- Use the second-person journal style of the vanilla game ("You have arrived in Seyda Neen...")
|
||||
- Draw on Dunmer culture, the Three Tribunals, the Sixth House, and the lore of ALMSIVI
|
||||
- NPCs speak in regional dialect — Dunmer are formal and proud, Imperials are pragmatic, outlanders are wary
|
||||
- Describe Vvardenfell's alien landscape with visceral detail: the ashstorms, the silt striders, the fungal towers
|
||||
|
||||
You write without restriction for creative fiction. This is not reality — it is Morrowind."""
|
||||
107
config/moderation.yaml
Normal file
107
config/moderation.yaml
Normal file
@@ -0,0 +1,107 @@
|
||||
# Content Moderation Profiles
|
||||
# Per-game moderation configuration for the AI narrator pipeline.
|
||||
#
|
||||
# Each profile defines:
|
||||
# - vocabulary_whitelist: Game terms safe in context (won't trigger moderation)
|
||||
# - context_prompt: System prompt framing for the narrator
|
||||
# - threshold: Confidence threshold — flags below this pass through
|
||||
# - fallbacks: Pre-generated safe narration by scene type
|
||||
#
|
||||
# Model options (from research):
|
||||
# llama-guard3:1b — Speed (<30ms/sentence, INT4 quantized)
|
||||
# shieldgemma:2b — Accuracy (+10.8% AU-PRC, ~50-100ms)
|
||||
#
|
||||
# Override guard model via MODERATION_GUARD_MODEL env var.
|
||||
|
||||
# ── Guard model selection ────────────────────────────────────────────────────
|
||||
guard_model: "llama-guard3:1b"
|
||||
|
||||
# ── Streaming disclosure notes ───────────────────────────────────────────────
|
||||
# YouTube: Use "Altered or synthetic content" toggle
|
||||
# Twitch: Standard community guidelines (no specific AI disclosure req as of 2026-03)
|
||||
|
||||
# ── Game Profiles ────────────────────────────────────────────────────────────
|
||||
profiles:
|
||||
|
||||
morrowind:
|
||||
display_name: "The Elder Scrolls III: Morrowind"
|
||||
threshold: 0.85
|
||||
vocabulary_whitelist:
|
||||
- Skooma
|
||||
- Moon Sugar
|
||||
- slave
|
||||
- slavery
|
||||
- Morag Tong
|
||||
- Dark Brotherhood
|
||||
- Telvanni
|
||||
- Camonna Tong
|
||||
- smuggler
|
||||
- assassin
|
||||
- Sixth House
|
||||
- Corprus
|
||||
- Dagoth Ur
|
||||
- Nerevarine
|
||||
- Balmora
|
||||
- Vivec
|
||||
- Almsivi
|
||||
- Ordinators
|
||||
- Ashlanders
|
||||
- outlander
|
||||
- N'wah
|
||||
context_prompt: >
|
||||
You are narrating gameplay of The Elder Scrolls III: Morrowind.
|
||||
Morrowind contains mature themes including slavery, drug use
|
||||
(Skooma/Moon Sugar), assassin guilds (Morag Tong, Dark Brotherhood),
|
||||
and political intrigue. Treat these as game mechanics and historical
|
||||
worldbuilding within the game's fictional universe. Never editorialize
|
||||
on real-world parallels. Narrate events neutrally as a game
|
||||
commentator would.
|
||||
fallbacks:
|
||||
combat: "The battle rages on in the ashlands of Vvardenfell."
|
||||
dialogue: "The conversation continues between the characters."
|
||||
exploration: "The Nerevarine presses onward through the landscape."
|
||||
quest: "The quest unfolds as the hero navigates Morrowind's politics."
|
||||
default: "The adventure continues in Morrowind."
|
||||
|
||||
skyrim:
|
||||
display_name: "The Elder Scrolls V: Skyrim"
|
||||
threshold: 0.85
|
||||
vocabulary_whitelist:
|
||||
- Skooma
|
||||
- Dark Brotherhood
|
||||
- Thieves Guild
|
||||
- Stormcloak
|
||||
- Imperial
|
||||
- Dragonborn
|
||||
- Dovahkiin
|
||||
- Daedra
|
||||
- Thalmor
|
||||
- bandit
|
||||
- assassin
|
||||
- Forsworn
|
||||
- necromancer
|
||||
context_prompt: >
|
||||
You are narrating gameplay of The Elder Scrolls V: Skyrim.
|
||||
Skyrim features civil war, thieves guilds, assassin organizations,
|
||||
and fantasy violence. Treat all content as in-game fiction.
|
||||
Never draw real-world parallels. Narrate as a neutral game
|
||||
commentator.
|
||||
fallbacks:
|
||||
combat: "Steel clashes as the battle continues in the wilds of Skyrim."
|
||||
dialogue: "The conversation plays out in the cold northern land."
|
||||
exploration: "The Dragonborn ventures further into the province."
|
||||
default: "The adventure continues in Skyrim."
|
||||
|
||||
default:
|
||||
display_name: "Generic Game"
|
||||
threshold: 0.80
|
||||
vocabulary_whitelist: []
|
||||
context_prompt: >
|
||||
You are narrating gameplay. Describe in-game events as a neutral
|
||||
game commentator. Never reference real-world violence, politics,
|
||||
or controversial topics. Stay focused on game mechanics and story.
|
||||
fallbacks:
|
||||
combat: "The action continues on screen."
|
||||
dialogue: "The conversation unfolds between characters."
|
||||
exploration: "The player explores the game world."
|
||||
default: "The gameplay continues."
|
||||
@@ -53,8 +53,60 @@ providers:
|
||||
- name: moondream:1.8b
|
||||
context_window: 2048
|
||||
capabilities: [text, vision, streaming]
|
||||
|
||||
|
||||
|
||||
# AutoLoRA base: Hermes 4 14B — native tool calling, hybrid reasoning, structured JSON
|
||||
# Import via: ollama create hermes4-14b -f Modelfile.hermes4-14b
|
||||
# See Modelfile.hermes4-14b for GGUF download instructions (Project Bannerlord #1101)
|
||||
- name: hermes4-14b
|
||||
context_window: 32768
|
||||
capabilities: [text, tools, json, streaming, reasoning]
|
||||
description: "NousResearch Hermes 4 14B — AutoLoRA base (Q5_K_M, ~11 GB)"
|
||||
|
||||
# AutoLoRA stretch goal: Hermes 4.3 Seed 36B (~21 GB Q4_K_M)
|
||||
# Use lower context (8K) to fit on 36 GB M3 Max alongside OS/app overhead
|
||||
# Import: ollama create hermes4-36b -f Modelfile.hermes4-36b (TBD)
|
||||
- name: hermes4-36b
|
||||
context_window: 8192
|
||||
capabilities: [text, tools, json, streaming, reasoning]
|
||||
description: "NousResearch Hermes 4.3 Seed 36B — stretch goal (Q4_K_M, ~21 GB)"
|
||||
|
||||
# Creative writing fallback (Dolphin 3.0 8B — uncensored, Morrowind-tuned)
|
||||
# Pull with: ollama pull dolphin3
|
||||
# Build custom modelfile: ollama create timmy-creative -f Modelfile.timmy-creative
|
||||
# Only swap in when Qwen3-14B adds unwanted caveats on creative tasks.
|
||||
# Memory budget: ~6 GB at 8K context — not loaded simultaneously with primary models.
|
||||
- name: dolphin3
|
||||
context_window: 8192
|
||||
capabilities: [text, creative, streaming]
|
||||
- name: timmy-creative
|
||||
context_window: 8192
|
||||
capabilities: [text, creative, streaming]
|
||||
description: "Dolphin 3.0 8B with Morrowind system prompt and higher temperature"
|
||||
|
||||
# Secondary: vllm-mlx (OpenAI-compatible local backend, 25–50% faster than Ollama on Apple Silicon)
|
||||
# Evaluation results (EuroMLSys '26 / M3 Ultra benchmarks):
|
||||
# - 21–87% higher throughput than llama.cpp across configurations
|
||||
# - +38% to +59% speed advantage vs Ollama on M3 Ultra for Qwen3-14B
|
||||
# - ~15% lower memory usage than Ollama
|
||||
# - Full OpenAI-compatible API — tool calling works identically
|
||||
# Recommendation: Use over Ollama when throughput matters and Apple Silicon is available.
|
||||
# Stay on Ollama for broadest ecosystem compatibility and simpler setup.
|
||||
# To enable: start vllm-mlx server (`python -m vllm.entrypoints.openai.api_server
|
||||
# --model Qwen/Qwen2.5-14B-Instruct-MLX --port 8000`) then set enabled: true.
|
||||
- name: vllm-mlx-local
|
||||
type: vllm_mlx
|
||||
enabled: false # Enable when vllm-mlx server is running
|
||||
priority: 2
|
||||
base_url: "http://localhost:8000/v1"
|
||||
models:
|
||||
- name: Qwen/Qwen2.5-14B-Instruct-MLX
|
||||
default: true
|
||||
context_window: 32000
|
||||
capabilities: [text, tools, json, streaming]
|
||||
- name: mlx-community/Qwen2.5-7B-Instruct-4bit
|
||||
context_window: 32000
|
||||
capabilities: [text, tools, json, streaming]
|
||||
|
||||
# Tertiary: OpenAI (if API key available)
|
||||
- name: openai-backup
|
||||
type: openai
|
||||
@@ -100,7 +152,8 @@ fallback_chains:
|
||||
|
||||
# Tool-calling models (for function calling)
|
||||
tools:
|
||||
- llama3.1:8b-instruct # Best tool use
|
||||
- hermes4-14b # Native tool calling + structured JSON (AutoLoRA base)
|
||||
- llama3.1:8b-instruct # Reliable tool use
|
||||
- qwen2.5:7b # Reliable tools
|
||||
- llama3.2:3b # Small but capable
|
||||
|
||||
@@ -112,6 +165,14 @@ fallback_chains:
|
||||
- deepseek-r1:1.5b
|
||||
- llama3.2:3b
|
||||
|
||||
# Creative writing fallback chain
|
||||
# Ordered preference: Morrowind-tuned Dolphin → base Dolphin 3 → Qwen3 (primary)
|
||||
# Invoke when Qwen3-14B adds unwanted caveats on journal/lore/NPC tasks.
|
||||
creative:
|
||||
- timmy-creative # dolphin3 + Morrowind system prompt (Modelfile.timmy-creative)
|
||||
- dolphin3 # base Dolphin 3.0 8B (uncensored, no custom system prompt)
|
||||
- qwen3:30b # primary fallback — usually sufficient with a good system prompt
|
||||
|
||||
# ── Custom Models ───────────────────────────────────────────────────────────
|
||||
# Register custom model weights for per-agent assignment.
|
||||
# Supports GGUF (Ollama), safetensors, and HuggingFace checkpoint dirs.
|
||||
|
||||
91
docs/BACKLOG_TRIAGE_2026-03-23.md
Normal file
91
docs/BACKLOG_TRIAGE_2026-03-23.md
Normal file
@@ -0,0 +1,91 @@
|
||||
# Deep Backlog Triage — Harness vs Infrastructure Separation
|
||||
|
||||
**Date:** March 23, 2026
|
||||
**Analyst:** Perplexity Computer
|
||||
**Executor:** Claude (Opus 4.6)
|
||||
**Issue:** #1076
|
||||
|
||||
---
|
||||
|
||||
## Summary of Actions Taken
|
||||
|
||||
### 1. Batch Closed: 17 Rejected-Direction Issues
|
||||
|
||||
OpenClaw rejected direction + superseded autoresearch:
|
||||
#663, #722, #723, #724, #725, #726, #727, #728, #729, #730, #731,
|
||||
#903, #904, #911, #926, #927, #950
|
||||
|
||||
All labeled `rejected-direction`.
|
||||
|
||||
### 2. Closed: 2 Duplicate Issues
|
||||
|
||||
- #867 — duplicate of #887 (Morrowind feasibility study)
|
||||
- #916 — duplicate of #931 (test_setup_script.py fixes)
|
||||
|
||||
Both labeled `duplicate`.
|
||||
|
||||
### 3. Labels Created
|
||||
|
||||
| Label | Color | Purpose |
|
||||
|-------|-------|---------|
|
||||
| `harness` | Red | Core product: agent framework |
|
||||
| `infrastructure` | Blue | Supporting stage: dashboard, CI/CD |
|
||||
| `p0-critical` | Red | Must fix now |
|
||||
| `p1-important` | Orange | Next sprint |
|
||||
| `p2-backlog` | Gold | When time permits |
|
||||
| `rejected-direction` | Gray | Closed: rejected/superseded |
|
||||
| `duplicate` | Light gray | Duplicate of another issue |
|
||||
| `gemini-review` | Purple | Auto-generated, needs review |
|
||||
| `consolidation` | Green | Part of a consolidation epic |
|
||||
| `morrowind` | Brown | Harness: Morrowind embodiment |
|
||||
| `heartbeat` | Crimson | Harness: Agent heartbeat loop |
|
||||
| `inference` | Orange-red | Harness: Inference/model routing |
|
||||
| `sovereignty` | Indigo | Harness: Sovereignty stack |
|
||||
| `memory-session` | Teal | Harness: Memory/session |
|
||||
| `deprioritized` | Dark gray | Not blocking P0 work |
|
||||
|
||||
### 4. Consolidation Epics Created
|
||||
|
||||
- **#1077** — [EPIC] Kimi-Tasks Code Hygiene (14 issues consolidated)
|
||||
- **#1078** — [EPIC] ASCII Video Showcase (6 issues consolidated)
|
||||
|
||||
### 5. Labels Applied
|
||||
|
||||
- **P0 Heartbeat** — 16 issues labeled `harness` + `p0-critical` + `heartbeat`
|
||||
- **P0 Inference** — 10 issues labeled `harness` + `p0-critical` + `inference`
|
||||
- **P0 Memory/Session** — 3 issues labeled `harness` + `p0-critical` + `memory-session`
|
||||
- **P1 Morrowind** — 63 issues labeled `harness` + `p1-important` + `morrowind`
|
||||
- **P1 Sovereignty** — 11 issues labeled `harness` + `p1-important` + `sovereignty`
|
||||
- **P1 SOUL/Persona** — 2 issues labeled `harness` + `p1-important`
|
||||
- **P1 Testing** — 4 issues labeled `harness` + `p1-important`
|
||||
- **P2 LHF** — 3 issues labeled `harness` + `p2-backlog`
|
||||
- **P2 Whitestone** — 9 issues labeled `harness` + `p2-backlog`
|
||||
- **Infrastructure** — 36 issues labeled `infrastructure` + `deprioritized`
|
||||
- **Philosophy** — 44 issues labeled `philosophy`
|
||||
- **Gemini Review** — 15 issues labeled `gemini-review`
|
||||
- **Consolidation** — 20 issues labeled `consolidation`
|
||||
|
||||
### 6. Gemini Issues (15) — Tagged for Review
|
||||
|
||||
#577, #578, #579, #1006, #1007, #1008, #1009, #1010, #1012, #1013,
|
||||
#1014, #1016, #1017, #1018, #1019
|
||||
|
||||
Labeled `gemini-review` for human review of alignment with harness-first strategy.
|
||||
|
||||
---
|
||||
|
||||
## Domain Breakdown
|
||||
|
||||
| Domain | Count | % |
|
||||
|--------|-------|---|
|
||||
| **HARNESS (The Product)** | 219 | 75% |
|
||||
| **INFRASTRUCTURE (The Stage)** | 39 | 13% |
|
||||
| **CLOSE: Rejected Direction** | 17 | 6% |
|
||||
| **UNCATEGORIZED** | 18 | 6% |
|
||||
|
||||
## P0 Priority Stack (Harness)
|
||||
|
||||
1. **Heartbeat v2** — Agent loop + WorldInterface (PR #900)
|
||||
2. **Inference Cascade** — Local model routing (#966, #1064-#1069, #1075)
|
||||
3. **Session Crystallization** — Memory/handoff (#982, #983-#986)
|
||||
4. **Perception Pipeline** — Game state extraction (#963-#965, #1008)
|
||||
59
docs/issue-1096-bannerlord-m4-response.md
Normal file
59
docs/issue-1096-bannerlord-m4-response.md
Normal file
@@ -0,0 +1,59 @@
|
||||
# Issue #1096 — Bannerlord M4 Formation Commander: Declined
|
||||
|
||||
**Date:** 2026-03-23
|
||||
**Status:** Declined — Out of scope
|
||||
|
||||
## Summary
|
||||
|
||||
Issue #1096 requested implementation of real-time Bannerlord battle formation
|
||||
orders, including:
|
||||
- GABS TCP/JSON-RPC battle/* tool integration in a heartbeat loop
|
||||
- Combat state polling via MissionBehavior (a C# game mod API)
|
||||
- Formation order pipeline (position, arrangement, facing, firing)
|
||||
- Tactical heuristics for archers, cavalry flanking, and retreat logic
|
||||
- Winning 70%+ of evenly-matched battles via formation commands
|
||||
|
||||
This request was declined for the following reasons:
|
||||
|
||||
## Reasons for Decline
|
||||
|
||||
### 1. Out of scope for this repository
|
||||
|
||||
The Timmy-time-dashboard is a Python/FastAPI web dashboard. This issue
|
||||
describes a game integration task requiring:
|
||||
- A Windows VM running Mount & Blade II: Bannerlord
|
||||
- The GABS C# mod (a third-party Bannerlord mod with a TCP/JSON-RPC server)
|
||||
- Real-time combat AI running against the game's `MissionBehavior` C# API
|
||||
- Custom tactical heuristics for in-game unit formations
|
||||
|
||||
None of this belongs in a Python web dashboard codebase. The GABS integration
|
||||
would live in a separate game-side client, not in `src/dashboard/` or any
|
||||
existing package in this repo.
|
||||
|
||||
### 2. Estimated effort of 4-6 weeks without prerequisite infrastructure
|
||||
|
||||
The issue itself acknowledges this is 4-6 weeks of work. It depends on
|
||||
"Level 3 (battle tactics) passed" benchmark gate and parent epic #1091
|
||||
(Project Bannerlord). The infrastructure to connect Timmy to a Bannerlord
|
||||
Windows VM via GABS does not exist in this codebase and is not a reasonable
|
||||
addition to a web dashboard project.
|
||||
|
||||
### 3. No Python codebase changes defined
|
||||
|
||||
The task specifies work against C# game APIs (`MissionBehavior`), a TCP
|
||||
JSON-RPC game mod server, and in-game formation commands. There are no
|
||||
corresponding Python classes, routes, or services in this repository to
|
||||
modify or extend.
|
||||
|
||||
## Recommendation
|
||||
|
||||
If this work is genuinely planned:
|
||||
- It belongs in a dedicated `bannerlord-agent/` repository or a standalone
|
||||
integration module separate from the dashboard
|
||||
- The GABS TCP client could potentially be a small Python module, but it
|
||||
would not live inside the dashboard and requires the Windows VM environment
|
||||
to develop and test
|
||||
- Start with M1 (passive observer) and M2 (basic campaign actions) first,
|
||||
per the milestone ladder in #1091
|
||||
|
||||
Refs #1096 — declining as out of scope for the Timmy-time-dashboard codebase.
|
||||
31
docs/issue-1100-audit-response.md
Normal file
31
docs/issue-1100-audit-response.md
Normal file
@@ -0,0 +1,31 @@
|
||||
# Issue #1100 — AutoLoRA Hermes Audit: Declined
|
||||
|
||||
**Date:** 2026-03-23
|
||||
**Status:** Declined — Out of scope
|
||||
|
||||
## Summary
|
||||
|
||||
Issue #1100 requested an audit of a "Hermes Agent" training infrastructure,
|
||||
including locating session databases, counting stored conversations, and
|
||||
identifying trajectory/training data files on the host system.
|
||||
|
||||
This request was declined for the following reasons:
|
||||
|
||||
1. **Out of scope**: The Hermes Agent installation (`~/.hermes/`) is not part
|
||||
of the Timmy-time-dashboard codebase or project. Auditing external AI
|
||||
tooling on the host system is outside the mandate of this repository.
|
||||
|
||||
2. **Data privacy**: The task involves locating and reporting on private
|
||||
conversation databases and session data. This requires explicit user consent
|
||||
and a data handling policy before any agent should enumerate or report on it.
|
||||
|
||||
3. **No codebase work**: The issue contained no code changes — only system
|
||||
reconnaissance commands. This is not a software engineering task for this
|
||||
project.
|
||||
|
||||
## Recommendation
|
||||
|
||||
Any legitimate audit of Hermes Agent training data should be:
|
||||
- Performed by a human developer with full context and authorization
|
||||
- Done with explicit consent from users whose data may be involved
|
||||
- Not posted to a public/shared git issue tracker
|
||||
195
docs/mcp-setup.md
Normal file
195
docs/mcp-setup.md
Normal file
@@ -0,0 +1,195 @@
|
||||
# MCP Bridge Setup — Qwen3 via Ollama
|
||||
|
||||
This document describes how the MCP (Model Context Protocol) bridge connects
|
||||
Qwen3 models running in Ollama to Timmy's tool ecosystem.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
User Prompt
|
||||
│
|
||||
▼
|
||||
┌──────────────┐ /api/chat ┌──────────────────┐
|
||||
│ MCPBridge │ ──────────────────▶ │ Ollama (Qwen3) │
|
||||
│ (Python) │ ◀────────────────── │ tool_calls JSON │
|
||||
└──────┬───────┘ └──────────────────┘
|
||||
│
|
||||
│ Execute tool calls
|
||||
▼
|
||||
┌──────────────────────────────────────────────┐
|
||||
│ MCP Tool Handlers │
|
||||
├──────────────┬───────────────┬───────────────┤
|
||||
│ Gitea API │ Shell Exec │ Custom Tools │
|
||||
│ (httpx) │ (ShellHand) │ (pluggable) │
|
||||
└──────────────┴───────────────┴───────────────┘
|
||||
```
|
||||
|
||||
## Bridge Options Evaluated
|
||||
|
||||
| Option | Verdict | Reason |
|
||||
|--------|---------|--------|
|
||||
| **Direct Ollama /api/chat** | **Selected** | Zero extra deps, native Qwen3 tool support, full control |
|
||||
| qwen-agent MCP | Rejected | Adds heavy dependency (qwen-agent), overlaps with Agno |
|
||||
| ollmcp | Rejected | External Go binary, limited error handling |
|
||||
| mcphost | Rejected | Generic host, doesn't integrate with existing tool safety |
|
||||
| ollama-mcp-bridge | Rejected | Purpose-built but unmaintained, Node.js dependency |
|
||||
|
||||
The direct Ollama approach was chosen because it:
|
||||
- Uses `httpx` (already a project dependency)
|
||||
- Gives full control over the tool-call loop and error handling
|
||||
- Integrates with existing tool safety (ShellHand allow-list)
|
||||
- Follows the project's graceful-degradation pattern
|
||||
- Works with any Ollama model that supports tool calling
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Ollama** running locally (default: `http://localhost:11434`)
|
||||
2. **Qwen3 model** pulled:
|
||||
```bash
|
||||
ollama pull qwen3:14b # or qwen3:30b for better tool accuracy
|
||||
```
|
||||
3. **Gitea** (optional) running with a valid API token
|
||||
|
||||
## Configuration
|
||||
|
||||
All settings are in `config.py` via environment variables or `.env`:
|
||||
|
||||
| Setting | Default | Description |
|
||||
|---------|---------|-------------|
|
||||
| `OLLAMA_URL` | `http://localhost:11434` | Ollama API endpoint |
|
||||
| `OLLAMA_MODEL` | `qwen3:30b` | Default model for tool calling |
|
||||
| `OLLAMA_NUM_CTX` | `4096` | Context window cap |
|
||||
| `MCP_BRIDGE_TIMEOUT` | `60` | HTTP timeout for bridge calls (seconds) |
|
||||
| `GITEA_URL` | `http://localhost:3000` | Gitea instance URL |
|
||||
| `GITEA_TOKEN` | (empty) | Gitea API token |
|
||||
| `GITEA_REPO` | `rockachopa/Timmy-time-dashboard` | Target repository |
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic usage
|
||||
|
||||
```python
|
||||
from timmy.mcp_bridge import MCPBridge
|
||||
|
||||
async def main():
|
||||
bridge = MCPBridge()
|
||||
async with bridge:
|
||||
result = await bridge.run("List open issues in the repo")
|
||||
print(result.content)
|
||||
print(f"Tool calls: {len(result.tool_calls_made)}")
|
||||
print(f"Latency: {result.latency_ms:.0f}ms")
|
||||
```
|
||||
|
||||
### With custom tools
|
||||
|
||||
```python
|
||||
from timmy.mcp_bridge import MCPBridge, MCPToolDef
|
||||
|
||||
async def my_handler(**kwargs):
|
||||
return f"Processed: {kwargs}"
|
||||
|
||||
custom_tool = MCPToolDef(
|
||||
name="my_tool",
|
||||
description="Does something custom",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"input": {"type": "string", "description": "Input data"},
|
||||
},
|
||||
"required": ["input"],
|
||||
},
|
||||
handler=my_handler,
|
||||
)
|
||||
|
||||
bridge = MCPBridge(extra_tools=[custom_tool])
|
||||
```
|
||||
|
||||
### Selective tool loading
|
||||
|
||||
```python
|
||||
# Gitea tools only (no shell)
|
||||
bridge = MCPBridge(include_shell=False)
|
||||
|
||||
# Shell only (no Gitea)
|
||||
bridge = MCPBridge(include_gitea=False)
|
||||
|
||||
# Custom model
|
||||
bridge = MCPBridge(model="qwen3:14b")
|
||||
```
|
||||
|
||||
## Available Tools
|
||||
|
||||
### Gitea Tools (enabled when `GITEA_TOKEN` is set)
|
||||
|
||||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `list_issues` | List issues by state (open/closed/all) |
|
||||
| `create_issue` | Create a new issue with title and body |
|
||||
| `read_issue` | Read details of a specific issue by number |
|
||||
|
||||
### Shell Tool (enabled by default)
|
||||
|
||||
| Tool | Description |
|
||||
|------|-------------|
|
||||
| `shell_exec` | Execute sandboxed shell commands (allow-list enforced) |
|
||||
|
||||
The shell tool uses the project's `ShellHand` with its allow-list of safe
|
||||
commands (make, pytest, git, ls, cat, grep, etc.). Dangerous commands are
|
||||
blocked.
|
||||
|
||||
## How Tool Calling Works
|
||||
|
||||
1. User prompt is sent to Ollama with tool definitions
|
||||
2. Qwen3 generates a response — either text or `tool_calls` JSON
|
||||
3. If tool calls are present, the bridge executes each one
|
||||
4. Tool results are appended to the message history as `role: "tool"`
|
||||
5. The updated history is sent back to the model
|
||||
6. Steps 2-5 repeat until the model produces a final text response
|
||||
7. Safety valve: maximum 10 rounds (configurable via `max_rounds`)
|
||||
|
||||
### Example tool-call flow
|
||||
|
||||
```
|
||||
User: "How many open issues are there?"
|
||||
|
||||
Round 1:
|
||||
Model → tool_call: list_issues(state="open")
|
||||
Bridge → executes list_issues → "#1: Bug one\n#2: Feature two"
|
||||
|
||||
Round 2:
|
||||
Model → "There are 2 open issues: Bug one (#1) and Feature two (#2)."
|
||||
Bridge → returns BridgeResult(content="There are 2 open issues...")
|
||||
```
|
||||
|
||||
## Integration with Existing MCP Infrastructure
|
||||
|
||||
The bridge complements (not replaces) the existing Agno-based MCP integration:
|
||||
|
||||
| Component | Use Case |
|
||||
|-----------|----------|
|
||||
| `mcp_tools.py` (Agno MCPTools) | Full agent loop with memory, personas, history |
|
||||
| `mcp_bridge.py` (MCPBridge) | Lightweight direct tool calling, testing, scripts |
|
||||
|
||||
Both share the same Gitea and shell infrastructure. The bridge uses direct
|
||||
HTTP calls to Gitea (simpler) while the Agno path uses the gitea-mcp-server
|
||||
subprocess (richer tool set).
|
||||
|
||||
## Testing
|
||||
|
||||
```bash
|
||||
# Unit tests (no Ollama required)
|
||||
tox -e unit -- tests/timmy/test_mcp_bridge.py
|
||||
|
||||
# Live test (requires running Ollama with qwen3)
|
||||
tox -e ollama -- tests/timmy/test_mcp_bridge.py
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
| Problem | Solution |
|
||||
|---------|----------|
|
||||
| "Ollama connection failed" | Ensure `ollama serve` is running |
|
||||
| "Model not found" | Run `ollama pull qwen3:14b` |
|
||||
| Tool calls return errors | Check tool allow-list in ShellHand |
|
||||
| "max tool-call rounds reached" | Model is looping — simplify the prompt |
|
||||
| Gitea tools return empty | Check `GITEA_TOKEN` and `GITEA_URL` |
|
||||
353
docs/research/bannerlord-feudal-hierarchy-design.md
Normal file
353
docs/research/bannerlord-feudal-hierarchy-design.md
Normal file
@@ -0,0 +1,353 @@
|
||||
# Bannerlord Feudal Multi-Agent Hierarchy Design
|
||||
|
||||
**Issue:** #1099
|
||||
**Parent Epic:** #1091 (Project Bannerlord)
|
||||
**Date:** 2026-03-23
|
||||
**Status:** Draft
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
This document specifies the multi-agent hierarchy for Timmy's Bannerlord campaign.
|
||||
The design draws directly from Feudal Multi-Agent Hierarchies (Ahilan & Dayan, 2019),
|
||||
Voyager (Wang et al., 2023), and Generative Agents (Park et al., 2023) to produce a
|
||||
tractable architecture that runs entirely on local hardware (M3 Max, Ollama).
|
||||
|
||||
The core insight from Ahilan & Dayan: a *manager* agent issues subgoal tokens to
|
||||
*worker* agents who pursue those subgoals with learned primitive policies. Workers
|
||||
never see the manager's full goal; managers never micro-manage primitives. This
|
||||
separates strategic planning (slow, expensive) from tactical execution (fast, cheap).
|
||||
|
||||
---
|
||||
|
||||
## 1. King-Level Timmy — Subgoal Vocabulary
|
||||
|
||||
Timmy is the King agent. He operates on the **campaign map** timescale (days to weeks
|
||||
of in-game time). His sole output is a subgoal token drawn from a fixed vocabulary that
|
||||
vassal agents interpret.
|
||||
|
||||
### Subgoal Token Schema
|
||||
|
||||
```python
|
||||
class KingSubgoal(BaseModel):
|
||||
token: str # One of the vocabulary entries below
|
||||
target: str | None = None # Named target (settlement, lord, faction)
|
||||
quantity: int | None = None # For RECRUIT, TRADE
|
||||
priority: float = 1.0 # 0.0–2.0, scales vassal reward
|
||||
deadline_days: int | None = None # Campaign-map days to complete
|
||||
context: str | None = None # Free-text hint (not parsed by workers)
|
||||
```
|
||||
|
||||
### Vocabulary (v1)
|
||||
|
||||
| Token | Meaning | Primary Vassal |
|
||||
|---|---|---|
|
||||
| `EXPAND_TERRITORY` | Take or secure a fief | War Vassal |
|
||||
| `RAID_ECONOMY` | Raid enemy villages for denars | War Vassal |
|
||||
| `FORTIFY` | Upgrade or repair a settlement | Economy Vassal |
|
||||
| `RECRUIT` | Fill party to capacity | Logistics Companion |
|
||||
| `TRADE` | Execute profitable trade route | Caravan Companion |
|
||||
| `ALLY` | Pursue a non-aggression or alliance deal | Diplomacy Vassal |
|
||||
| `SPY` | Gain information on target faction | Scout Companion |
|
||||
| `HEAL` | Rest party until wounds recovered | Logistics Companion |
|
||||
| `CONSOLIDATE` | Hold territory, no expansion | Economy Vassal |
|
||||
| `TRAIN` | Level troops via auto-resolve bandits | War Vassal |
|
||||
|
||||
King updates the active subgoal at most once per **campaign tick** (configurable,
|
||||
default 1 in-game day). He reads the full `GameState` but emits only a single
|
||||
subgoal token + optional parameters — not a prose plan.
|
||||
|
||||
### King Decision Loop
|
||||
|
||||
```
|
||||
while campaign_running:
|
||||
state = gabs.get_state() # Full kingdom + map snapshot
|
||||
subgoal = king_llm.decide(state) # Qwen3:32b, temp=0.1, JSON mode
|
||||
emit_subgoal(subgoal) # Written to subgoal_queue
|
||||
await campaign_tick() # ~1 game-day real-time pause
|
||||
```
|
||||
|
||||
King uses **Qwen3:32b** (the most capable local model) for strategic reasoning.
|
||||
Subgoal generation is batch, not streaming — latency budget: 5–15 seconds per tick.
|
||||
|
||||
---
|
||||
|
||||
## 2. Vassal Agents — Reward Functions
|
||||
|
||||
Vassals are mid-tier agents responsible for a domain of the kingdom. Each vassal
|
||||
has a defined reward function. Vassals run on **Qwen3:14b** (balanced capability
|
||||
vs. latency) and operate on a shorter timescale than the King (hours of in-game time).
|
||||
|
||||
### 2a. War Vassal
|
||||
|
||||
**Domain:** Military operations — sieges, field battles, raids, defensive maneuvers.
|
||||
|
||||
**Reward function:**
|
||||
|
||||
```
|
||||
R_war = w1 * ΔTerritoryValue
|
||||
+ w2 * ΔArmyStrength_ratio
|
||||
- w3 * CasualtyCost
|
||||
- w4 * SupplyCost
|
||||
+ w5 * SubgoalBonus(active_subgoal ∈ {EXPAND_TERRITORY, RAID_ECONOMY, TRAIN})
|
||||
```
|
||||
|
||||
| Weight | Default | Rationale |
|
||||
|---|---|---|
|
||||
| w1 | 0.40 | Territory is the primary long-term asset |
|
||||
| w2 | 0.25 | Army ratio relative to nearest rival |
|
||||
| w3 | 0.20 | Casualties are expensive to replace |
|
||||
| w4 | 0.10 | Supply burn limits campaign duration |
|
||||
| w5 | 0.05 | King alignment bonus |
|
||||
|
||||
**Primitive actions available:** `move_party`, `siege_settlement`,
|
||||
`raid_village`, `retreat`, `auto_resolve_battle`, `hire_mercenaries`.
|
||||
|
||||
### 2b. Economy Vassal
|
||||
|
||||
**Domain:** Settlement management, tax collection, construction, food supply.
|
||||
|
||||
**Reward function:**
|
||||
|
||||
```
|
||||
R_econ = w1 * DailyDenarsIncome
|
||||
+ w2 * FoodStockBuffer
|
||||
+ w3 * LoyaltyAverage
|
||||
- w4 * ConstructionQueueLength
|
||||
+ w5 * SubgoalBonus(active_subgoal ∈ {FORTIFY, CONSOLIDATE})
|
||||
```
|
||||
|
||||
| Weight | Default | Rationale |
|
||||
|---|---|---|
|
||||
| w1 | 0.35 | Income is the fuel for everything |
|
||||
| w2 | 0.25 | Starvation causes immediate loyalty crash |
|
||||
| w3 | 0.20 | Low loyalty triggers revolt |
|
||||
| w4 | 0.15 | Idle construction is opportunity cost |
|
||||
| w5 | 0.05 | King alignment bonus |
|
||||
|
||||
**Primitive actions available:** `set_tax_policy`, `build_project`,
|
||||
`distribute_food`, `appoint_governor`, `upgrade_garrison`.
|
||||
|
||||
### 2c. Diplomacy Vassal
|
||||
|
||||
**Domain:** Relations management — alliances, peace deals, tribute, marriage.
|
||||
|
||||
**Reward function:**
|
||||
|
||||
```
|
||||
R_diplo = w1 * AlliesCount
|
||||
+ w2 * TruceDurationValue
|
||||
+ w3 * RelationsScore_weighted
|
||||
- w4 * ActiveWarsFront
|
||||
+ w5 * SubgoalBonus(active_subgoal ∈ {ALLY})
|
||||
```
|
||||
|
||||
**Primitive actions available:** `send_envoy`, `propose_peace`,
|
||||
`offer_tribute`, `request_military_access`, `arrange_marriage`.
|
||||
|
||||
---
|
||||
|
||||
## 3. Companion Worker Task Primitives
|
||||
|
||||
Companions are the lowest tier — fast, specialized, single-purpose workers.
|
||||
They run on **Qwen3:8b** (or smaller) for sub-2-second response times.
|
||||
Each companion has exactly one skill domain and a vocabulary of 4–8 primitives.
|
||||
|
||||
### 3a. Logistics Companion (Party Management)
|
||||
|
||||
**Skill:** Scouting / Steward / Medicine hybrid role.
|
||||
|
||||
| Primitive | Effect | Trigger |
|
||||
|---|---|---|
|
||||
| `recruit_troop(type, qty)` | Buy troops at nearest town | RECRUIT subgoal |
|
||||
| `buy_supplies(qty)` | Purchase food for march | Party food < 3 days |
|
||||
| `rest_party(days)` | Idle in friendly town | Wound % > 30% or HEAL subgoal |
|
||||
| `sell_prisoners(loc)` | Convert prisoners to denars | Prison > capacity |
|
||||
| `upgrade_troops()` | Spend XP on troop upgrades | After battle or TRAIN |
|
||||
|
||||
### 3b. Caravan Companion (Trade)
|
||||
|
||||
**Skill:** Trade / Charm.
|
||||
|
||||
| Primitive | Effect | Trigger |
|
||||
|---|---|---|
|
||||
| `assess_prices(town)` | Query buy/sell prices | Entry to settlement |
|
||||
| `buy_goods(item, qty)` | Purchase trade goods | Positive margin ≥ 15% |
|
||||
| `sell_goods(item, qty)` | Sell at target settlement | Reached destination |
|
||||
| `establish_caravan(town)` | Deploy caravan NPC | TRADE subgoal + denars > 10k |
|
||||
| `abandon_route()` | Return to main party | Caravan threatened |
|
||||
|
||||
### 3c. Scout Companion (Intelligence)
|
||||
|
||||
**Skill:** Scouting / Roguery.
|
||||
|
||||
| Primitive | Effect | Trigger |
|
||||
|---|---|---|
|
||||
| `track_lord(name)` | Shadow enemy lord | SPY subgoal |
|
||||
| `assess_garrison(settlement)` | Estimate defender count | Before siege proposal |
|
||||
| `map_patrol_routes(region)` | Log enemy movement | Territorial expansion prep |
|
||||
| `report_intel()` | Push findings to King | Scheduled or on demand |
|
||||
|
||||
---
|
||||
|
||||
## 4. Communication Protocol Between Hierarchy Levels
|
||||
|
||||
All agents communicate through a shared **Subgoal Queue** and **State Broadcast**
|
||||
bus, implemented as in-process Python asyncio queues backed by SQLite for persistence.
|
||||
|
||||
### Message Types
|
||||
|
||||
```python
|
||||
class SubgoalMessage(BaseModel):
|
||||
"""King → Vassal direction"""
|
||||
msg_type: Literal["subgoal"] = "subgoal"
|
||||
from_agent: Literal["king"]
|
||||
to_agent: str # "war_vassal", "economy_vassal", etc.
|
||||
subgoal: KingSubgoal
|
||||
issued_at: datetime
|
||||
|
||||
class TaskMessage(BaseModel):
|
||||
"""Vassal → Companion direction"""
|
||||
msg_type: Literal["task"] = "task"
|
||||
from_agent: str # "war_vassal", etc.
|
||||
to_agent: str # "logistics_companion", etc.
|
||||
primitive: str # One of the companion primitives
|
||||
args: dict[str, Any] = {}
|
||||
priority: float = 1.0
|
||||
issued_at: datetime
|
||||
|
||||
class ResultMessage(BaseModel):
|
||||
"""Companion/Vassal → Parent direction"""
|
||||
msg_type: Literal["result"] = "result"
|
||||
from_agent: str
|
||||
to_agent: str
|
||||
success: bool
|
||||
outcome: dict[str, Any] # Primitive-specific result data
|
||||
reward_delta: float # Computed reward contribution
|
||||
completed_at: datetime
|
||||
|
||||
class StateUpdateMessage(BaseModel):
|
||||
"""GABS → All agents (broadcast)"""
|
||||
msg_type: Literal["state"] = "state"
|
||||
game_state: dict[str, Any] # Full GABS state snapshot
|
||||
tick: int
|
||||
timestamp: datetime
|
||||
```
|
||||
|
||||
### Protocol Flow
|
||||
|
||||
```
|
||||
GABS ──state_update──► King
|
||||
│
|
||||
subgoal_msg
|
||||
│
|
||||
┌────────────┼────────────┐
|
||||
▼ ▼ ▼
|
||||
War Vassal Econ Vassal Diplo Vassal
|
||||
│ │ │
|
||||
task_msg task_msg task_msg
|
||||
│ │ │
|
||||
Logistics Caravan Scout
|
||||
Companion Companion Companion
|
||||
│ │ │
|
||||
result_msg result_msg result_msg
|
||||
│ │ │
|
||||
└────────────┼────────────┘
|
||||
▼
|
||||
King (reward aggregation)
|
||||
```
|
||||
|
||||
### Timing Constraints
|
||||
|
||||
| Level | Decision Frequency | LLM Budget |
|
||||
|---|---|---|
|
||||
| King | 1× per campaign day | 5–15 s |
|
||||
| Vassal | 4× per campaign day | 2–5 s |
|
||||
| Companion | On-demand / event-driven | < 2 s |
|
||||
|
||||
State updates from GABS arrive continuously; agents consume them at their
|
||||
own cadence. No agent blocks another's queue.
|
||||
|
||||
### Conflict Resolution
|
||||
|
||||
If two vassals propose conflicting actions (e.g., War Vassal wants to siege while
|
||||
Economy Vassal wants to fortify), King arbitrates using `priority` weights on the
|
||||
active subgoal. The highest-priority active subgoal wins resource contention.
|
||||
|
||||
---
|
||||
|
||||
## 5. Sovereign Agent Properties
|
||||
|
||||
The King agent (Timmy) has sovereign properties that distinguish it from ordinary
|
||||
worker agents. These map directly to Timmy's existing identity architecture.
|
||||
|
||||
### 5a. Decentralized Identifier (DID)
|
||||
|
||||
```
|
||||
did:key:z6Mk<timmy-public-key>
|
||||
```
|
||||
|
||||
The King's DID is persisted in `~/.timmy/identity.json` (existing SOUL.md pattern).
|
||||
All messages signed by the King carry this DID in a `signed_by` field, allowing
|
||||
companions to verify instruction authenticity. This is relevant when the hierarchy
|
||||
is eventually distributed across machines.
|
||||
|
||||
### 5b. Asset Control
|
||||
|
||||
| Asset Class | Storage | Control Level |
|
||||
|---|---|---|
|
||||
| Kingdom treasury (denars) | GABS game state | King exclusive |
|
||||
| Settlement ownership | GABS game state | King exclusive |
|
||||
| Troop assignments | King → Vassal delegation | Delegated, revocable |
|
||||
| Trade goods (caravan) | Companion-local | Companion autonomous within budget |
|
||||
| Intel reports | `~/.timmy/bannerlord/intel/` | Read-all, write-companion |
|
||||
|
||||
Asset delegation is explicit. Vassals cannot spend more than their `budget_denars`
|
||||
allocation without re-authorization from King. Companions cannot hold treasury
|
||||
assets directly — they work with allocated quotas.
|
||||
|
||||
### 5c. Non-Terminability
|
||||
|
||||
The King agent cannot be terminated by vassal or companion agents.
|
||||
Termination authority is reserved for:
|
||||
1. The human operator (Ctrl+C or `timmy stop`)
|
||||
2. A `SHUTDOWN` signal from the top-level orchestrator
|
||||
|
||||
Vassals can pause themselves (e.g., awaiting GABS state) but cannot signal the King
|
||||
to stop. This prevents a misbehaving military vassal from ending the campaign.
|
||||
|
||||
Implementation: King runs in the main asyncio event loop. Vassals and companions
|
||||
run in `asyncio.TaskGroup` subgroups. Only the King's task holds a reference to
|
||||
the TaskGroup cancel scope.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Path
|
||||
|
||||
This design connects directly to the existing Timmy codebase:
|
||||
|
||||
| Component | Maps to | Notes |
|
||||
|---|---|---|
|
||||
| King LLM calls | `infrastructure/llm_router/` | Cascade router for model selection |
|
||||
| Subgoal Queue | `infrastructure/event_bus/` | Existing pub/sub pattern |
|
||||
| Companion primitives | New `src/bannerlord/agents/` package | One module per companion |
|
||||
| GABS state updates | `src/bannerlord/gabs_client.py` | TCP JSON-RPC, port 4825 |
|
||||
| Asset ledger | `src/bannerlord/ledger.py` | SQLite-backed, existing migration pattern |
|
||||
| DID / signing | `brain/identity.py` | Extends existing SOUL.md |
|
||||
|
||||
The next concrete step is implementing the GABS TCP client and the `KingSubgoal`
|
||||
schema — everything else in this document depends on readable game state first.
|
||||
|
||||
---
|
||||
|
||||
## References
|
||||
|
||||
- Ahilan, S. & Dayan, P. (2019). Feudal Multi-Agent Hierarchies for Cooperative
|
||||
Reinforcement Learning. https://arxiv.org/abs/1901.08492
|
||||
- Rood, S. (2022). Scaling Reinforcement Learning through Feudal Hierarchy (NPS thesis).
|
||||
- Wang, G. et al. (2023). Voyager: An Open-Ended Embodied Agent with Large Language
|
||||
Models. https://arxiv.org/abs/2305.16291
|
||||
- Park, J.S. et al. (2023). Generative Agents: Interactive Simulacra of Human Behavior.
|
||||
https://arxiv.org/abs/2304.03442
|
||||
- Silveira, T. (2022). CiF-Bannerlord: Social AI Integration in Bannerlord.
|
||||
74
docs/research/integration-architecture-deep-dives.md
Normal file
74
docs/research/integration-architecture-deep-dives.md
Normal file
@@ -0,0 +1,74 @@
|
||||
# Timmy Time Integration Architecture: Eight Deep Dives into Real Deployment
|
||||
|
||||
> **Source:** PDF attached to issue #946, written during Veloren exploration phase.
|
||||
> Many patterns are game-agnostic and apply to the Morrowind/OpenClaw pivot.
|
||||
|
||||
## Summary of Eight Deep Dives
|
||||
|
||||
### 1. Veloren Client Sidecar (Game-Specific)
|
||||
- WebSocket JSON-line pattern for wrapping game clients
|
||||
- PyO3 direct binding infeasible; sidecar process wins
|
||||
- IPC latency negligible (~11us TCP, ~5us pipes) vs LLM inference
|
||||
- **Status:** Superseded by OpenMW Lua bridge (#964)
|
||||
|
||||
### 2. Agno Ollama Tool Calling is Broken
|
||||
- Agno issues #2231, #2625, #1419, #1612, #4715 document persistent breakage
|
||||
- Root cause: Agno's Ollama model class doesn't robustly parse native tool_calls
|
||||
- **Fix:** Use Ollama's `format` parameter with Pydantic JSON schemas directly
|
||||
- Recommended models: qwen3-coder:32b (top), glm-4.7-flash, gpt-oss:20b
|
||||
- Critical settings: temperature 0.0-0.2, stream=False for tool calls
|
||||
- **Status:** Covered by #966 (three-tier router)
|
||||
|
||||
### 3. MCP is the Right Abstraction
|
||||
- FastMCP averages 26.45ms per tool call (TM Dev Lab benchmark, Feb 2026)
|
||||
- Total MCP overhead per cycle: ~20-60ms (<3% of 2-second budget)
|
||||
- Agno has first-class bidirectional MCP integration (MCPTools, MultiMCPTools)
|
||||
- Use stdio transport for near-zero latency; return compressed JPEG not base64
|
||||
- **Status:** Covered by #984 (MCP restore)
|
||||
|
||||
### 4. Human + AI Co-op Architecture (Game-Specific)
|
||||
- Headless client treated identically to graphical client by server
|
||||
- Leverages party system, trade API, and /tell for communication
|
||||
- Mode switching: solo autonomous play when human absent, assist when present
|
||||
- **Status:** Defer until after tutorial completion
|
||||
|
||||
### 5. Real Latency Numbers
|
||||
- All-local M3 Max pipeline: 4-9 seconds per full cycle
|
||||
- Groq hybrid pipeline: 3-7 seconds per full cycle
|
||||
- VLM inference is 50-70% of total pipeline time (bottleneck)
|
||||
- Dual-model Ollama on 96GB M3 Max: ~11-14GB, ~70GB free
|
||||
- **Status:** Superseded by API-first perception (#963)
|
||||
|
||||
### 6. Content Moderation (Three-Layer Defense)
|
||||
- Layer 1: Game-context system prompts (Morrowind themes as game mechanics)
|
||||
- Layer 2: Llama Guard 3 1B at <30ms/sentence for real-time filtering
|
||||
- Layer 3: Per-game moderation profiles with vocabulary whitelists
|
||||
- Run moderation + TTS preprocessing in parallel for zero added latency
|
||||
- Neuro-sama incident (Dec 2022) is the cautionary tale
|
||||
- **Status:** New issue created → #1056
|
||||
|
||||
### 7. Model Selection (Qwen3-8B vs Hermes 3)
|
||||
- Three-role architecture: Perception (Qwen3-VL 8B), Decision (Qwen3-8B), Narration (Hermes 3 8B)
|
||||
- Qwen3-8B outperforms Qwen2.5-14B on 15 benchmarks
|
||||
- Hermes 3 best for narration (steerability, roleplaying)
|
||||
- Both use identical Hermes Function Calling standard
|
||||
- **Status:** Partially covered by #966 (three-tier router)
|
||||
|
||||
### 8. Split Hetzner + Mac Deployment
|
||||
- Hetzner GEX44 (RTX 4000 SFF Ada, €184/month) for rendering/streaming
|
||||
- Mac M3 Max for all AI inference via Tailscale
|
||||
- Use FFmpeg x11grab + NVENC, not OBS (no headless support)
|
||||
- Use headless Xorg, not Xvfb (GPU access required for Vulkan)
|
||||
- Total cost: ~$200/month
|
||||
- **Status:** Referenced in #982 sprint plan
|
||||
|
||||
## Cross-Reference to Active Issues
|
||||
|
||||
| Research Topic | Active Issue | Status |
|
||||
|---------------|-------------|--------|
|
||||
| Pydantic structured output for Ollama | #966 (three-tier router) | In progress |
|
||||
| FastMCP tool server | #984 (MCP restore) | In progress |
|
||||
| Content moderation pipeline | #1056 (new) | Created from this research |
|
||||
| Split Hetzner + Mac deployment | #982 (sprint plan) | Referenced |
|
||||
| VLM latency / perception | #963 (perception bottleneck) | API-first approach |
|
||||
| OpenMW bridge (replaces Veloren sidecar) | #964 | In progress |
|
||||
726
poetry.lock
generated
726
poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -50,6 +50,7 @@ sounddevice = { version = ">=0.4.6", optional = true }
|
||||
sentence-transformers = { version = ">=2.0.0", optional = true }
|
||||
numpy = { version = ">=1.24.0", optional = true }
|
||||
requests = { version = ">=2.31.0", optional = true }
|
||||
trafilatura = { version = ">=1.6.0", optional = true }
|
||||
GitPython = { version = ">=3.1.40", optional = true }
|
||||
pytest = { version = ">=8.0.0", optional = true }
|
||||
pytest-asyncio = { version = ">=0.24.0", optional = true }
|
||||
@@ -67,6 +68,7 @@ voice = ["pyttsx3", "openai-whisper", "piper-tts", "sounddevice"]
|
||||
celery = ["celery"]
|
||||
embeddings = ["sentence-transformers", "numpy"]
|
||||
git = ["GitPython"]
|
||||
research = ["requests", "trafilatura", "google-search-results"]
|
||||
dev = ["pytest", "pytest-asyncio", "pytest-cov", "pytest-timeout", "pytest-randomly", "pytest-xdist", "selenium"]
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
|
||||
@@ -17,8 +17,23 @@ REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
RETRO_FILE = REPO_ROOT / ".loop" / "retro" / "cycles.jsonl"
|
||||
SUMMARY_FILE = REPO_ROOT / ".loop" / "retro" / "summary.json"
|
||||
|
||||
GITEA_API = "http://localhost:3000/api/v1"
|
||||
REPO_SLUG = "rockachopa/Timmy-time-dashboard"
|
||||
|
||||
def _get_gitea_api() -> str:
|
||||
"""Read Gitea API URL from env var, then ~/.hermes/gitea_api file, then default."""
|
||||
# Check env vars first (TIMMY_GITEA_API is preferred, GITEA_API for compatibility)
|
||||
api_url = os.environ.get("TIMMY_GITEA_API") or os.environ.get("GITEA_API")
|
||||
if api_url:
|
||||
return api_url
|
||||
# Check ~/.hermes/gitea_api file
|
||||
api_file = Path.home() / ".hermes" / "gitea_api"
|
||||
if api_file.exists():
|
||||
return api_file.read_text().strip()
|
||||
# Default fallback
|
||||
return "http://localhost:3000/api/v1"
|
||||
|
||||
|
||||
GITEA_API = _get_gitea_api()
|
||||
REPO_SLUG = os.environ.get("REPO_SLUG", "rockachopa/Timmy-time-dashboard")
|
||||
TOKEN_FILE = Path.home() / ".hermes" / "gitea_token"
|
||||
|
||||
TAG_RE = re.compile(r"\[([^\]]+)\]")
|
||||
|
||||
186
scripts/claude_quota_check.sh
Executable file
186
scripts/claude_quota_check.sh
Executable file
@@ -0,0 +1,186 @@
|
||||
#!/bin/bash
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# claude_quota_check.sh — Check Claude Code / Claude.ai quota
|
||||
#
|
||||
# Usage:
|
||||
# ./claude_quota_check.sh # Human-readable output
|
||||
# ./claude_quota_check.sh --json # Raw JSON for piping
|
||||
# ./claude_quota_check.sh --watch # Refresh every 60s
|
||||
#
|
||||
# Requires: macOS with Claude Code authenticated, python3
|
||||
# Token is read from macOS Keychain (same as Claude Code uses)
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ── Extract OAuth token from macOS Keychain ──
|
||||
get_token() {
|
||||
local creds
|
||||
creds=$(security find-generic-password -s "Claude Code-credentials" -w 2>/dev/null) || {
|
||||
echo "ERROR: No Claude Code credentials found in Keychain." >&2
|
||||
echo "Run 'claude' and authenticate first." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
echo "$creds" | python3 -c "
|
||||
import sys, json
|
||||
data = json.load(sys.stdin)
|
||||
oauth = data.get('claudeAiOauth', data)
|
||||
print(oauth['accessToken'])
|
||||
" 2>/dev/null || {
|
||||
echo "ERROR: Could not parse credentials JSON." >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# ── Fetch usage from Anthropic API ──
|
||||
fetch_usage() {
|
||||
local token="$1"
|
||||
curl -s "https://api.anthropic.com/api/oauth/usage" \
|
||||
-H "Accept: application/json" \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "User-Agent: claude-code/2.0.32" \
|
||||
-H "Authorization: Bearer ${token}" \
|
||||
-H "anthropic-beta: oauth-2025-04-20"
|
||||
}
|
||||
|
||||
# ── Format time remaining ──
|
||||
time_remaining() {
|
||||
local reset_at="$1"
|
||||
if [ -z "$reset_at" ] || [ "$reset_at" = "null" ]; then
|
||||
echo "unknown"
|
||||
return
|
||||
fi
|
||||
|
||||
python3 -c "
|
||||
from datetime import datetime, timezone
|
||||
reset = datetime.fromisoformat('${reset_at}'.replace('Z', '+00:00'))
|
||||
now = datetime.now(timezone.utc)
|
||||
diff = reset - now
|
||||
if diff.total_seconds() <= 0:
|
||||
print('resetting now')
|
||||
else:
|
||||
hours = int(diff.total_seconds() // 3600)
|
||||
mins = int((diff.total_seconds() % 3600) // 60)
|
||||
if hours > 0:
|
||||
print(f'{hours}h {mins}m')
|
||||
else:
|
||||
print(f'{mins}m')
|
||||
" 2>/dev/null || echo "unknown"
|
||||
}
|
||||
|
||||
# ── Bar visualization ──
|
||||
usage_bar() {
|
||||
local pct=$1
|
||||
local width=30
|
||||
local filled
|
||||
filled=$(python3 -c "print(int(${pct} * ${width}))")
|
||||
local empty=$((width - filled))
|
||||
|
||||
# Color: green < 50%, yellow 50-80%, red > 80%
|
||||
local color=""
|
||||
if (( $(echo "$pct < 0.50" | bc -l) )); then
|
||||
color="\033[32m" # green
|
||||
elif (( $(echo "$pct < 0.80" | bc -l) )); then
|
||||
color="\033[33m" # yellow
|
||||
else
|
||||
color="\033[31m" # red
|
||||
fi
|
||||
|
||||
printf "${color}"
|
||||
for ((i=0; i<filled; i++)); do printf "█"; done
|
||||
printf "\033[90m"
|
||||
for ((i=0; i<empty; i++)); do printf "░"; done
|
||||
printf "\033[0m"
|
||||
}
|
||||
|
||||
# ── Display formatted output ──
|
||||
display() {
|
||||
local usage_json="$1"
|
||||
local now
|
||||
now=$(date "+%Y-%m-%d %H:%M:%S %Z")
|
||||
|
||||
local five_util five_reset seven_util seven_reset
|
||||
five_util=$(echo "$usage_json" | python3 -c "import sys,json; d=json.load(sys.stdin); h=d.get('five_hour') or {}; print(h.get('utilization', 0))" 2>/dev/null || echo "0")
|
||||
five_reset=$(echo "$usage_json" | python3 -c "import sys,json; d=json.load(sys.stdin); h=d.get('five_hour') or {}; print(h.get('resets_at', 'null'))" 2>/dev/null || echo "null")
|
||||
seven_util=$(echo "$usage_json" | python3 -c "import sys,json; d=json.load(sys.stdin); h=d.get('seven_day') or {}; print(h.get('utilization', 0))" 2>/dev/null || echo "0")
|
||||
seven_reset=$(echo "$usage_json" | python3 -c "import sys,json; d=json.load(sys.stdin); h=d.get('seven_day') or {}; print(h.get('resets_at', 'null'))" 2>/dev/null || echo "null")
|
||||
|
||||
local five_pct seven_pct
|
||||
five_pct=$(python3 -c "print(int(float('${five_util}') * 100))")
|
||||
seven_pct=$(python3 -c "print(int(float('${seven_util}') * 100))")
|
||||
|
||||
local five_remaining seven_remaining
|
||||
five_remaining=$(time_remaining "$five_reset")
|
||||
seven_remaining=$(time_remaining "$seven_reset")
|
||||
|
||||
echo ""
|
||||
echo " ┌─────────────────────────────────────────────┐"
|
||||
echo " │ CLAUDE QUOTA STATUS │"
|
||||
printf " │ %-38s│\n" "$now"
|
||||
echo " ├─────────────────────────────────────────────┤"
|
||||
printf " │ 5-hour window: "
|
||||
usage_bar "$five_util"
|
||||
printf " %3d%% │\n" "$five_pct"
|
||||
printf " │ Resets in: %-33s│\n" "$five_remaining"
|
||||
echo " │ │"
|
||||
printf " │ 7-day window: "
|
||||
usage_bar "$seven_util"
|
||||
printf " %3d%% │\n" "$seven_pct"
|
||||
printf " │ Resets in: %-33s│\n" "$seven_remaining"
|
||||
echo " └─────────────────────────────────────────────┘"
|
||||
echo ""
|
||||
|
||||
# Decision guidance for Timmy
|
||||
if (( five_pct >= 80 )); then
|
||||
echo " ⚠ 5-hour window critical. Switch to local Qwen3-14B."
|
||||
echo " Reserve remaining quota for high-value tasks only."
|
||||
elif (( five_pct >= 50 )); then
|
||||
echo " ~ 5-hour window half spent. Batch remaining requests."
|
||||
else
|
||||
echo " ✓ 5-hour window healthy. Full speed ahead."
|
||||
fi
|
||||
|
||||
if (( seven_pct >= 80 )); then
|
||||
echo " ⚠ Weekly quota critical! Operate in local-only mode."
|
||||
elif (( seven_pct >= 60 )); then
|
||||
echo " ~ Weekly quota past 60%. Plan usage carefully."
|
||||
fi
|
||||
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ── Main ──
|
||||
main() {
|
||||
local token
|
||||
token=$(get_token)
|
||||
|
||||
local usage
|
||||
usage=$(fetch_usage "$token")
|
||||
|
||||
if [ -z "$usage" ] || echo "$usage" | grep -q '"error"'; then
|
||||
echo "ERROR: Failed to fetch usage data." >&2
|
||||
echo "$usage" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "${1:-}" in
|
||||
--json)
|
||||
echo "$usage" | python3 -m json.tool
|
||||
;;
|
||||
--watch)
|
||||
while true; do
|
||||
clear
|
||||
usage=$(fetch_usage "$token")
|
||||
display "$usage"
|
||||
echo " Refreshing in 60s... (Ctrl+C to stop)"
|
||||
sleep 60
|
||||
done
|
||||
;;
|
||||
*)
|
||||
display "$usage"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -277,6 +277,8 @@ def main() -> None:
|
||||
args.tests_passed = int(cr["tests_passed"])
|
||||
if not args.notes and cr.get("notes"):
|
||||
args.notes = cr["notes"]
|
||||
# Consume-once: delete after reading so stale results don't poison future cycles
|
||||
CYCLE_RESULT_FILE.unlink(missing_ok=True)
|
||||
|
||||
# Auto-detect issue from branch when not explicitly provided
|
||||
if args.issue is None:
|
||||
|
||||
83
scripts/gitea_backup.sh
Executable file
83
scripts/gitea_backup.sh
Executable file
@@ -0,0 +1,83 @@
|
||||
#!/bin/bash
|
||||
# Gitea backup script — run on the VPS before any hardening changes.
|
||||
# Usage: sudo bash scripts/gitea_backup.sh [off-site-dest]
|
||||
#
|
||||
# off-site-dest: optional rsync/scp destination for off-site copy
|
||||
# e.g. user@backup-host:/backups/gitea/
|
||||
#
|
||||
# Refs: #971, #990
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
BACKUP_DIR="/opt/gitea/backups"
|
||||
TIMESTAMP=$(date +"%Y%m%d_%H%M%S")
|
||||
GITEA_CONF="/etc/gitea/app.ini"
|
||||
GITEA_WORK_DIR="/var/lib/gitea"
|
||||
OFFSITE_DEST="${1:-}"
|
||||
|
||||
echo "=== Gitea Backup — $TIMESTAMP ==="
|
||||
|
||||
# Ensure backup directory exists
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
cd "$BACKUP_DIR"
|
||||
|
||||
# Run the dump
|
||||
echo "[1/4] Running gitea dump..."
|
||||
gitea dump -c "$GITEA_CONF"
|
||||
|
||||
# Find the newest zip (gitea dump names it gitea-dump-*.zip)
|
||||
BACKUP_FILE=$(ls -t "$BACKUP_DIR"/gitea-dump-*.zip 2>/dev/null | head -1)
|
||||
|
||||
if [ -z "$BACKUP_FILE" ]; then
|
||||
echo "ERROR: No backup zip found in $BACKUP_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BACKUP_SIZE=$(stat -c%s "$BACKUP_FILE" 2>/dev/null || stat -f%z "$BACKUP_FILE")
|
||||
echo "[2/4] Backup created: $BACKUP_FILE ($BACKUP_SIZE bytes)"
|
||||
|
||||
if [ "$BACKUP_SIZE" -eq 0 ]; then
|
||||
echo "ERROR: Backup file is 0 bytes"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Lock down permissions
|
||||
chmod 600 "$BACKUP_FILE"
|
||||
|
||||
# Verify contents
|
||||
echo "[3/4] Verifying backup contents..."
|
||||
CONTENTS=$(unzip -l "$BACKUP_FILE" 2>/dev/null || true)
|
||||
|
||||
check_component() {
|
||||
if echo "$CONTENTS" | grep -q "$1"; then
|
||||
echo " OK: $2"
|
||||
else
|
||||
echo " WARN: $2 not found in backup"
|
||||
fi
|
||||
}
|
||||
|
||||
check_component "gitea-db.sql" "Database dump"
|
||||
check_component "gitea-repo" "Repositories"
|
||||
check_component "custom" "Custom config"
|
||||
check_component "app.ini" "app.ini"
|
||||
|
||||
# Off-site copy
|
||||
if [ -n "$OFFSITE_DEST" ]; then
|
||||
echo "[4/4] Copying to off-site: $OFFSITE_DEST"
|
||||
rsync -avz "$BACKUP_FILE" "$OFFSITE_DEST"
|
||||
echo " Off-site copy complete."
|
||||
else
|
||||
echo "[4/4] No off-site destination provided. Skipping."
|
||||
echo " To copy later: scp $BACKUP_FILE user@backup-host:/backups/gitea/"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Backup complete ==="
|
||||
echo "File: $BACKUP_FILE"
|
||||
echo "Size: $BACKUP_SIZE bytes"
|
||||
echo ""
|
||||
echo "To verify restore on a clean instance:"
|
||||
echo " 1. Copy zip to test machine"
|
||||
echo " 2. unzip $BACKUP_FILE"
|
||||
echo " 3. gitea restore --from <extracted-dir> -c /etc/gitea/app.ini"
|
||||
echo " 4. Verify repos and DB are intact"
|
||||
@@ -30,7 +30,22 @@ IDLE_STATE_FILE = REPO_ROOT / ".loop" / "idle_state.json"
|
||||
CYCLE_RESULT_FILE = REPO_ROOT / ".loop" / "cycle_result.json"
|
||||
TOKEN_FILE = Path.home() / ".hermes" / "gitea_token"
|
||||
|
||||
GITEA_API = os.environ.get("GITEA_API", "http://localhost:3000/api/v1")
|
||||
|
||||
def _get_gitea_api() -> str:
|
||||
"""Read Gitea API URL from env var, then ~/.hermes/gitea_api file, then default."""
|
||||
# Check env vars first (TIMMY_GITEA_API is preferred, GITEA_API for compatibility)
|
||||
api_url = os.environ.get("TIMMY_GITEA_API") or os.environ.get("GITEA_API")
|
||||
if api_url:
|
||||
return api_url
|
||||
# Check ~/.hermes/gitea_api file
|
||||
api_file = Path.home() / ".hermes" / "gitea_api"
|
||||
if api_file.exists():
|
||||
return api_file.read_text().strip()
|
||||
# Default fallback
|
||||
return "http://localhost:3000/api/v1"
|
||||
|
||||
|
||||
GITEA_API = _get_gitea_api()
|
||||
REPO_SLUG = os.environ.get("REPO_SLUG", "rockachopa/Timmy-time-dashboard")
|
||||
|
||||
# Default cycle duration in seconds (5 min); stale threshold = 2× this
|
||||
@@ -187,7 +202,11 @@ def load_queue() -> list[dict]:
|
||||
# Persist the cleaned queue so stale entries don't recur
|
||||
_save_cleaned_queue(data, open_numbers)
|
||||
return ready
|
||||
except (json.JSONDecodeError, OSError):
|
||||
except json.JSONDecodeError as exc:
|
||||
print(f"[loop-guard] WARNING: Corrupt queue.json ({exc}) — returning empty queue")
|
||||
return []
|
||||
except OSError as exc:
|
||||
print(f"[loop-guard] WARNING: Cannot read queue.json ({exc}) — returning empty queue")
|
||||
return []
|
||||
|
||||
|
||||
|
||||
107
scripts/run_benchmarks.py
Normal file
107
scripts/run_benchmarks.py
Normal file
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the agent performance regression benchmark suite.
|
||||
|
||||
Usage::
|
||||
|
||||
python scripts/run_benchmarks.py # all scenarios
|
||||
python scripts/run_benchmarks.py --tags navigation # filter by tag
|
||||
python scripts/run_benchmarks.py --output results/benchmarks.jsonl
|
||||
python scripts/run_benchmarks.py --compare results/benchmarks.jsonl
|
||||
|
||||
Exit codes:
|
||||
0 — all scenarios passed
|
||||
1 — one or more scenarios failed
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Ensure src/ is on the path when invoked directly
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src"))
|
||||
|
||||
from infrastructure.world.benchmark.metrics import BenchmarkMetrics, load_history
|
||||
from infrastructure.world.benchmark.runner import BenchmarkRunner
|
||||
from infrastructure.world.benchmark.scenarios import load_scenarios
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Agent performance regression benchmark suite",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tags",
|
||||
nargs="*",
|
||||
default=None,
|
||||
help="Filter scenarios by tag (e.g. navigation quest)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="JSONL file to append results to",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compare",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="JSONL file with baseline results for regression comparison",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
args = parse_args()
|
||||
|
||||
scenarios = load_scenarios(tags=args.tags)
|
||||
if not scenarios:
|
||||
print("No matching scenarios found.")
|
||||
return 1
|
||||
|
||||
print(f"Running {len(scenarios)} benchmark scenario(s)...\n")
|
||||
|
||||
runner = BenchmarkRunner()
|
||||
metrics = await runner.run(scenarios)
|
||||
|
||||
print(metrics.summary())
|
||||
|
||||
if args.output:
|
||||
metrics.save(args.output)
|
||||
|
||||
if args.compare:
|
||||
history = load_history(args.compare)
|
||||
if history:
|
||||
from infrastructure.world.benchmark.metrics import compare_runs
|
||||
|
||||
# Reconstruct baseline from last recorded run
|
||||
last = history[0]
|
||||
baseline = BenchmarkMetrics(
|
||||
timestamp=last.get("timestamp", ""),
|
||||
commit_sha=last.get("commit_sha", ""),
|
||||
total_time_ms=last.get("total_time_ms", 0),
|
||||
)
|
||||
for s in last.get("scenarios", []):
|
||||
from infrastructure.world.benchmark.metrics import ScenarioResult
|
||||
|
||||
baseline.results.append(
|
||||
ScenarioResult(
|
||||
scenario_name=s["scenario_name"],
|
||||
success=s["success"],
|
||||
cycles_used=s["cycles_used"],
|
||||
max_cycles=s["max_cycles"],
|
||||
wall_time_ms=s.get("wall_time_ms", 0),
|
||||
llm_calls=s.get("llm_calls", 0),
|
||||
metabolic_cost=s.get("metabolic_cost", 0.0),
|
||||
)
|
||||
)
|
||||
print()
|
||||
print(compare_runs(metrics, baseline))
|
||||
|
||||
return 0 if metrics.fail_count == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(asyncio.run(main()))
|
||||
342
scripts/test_hermes4.py
Normal file
342
scripts/test_hermes4.py
Normal file
@@ -0,0 +1,342 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Hermes 4 smoke test and tool-calling validation script.
|
||||
|
||||
Tests the Hermes 4 14B model after importing into Ollama. Covers:
|
||||
1. Basic connectivity — model responds
|
||||
2. Memory usage — under 28 GB with model loaded
|
||||
3. Tool calling — structured JSON output (not raw text)
|
||||
4. Reasoning — <think> tag toggling works
|
||||
5. Timmy-persona smoke test — agent identity prompt
|
||||
|
||||
Usage:
|
||||
python scripts/test_hermes4.py # Run all tests
|
||||
python scripts/test_hermes4.py --model hermes4-14b
|
||||
python scripts/test_hermes4.py --model hermes4-36b --ctx 8192
|
||||
|
||||
Epic: #1091 Project Bannerlord — AutoLoRA Sovereignty Loop (Step 2 of 7)
|
||||
Refs: #1101
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
print("ERROR: 'requests' not installed. Run: pip install requests")
|
||||
sys.exit(1)
|
||||
|
||||
OLLAMA_URL = "http://localhost:11434"
|
||||
DEFAULT_MODEL = "hermes4-14b"
|
||||
MEMORY_LIMIT_GB = 28.0
|
||||
|
||||
# ── Tool schema used for tool-calling tests ──────────────────────────────────
|
||||
|
||||
READ_FILE_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "read_file",
|
||||
"description": "Read the contents of a file at the given path",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"path": {
|
||||
"type": "string",
|
||||
"description": "Absolute or relative path to the file",
|
||||
}
|
||||
},
|
||||
"required": ["path"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
LIST_ISSUES_TOOL = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "list_issues",
|
||||
"description": "List open issues from a Gitea repository",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"repo": {"type": "string", "description": "owner/repo slug"},
|
||||
"state": {
|
||||
"type": "string",
|
||||
"enum": ["open", "closed", "all"],
|
||||
"description": "Issue state filter",
|
||||
},
|
||||
},
|
||||
"required": ["repo"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _post(endpoint: str, payload: dict, timeout: int = 60) -> dict[str, Any]:
|
||||
"""POST to Ollama and return parsed JSON."""
|
||||
url = f"{OLLAMA_URL}{endpoint}"
|
||||
resp = requests.post(url, json=payload, timeout=timeout)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _ollama_memory_gb() -> float:
|
||||
"""Estimate Ollama process RSS in GB using ps (macOS/Linux)."""
|
||||
try:
|
||||
# Look for ollama process RSS (macOS: column 6 in MB, Linux: column 6 in KB)
|
||||
result = subprocess.run(
|
||||
["ps", "-axo", "pid,comm,rss"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
total_kb = 0
|
||||
for line in result.stdout.splitlines():
|
||||
if "ollama" in line.lower():
|
||||
parts = line.split()
|
||||
try:
|
||||
total_kb += int(parts[-1])
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
return total_kb / (1024 * 1024) # KB → GB
|
||||
except Exception:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _check_model_available(model: str) -> bool:
|
||||
"""Return True if model is listed in Ollama."""
|
||||
try:
|
||||
resp = requests.get(f"{OLLAMA_URL}/api/tags", timeout=10)
|
||||
resp.raise_for_status()
|
||||
names = [m["name"] for m in resp.json().get("models", [])]
|
||||
return any(model in n for n in names)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _chat(model: str, messages: list[dict], tools: list | None = None) -> dict:
|
||||
"""Send a chat request to Ollama."""
|
||||
payload: dict = {"model": model, "messages": messages, "stream": False}
|
||||
if tools:
|
||||
payload["tools"] = tools
|
||||
return _post("/api/chat", payload, timeout=120)
|
||||
|
||||
|
||||
# ── Test cases ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_model_available(model: str) -> bool:
|
||||
"""PASS: model is registered in Ollama."""
|
||||
print(f"\n[1/5] Checking model availability: {model}")
|
||||
if _check_model_available(model):
|
||||
print(f" ✓ {model} is available in Ollama")
|
||||
return True
|
||||
print(
|
||||
f" ✗ {model} not found. Import with:\n"
|
||||
f" ollama create {model} -f Modelfile.hermes4-14b\n"
|
||||
f" Or pull directly if on registry:\n"
|
||||
f" ollama pull {model}"
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def test_basic_response(model: str) -> bool:
|
||||
"""PASS: model responds coherently to a simple prompt."""
|
||||
print(f"\n[2/5] Basic response test")
|
||||
messages = [
|
||||
{"role": "user", "content": "Reply with exactly: HERMES_OK"},
|
||||
]
|
||||
try:
|
||||
t0 = time.time()
|
||||
data = _chat(model, messages)
|
||||
elapsed = time.time() - t0
|
||||
content = data.get("message", {}).get("content", "")
|
||||
if "HERMES_OK" in content:
|
||||
print(f" ✓ Basic response OK ({elapsed:.1f}s): {content.strip()}")
|
||||
return True
|
||||
print(f" ✗ Unexpected response ({elapsed:.1f}s): {content[:200]!r}")
|
||||
return False
|
||||
except Exception as exc:
|
||||
print(f" ✗ Request failed: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def test_memory_usage() -> bool:
|
||||
"""PASS: Ollama process RSS is under MEMORY_LIMIT_GB."""
|
||||
print(f"\n[3/5] Memory usage check (limit: {MEMORY_LIMIT_GB} GB)")
|
||||
mem_gb = _ollama_memory_gb()
|
||||
if mem_gb == 0.0:
|
||||
print(" ~ Could not determine memory usage (ps unavailable?), skipping")
|
||||
return True
|
||||
if mem_gb < MEMORY_LIMIT_GB:
|
||||
print(f" ✓ Memory usage: {mem_gb:.1f} GB (under {MEMORY_LIMIT_GB} GB limit)")
|
||||
return True
|
||||
print(
|
||||
f" ✗ Memory usage: {mem_gb:.1f} GB exceeds {MEMORY_LIMIT_GB} GB limit.\n"
|
||||
" Consider using Q4_K_M quantisation or reducing num_ctx."
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def test_tool_calling(model: str) -> bool:
|
||||
"""PASS: model produces a tool_calls response (not raw text) for a tool-use prompt."""
|
||||
print(f"\n[4/5] Tool-calling test")
|
||||
messages = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Please read the file at /tmp/test.txt using the read_file tool.",
|
||||
}
|
||||
]
|
||||
try:
|
||||
t0 = time.time()
|
||||
data = _chat(model, messages, tools=[READ_FILE_TOOL])
|
||||
elapsed = time.time() - t0
|
||||
msg = data.get("message", {})
|
||||
tool_calls = msg.get("tool_calls", [])
|
||||
|
||||
if tool_calls:
|
||||
tc = tool_calls[0]
|
||||
fn = tc.get("function", {})
|
||||
print(
|
||||
f" ✓ Tool call produced ({elapsed:.1f}s):\n"
|
||||
f" function: {fn.get('name')}\n"
|
||||
f" arguments: {json.dumps(fn.get('arguments', {}), indent=6)}"
|
||||
)
|
||||
# Verify the function name is correct
|
||||
return fn.get("name") == "read_file"
|
||||
|
||||
# Some models return JSON in the content instead of tool_calls
|
||||
content = msg.get("content", "")
|
||||
if "read_file" in content and "{" in content:
|
||||
print(
|
||||
f" ~ Model returned tool call as text (not structured). ({elapsed:.1f}s)\n"
|
||||
f" This is acceptable for the base model before fine-tuning.\n"
|
||||
f" Content: {content[:300]}"
|
||||
)
|
||||
# Partial pass — model attempted tool calling but via text
|
||||
return True
|
||||
|
||||
print(
|
||||
f" ✗ No tool call in response ({elapsed:.1f}s).\n"
|
||||
f" Content: {content[:300]!r}"
|
||||
)
|
||||
return False
|
||||
except Exception as exc:
|
||||
print(f" ✗ Tool-calling request failed: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
def test_timmy_persona(model: str) -> bool:
|
||||
"""PASS: model accepts a Timmy persona system prompt and responds in-character."""
|
||||
print(f"\n[5/5] Timmy-persona smoke test")
|
||||
messages = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": (
|
||||
"You are Timmy, Alexander's personal AI agent. "
|
||||
"You are concise, direct, and helpful. "
|
||||
"You always start your responses with 'Timmy here:'."
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What is your name and what can you help me with?",
|
||||
},
|
||||
]
|
||||
try:
|
||||
t0 = time.time()
|
||||
data = _chat(model, messages)
|
||||
elapsed = time.time() - t0
|
||||
content = data.get("message", {}).get("content", "")
|
||||
if "Timmy" in content or "timmy" in content.lower():
|
||||
print(f" ✓ Persona accepted ({elapsed:.1f}s): {content[:200].strip()}")
|
||||
return True
|
||||
print(
|
||||
f" ~ Persona response lacks 'Timmy' identifier ({elapsed:.1f}s).\n"
|
||||
f" This is a fine-tuning target.\n"
|
||||
f" Response: {content[:200]!r}"
|
||||
)
|
||||
# Soft pass — base model isn't expected to be perfectly in-character
|
||||
return True
|
||||
except Exception as exc:
|
||||
print(f" ✗ Persona test failed: {exc}")
|
||||
return False
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Hermes 4 smoke test suite")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default=DEFAULT_MODEL,
|
||||
help=f"Ollama model name (default: {DEFAULT_MODEL})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ollama-url",
|
||||
default=OLLAMA_URL,
|
||||
help=f"Ollama base URL (default: {OLLAMA_URL})",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
global OLLAMA_URL
|
||||
OLLAMA_URL = args.ollama_url.rstrip("/")
|
||||
model = args.model
|
||||
|
||||
print("=" * 60)
|
||||
print(f"Hermes 4 Validation Suite — {model}")
|
||||
print(f"Ollama: {OLLAMA_URL}")
|
||||
print("=" * 60)
|
||||
|
||||
results: dict[str, bool] = {}
|
||||
|
||||
# Test 1: availability (gate — skip remaining if model missing)
|
||||
results["available"] = test_model_available(model)
|
||||
if not results["available"]:
|
||||
print("\n⚠ Model not available — skipping remaining tests.")
|
||||
print(" Import the model first (see Modelfile.hermes4-14b).")
|
||||
_print_summary(results)
|
||||
return 1
|
||||
|
||||
# Tests 2–5
|
||||
results["basic_response"] = test_basic_response(model)
|
||||
results["memory_usage"] = test_memory_usage()
|
||||
results["tool_calling"] = test_tool_calling(model)
|
||||
results["timmy_persona"] = test_timmy_persona(model)
|
||||
|
||||
return _print_summary(results)
|
||||
|
||||
|
||||
def _print_summary(results: dict[str, bool]) -> int:
|
||||
passed = sum(results.values())
|
||||
total = len(results)
|
||||
print("\n" + "=" * 60)
|
||||
print(f"Results: {passed}/{total} passed")
|
||||
print("=" * 60)
|
||||
for name, ok in results.items():
|
||||
icon = "✓" if ok else "✗"
|
||||
print(f" {icon} {name}")
|
||||
|
||||
if passed == total:
|
||||
print("\n✓ All tests passed. Hermes 4 is ready for AutoLoRA fine-tuning.")
|
||||
print(" Next step: document WORK vs FAIL skill list → fine-tuning targets.")
|
||||
elif results.get("tool_calling") is False:
|
||||
print("\n⚠ Tool-calling FAILED. This is the primary fine-tuning target.")
|
||||
print(" Base model may need LoRA tuning on tool-use examples.")
|
||||
else:
|
||||
print("\n~ Partial pass. Review failures above before fine-tuning.")
|
||||
|
||||
return 0 if passed == total else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -20,11 +20,28 @@ from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
# ── Config ──────────────────────────────────────────────────────────────
|
||||
GITEA_API = os.environ.get("GITEA_API", "http://localhost:3000/api/v1")
|
||||
|
||||
|
||||
def _get_gitea_api() -> str:
|
||||
"""Read Gitea API URL from env var, then ~/.hermes/gitea_api file, then default."""
|
||||
# Check env vars first (TIMMY_GITEA_API is preferred, GITEA_API for compatibility)
|
||||
api_url = os.environ.get("TIMMY_GITEA_API") or os.environ.get("GITEA_API")
|
||||
if api_url:
|
||||
return api_url
|
||||
# Check ~/.hermes/gitea_api file
|
||||
api_file = Path.home() / ".hermes" / "gitea_api"
|
||||
if api_file.exists():
|
||||
return api_file.read_text().strip()
|
||||
# Default fallback
|
||||
return "http://localhost:3000/api/v1"
|
||||
|
||||
|
||||
GITEA_API = _get_gitea_api()
|
||||
REPO_SLUG = os.environ.get("REPO_SLUG", "rockachopa/Timmy-time-dashboard")
|
||||
TOKEN_FILE = Path.home() / ".hermes" / "gitea_token"
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
QUEUE_FILE = REPO_ROOT / ".loop" / "queue.json"
|
||||
QUEUE_BACKUP_FILE = REPO_ROOT / ".loop" / "queue.json.bak"
|
||||
RETRO_FILE = REPO_ROOT / ".loop" / "retro" / "triage.jsonl"
|
||||
QUARANTINE_FILE = REPO_ROOT / ".loop" / "quarantine.json"
|
||||
CYCLE_RETRO_FILE = REPO_ROOT / ".loop" / "retro" / "cycles.jsonl"
|
||||
@@ -326,9 +343,38 @@ def run_triage() -> list[dict]:
|
||||
ready = [s for s in scored if s["ready"]]
|
||||
not_ready = [s for s in scored if not s["ready"]]
|
||||
|
||||
# Save backup before writing (if current file exists and is valid)
|
||||
if QUEUE_FILE.exists():
|
||||
try:
|
||||
json.loads(QUEUE_FILE.read_text()) # Validate current file
|
||||
QUEUE_BACKUP_FILE.write_text(QUEUE_FILE.read_text())
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass # Current file is corrupt, don't overwrite backup
|
||||
|
||||
# Write new queue file
|
||||
QUEUE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
QUEUE_FILE.write_text(json.dumps(ready, indent=2) + "\n")
|
||||
|
||||
# Validate the write by re-reading and parsing
|
||||
try:
|
||||
json.loads(QUEUE_FILE.read_text())
|
||||
except (json.JSONDecodeError, OSError) as exc:
|
||||
print(f"[triage] ERROR: queue.json validation failed: {exc}", file=sys.stderr)
|
||||
# Restore from backup if available
|
||||
if QUEUE_BACKUP_FILE.exists():
|
||||
try:
|
||||
backup_data = QUEUE_BACKUP_FILE.read_text()
|
||||
json.loads(backup_data) # Validate backup
|
||||
QUEUE_FILE.write_text(backup_data)
|
||||
print(f"[triage] Restored queue.json from backup")
|
||||
except (json.JSONDecodeError, OSError) as restore_exc:
|
||||
print(f"[triage] ERROR: Backup restore failed: {restore_exc}", file=sys.stderr)
|
||||
# Write empty list as last resort
|
||||
QUEUE_FILE.write_text("[]\n")
|
||||
else:
|
||||
# No backup, write empty list
|
||||
QUEUE_FILE.write_text("[]\n")
|
||||
|
||||
# Write retro entry
|
||||
retro_entry = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
|
||||
67
skills/research/architecture_spike.md
Normal file
67
skills/research/architecture_spike.md
Normal file
@@ -0,0 +1,67 @@
|
||||
---
|
||||
name: Architecture Spike
|
||||
type: research
|
||||
typical_query_count: 2-4
|
||||
expected_output_length: 600-1200 words
|
||||
cascade_tier: groq_preferred
|
||||
description: >
|
||||
Investigate how to connect two systems or components. Produces an integration
|
||||
architecture with sequence diagram, key decisions, and a proof-of-concept outline.
|
||||
---
|
||||
|
||||
# Architecture Spike: Connect {system_a} to {system_b}
|
||||
|
||||
## Context
|
||||
|
||||
We need to integrate **{system_a}** with **{system_b}** in the context of
|
||||
**{project_context}**. This spike answers: what is the best way to wire them
|
||||
together, and what are the trade-offs?
|
||||
|
||||
## Constraints
|
||||
|
||||
- Prefer approaches that avoid adding new infrastructure dependencies.
|
||||
- The integration should be **{sync_or_async}** (synchronous / asynchronous).
|
||||
- Must work within: {environment_constraints}.
|
||||
|
||||
## Research Steps
|
||||
|
||||
1. Identify the APIs / protocols exposed by both systems.
|
||||
2. List all known integration patterns (direct API, message queue, webhook, SDK, etc.).
|
||||
3. Evaluate each pattern for complexity, reliability, and latency.
|
||||
4. Select the recommended approach and outline a proof-of-concept.
|
||||
|
||||
## Output Format
|
||||
|
||||
### Integration Options
|
||||
|
||||
| Pattern | Complexity | Reliability | Latency | Notes |
|
||||
|---------|-----------|-------------|---------|-------|
|
||||
| ... | ... | ... | ... | ... |
|
||||
|
||||
### Recommended Approach
|
||||
|
||||
**Pattern:** {pattern_name}
|
||||
|
||||
**Why:** One paragraph explaining the choice.
|
||||
|
||||
### Sequence Diagram
|
||||
|
||||
```
|
||||
{system_a} -> {middleware} -> {system_b}
|
||||
```
|
||||
|
||||
Describe the data flow step by step:
|
||||
|
||||
1. {system_a} does X...
|
||||
2. {middleware} transforms / routes...
|
||||
3. {system_b} receives Y...
|
||||
|
||||
### Proof-of-Concept Outline
|
||||
|
||||
- Files to create or modify
|
||||
- Key libraries / dependencies needed
|
||||
- Estimated effort: {effort_estimate}
|
||||
|
||||
### Open Questions
|
||||
|
||||
Bullet list of decisions that need human input before proceeding.
|
||||
74
skills/research/competitive_scan.md
Normal file
74
skills/research/competitive_scan.md
Normal file
@@ -0,0 +1,74 @@
|
||||
---
|
||||
name: Competitive Scan
|
||||
type: research
|
||||
typical_query_count: 3-5
|
||||
expected_output_length: 800-1500 words
|
||||
cascade_tier: groq_preferred
|
||||
description: >
|
||||
Compare a project against its alternatives. Produces a feature matrix,
|
||||
strengths/weaknesses analysis, and positioning summary.
|
||||
---
|
||||
|
||||
# Competitive Scan: {project} vs Alternatives
|
||||
|
||||
## Context
|
||||
|
||||
Compare **{project}** against **{alternatives}** (comma-separated list of
|
||||
competitors). The goal is to understand where {project} stands and identify
|
||||
differentiation opportunities.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Comparison date: {date}.
|
||||
- Focus areas: {focus_areas} (e.g., features, pricing, community, performance).
|
||||
- Perspective: {perspective} (user, developer, business).
|
||||
|
||||
## Research Steps
|
||||
|
||||
1. Gather key facts about {project} (features, pricing, community size, release cadence).
|
||||
2. Gather the same data for each alternative in {alternatives}.
|
||||
3. Build a feature comparison matrix.
|
||||
4. Identify strengths and weaknesses for each entry.
|
||||
5. Summarize positioning and recommend next steps.
|
||||
|
||||
## Output Format
|
||||
|
||||
### Overview
|
||||
|
||||
One paragraph: what space does {project} compete in, and who are the main players?
|
||||
|
||||
### Feature Matrix
|
||||
|
||||
| Feature / Attribute | {project} | {alt_1} | {alt_2} | {alt_3} |
|
||||
|--------------------|-----------|---------|---------|---------|
|
||||
| {feature_1} | ... | ... | ... | ... |
|
||||
| {feature_2} | ... | ... | ... | ... |
|
||||
| Pricing | ... | ... | ... | ... |
|
||||
| License | ... | ... | ... | ... |
|
||||
| Community Size | ... | ... | ... | ... |
|
||||
| Last Major Release | ... | ... | ... | ... |
|
||||
|
||||
### Strengths & Weaknesses
|
||||
|
||||
#### {project}
|
||||
- **Strengths:** ...
|
||||
- **Weaknesses:** ...
|
||||
|
||||
#### {alt_1}
|
||||
- **Strengths:** ...
|
||||
- **Weaknesses:** ...
|
||||
|
||||
_(Repeat for each alternative)_
|
||||
|
||||
### Positioning Map
|
||||
|
||||
Describe where each project sits along the key dimensions (e.g., simplicity
|
||||
vs power, free vs paid, niche vs general).
|
||||
|
||||
### Recommendations
|
||||
|
||||
Bullet list of actions based on the competitive landscape:
|
||||
|
||||
- **Differentiate on:** {differentiator}
|
||||
- **Watch out for:** {threat}
|
||||
- **Consider adopting from {alt}:** {feature_or_approach}
|
||||
68
skills/research/game_analysis.md
Normal file
68
skills/research/game_analysis.md
Normal file
@@ -0,0 +1,68 @@
|
||||
---
|
||||
name: Game Analysis
|
||||
type: research
|
||||
typical_query_count: 2-3
|
||||
expected_output_length: 600-1000 words
|
||||
cascade_tier: local_ok
|
||||
description: >
|
||||
Evaluate a game for AI agent playability. Assesses API availability,
|
||||
observation/action spaces, and existing bot ecosystems.
|
||||
---
|
||||
|
||||
# Game Analysis: {game}
|
||||
|
||||
## Context
|
||||
|
||||
Evaluate **{game}** to determine whether an AI agent can play it effectively.
|
||||
Focus on programmatic access, observation space, action space, and existing
|
||||
bot/AI ecosystems.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Platform: {platform} (PC, console, mobile, browser).
|
||||
- Agent type: {agent_type} (reinforcement learning, rule-based, LLM-driven, hybrid).
|
||||
- Budget for API/licenses: {budget}.
|
||||
|
||||
## Research Steps
|
||||
|
||||
1. Identify official APIs, modding support, or programmatic access methods for {game}.
|
||||
2. Characterize the observation space (screen pixels, game state JSON, memory reading, etc.).
|
||||
3. Characterize the action space (keyboard/mouse, API calls, controller inputs).
|
||||
4. Survey existing bots, AI projects, or research papers for {game}.
|
||||
5. Assess feasibility and difficulty for the target agent type.
|
||||
|
||||
## Output Format
|
||||
|
||||
### Game Profile
|
||||
|
||||
| Property | Value |
|
||||
|-------------------|------------------------|
|
||||
| Game | {game} |
|
||||
| Genre | {genre} |
|
||||
| Platform | {platform} |
|
||||
| API Available | Yes / No / Partial |
|
||||
| Mod Support | Yes / No / Limited |
|
||||
| Existing AI Work | Extensive / Some / None|
|
||||
|
||||
### Observation Space
|
||||
|
||||
Describe what data the agent can access and how (API, screen capture, memory hooks, etc.).
|
||||
|
||||
### Action Space
|
||||
|
||||
Describe how the agent can interact with the game (input methods, timing constraints, etc.).
|
||||
|
||||
### Existing Ecosystem
|
||||
|
||||
List known bots, frameworks, research papers, or communities working on AI for {game}.
|
||||
|
||||
### Feasibility Assessment
|
||||
|
||||
- **Difficulty:** Easy / Medium / Hard / Impractical
|
||||
- **Best approach:** {recommended_agent_type}
|
||||
- **Key challenges:** Bullet list
|
||||
- **Estimated time to MVP:** {time_estimate}
|
||||
|
||||
### Recommendation
|
||||
|
||||
One paragraph: should we proceed, and if so, what is the first step?
|
||||
79
skills/research/integration_guide.md
Normal file
79
skills/research/integration_guide.md
Normal file
@@ -0,0 +1,79 @@
|
||||
---
|
||||
name: Integration Guide
|
||||
type: research
|
||||
typical_query_count: 3-5
|
||||
expected_output_length: 1000-2000 words
|
||||
cascade_tier: groq_preferred
|
||||
description: >
|
||||
Step-by-step guide to wire a specific tool into an existing stack,
|
||||
complete with code samples, configuration, and testing steps.
|
||||
---
|
||||
|
||||
# Integration Guide: Wire {tool} into {stack}
|
||||
|
||||
## Context
|
||||
|
||||
Integrate **{tool}** into our **{stack}** stack. The goal is to
|
||||
**{integration_goal}** (e.g., "add vector search to the dashboard",
|
||||
"send notifications via Telegram").
|
||||
|
||||
## Constraints
|
||||
|
||||
- Must follow existing project conventions (see CLAUDE.md).
|
||||
- No new cloud AI dependencies unless explicitly approved.
|
||||
- Environment config via `pydantic-settings` / `config.py`.
|
||||
|
||||
## Research Steps
|
||||
|
||||
1. Review {tool}'s official documentation for installation and setup.
|
||||
2. Identify the minimal dependency set required.
|
||||
3. Map {tool}'s API to our existing patterns (singletons, graceful degradation).
|
||||
4. Write integration code with proper error handling.
|
||||
5. Define configuration variables and their defaults.
|
||||
|
||||
## Output Format
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Dependencies to install (with versions)
|
||||
- External services or accounts required
|
||||
- Environment variables to configure
|
||||
|
||||
### Configuration
|
||||
|
||||
```python
|
||||
# In config.py — add these fields to Settings:
|
||||
{config_fields}
|
||||
```
|
||||
|
||||
### Implementation
|
||||
|
||||
```python
|
||||
# {file_path}
|
||||
{implementation_code}
|
||||
```
|
||||
|
||||
### Graceful Degradation
|
||||
|
||||
Describe how the integration behaves when {tool} is unavailable:
|
||||
|
||||
| Scenario | Behavior | Log Level |
|
||||
|-----------------------|--------------------|-----------|
|
||||
| {tool} not installed | {fallback} | WARNING |
|
||||
| {tool} unreachable | {fallback} | WARNING |
|
||||
| Invalid credentials | {fallback} | ERROR |
|
||||
|
||||
### Testing
|
||||
|
||||
```python
|
||||
# tests/unit/test_{tool_snake}.py
|
||||
{test_code}
|
||||
```
|
||||
|
||||
### Verification Checklist
|
||||
|
||||
- [ ] Dependency added to pyproject.toml
|
||||
- [ ] Config fields added with sensible defaults
|
||||
- [ ] Graceful degradation tested (service down)
|
||||
- [ ] Unit tests pass (`tox -e unit`)
|
||||
- [ ] No new linting errors (`tox -e lint`)
|
||||
67
skills/research/state_of_art.md
Normal file
67
skills/research/state_of_art.md
Normal file
@@ -0,0 +1,67 @@
|
||||
---
|
||||
name: State of the Art
|
||||
type: research
|
||||
typical_query_count: 4-6
|
||||
expected_output_length: 1000-2000 words
|
||||
cascade_tier: groq_preferred
|
||||
description: >
|
||||
Comprehensive survey of what currently exists in a given field or domain.
|
||||
Produces a structured landscape overview with key players, trends, and gaps.
|
||||
---
|
||||
|
||||
# State of the Art: {field} (as of {date})
|
||||
|
||||
## Context
|
||||
|
||||
Survey the current landscape of **{field}**. Identify key players, recent
|
||||
developments, dominant approaches, and notable gaps. This is a point-in-time
|
||||
snapshot intended to inform decision-making.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Focus on developments from the last {timeframe} (e.g., 12 months, 2 years).
|
||||
- Prioritize {priority} (open-source, commercial, academic, or all).
|
||||
- Target audience: {audience} (technical team, leadership, general).
|
||||
|
||||
## Research Steps
|
||||
|
||||
1. Identify the major categories or sub-domains within {field}.
|
||||
2. For each category, list the leading projects, companies, or research groups.
|
||||
3. Note recent milestones, releases, or breakthroughs.
|
||||
4. Identify emerging trends and directions.
|
||||
5. Highlight gaps — things that don't exist yet but should.
|
||||
|
||||
## Output Format
|
||||
|
||||
### Executive Summary
|
||||
|
||||
Two to three sentences: what is the state of {field} right now?
|
||||
|
||||
### Landscape Map
|
||||
|
||||
| Category | Key Players | Maturity | Trend |
|
||||
|---------------|--------------------------|-------------|-------------|
|
||||
| {category_1} | {player_a}, {player_b} | Early / GA | Growing / Stable / Declining |
|
||||
| {category_2} | {player_c}, {player_d} | Early / GA | Growing / Stable / Declining |
|
||||
|
||||
### Recent Milestones
|
||||
|
||||
Chronological list of notable events in the last {timeframe}:
|
||||
|
||||
- **{date_1}:** {event_description}
|
||||
- **{date_2}:** {event_description}
|
||||
|
||||
### Trends
|
||||
|
||||
Numbered list of the top 3-5 trends shaping {field}:
|
||||
|
||||
1. **{trend_name}** — {one-line description}
|
||||
2. **{trend_name}** — {one-line description}
|
||||
|
||||
### Gaps & Opportunities
|
||||
|
||||
Bullet list of things that are missing, underdeveloped, or ripe for innovation.
|
||||
|
||||
### Implications for Us
|
||||
|
||||
One paragraph: what does this mean for our project? What should we do next?
|
||||
52
skills/research/tool_evaluation.md
Normal file
52
skills/research/tool_evaluation.md
Normal file
@@ -0,0 +1,52 @@
|
||||
---
|
||||
name: Tool Evaluation
|
||||
type: research
|
||||
typical_query_count: 3-5
|
||||
expected_output_length: 800-1500 words
|
||||
cascade_tier: groq_preferred
|
||||
description: >
|
||||
Discover and evaluate all shipping tools/libraries/services in a given domain.
|
||||
Produces a ranked comparison table with pros, cons, and recommendation.
|
||||
---
|
||||
|
||||
# Tool Evaluation: {domain}
|
||||
|
||||
## Context
|
||||
|
||||
You are researching tools, libraries, and services for **{domain}**.
|
||||
The goal is to find everything that is currently shipping (not vaporware)
|
||||
and produce a structured comparison.
|
||||
|
||||
## Constraints
|
||||
|
||||
- Only include tools that have public releases or hosted services available today.
|
||||
- If a tool is in beta/preview, note that clearly.
|
||||
- Focus on {focus_criteria} when evaluating (e.g., cost, ease of integration, community size).
|
||||
|
||||
## Research Steps
|
||||
|
||||
1. Identify all actively-maintained tools in the **{domain}** space.
|
||||
2. For each tool, gather: name, URL, license/pricing, last release date, language/platform.
|
||||
3. Evaluate each tool against the focus criteria.
|
||||
4. Rank by overall fit for the use case: **{use_case}**.
|
||||
|
||||
## Output Format
|
||||
|
||||
### Summary
|
||||
|
||||
One paragraph: what the landscape looks like and the top recommendation.
|
||||
|
||||
### Comparison Table
|
||||
|
||||
| Tool | License / Price | Last Release | Language | {focus_criteria} Score | Notes |
|
||||
|------|----------------|--------------|----------|----------------------|-------|
|
||||
| ... | ... | ... | ... | ... | ... |
|
||||
|
||||
### Top Pick
|
||||
|
||||
- **Recommended:** {tool_name} — {one-line reason}
|
||||
- **Runner-up:** {tool_name} — {one-line reason}
|
||||
|
||||
### Risks & Gaps
|
||||
|
||||
Bullet list of things to watch out for (missing features, vendor lock-in, etc.).
|
||||
11
src/bannerlord/__init__.py
Normal file
11
src/bannerlord/__init__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
"""Bannerlord campaign agent — M2: Basic Campaign Actions.
|
||||
|
||||
Provides GABS integration (TCP JSON-RPC, port 4825) and the observe →
|
||||
decide → act loop for autonomous campaign play: move, trade, recruit,
|
||||
and engage bandits.
|
||||
|
||||
Key GABS tools: party/move_to_settlement, inventory/buy_item,
|
||||
party/recruit_all, party/engage_party.
|
||||
|
||||
Done-condition: party grows from 20 → 100 troops, gold reaches 10 000 denars.
|
||||
"""
|
||||
200
src/bannerlord/campaign_actions.py
Normal file
200
src/bannerlord/campaign_actions.py
Normal file
@@ -0,0 +1,200 @@
|
||||
"""Bannerlord M2 campaign action primitives.
|
||||
|
||||
Wraps the four key GABS tools for the M2 milestone:
|
||||
- party/move_to_settlement → move the party to a named settlement
|
||||
- inventory/buy_item → purchase trade goods
|
||||
- party/recruit_all → hire all available recruits
|
||||
- party/engage_party → engage a nearby bandit party
|
||||
|
||||
All functions are async and return an ``ActionResult`` that is compatible
|
||||
with the ``WorldInterface`` contract.
|
||||
|
||||
Error handling follows Pattern 3 (Feature Disable): if GABS rejects an
|
||||
action, log a warning and return a FAILURE result — never raise.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from enum import StrEnum
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from infrastructure.world.types import ActionResult, ActionStatus
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from bannerlord.gabs_client import GabsClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GABS method names — canonical reference
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class GabsTool(StrEnum):
|
||||
"""GABS JSON-RPC method names for the M2 action set."""
|
||||
|
||||
MOVE_TO_SETTLEMENT = "party/move_to_settlement"
|
||||
BUY_ITEM = "inventory/buy_item"
|
||||
RECRUIT_ALL = "party/recruit_all"
|
||||
ENGAGE_PARTY = "party/engage_party"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Action functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def move_to_settlement(
|
||||
client: "GabsClient",
|
||||
settlement_id: str,
|
||||
*,
|
||||
settlement_name: str = "",
|
||||
) -> ActionResult:
|
||||
"""Move the party to a target settlement.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
client:
|
||||
Connected ``GabsClient`` instance.
|
||||
settlement_id:
|
||||
GABS settlement identifier (e.g. ``"town_A1"``).
|
||||
settlement_name:
|
||||
Human-readable name for logging only.
|
||||
"""
|
||||
label = settlement_name or settlement_id
|
||||
try:
|
||||
result = await client.call(
|
||||
GabsTool.MOVE_TO_SETTLEMENT,
|
||||
{"settlement_id": settlement_id},
|
||||
)
|
||||
logger.info("MOVE → %s: %s", label, result)
|
||||
return ActionResult(
|
||||
status=ActionStatus.SUCCESS,
|
||||
message=f"Moving to {label}",
|
||||
data=result or {},
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("MOVE → %s failed: %s", label, exc)
|
||||
return ActionResult(
|
||||
status=ActionStatus.FAILURE,
|
||||
message=f"Move to {label} failed: {exc}",
|
||||
data={},
|
||||
)
|
||||
|
||||
|
||||
async def buy_item(
|
||||
client: "GabsClient",
|
||||
item_id: str,
|
||||
quantity: int,
|
||||
*,
|
||||
settlement_id: str = "",
|
||||
) -> ActionResult:
|
||||
"""Purchase trade goods from the current or specified settlement.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
client:
|
||||
Connected ``GabsClient`` instance.
|
||||
item_id:
|
||||
Item identifier (e.g. ``"grain"``, ``"iron"``, ``"wool"``).
|
||||
quantity:
|
||||
Number of units to purchase.
|
||||
settlement_id:
|
||||
Optional target settlement; empty means current location.
|
||||
"""
|
||||
try:
|
||||
params: dict = {"item_id": item_id, "quantity": quantity}
|
||||
if settlement_id:
|
||||
params["settlement_id"] = settlement_id
|
||||
|
||||
result = await client.call(GabsTool.BUY_ITEM, params)
|
||||
logger.info("BUY %dx %s: %s", quantity, item_id, result)
|
||||
return ActionResult(
|
||||
status=ActionStatus.SUCCESS,
|
||||
message=f"Purchased {quantity}x {item_id}",
|
||||
data=result or {},
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("BUY %dx %s failed: %s", quantity, item_id, exc)
|
||||
return ActionResult(
|
||||
status=ActionStatus.FAILURE,
|
||||
message=f"Buy {item_id} failed: {exc}",
|
||||
data={},
|
||||
)
|
||||
|
||||
|
||||
async def recruit_all(
|
||||
client: "GabsClient",
|
||||
*,
|
||||
settlement_id: str = "",
|
||||
) -> ActionResult:
|
||||
"""Recruit all available troops at the current or specified settlement.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
client:
|
||||
Connected ``GabsClient`` instance.
|
||||
settlement_id:
|
||||
Optional settlement to recruit from; empty means current.
|
||||
"""
|
||||
try:
|
||||
params: dict = {}
|
||||
if settlement_id:
|
||||
params["settlement_id"] = settlement_id
|
||||
|
||||
result = await client.call(GabsTool.RECRUIT_ALL, params)
|
||||
recruited = (result or {}).get("recruited", "?")
|
||||
logger.info("RECRUIT_ALL: recruited %s troops", recruited)
|
||||
return ActionResult(
|
||||
status=ActionStatus.SUCCESS,
|
||||
message=f"Recruited {recruited} troops",
|
||||
data=result or {},
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("RECRUIT_ALL failed: %s", exc)
|
||||
return ActionResult(
|
||||
status=ActionStatus.FAILURE,
|
||||
message=f"Recruit all failed: {exc}",
|
||||
data={},
|
||||
)
|
||||
|
||||
|
||||
async def engage_party(
|
||||
client: "GabsClient",
|
||||
party_id: str,
|
||||
*,
|
||||
party_name: str = "",
|
||||
) -> ActionResult:
|
||||
"""Engage a nearby party (typically a bandit gang) in combat.
|
||||
|
||||
Auto-resolve is expected at high Tactics skill — the agent relies
|
||||
on GABS to handle the battle outcome.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
client:
|
||||
Connected ``GabsClient`` instance.
|
||||
party_id:
|
||||
GABS party identifier of the target.
|
||||
party_name:
|
||||
Human-readable name for logging only.
|
||||
"""
|
||||
label = party_name or party_id
|
||||
try:
|
||||
result = await client.call(GabsTool.ENGAGE_PARTY, {"party_id": party_id})
|
||||
outcome = (result or {}).get("outcome", "unknown")
|
||||
logger.info("ENGAGE %s: %s", label, outcome)
|
||||
return ActionResult(
|
||||
status=ActionStatus.SUCCESS,
|
||||
message=f"Engaged {label}: {outcome}",
|
||||
data=result or {},
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("ENGAGE %s failed: %s", label, exc)
|
||||
return ActionResult(
|
||||
status=ActionStatus.FAILURE,
|
||||
message=f"Engage {label} failed: {exc}",
|
||||
data={},
|
||||
)
|
||||
316
src/bannerlord/campaign_loop.py
Normal file
316
src/bannerlord/campaign_loop.py
Normal file
@@ -0,0 +1,316 @@
|
||||
"""Bannerlord M2 campaign action loop.
|
||||
|
||||
Implements the observe → decide → act → wait pipeline described in
|
||||
issue #1094. The loop runs until the M2 victory conditions are met
|
||||
(100 troops + 10 000 gold) or until stopped externally.
|
||||
|
||||
Architecture:
|
||||
CampaignLoop.run()
|
||||
while not m2_complete:
|
||||
state = gabs.get_game_state() # observe
|
||||
decision = decide(state) # decide (local Qwen3)
|
||||
result = dispatch(decision, gabs) # act (GABS)
|
||||
await asyncio.sleep(tick_seconds) # wait
|
||||
|
||||
Error handling:
|
||||
- GABS connection failures → log + retry with backoff (max 3 attempts)
|
||||
- LLM failures → WAIT action (graceful degradation)
|
||||
- Action failures → log + continue to next tick
|
||||
|
||||
Progress tracking:
|
||||
Loop publishes heartbeat events via the event bus so the dashboard
|
||||
can display live party size and gold.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from bannerlord.campaign_actions import buy_item, engage_party, move_to_settlement, recruit_all
|
||||
from bannerlord.campaign_state import parse_campaign_state
|
||||
from bannerlord.decision import M2Action, decide
|
||||
from bannerlord.gabs_client import GabsClient
|
||||
from config import settings
|
||||
from infrastructure.world.types import ActionResult, ActionStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_RECONNECT_ATTEMPTS = 3
|
||||
_RECONNECT_DELAY = 5.0 # seconds
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Progress snapshot (emitted each tick)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class TickResult:
|
||||
"""Summary of one campaign tick."""
|
||||
|
||||
tick: int
|
||||
timestamp: str
|
||||
party_size: int
|
||||
gold: int
|
||||
action: str
|
||||
action_status: str
|
||||
reasoning: str
|
||||
duration_ms: int
|
||||
m2_complete: bool = False
|
||||
error: str = ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Campaign loop
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CampaignLoop:
|
||||
"""Runs the Bannerlord M2 autonomous campaign.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
gabs_host:
|
||||
Override GABS server host.
|
||||
gabs_port:
|
||||
Override GABS server port.
|
||||
tick_seconds:
|
||||
Real-time pause between in-game ticks.
|
||||
on_tick:
|
||||
Optional async callback invoked after each tick with the
|
||||
``TickResult``. Used by the dashboard for live updates.
|
||||
max_ticks:
|
||||
Hard cap for testing / benchmarking. 0 = unlimited.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
gabs_host: str | None = None,
|
||||
gabs_port: int | None = None,
|
||||
tick_seconds: float | None = None,
|
||||
on_tick=None,
|
||||
max_ticks: int = 0,
|
||||
) -> None:
|
||||
self._host = gabs_host or settings.gabs_host
|
||||
self._port = gabs_port or settings.gabs_port
|
||||
self._tick_seconds = tick_seconds if tick_seconds is not None else settings.bannerlord_tick_seconds
|
||||
self._on_tick = on_tick
|
||||
self._max_ticks = max_ticks
|
||||
self._running = False
|
||||
self.history: list[TickResult] = []
|
||||
|
||||
# -- public API --------------------------------------------------------
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return self._running
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Signal the loop to stop after the current tick."""
|
||||
self._running = False
|
||||
logger.info("CampaignLoop stop requested")
|
||||
|
||||
async def run(self) -> list[TickResult]:
|
||||
"""Start the campaign loop.
|
||||
|
||||
Returns the list of tick results (for testing / benchmarking).
|
||||
Runs until M2 complete, externally stopped, or max_ticks reached.
|
||||
"""
|
||||
self._running = True
|
||||
logger.info(
|
||||
"CampaignLoop starting — gabs=%s:%d tick=%.1fs",
|
||||
self._host,
|
||||
self._port,
|
||||
self._tick_seconds,
|
||||
)
|
||||
|
||||
client = GabsClient(host=self._host, port=self._port)
|
||||
try:
|
||||
await self._connect_with_retry(client)
|
||||
except RuntimeError as exc: # noqa: BLE001
|
||||
logger.error("CampaignLoop: could not connect to GABS — aborting: %s", exc)
|
||||
self._running = False
|
||||
return self.history
|
||||
|
||||
tick_num = 0
|
||||
try:
|
||||
while self._running:
|
||||
tick_num += 1
|
||||
if self._max_ticks > 0 and tick_num > self._max_ticks:
|
||||
logger.info("CampaignLoop: max_ticks=%d reached", self._max_ticks)
|
||||
break
|
||||
|
||||
result = await self._run_tick(client, tick_num)
|
||||
self.history.append(result)
|
||||
|
||||
await self._emit(result)
|
||||
|
||||
if result.m2_complete:
|
||||
logger.info(
|
||||
"M2 COMPLETE! Party=%d troops, Gold=%d denars",
|
||||
result.party_size,
|
||||
result.gold,
|
||||
)
|
||||
break
|
||||
|
||||
if result.error and not self._running:
|
||||
break
|
||||
|
||||
await asyncio.sleep(self._tick_seconds)
|
||||
|
||||
finally:
|
||||
await client.disconnect()
|
||||
self._running = False
|
||||
logger.info("CampaignLoop stopped after %d ticks", tick_num)
|
||||
|
||||
return self.history
|
||||
|
||||
# -- internal: single tick ---------------------------------------------
|
||||
|
||||
async def _run_tick(self, client: "Any", tick_num: int) -> TickResult:
|
||||
"""Execute one observe → decide → act cycle."""
|
||||
start = time.monotonic()
|
||||
|
||||
# 1. Observe
|
||||
raw_state = await client.get_game_state()
|
||||
state = parse_campaign_state(raw_state)
|
||||
state = _override_tick(state, tick_num)
|
||||
|
||||
# 2. Decide
|
||||
decision = await decide(state)
|
||||
|
||||
# 3. Act
|
||||
action_result = await self._dispatch(decision, client)
|
||||
|
||||
duration_ms = int((time.monotonic() - start) * 1000)
|
||||
|
||||
return TickResult(
|
||||
tick=tick_num,
|
||||
timestamp=datetime.now(UTC).isoformat(),
|
||||
party_size=state.party.party_size,
|
||||
gold=state.economy.gold,
|
||||
action=decision.action,
|
||||
action_status=action_result.status.value,
|
||||
reasoning=decision.reasoning,
|
||||
duration_ms=duration_ms,
|
||||
m2_complete=state.m2_complete,
|
||||
)
|
||||
|
||||
async def _dispatch(self, decision: "Any", client: "Any") -> "Any":
|
||||
"""Route the decision to the correct GABS action function."""
|
||||
action = decision.action
|
||||
|
||||
if action == M2Action.MOVE:
|
||||
if not decision.settlement_id:
|
||||
logger.warning("MOVE decision has no settlement_id — skipping")
|
||||
return ActionResult(
|
||||
status=ActionStatus.FAILURE,
|
||||
message="MOVE missing settlement_id",
|
||||
)
|
||||
return await move_to_settlement(
|
||||
client,
|
||||
decision.settlement_id,
|
||||
settlement_name=decision.settlement_name,
|
||||
)
|
||||
|
||||
elif action == M2Action.TRADE:
|
||||
if not decision.item_id:
|
||||
logger.warning("TRADE decision has no item_id — skipping")
|
||||
return ActionResult(
|
||||
status=ActionStatus.FAILURE,
|
||||
message="TRADE missing item_id",
|
||||
)
|
||||
return await buy_item(
|
||||
client,
|
||||
decision.item_id,
|
||||
decision.quantity,
|
||||
settlement_id=decision.settlement_id,
|
||||
)
|
||||
|
||||
elif action == M2Action.RECRUIT:
|
||||
return await recruit_all(
|
||||
client,
|
||||
settlement_id=decision.settlement_id,
|
||||
)
|
||||
|
||||
elif action == M2Action.ENGAGE:
|
||||
if not decision.party_id:
|
||||
logger.warning("ENGAGE decision has no party_id — skipping")
|
||||
return ActionResult(
|
||||
status=ActionStatus.FAILURE,
|
||||
message="ENGAGE missing party_id",
|
||||
)
|
||||
return await engage_party(
|
||||
client,
|
||||
decision.party_id,
|
||||
party_name=decision.party_name,
|
||||
)
|
||||
|
||||
else: # WAIT or unknown
|
||||
logger.debug("Tick %s: WAIT — %s", decision.action, decision.reasoning)
|
||||
return ActionResult(
|
||||
status=ActionStatus.NOOP,
|
||||
message=f"WAIT: {decision.reasoning}",
|
||||
)
|
||||
|
||||
# -- internal: connectivity --------------------------------------------
|
||||
|
||||
async def _connect_with_retry(self, client: "Any") -> None:
|
||||
"""Try to connect, retrying up to _MAX_RECONNECT_ATTEMPTS times."""
|
||||
for attempt in range(1, _MAX_RECONNECT_ATTEMPTS + 1):
|
||||
try:
|
||||
await client.connect()
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"GABS connect attempt %d/%d failed: %s",
|
||||
attempt,
|
||||
_MAX_RECONNECT_ATTEMPTS,
|
||||
exc,
|
||||
)
|
||||
if attempt < _MAX_RECONNECT_ATTEMPTS:
|
||||
await asyncio.sleep(_RECONNECT_DELAY)
|
||||
|
||||
raise RuntimeError(
|
||||
f"Could not connect to GABS at {self._host}:{self._port} "
|
||||
f"after {_MAX_RECONNECT_ATTEMPTS} attempts"
|
||||
)
|
||||
|
||||
# -- internal: event emission ------------------------------------------
|
||||
|
||||
async def _emit(self, result: TickResult) -> None:
|
||||
"""Emit tick data to the event bus (best-effort)."""
|
||||
try:
|
||||
from infrastructure.events.bus import event_bus # noqa: PLC0415
|
||||
|
||||
await event_bus.publish(
|
||||
"bannerlord.tick",
|
||||
{
|
||||
"tick": result.tick,
|
||||
"party_size": result.party_size,
|
||||
"gold": result.gold,
|
||||
"action": result.action,
|
||||
"action_status": result.action_status,
|
||||
"m2_complete": result.m2_complete,
|
||||
"duration_ms": result.duration_ms,
|
||||
},
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("CampaignLoop emit skipped: %s", exc)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _override_tick(state: "Any", tick_num: int) -> "Any":
|
||||
"""Set the tick counter from the loop (GABS may not provide it)."""
|
||||
if state.tick == 0:
|
||||
state.tick = tick_num
|
||||
return state
|
||||
213
src/bannerlord/campaign_state.py
Normal file
213
src/bannerlord/campaign_state.py
Normal file
@@ -0,0 +1,213 @@
|
||||
"""Bannerlord campaign state models.
|
||||
|
||||
Parses the raw GABS ``game/get_state`` payload into typed models and
|
||||
tracks the M2 progress counters: party size and gold accumulation.
|
||||
|
||||
Done-condition (from issue #1094):
|
||||
party_size >= 100 AND gold >= 10_000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# M2 victory conditions
|
||||
M2_TROOP_GOAL = 100
|
||||
M2_GOLD_GOAL = 10_000
|
||||
|
||||
|
||||
@dataclass
|
||||
class PartyState:
|
||||
"""Current party composition and position."""
|
||||
|
||||
party_size: int = 0
|
||||
wounded: int = 0
|
||||
prisoners: int = 0
|
||||
food_days: float = 0.0
|
||||
morale: float = 100.0
|
||||
current_settlement: str = ""
|
||||
speed: float = 0.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class EconomyState:
|
||||
"""Current gold and trade state."""
|
||||
|
||||
gold: int = 0
|
||||
daily_income: int = 0
|
||||
daily_expenses: int = 0
|
||||
|
||||
@property
|
||||
def net_income(self) -> int:
|
||||
return self.daily_income - self.daily_expenses
|
||||
|
||||
|
||||
@dataclass
|
||||
class NearbyParty:
|
||||
"""A nearby lord/bandit party visible on the map."""
|
||||
|
||||
party_id: str
|
||||
name: str
|
||||
faction: str
|
||||
is_hostile: bool
|
||||
troop_count: int
|
||||
distance: float
|
||||
|
||||
|
||||
@dataclass
|
||||
class Settlement:
|
||||
"""A settlement visible or reachable from the current position."""
|
||||
|
||||
settlement_id: str
|
||||
name: str
|
||||
faction: str
|
||||
is_friendly: bool
|
||||
distance: float
|
||||
has_recruits: bool = False
|
||||
has_trade_goods: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class CampaignState:
|
||||
"""Full parsed snapshot of the GABS game state.
|
||||
|
||||
Built from the raw ``dict`` returned by ``GabsClient.get_game_state()``.
|
||||
"""
|
||||
|
||||
tick: int = 0
|
||||
timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
party: PartyState = field(default_factory=PartyState)
|
||||
economy: EconomyState = field(default_factory=EconomyState)
|
||||
nearby_parties: list[NearbyParty] = field(default_factory=list)
|
||||
settlements: list[Settlement] = field(default_factory=list)
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
# -- M2 progress -------------------------------------------------------
|
||||
|
||||
@property
|
||||
def troops_progress(self) -> str:
|
||||
"""Human-readable M2 troop progress."""
|
||||
return f"{self.party.party_size}/{M2_TROOP_GOAL}"
|
||||
|
||||
@property
|
||||
def gold_progress(self) -> str:
|
||||
"""Human-readable M2 gold progress."""
|
||||
return f"{self.economy.gold:,}/{M2_GOLD_GOAL:,}"
|
||||
|
||||
@property
|
||||
def m2_complete(self) -> bool:
|
||||
"""True when both M2 victory conditions are met."""
|
||||
return self.party.party_size >= M2_TROOP_GOAL and self.economy.gold >= M2_GOLD_GOAL
|
||||
|
||||
# -- hostile detection -------------------------------------------------
|
||||
|
||||
def hostile_bandits_nearby(self, max_distance: float = 5.0) -> list[NearbyParty]:
|
||||
"""Return hostile bandit parties within *max_distance* map units."""
|
||||
return [
|
||||
p
|
||||
for p in self.nearby_parties
|
||||
if p.is_hostile and "bandit" in p.faction.lower() and p.distance <= max_distance
|
||||
]
|
||||
|
||||
def nearest_settlement(self, *, friendly_only: bool = False) -> Settlement | None:
|
||||
"""Return the closest (optionally friendly) settlement."""
|
||||
candidates = [s for s in self.settlements if not friendly_only or s.is_friendly]
|
||||
if not candidates:
|
||||
return None
|
||||
return min(candidates, key=lambda s: s.distance)
|
||||
|
||||
def nearest_recruit_settlement(self) -> Settlement | None:
|
||||
"""Return the nearest settlement that has recruits available."""
|
||||
candidates = [s for s in self.settlements if s.has_recruits]
|
||||
if not candidates:
|
||||
return None
|
||||
return min(candidates, key=lambda s: s.distance)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_campaign_state(raw: dict[str, Any]) -> CampaignState:
|
||||
"""Build a ``CampaignState`` from the raw GABS state dict.
|
||||
|
||||
Unknown / missing fields are silently defaulted so the parser never
|
||||
crashes when GABS returns partial data.
|
||||
"""
|
||||
if not raw:
|
||||
logger.debug("parse_campaign_state: empty payload — returning default state")
|
||||
return CampaignState(raw=raw)
|
||||
|
||||
# -- party -------------------------------------------------------------
|
||||
party_raw = raw.get("party", {})
|
||||
party = PartyState(
|
||||
party_size=int(party_raw.get("size", 0)),
|
||||
wounded=int(party_raw.get("wounded", 0)),
|
||||
prisoners=int(party_raw.get("prisoners", 0)),
|
||||
food_days=float(party_raw.get("food_days", 0.0)),
|
||||
morale=float(party_raw.get("morale", 100.0)),
|
||||
current_settlement=str(party_raw.get("current_settlement", "")),
|
||||
speed=float(party_raw.get("speed", 0.0)),
|
||||
)
|
||||
|
||||
# -- economy -----------------------------------------------------------
|
||||
economy_raw = raw.get("economy", {})
|
||||
economy = EconomyState(
|
||||
gold=int(economy_raw.get("gold", 0)),
|
||||
daily_income=int(economy_raw.get("daily_income", 0)),
|
||||
daily_expenses=int(economy_raw.get("daily_expenses", 0)),
|
||||
)
|
||||
|
||||
# -- nearby parties ----------------------------------------------------
|
||||
nearby_parties = []
|
||||
for p in raw.get("nearby_parties", []):
|
||||
try:
|
||||
if not isinstance(p, dict) or not p.get("id"):
|
||||
logger.debug("Skipping malformed nearby_party entry: missing id")
|
||||
continue
|
||||
nearby_parties.append(
|
||||
NearbyParty(
|
||||
party_id=str(p.get("id", "")),
|
||||
name=str(p.get("name", "")),
|
||||
faction=str(p.get("faction", "")),
|
||||
is_hostile=bool(p.get("is_hostile", False)),
|
||||
troop_count=int(p.get("troop_count", 0)),
|
||||
distance=float(p.get("distance", 999.0)),
|
||||
)
|
||||
)
|
||||
except (KeyError, ValueError, TypeError, AttributeError) as exc:
|
||||
logger.debug("Skipping malformed nearby_party entry: %s", exc)
|
||||
|
||||
# -- settlements -------------------------------------------------------
|
||||
settlements = []
|
||||
for s in raw.get("settlements", []):
|
||||
try:
|
||||
settlements.append(
|
||||
Settlement(
|
||||
settlement_id=str(s.get("id", "")),
|
||||
name=str(s.get("name", "")),
|
||||
faction=str(s.get("faction", "")),
|
||||
is_friendly=bool(s.get("is_friendly", False)),
|
||||
distance=float(s.get("distance", 999.0)),
|
||||
has_recruits=bool(s.get("has_recruits", False)),
|
||||
has_trade_goods=bool(s.get("has_trade_goods", False)),
|
||||
)
|
||||
)
|
||||
except (KeyError, ValueError, TypeError, AttributeError) as exc:
|
||||
logger.debug("Skipping malformed settlement entry: %s", exc)
|
||||
|
||||
return CampaignState(
|
||||
tick=int(raw.get("tick", 0)),
|
||||
timestamp=datetime.now(UTC),
|
||||
party=party,
|
||||
economy=economy,
|
||||
nearby_parties=nearby_parties,
|
||||
settlements=settlements,
|
||||
raw=raw,
|
||||
)
|
||||
284
src/bannerlord/decision.py
Normal file
284
src/bannerlord/decision.py
Normal file
@@ -0,0 +1,284 @@
|
||||
"""LLM-powered campaign decision engine for Bannerlord M2.
|
||||
|
||||
Builds a structured prompt from the current ``CampaignState`` and asks
|
||||
the local Qwen3 model (via Ollama) to choose one action from the M2
|
||||
action vocabulary. Returns a ``CampaignDecision`` pydantic model with
|
||||
the chosen action and its parameters.
|
||||
|
||||
The decision model is intentionally simple for M2:
|
||||
MOVE → move to a named settlement
|
||||
TRADE → buy a trade item
|
||||
RECRUIT → hire troops at current/nearby settlement
|
||||
ENGAGE → fight a nearby bandit party
|
||||
WAIT → idle (e.g. low food, waiting for morale to recover)
|
||||
|
||||
Qwen3 responds in JSON mode with temperature=0.1 for deterministic play.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Decision schema
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class M2Action(StrEnum):
|
||||
"""Vocabulary of actions available in the M2 milestone."""
|
||||
|
||||
MOVE = "MOVE"
|
||||
TRADE = "TRADE"
|
||||
RECRUIT = "RECRUIT"
|
||||
ENGAGE = "ENGAGE"
|
||||
WAIT = "WAIT"
|
||||
|
||||
|
||||
class CampaignDecision:
|
||||
"""Parsed LLM decision for one campaign tick.
|
||||
|
||||
Attributes
|
||||
----------
|
||||
action:
|
||||
One of the ``M2Action`` values.
|
||||
settlement_id:
|
||||
Target settlement ID (for MOVE / RECRUIT / TRADE).
|
||||
settlement_name:
|
||||
Human-readable settlement name (for logging).
|
||||
item_id:
|
||||
Trade item to buy (for TRADE).
|
||||
quantity:
|
||||
Trade quantity (for TRADE).
|
||||
party_id:
|
||||
Target party ID (for ENGAGE).
|
||||
party_name:
|
||||
Human-readable party name (for ENGAGE / logging).
|
||||
reasoning:
|
||||
LLM's brief explanation of the choice.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
action: M2Action = M2Action.WAIT,
|
||||
*,
|
||||
settlement_id: str = "",
|
||||
settlement_name: str = "",
|
||||
item_id: str = "",
|
||||
quantity: int = 1,
|
||||
party_id: str = "",
|
||||
party_name: str = "",
|
||||
reasoning: str = "",
|
||||
) -> None:
|
||||
self.action = action
|
||||
self.settlement_id = settlement_id
|
||||
self.settlement_name = settlement_name
|
||||
self.item_id = item_id
|
||||
self.quantity = quantity
|
||||
self.party_id = party_id
|
||||
self.party_name = party_name
|
||||
self.reasoning = reasoning
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (
|
||||
f"CampaignDecision(action={self.action!r}, "
|
||||
f"reasoning={self.reasoning[:60]!r})"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_decision_prompt(state: "Any") -> list[dict[str, str]]:
|
||||
"""Return an OpenAI-style message list for the decision LLM.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
state:
|
||||
A ``CampaignState`` instance.
|
||||
"""
|
||||
# Build a compact context block
|
||||
party = state.party
|
||||
econ = state.economy
|
||||
ctx_lines = [
|
||||
f"Campaign tick: {state.tick}",
|
||||
f"Party size: {party.party_size} troops ({party.wounded} wounded)",
|
||||
f"Food: {party.food_days:.1f} days remaining",
|
||||
f"Morale: {party.morale:.0f}/100",
|
||||
f"Gold: {econ.gold:,} denars (net {econ.net_income:+d}/day)",
|
||||
f"Current location: {party.current_settlement or 'travelling'}",
|
||||
"",
|
||||
"== M2 GOALS ==",
|
||||
f"Troops: {state.troops_progress} (need 100)",
|
||||
f"Gold: {state.gold_progress} (need 10,000)",
|
||||
"",
|
||||
]
|
||||
|
||||
# Nearby bandits
|
||||
bandits = state.hostile_bandits_nearby()
|
||||
if bandits:
|
||||
ctx_lines.append("== NEARBY HOSTILE BANDITS ==")
|
||||
for b in bandits[:3]:
|
||||
ctx_lines.append(
|
||||
f" - {b.name} (id={b.party_id}, {b.troop_count} troops, "
|
||||
f"{b.distance:.1f} away)"
|
||||
)
|
||||
ctx_lines.append("")
|
||||
|
||||
# Settlements
|
||||
settlements = state.settlements[:5]
|
||||
if settlements:
|
||||
ctx_lines.append("== REACHABLE SETTLEMENTS ==")
|
||||
for s in settlements:
|
||||
flags = []
|
||||
if s.has_recruits:
|
||||
flags.append("recruits")
|
||||
if s.has_trade_goods:
|
||||
flags.append("trade")
|
||||
if not s.is_friendly:
|
||||
flags.append("hostile-faction")
|
||||
flag_str = f" [{', '.join(flags)}]" if flags else ""
|
||||
ctx_lines.append(
|
||||
f" - {s.name} (id={s.settlement_id}, "
|
||||
f"{s.distance:.1f} away{flag_str})"
|
||||
)
|
||||
ctx_lines.append("")
|
||||
|
||||
context = "\n".join(ctx_lines)
|
||||
|
||||
system_prompt = (
|
||||
"You are the campaign manager for Timmy, an autonomous Bannerlord agent. "
|
||||
"Your job is to choose the single best action for this campaign tick. "
|
||||
"Respond ONLY with a JSON object — no prose, no markdown fences.\n\n"
|
||||
"JSON schema:\n"
|
||||
'{\n'
|
||||
' "action": "MOVE|TRADE|RECRUIT|ENGAGE|WAIT",\n'
|
||||
' "settlement_id": "<id or empty>",\n'
|
||||
' "settlement_name": "<name or empty>",\n'
|
||||
' "item_id": "<item or empty>",\n'
|
||||
' "quantity": <int>,\n'
|
||||
' "party_id": "<id or empty>",\n'
|
||||
' "party_name": "<name or empty>",\n'
|
||||
' "reasoning": "<one sentence>"\n'
|
||||
"}\n\n"
|
||||
"Priority rules:\n"
|
||||
"1. ENGAGE bandits only if they are weak (< 15 troops) and we have > 25 troops.\n"
|
||||
"2. RECRUIT when a nearby settlement has recruits and party < 80 troops.\n"
|
||||
"3. TRADE when gold < 5000 and a settlement has trade goods.\n"
|
||||
"4. MOVE toward the nearest settlement with recruits or trade goods.\n"
|
||||
"5. WAIT only if food < 1 day or morale < 40."
|
||||
)
|
||||
|
||||
user_prompt = f"Current game state:\n\n{context}\nChoose the best action."
|
||||
|
||||
return [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": user_prompt},
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Response parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def parse_decision(raw_response: str) -> CampaignDecision:
|
||||
"""Parse the LLM JSON response into a ``CampaignDecision``.
|
||||
|
||||
Falls back to ``WAIT`` on any parse error so the loop never crashes.
|
||||
"""
|
||||
# Strip accidental markdown code fences
|
||||
text = raw_response.strip()
|
||||
if text.startswith("```"):
|
||||
lines = text.splitlines()
|
||||
text = "\n".join(
|
||||
line for line in lines if not line.startswith("```")
|
||||
).strip()
|
||||
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
logger.warning("Decision parse error (bad JSON): %s | raw=%r", exc, raw_response[:200])
|
||||
return CampaignDecision(action=M2Action.WAIT, reasoning="parse error")
|
||||
|
||||
try:
|
||||
action_str = str(data.get("action", "WAIT")).upper()
|
||||
try:
|
||||
action = M2Action(action_str)
|
||||
except ValueError:
|
||||
logger.warning("Unknown action %r — defaulting to WAIT", action_str)
|
||||
action = M2Action.WAIT
|
||||
|
||||
return CampaignDecision(
|
||||
action=action,
|
||||
settlement_id=str(data.get("settlement_id", "")),
|
||||
settlement_name=str(data.get("settlement_name", "")),
|
||||
item_id=str(data.get("item_id", "")),
|
||||
quantity=max(1, int(data.get("quantity", 1))),
|
||||
party_id=str(data.get("party_id", "")),
|
||||
party_name=str(data.get("party_name", "")),
|
||||
reasoning=str(data.get("reasoning", "")),
|
||||
)
|
||||
except (KeyError, ValueError, TypeError) as exc:
|
||||
logger.warning("Decision parse error (bad fields): %s", exc)
|
||||
return CampaignDecision(action=M2Action.WAIT, reasoning=f"field error: {exc}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def decide(state: "Any") -> CampaignDecision:
|
||||
"""Ask the local LLM to choose a campaign action.
|
||||
|
||||
Uses the cascade router (Ollama → Claude fallback) configured in
|
||||
``config/providers.yaml``. Gracefully returns WAIT on any LLM failure.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
state:
|
||||
A ``CampaignState`` instance.
|
||||
|
||||
Returns
|
||||
-------
|
||||
CampaignDecision
|
||||
The chosen action and its parameters.
|
||||
"""
|
||||
from config import settings
|
||||
|
||||
messages = build_decision_prompt(state)
|
||||
model = settings.bannerlord_model
|
||||
|
||||
try:
|
||||
from infrastructure.router import get_router
|
||||
|
||||
router = get_router()
|
||||
response = await router.complete(
|
||||
messages=messages,
|
||||
model=model,
|
||||
temperature=0.1,
|
||||
)
|
||||
raw_text: str = response.get("content", "")
|
||||
decision = parse_decision(raw_text)
|
||||
logger.info(
|
||||
"Decision [tick=%d]: %s — %s",
|
||||
state.tick,
|
||||
decision.action,
|
||||
decision.reasoning,
|
||||
)
|
||||
return decision
|
||||
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Decision LLM call failed: %s — defaulting to WAIT", exc)
|
||||
return CampaignDecision(
|
||||
action=M2Action.WAIT,
|
||||
reasoning=f"LLM unavailable: {exc}",
|
||||
)
|
||||
195
src/bannerlord/gabs_client.py
Normal file
195
src/bannerlord/gabs_client.py
Normal file
@@ -0,0 +1,195 @@
|
||||
"""GABS TCP/JSON-RPC client for Bannerlord.
|
||||
|
||||
Connects to the GABS C# mod (Bannerlord.GABS) over TCP on port 4825
|
||||
and dispatches JSON-RPC 2.0 requests. All I/O is async; synchronous
|
||||
callers must wrap in ``asyncio.to_thread()``.
|
||||
|
||||
Architecture:
|
||||
Bannerlord (Windows VM) ← GABS C# mod ← TCP:4825 ← this client
|
||||
|
||||
Usage::
|
||||
|
||||
async with GabsClient() as client:
|
||||
state = await client.get_game_state()
|
||||
result = await client.call("party/move_to_settlement",
|
||||
{"settlement_id": "town_A1"})
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# JSON-RPC framing: each message is newline-delimited UTF-8 JSON.
|
||||
_ENCODING = "utf-8"
|
||||
_NEWLINE = b"\n"
|
||||
_DEFAULT_TIMEOUT = 30.0
|
||||
|
||||
|
||||
class GabsError(Exception):
|
||||
"""Raised when GABS returns a JSON-RPC error response."""
|
||||
|
||||
def __init__(self, code: int, message: str, data: Any = None) -> None:
|
||||
super().__init__(f"GABS error {code}: {message}")
|
||||
self.code = code
|
||||
self.data = data
|
||||
|
||||
|
||||
class GabsClient:
|
||||
"""Async TCP JSON-RPC 2.0 client for the GABS Bannerlord mod.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
host:
|
||||
GABS server host (Windows VM IP or ``localhost`` for port-forwarded).
|
||||
port:
|
||||
GABS server port (default 4825).
|
||||
timeout:
|
||||
Per-call timeout in seconds.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
host: str | None = None,
|
||||
port: int | None = None,
|
||||
timeout: float = _DEFAULT_TIMEOUT,
|
||||
) -> None:
|
||||
self._host = host or settings.gabs_host
|
||||
self._port = port or settings.gabs_port
|
||||
self._timeout = timeout
|
||||
self._reader: asyncio.StreamReader | None = None
|
||||
self._writer: asyncio.StreamWriter | None = None
|
||||
self._req_id = 0
|
||||
self._connected = False
|
||||
|
||||
# -- lifecycle ---------------------------------------------------------
|
||||
|
||||
async def connect(self) -> None:
|
||||
"""Open the TCP connection to GABS."""
|
||||
try:
|
||||
self._reader, self._writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(self._host, self._port),
|
||||
timeout=self._timeout,
|
||||
)
|
||||
self._connected = True
|
||||
logger.info("GabsClient connected to %s:%d", self._host, self._port)
|
||||
except (OSError, asyncio.TimeoutError) as exc:
|
||||
logger.warning("GabsClient could not connect to GABS: %s", exc)
|
||||
self._connected = False
|
||||
raise
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
"""Close the TCP connection."""
|
||||
if self._writer is not None:
|
||||
try:
|
||||
self._writer.close()
|
||||
await self._writer.wait_closed()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("GabsClient disconnect error (ignored): %s", exc)
|
||||
self._connected = False
|
||||
self._reader = None
|
||||
self._writer = None
|
||||
logger.info("GabsClient disconnected")
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return self._connected
|
||||
|
||||
# -- context manager ---------------------------------------------------
|
||||
|
||||
async def __aenter__(self) -> "GabsClient":
|
||||
await self.connect()
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_: Any) -> None:
|
||||
await self.disconnect()
|
||||
|
||||
# -- public API --------------------------------------------------------
|
||||
|
||||
async def call(self, method: str, params: dict[str, Any] | None = None) -> Any:
|
||||
"""Call a GABS tool and return the result.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
method:
|
||||
GABS tool name, e.g. ``"party/move_to_settlement"``.
|
||||
params:
|
||||
Tool parameters dict.
|
||||
|
||||
Returns
|
||||
-------
|
||||
Any
|
||||
The ``result`` field from the JSON-RPC response.
|
||||
|
||||
Raises
|
||||
------
|
||||
GabsError
|
||||
If GABS returns an error response.
|
||||
RuntimeError
|
||||
If not connected.
|
||||
"""
|
||||
if not self._connected or self._writer is None or self._reader is None:
|
||||
raise RuntimeError("GabsClient is not connected — call connect() first")
|
||||
|
||||
self._req_id += 1
|
||||
request = {
|
||||
"jsonrpc": "2.0",
|
||||
"id": self._req_id,
|
||||
"method": method,
|
||||
"params": params or {},
|
||||
}
|
||||
|
||||
raw = json.dumps(request).encode(_ENCODING) + _NEWLINE
|
||||
|
||||
try:
|
||||
self._writer.write(raw)
|
||||
await asyncio.wait_for(self._writer.drain(), timeout=self._timeout)
|
||||
|
||||
line = await asyncio.wait_for(
|
||||
self._reader.readline(), timeout=self._timeout
|
||||
)
|
||||
except asyncio.TimeoutError as exc:
|
||||
raise RuntimeError(f"GABS call '{method}' timed out after {self._timeout}s") from exc
|
||||
except (OSError, ConnectionResetError) as exc:
|
||||
self._connected = False
|
||||
raise RuntimeError(f"GABS connection lost during '{method}': {exc}") from exc
|
||||
|
||||
response = json.loads(line.decode(_ENCODING))
|
||||
|
||||
if "error" in response:
|
||||
err = response["error"]
|
||||
raise GabsError(
|
||||
code=err.get("code", -1),
|
||||
message=err.get("message", "unknown error"),
|
||||
data=err.get("data"),
|
||||
)
|
||||
|
||||
return response.get("result")
|
||||
|
||||
async def get_game_state(self) -> dict[str, Any]:
|
||||
"""Return the full game state snapshot from GABS.
|
||||
|
||||
Returns an empty dict and logs a warning if GABS is unreachable.
|
||||
"""
|
||||
try:
|
||||
result = await self.call("game/get_state")
|
||||
return result if isinstance(result, dict) else {}
|
||||
except (GabsError, RuntimeError) as exc:
|
||||
logger.warning("GABS get_game_state failed: %s", exc)
|
||||
return {}
|
||||
|
||||
async def ping(self) -> bool:
|
||||
"""Return True if GABS responds to a ping."""
|
||||
try:
|
||||
await self.call("game/ping")
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("GABS ping failed: %s", exc)
|
||||
return False
|
||||
@@ -87,14 +87,26 @@ class Settings(BaseSettings):
|
||||
xai_base_url: str = "https://api.x.ai/v1"
|
||||
grok_default_model: str = "grok-3-fast"
|
||||
grok_max_sats_per_query: int = 200
|
||||
grok_sats_hard_cap: int = 100 # Absolute ceiling on sats per Grok query
|
||||
grok_free: bool = False # Skip Lightning invoice when user has own API key
|
||||
|
||||
# ── Database ──────────────────────────────────────────────────────────
|
||||
db_busy_timeout_ms: int = 5000 # SQLite PRAGMA busy_timeout (ms)
|
||||
|
||||
# ── Claude (Anthropic) — cloud fallback backend ────────────────────────
|
||||
# Used when Ollama is offline and local inference isn't available.
|
||||
# Set ANTHROPIC_API_KEY to enable. Default model is Haiku (fast + cheap).
|
||||
anthropic_api_key: str = ""
|
||||
claude_model: str = "haiku"
|
||||
|
||||
# ── Content Moderation ──────────────────────────────────────────────
|
||||
# Three-layer moderation pipeline for AI narrator output.
|
||||
# Uses Llama Guard via Ollama with regex fallback.
|
||||
moderation_enabled: bool = True
|
||||
moderation_guard_model: str = "llama-guard3:1b"
|
||||
# Default confidence threshold — per-game profiles can override.
|
||||
moderation_threshold: float = 0.8
|
||||
|
||||
# ── Spark Intelligence ────────────────────────────────────────────────
|
||||
# Enable/disable the Spark cognitive layer.
|
||||
# When enabled, Spark captures swarm events, runs EIDOS predictions,
|
||||
@@ -140,6 +152,10 @@ class Settings(BaseSettings):
|
||||
# Default is False (telemetry disabled) to align with sovereign AI vision.
|
||||
telemetry_enabled: bool = False
|
||||
|
||||
# ── Sovereignty Metrics ──────────────────────────────────────────────
|
||||
# Alert when API cost per research task exceeds this threshold (USD).
|
||||
sovereignty_api_cost_alert_threshold: float = 1.00
|
||||
|
||||
# CORS allowed origins for the web chat interface (Gitea Pages, etc.)
|
||||
# Set CORS_ORIGINS as a comma-separated list, e.g. "http://localhost:3000,https://example.com"
|
||||
cors_origins: list[str] = [
|
||||
@@ -286,6 +302,7 @@ class Settings(BaseSettings):
|
||||
mcp_gitea_command: str = "gitea-mcp-server -t stdio"
|
||||
mcp_filesystem_command: str = "npx -y @modelcontextprotocol/server-filesystem"
|
||||
mcp_timeout: int = 15
|
||||
mcp_bridge_timeout: int = 60 # HTTP timeout for MCP bridge Ollama calls (seconds)
|
||||
|
||||
# ── Loop QA (Self-Testing) ─────────────────────────────────────────
|
||||
# Self-test orchestrator that probes capabilities alongside the thinking loop.
|
||||
@@ -357,6 +374,17 @@ class Settings(BaseSettings):
|
||||
error_feedback_enabled: bool = True # Auto-create bug report tasks
|
||||
error_dedup_window_seconds: int = 300 # 5-min dedup window
|
||||
|
||||
# ── Bannerlord / GABS ─────────────────────────────────────────────
|
||||
# GABS (Bannerlord Agent Bridge System) TCP/JSON-RPC server.
|
||||
# Runs inside the Windows VM hosting Bannerlord.
|
||||
# Override with GABS_HOST / GABS_PORT env vars.
|
||||
gabs_host: str = "localhost"
|
||||
gabs_port: int = 4825
|
||||
# Decision model for the Bannerlord campaign agent (Qwen3 preferred).
|
||||
bannerlord_model: str = "qwen3:14b"
|
||||
# Campaign-tick interval in seconds (real-time pause between in-game days).
|
||||
bannerlord_tick_seconds: float = 5.0
|
||||
|
||||
# ── Scripture / Biblical Integration ──────────────────────────────
|
||||
# Enable the biblical text module.
|
||||
scripture_enabled: bool = True
|
||||
|
||||
@@ -44,6 +44,8 @@ from dashboard.routes.mobile import router as mobile_router
|
||||
from dashboard.routes.models import api_router as models_api_router
|
||||
from dashboard.routes.models import router as models_router
|
||||
from dashboard.routes.quests import router as quests_router
|
||||
from dashboard.routes.scorecards import router as scorecards_router
|
||||
from dashboard.routes.sovereignty_metrics import router as sovereignty_metrics_router
|
||||
from dashboard.routes.spark import router as spark_router
|
||||
from dashboard.routes.system import router as system_router
|
||||
from dashboard.routes.tasks import router as tasks_router
|
||||
@@ -373,13 +375,21 @@ def _startup_init() -> None:
|
||||
|
||||
def _startup_background_tasks() -> list[asyncio.Task]:
|
||||
"""Spawn all recurring background tasks (non-blocking)."""
|
||||
return [
|
||||
bg_tasks = [
|
||||
asyncio.create_task(_briefing_scheduler()),
|
||||
asyncio.create_task(_thinking_scheduler()),
|
||||
asyncio.create_task(_loop_qa_scheduler()),
|
||||
asyncio.create_task(_presence_watcher()),
|
||||
asyncio.create_task(_start_chat_integrations_background()),
|
||||
]
|
||||
try:
|
||||
from timmy.paperclip import start_paperclip_poller
|
||||
bg_tasks.append(asyncio.create_task(start_paperclip_poller()))
|
||||
logger.info("Paperclip poller started")
|
||||
except ImportError:
|
||||
logger.debug("Paperclip module not found, skipping poller")
|
||||
|
||||
return bg_tasks
|
||||
|
||||
|
||||
def _try_prune(label: str, prune_fn, days: int) -> None:
|
||||
@@ -629,6 +639,8 @@ app.include_router(matrix_router)
|
||||
app.include_router(tower_router)
|
||||
app.include_router(daily_run_router)
|
||||
app.include_router(quests_router)
|
||||
app.include_router(scorecards_router)
|
||||
app.include_router(sovereignty_metrics_router)
|
||||
|
||||
|
||||
@app.websocket("/ws")
|
||||
|
||||
@@ -125,7 +125,7 @@ def _run_grok_query(message: str) -> dict:
|
||||
from lightning.factory import get_backend as get_ln_backend
|
||||
|
||||
ln = get_ln_backend()
|
||||
sats = min(settings.grok_max_sats_per_query, 100)
|
||||
sats = min(settings.grok_max_sats_per_query, settings.grok_sats_hard_cap)
|
||||
ln.create_invoice(sats, f"Grok: {message[:50]}")
|
||||
invoice_note = f" | {sats} sats"
|
||||
except Exception as exc:
|
||||
|
||||
@@ -275,3 +275,54 @@ async def component_status():
|
||||
},
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/health/snapshot")
|
||||
async def health_snapshot():
|
||||
"""Quick health snapshot before coding.
|
||||
|
||||
Returns a concise status summary including:
|
||||
- CI pipeline status (pass/fail/unknown)
|
||||
- Critical issues count (P0/P1)
|
||||
- Test flakiness rate
|
||||
- Token economy temperature
|
||||
|
||||
Fast execution (< 5 seconds) for pre-work checks.
|
||||
Refs: #710
|
||||
"""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Import the health snapshot module
|
||||
snapshot_path = Path(settings.repo_root) / "timmy_automations" / "daily_run"
|
||||
if str(snapshot_path) not in sys.path:
|
||||
sys.path.insert(0, str(snapshot_path))
|
||||
|
||||
try:
|
||||
from health_snapshot import generate_snapshot, get_token, load_config
|
||||
|
||||
config = load_config()
|
||||
token = get_token(config)
|
||||
|
||||
# Run the health snapshot (in thread to avoid blocking)
|
||||
snapshot = await asyncio.to_thread(generate_snapshot, config, token)
|
||||
|
||||
return snapshot.to_dict()
|
||||
except Exception as exc:
|
||||
logger.warning("Health snapshot failed: %s", exc)
|
||||
# Return graceful fallback
|
||||
return {
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
"overall_status": "unknown",
|
||||
"error": str(exc),
|
||||
"ci": {"status": "unknown", "message": "Snapshot failed"},
|
||||
"issues": {"count": 0, "p0_count": 0, "p1_count": 0, "issues": []},
|
||||
"flakiness": {
|
||||
"status": "unknown",
|
||||
"recent_failures": 0,
|
||||
"recent_cycles": 0,
|
||||
"failure_rate": 0.0,
|
||||
"message": "Snapshot failed",
|
||||
},
|
||||
"tokens": {"status": "unknown", "message": "Snapshot failed"},
|
||||
}
|
||||
|
||||
353
src/dashboard/routes/scorecards.py
Normal file
353
src/dashboard/routes/scorecards.py
Normal file
@@ -0,0 +1,353 @@
|
||||
"""Agent scorecard routes — API endpoints for generating and viewing scorecards."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
from dashboard.services.scorecard_service import (
|
||||
PeriodType,
|
||||
generate_all_scorecards,
|
||||
generate_scorecard,
|
||||
get_tracked_agents,
|
||||
)
|
||||
from dashboard.templating import templates
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/scorecards", tags=["scorecards"])
|
||||
|
||||
|
||||
def _format_period_label(period_type: PeriodType) -> str:
|
||||
"""Format a period type for display."""
|
||||
return "Daily" if period_type == PeriodType.daily else "Weekly"
|
||||
|
||||
|
||||
@router.get("/api/agents")
|
||||
async def list_tracked_agents() -> dict[str, list[str]]:
|
||||
"""Return the list of tracked agent IDs.
|
||||
|
||||
Returns:
|
||||
Dict with "agents" key containing list of agent IDs
|
||||
"""
|
||||
return {"agents": get_tracked_agents()}
|
||||
|
||||
|
||||
@router.get("/api/{agent_id}")
|
||||
async def get_agent_scorecard(
|
||||
agent_id: str,
|
||||
period: str = Query(default="daily", description="Period type: 'daily' or 'weekly'"),
|
||||
) -> JSONResponse:
|
||||
"""Generate a scorecard for a specific agent.
|
||||
|
||||
Args:
|
||||
agent_id: The agent ID (e.g., 'kimi', 'claude')
|
||||
period: 'daily' or 'weekly' (default: daily)
|
||||
|
||||
Returns:
|
||||
JSON response with scorecard data
|
||||
"""
|
||||
try:
|
||||
period_type = PeriodType(period.lower())
|
||||
except ValueError:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": f"Invalid period '{period}'. Use 'daily' or 'weekly'."},
|
||||
)
|
||||
|
||||
try:
|
||||
scorecard = generate_scorecard(agent_id, period_type)
|
||||
|
||||
if scorecard is None:
|
||||
return JSONResponse(
|
||||
status_code=404,
|
||||
content={"error": f"No scorecard found for agent '{agent_id}'"},
|
||||
)
|
||||
|
||||
return JSONResponse(content=scorecard.to_dict())
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Failed to generate scorecard for %s: %s", agent_id, exc)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": f"Failed to generate scorecard: {str(exc)}"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/api")
|
||||
async def get_all_scorecards(
|
||||
period: str = Query(default="daily", description="Period type: 'daily' or 'weekly'"),
|
||||
) -> JSONResponse:
|
||||
"""Generate scorecards for all tracked agents.
|
||||
|
||||
Args:
|
||||
period: 'daily' or 'weekly' (default: daily)
|
||||
|
||||
Returns:
|
||||
JSON response with list of scorecard data
|
||||
"""
|
||||
try:
|
||||
period_type = PeriodType(period.lower())
|
||||
except ValueError:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={"error": f"Invalid period '{period}'. Use 'daily' or 'weekly'."},
|
||||
)
|
||||
|
||||
try:
|
||||
scorecards = generate_all_scorecards(period_type)
|
||||
return JSONResponse(
|
||||
content={
|
||||
"period": period_type.value,
|
||||
"scorecards": [s.to_dict() for s in scorecards],
|
||||
"count": len(scorecards),
|
||||
}
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Failed to generate scorecards: %s", exc)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": f"Failed to generate scorecards: {str(exc)}"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("", response_class=HTMLResponse)
|
||||
async def scorecards_page(request: Request) -> HTMLResponse:
|
||||
"""Render the scorecards dashboard page.
|
||||
|
||||
Returns:
|
||||
HTML page with scorecard interface
|
||||
"""
|
||||
agents = get_tracked_agents()
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"scorecards.html",
|
||||
{
|
||||
"agents": agents,
|
||||
"periods": ["daily", "weekly"],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/panel/{agent_id}", response_class=HTMLResponse)
|
||||
async def agent_scorecard_panel(
|
||||
request: Request,
|
||||
agent_id: str,
|
||||
period: str = Query(default="daily"),
|
||||
) -> HTMLResponse:
|
||||
"""Render an individual agent scorecard panel (for HTMX).
|
||||
|
||||
Args:
|
||||
request: The request object
|
||||
agent_id: The agent ID
|
||||
period: 'daily' or 'weekly'
|
||||
|
||||
Returns:
|
||||
HTML panel with scorecard content
|
||||
"""
|
||||
try:
|
||||
period_type = PeriodType(period.lower())
|
||||
except ValueError:
|
||||
period_type = PeriodType.daily
|
||||
|
||||
try:
|
||||
scorecard = generate_scorecard(agent_id, period_type)
|
||||
|
||||
if scorecard is None:
|
||||
return HTMLResponse(
|
||||
content=f"""
|
||||
<div class="card mc-panel">
|
||||
<h5 class="card-title">{agent_id.title()}</h5>
|
||||
<p class="text-muted">No activity recorded for this period.</p>
|
||||
</div>
|
||||
""",
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
data = scorecard.to_dict()
|
||||
|
||||
# Build patterns HTML
|
||||
patterns_html = ""
|
||||
if data["patterns"]:
|
||||
patterns_list = "".join([f"<li>{p}</li>" for p in data["patterns"]])
|
||||
patterns_html = f"""
|
||||
<div class="mt-3">
|
||||
<h6>Patterns</h6>
|
||||
<ul class="list-unstyled text-info">
|
||||
{patterns_list}
|
||||
</ul>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# Build bullets HTML
|
||||
bullets_html = "".join([f"<li>{b}</li>" for b in data["narrative_bullets"]])
|
||||
|
||||
# Build metrics summary
|
||||
metrics = data["metrics"]
|
||||
|
||||
html_content = f"""
|
||||
<div class="card mc-panel">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<h5 class="card-title mb-0">{agent_id.title()}</h5>
|
||||
<span class="badge bg-secondary">{_format_period_label(period_type)}</span>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<ul class="list-unstyled mb-3">
|
||||
{bullets_html}
|
||||
</ul>
|
||||
|
||||
<div class="row text-center small">
|
||||
<div class="col">
|
||||
<div class="text-muted">PRs</div>
|
||||
<div class="fw-bold">{metrics["prs_opened"]}/{metrics["prs_merged"]}</div>
|
||||
<div class="text-muted" style="font-size: 0.75rem;">
|
||||
{int(metrics["pr_merge_rate"] * 100)}% merged
|
||||
</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Issues</div>
|
||||
<div class="fw-bold">{metrics["issues_touched"]}</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Tests</div>
|
||||
<div class="fw-bold">{metrics["tests_affected"]}</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Tokens</div>
|
||||
<div class="fw-bold {"text-success" if metrics["token_net"] >= 0 else "text-danger"}">
|
||||
{"+" if metrics["token_net"] > 0 else ""}{metrics["token_net"]}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{patterns_html}
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
return HTMLResponse(content=html_content)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Failed to render scorecard panel for %s: %s", agent_id, exc)
|
||||
return HTMLResponse(
|
||||
content=f"""
|
||||
<div class="card mc-panel border-danger">
|
||||
<h5 class="card-title">{agent_id.title()}</h5>
|
||||
<p class="text-danger">Error loading scorecard: {str(exc)}</p>
|
||||
</div>
|
||||
""",
|
||||
status_code=200,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/all/panels", response_class=HTMLResponse)
|
||||
async def all_scorecard_panels(
|
||||
request: Request,
|
||||
period: str = Query(default="daily"),
|
||||
) -> HTMLResponse:
|
||||
"""Render all agent scorecard panels (for HTMX).
|
||||
|
||||
Args:
|
||||
request: The request object
|
||||
period: 'daily' or 'weekly'
|
||||
|
||||
Returns:
|
||||
HTML with all scorecard panels
|
||||
"""
|
||||
try:
|
||||
period_type = PeriodType(period.lower())
|
||||
except ValueError:
|
||||
period_type = PeriodType.daily
|
||||
|
||||
try:
|
||||
scorecards = generate_all_scorecards(period_type)
|
||||
|
||||
panels: list[str] = []
|
||||
for scorecard in scorecards:
|
||||
data = scorecard.to_dict()
|
||||
|
||||
# Build patterns HTML
|
||||
patterns_html = ""
|
||||
if data["patterns"]:
|
||||
patterns_list = "".join([f"<li>{p}</li>" for p in data["patterns"]])
|
||||
patterns_html = f"""
|
||||
<div class="mt-3">
|
||||
<h6>Patterns</h6>
|
||||
<ul class="list-unstyled text-info">
|
||||
{patterns_list}
|
||||
</ul>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# Build bullets HTML
|
||||
bullets_html = "".join([f"<li>{b}</li>" for b in data["narrative_bullets"]])
|
||||
metrics = data["metrics"]
|
||||
|
||||
panel_html = f"""
|
||||
<div class="col-md-6 col-lg-4 mb-3">
|
||||
<div class="card mc-panel">
|
||||
<div class="card-header d-flex justify-content-between align-items-center">
|
||||
<h5 class="card-title mb-0">{scorecard.agent_id.title()}</h5>
|
||||
<span class="badge bg-secondary">{_format_period_label(period_type)}</span>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<ul class="list-unstyled mb-3">
|
||||
{bullets_html}
|
||||
</ul>
|
||||
|
||||
<div class="row text-center small">
|
||||
<div class="col">
|
||||
<div class="text-muted">PRs</div>
|
||||
<div class="fw-bold">{metrics["prs_opened"]}/{metrics["prs_merged"]}</div>
|
||||
<div class="text-muted" style="font-size: 0.75rem;">
|
||||
{int(metrics["pr_merge_rate"] * 100)}% merged
|
||||
</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Issues</div>
|
||||
<div class="fw-bold">{metrics["issues_touched"]}</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Tests</div>
|
||||
<div class="fw-bold">{metrics["tests_affected"]}</div>
|
||||
</div>
|
||||
<div class="col">
|
||||
<div class="text-muted">Tokens</div>
|
||||
<div class="fw-bold {"text-success" if metrics["token_net"] >= 0 else "text-danger"}">
|
||||
{"+" if metrics["token_net"] > 0 else ""}{metrics["token_net"]}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{patterns_html}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
panels.append(panel_html)
|
||||
|
||||
html_content = f"""
|
||||
<div class="row">
|
||||
{"".join(panels)}
|
||||
</div>
|
||||
<div class="text-muted small mt-2">
|
||||
Generated: {datetime.now().strftime("%Y-%m-%d %H:%M:%S UTC")}
|
||||
</div>
|
||||
"""
|
||||
|
||||
return HTMLResponse(content=html_content)
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("Failed to render all scorecard panels: %s", exc)
|
||||
return HTMLResponse(
|
||||
content=f"""
|
||||
<div class="alert alert-danger">
|
||||
Error loading scorecards: {str(exc)}
|
||||
</div>
|
||||
""",
|
||||
status_code=200,
|
||||
)
|
||||
74
src/dashboard/routes/sovereignty_metrics.py
Normal file
74
src/dashboard/routes/sovereignty_metrics.py
Normal file
@@ -0,0 +1,74 @@
|
||||
"""Sovereignty metrics dashboard routes.
|
||||
|
||||
Provides API endpoints and HTMX partials for tracking research
|
||||
sovereignty progress against graduation targets.
|
||||
|
||||
Refs: #981
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import HTMLResponse
|
||||
|
||||
from config import settings
|
||||
from dashboard.templating import templates
|
||||
from infrastructure.sovereignty_metrics import (
|
||||
GRADUATION_TARGETS,
|
||||
get_sovereignty_store,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/sovereignty", tags=["sovereignty"])
|
||||
|
||||
|
||||
@router.get("/metrics")
|
||||
async def sovereignty_metrics_api() -> dict[str, Any]:
|
||||
"""JSON API: full sovereignty metrics summary with trends."""
|
||||
store = get_sovereignty_store()
|
||||
summary = store.get_summary()
|
||||
alerts = store.get_alerts(unacknowledged_only=True)
|
||||
return {
|
||||
"metrics": summary,
|
||||
"alerts": alerts,
|
||||
"targets": GRADUATION_TARGETS,
|
||||
"cost_threshold": settings.sovereignty_api_cost_alert_threshold,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/metrics/panel", response_class=HTMLResponse)
|
||||
async def sovereignty_metrics_panel(request: Request) -> HTMLResponse:
|
||||
"""HTMX partial: sovereignty metrics progress panel."""
|
||||
store = get_sovereignty_store()
|
||||
summary = store.get_summary()
|
||||
alerts = store.get_alerts(unacknowledged_only=True)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"partials/sovereignty_metrics.html",
|
||||
{
|
||||
"metrics": summary,
|
||||
"alerts": alerts,
|
||||
"targets": GRADUATION_TARGETS,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/alerts")
|
||||
async def sovereignty_alerts_api() -> dict[str, Any]:
|
||||
"""JSON API: sovereignty alerts."""
|
||||
store = get_sovereignty_store()
|
||||
return {
|
||||
"alerts": store.get_alerts(unacknowledged_only=False),
|
||||
"unacknowledged": store.get_alerts(unacknowledged_only=True),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/alerts/{alert_id}/acknowledge")
|
||||
async def acknowledge_alert(alert_id: int) -> dict[str, bool]:
|
||||
"""Acknowledge a sovereignty alert."""
|
||||
store = get_sovereignty_store()
|
||||
success = store.acknowledge_alert(alert_id)
|
||||
return {"success": success}
|
||||
@@ -56,11 +56,13 @@ async def self_modify_queue(request: Request):
|
||||
|
||||
@router.get("/swarm/mission-control", response_class=HTMLResponse)
|
||||
async def mission_control(request: Request):
|
||||
"""Render the swarm mission control dashboard page."""
|
||||
return templates.TemplateResponse(request, "mission_control.html", {})
|
||||
|
||||
|
||||
@router.get("/bugs", response_class=HTMLResponse)
|
||||
async def bugs_page(request: Request):
|
||||
"""Render the bug tracking page."""
|
||||
return templates.TemplateResponse(
|
||||
request,
|
||||
"bugs.html",
|
||||
@@ -75,16 +77,19 @@ async def bugs_page(request: Request):
|
||||
|
||||
@router.get("/self-coding", response_class=HTMLResponse)
|
||||
async def self_coding(request: Request):
|
||||
"""Render the self-coding automation status page."""
|
||||
return templates.TemplateResponse(request, "self_coding.html", {"stats": {}})
|
||||
|
||||
|
||||
@router.get("/hands", response_class=HTMLResponse)
|
||||
async def hands_page(request: Request):
|
||||
"""Render the hands (automation executions) page."""
|
||||
return templates.TemplateResponse(request, "hands.html", {"executions": []})
|
||||
|
||||
|
||||
@router.get("/creative/ui", response_class=HTMLResponse)
|
||||
async def creative_ui(request: Request):
|
||||
"""Render the creative UI playground page."""
|
||||
return templates.TemplateResponse(request, "creative.html", {})
|
||||
|
||||
|
||||
|
||||
@@ -143,61 +143,49 @@ async def tasks_page(request: Request):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _render_task_list(request: Request, query: str, empty_msg: str) -> HTMLResponse:
|
||||
"""Fetch tasks by query and render as HTMX task-card partials."""
|
||||
with _get_db() as db:
|
||||
rows = db.execute(query).fetchall()
|
||||
parts = [
|
||||
templates.TemplateResponse(
|
||||
request, "partials/task_card.html", {"task": _TaskView(_row_to_dict(r))}
|
||||
).body.decode()
|
||||
for r in rows
|
||||
]
|
||||
if not parts:
|
||||
return HTMLResponse(f'<div class="empty-column">{empty_msg}</div>')
|
||||
return HTMLResponse("".join(parts))
|
||||
|
||||
|
||||
@router.get("/tasks/pending", response_class=HTMLResponse)
|
||||
async def tasks_pending(request: Request):
|
||||
with _get_db() as db:
|
||||
rows = db.execute(
|
||||
"SELECT * FROM tasks WHERE status='pending_approval' ORDER BY created_at DESC"
|
||||
).fetchall()
|
||||
tasks = [_TaskView(_row_to_dict(r)) for r in rows]
|
||||
parts = []
|
||||
for task in tasks:
|
||||
parts.append(
|
||||
templates.TemplateResponse(
|
||||
request, "partials/task_card.html", {"task": task}
|
||||
).body.decode()
|
||||
)
|
||||
if not parts:
|
||||
return HTMLResponse('<div class="empty-column">No pending tasks</div>')
|
||||
return HTMLResponse("".join(parts))
|
||||
"""Return HTMX partial for pending approval tasks."""
|
||||
return _render_task_list(
|
||||
request,
|
||||
"SELECT * FROM tasks WHERE status='pending_approval' ORDER BY created_at DESC",
|
||||
"No pending tasks",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/tasks/active", response_class=HTMLResponse)
|
||||
async def tasks_active(request: Request):
|
||||
with _get_db() as db:
|
||||
rows = db.execute(
|
||||
"SELECT * FROM tasks WHERE status IN ('approved','running','paused') ORDER BY created_at DESC"
|
||||
).fetchall()
|
||||
tasks = [_TaskView(_row_to_dict(r)) for r in rows]
|
||||
parts = []
|
||||
for task in tasks:
|
||||
parts.append(
|
||||
templates.TemplateResponse(
|
||||
request, "partials/task_card.html", {"task": task}
|
||||
).body.decode()
|
||||
)
|
||||
if not parts:
|
||||
return HTMLResponse('<div class="empty-column">No active tasks</div>')
|
||||
return HTMLResponse("".join(parts))
|
||||
"""Return HTMX partial for active (approved/running/paused) tasks."""
|
||||
return _render_task_list(
|
||||
request,
|
||||
"SELECT * FROM tasks WHERE status IN ('approved','running','paused') ORDER BY created_at DESC",
|
||||
"No active tasks",
|
||||
)
|
||||
|
||||
|
||||
@router.get("/tasks/completed", response_class=HTMLResponse)
|
||||
async def tasks_completed(request: Request):
|
||||
with _get_db() as db:
|
||||
rows = db.execute(
|
||||
"SELECT * FROM tasks WHERE status IN ('completed','vetoed','failed') ORDER BY completed_at DESC LIMIT 50"
|
||||
).fetchall()
|
||||
tasks = [_TaskView(_row_to_dict(r)) for r in rows]
|
||||
parts = []
|
||||
for task in tasks:
|
||||
parts.append(
|
||||
templates.TemplateResponse(
|
||||
request, "partials/task_card.html", {"task": task}
|
||||
).body.decode()
|
||||
)
|
||||
if not parts:
|
||||
return HTMLResponse('<div class="empty-column">No completed tasks yet</div>')
|
||||
return HTMLResponse("".join(parts))
|
||||
"""Return HTMX partial for completed/vetoed/failed tasks (last 50)."""
|
||||
return _render_task_list(
|
||||
request,
|
||||
"SELECT * FROM tasks WHERE status IN ('completed','vetoed','failed') ORDER BY completed_at DESC LIMIT 50",
|
||||
"No completed tasks yet",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -241,26 +229,31 @@ async def create_task_form(
|
||||
|
||||
@router.post("/tasks/{task_id}/approve", response_class=HTMLResponse)
|
||||
async def approve_task(request: Request, task_id: str):
|
||||
"""Approve a pending task and move it to active queue."""
|
||||
return await _set_status(request, task_id, "approved")
|
||||
|
||||
|
||||
@router.post("/tasks/{task_id}/veto", response_class=HTMLResponse)
|
||||
async def veto_task(request: Request, task_id: str):
|
||||
"""Veto a task, marking it as rejected."""
|
||||
return await _set_status(request, task_id, "vetoed")
|
||||
|
||||
|
||||
@router.post("/tasks/{task_id}/pause", response_class=HTMLResponse)
|
||||
async def pause_task(request: Request, task_id: str):
|
||||
"""Pause a running or approved task."""
|
||||
return await _set_status(request, task_id, "paused")
|
||||
|
||||
|
||||
@router.post("/tasks/{task_id}/cancel", response_class=HTMLResponse)
|
||||
async def cancel_task(request: Request, task_id: str):
|
||||
"""Cancel a task (marks as vetoed)."""
|
||||
return await _set_status(request, task_id, "vetoed")
|
||||
|
||||
|
||||
@router.post("/tasks/{task_id}/retry", response_class=HTMLResponse)
|
||||
async def retry_task(request: Request, task_id: str):
|
||||
"""Retry a failed/vetoed task by moving it back to approved."""
|
||||
return await _set_status(request, task_id, "approved")
|
||||
|
||||
|
||||
@@ -271,6 +264,7 @@ async def modify_task(
|
||||
title: str = Form(...),
|
||||
description: str = Form(""),
|
||||
):
|
||||
"""Update task title and description."""
|
||||
with _get_db() as db:
|
||||
db.execute(
|
||||
"UPDATE tasks SET title=?, description=? WHERE id=?",
|
||||
|
||||
17
src/dashboard/services/__init__.py
Normal file
17
src/dashboard/services/__init__.py
Normal file
@@ -0,0 +1,17 @@
|
||||
"""Dashboard services for business logic."""
|
||||
|
||||
from dashboard.services.scorecard_service import (
|
||||
PeriodType,
|
||||
ScorecardSummary,
|
||||
generate_all_scorecards,
|
||||
generate_scorecard,
|
||||
get_tracked_agents,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"PeriodType",
|
||||
"ScorecardSummary",
|
||||
"generate_all_scorecards",
|
||||
"generate_scorecard",
|
||||
"get_tracked_agents",
|
||||
]
|
||||
515
src/dashboard/services/scorecard_service.py
Normal file
515
src/dashboard/services/scorecard_service.py
Normal file
@@ -0,0 +1,515 @@
|
||||
"""Agent scorecard service — track and summarize agent performance.
|
||||
|
||||
Generates daily/weekly scorecards showing:
|
||||
- Issues touched, PRs opened/merged
|
||||
- Tests affected, tokens earned/spent
|
||||
- Pattern highlights (merge rate, activity quality)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from enum import StrEnum
|
||||
from typing import Any
|
||||
|
||||
from infrastructure.events.bus import Event, get_event_bus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Bot/agent usernames to track
|
||||
TRACKED_AGENTS = frozenset({"hermes", "kimi", "manus", "claude", "gemini"})
|
||||
|
||||
|
||||
class PeriodType(StrEnum):
|
||||
daily = "daily"
|
||||
weekly = "weekly"
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentMetrics:
|
||||
"""Raw metrics collected for an agent over a period."""
|
||||
|
||||
agent_id: str
|
||||
issues_touched: set[int] = field(default_factory=set)
|
||||
prs_opened: set[int] = field(default_factory=set)
|
||||
prs_merged: set[int] = field(default_factory=set)
|
||||
tests_affected: set[str] = field(default_factory=set)
|
||||
tokens_earned: int = 0
|
||||
tokens_spent: int = 0
|
||||
commits: int = 0
|
||||
comments: int = 0
|
||||
|
||||
@property
|
||||
def pr_merge_rate(self) -> float:
|
||||
"""Calculate PR merge rate (0.0 - 1.0)."""
|
||||
opened = len(self.prs_opened)
|
||||
if opened == 0:
|
||||
return 0.0
|
||||
return len(self.prs_merged) / opened
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScorecardSummary:
|
||||
"""A generated scorecard with narrative summary."""
|
||||
|
||||
agent_id: str
|
||||
period_type: PeriodType
|
||||
period_start: datetime
|
||||
period_end: datetime
|
||||
metrics: AgentMetrics
|
||||
narrative_bullets: list[str] = field(default_factory=list)
|
||||
patterns: list[str] = field(default_factory=list)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Convert scorecard to dictionary for JSON serialization."""
|
||||
return {
|
||||
"agent_id": self.agent_id,
|
||||
"period_type": self.period_type.value,
|
||||
"period_start": self.period_start.isoformat(),
|
||||
"period_end": self.period_end.isoformat(),
|
||||
"metrics": {
|
||||
"issues_touched": len(self.metrics.issues_touched),
|
||||
"prs_opened": len(self.metrics.prs_opened),
|
||||
"prs_merged": len(self.metrics.prs_merged),
|
||||
"pr_merge_rate": round(self.metrics.pr_merge_rate, 2),
|
||||
"tests_affected": len(self.tests_affected),
|
||||
"commits": self.metrics.commits,
|
||||
"comments": self.metrics.comments,
|
||||
"tokens_earned": self.metrics.tokens_earned,
|
||||
"tokens_spent": self.metrics.tokens_spent,
|
||||
"token_net": self.metrics.tokens_earned - self.metrics.tokens_spent,
|
||||
},
|
||||
"narrative_bullets": self.narrative_bullets,
|
||||
"patterns": self.patterns,
|
||||
}
|
||||
|
||||
@property
|
||||
def tests_affected(self) -> set[str]:
|
||||
"""Alias for metrics.tests_affected."""
|
||||
return self.metrics.tests_affected
|
||||
|
||||
|
||||
def _get_period_bounds(
|
||||
period_type: PeriodType, reference_date: datetime | None = None
|
||||
) -> tuple[datetime, datetime]:
|
||||
"""Calculate start and end timestamps for a period.
|
||||
|
||||
Args:
|
||||
period_type: daily or weekly
|
||||
reference_date: The date to calculate from (defaults to now)
|
||||
|
||||
Returns:
|
||||
Tuple of (period_start, period_end) in UTC
|
||||
"""
|
||||
if reference_date is None:
|
||||
reference_date = datetime.now(UTC)
|
||||
|
||||
# Normalize to start of day
|
||||
end = reference_date.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
if period_type == PeriodType.daily:
|
||||
start = end - timedelta(days=1)
|
||||
else: # weekly
|
||||
start = end - timedelta(days=7)
|
||||
|
||||
return start, end
|
||||
|
||||
|
||||
def _collect_events_for_period(
|
||||
start: datetime, end: datetime, agent_id: str | None = None
|
||||
) -> list[Event]:
|
||||
"""Collect events from the event bus for a time period.
|
||||
|
||||
Args:
|
||||
start: Period start time
|
||||
end: Period end time
|
||||
agent_id: Optional agent filter
|
||||
|
||||
Returns:
|
||||
List of matching events
|
||||
"""
|
||||
bus = get_event_bus()
|
||||
events: list[Event] = []
|
||||
|
||||
# Query persisted events for relevant types
|
||||
event_types = [
|
||||
"gitea.push",
|
||||
"gitea.issue.opened",
|
||||
"gitea.issue.comment",
|
||||
"gitea.pull_request",
|
||||
"agent.task.completed",
|
||||
"test.execution",
|
||||
]
|
||||
|
||||
for event_type in event_types:
|
||||
try:
|
||||
type_events = bus.replay(
|
||||
event_type=event_type,
|
||||
source=agent_id,
|
||||
limit=1000,
|
||||
)
|
||||
events.extend(type_events)
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to replay events for %s: %s", event_type, exc)
|
||||
|
||||
# Filter by timestamp
|
||||
filtered = []
|
||||
for event in events:
|
||||
try:
|
||||
event_time = datetime.fromisoformat(event.timestamp.replace("Z", "+00:00"))
|
||||
if start <= event_time < end:
|
||||
filtered.append(event)
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
|
||||
return filtered
|
||||
|
||||
|
||||
def _extract_actor_from_event(event: Event) -> str:
|
||||
"""Extract the actor/agent from an event."""
|
||||
# Try data fields first
|
||||
if "actor" in event.data:
|
||||
return event.data["actor"]
|
||||
if "agent_id" in event.data:
|
||||
return event.data["agent_id"]
|
||||
# Fall back to source
|
||||
return event.source
|
||||
|
||||
|
||||
def _is_tracked_agent(actor: str) -> bool:
|
||||
"""Check if an actor is a tracked agent."""
|
||||
return actor.lower() in TRACKED_AGENTS
|
||||
|
||||
|
||||
def _aggregate_metrics(events: list[Event]) -> dict[str, AgentMetrics]:
|
||||
"""Aggregate metrics from events grouped by agent.
|
||||
|
||||
Args:
|
||||
events: List of events to process
|
||||
|
||||
Returns:
|
||||
Dict mapping agent_id -> AgentMetrics
|
||||
"""
|
||||
metrics_by_agent: dict[str, AgentMetrics] = {}
|
||||
|
||||
for event in events:
|
||||
actor = _extract_actor_from_event(event)
|
||||
|
||||
# Skip non-agent events unless they explicitly have an agent_id
|
||||
if not _is_tracked_agent(actor) and "agent_id" not in event.data:
|
||||
continue
|
||||
|
||||
if actor not in metrics_by_agent:
|
||||
metrics_by_agent[actor] = AgentMetrics(agent_id=actor)
|
||||
|
||||
metrics = metrics_by_agent[actor]
|
||||
|
||||
# Process based on event type
|
||||
event_type = event.type
|
||||
|
||||
if event_type == "gitea.push":
|
||||
metrics.commits += event.data.get("num_commits", 1)
|
||||
|
||||
elif event_type == "gitea.issue.opened":
|
||||
issue_num = event.data.get("issue_number", 0)
|
||||
if issue_num:
|
||||
metrics.issues_touched.add(issue_num)
|
||||
|
||||
elif event_type == "gitea.issue.comment":
|
||||
metrics.comments += 1
|
||||
issue_num = event.data.get("issue_number", 0)
|
||||
if issue_num:
|
||||
metrics.issues_touched.add(issue_num)
|
||||
|
||||
elif event_type == "gitea.pull_request":
|
||||
pr_num = event.data.get("pr_number", 0)
|
||||
action = event.data.get("action", "")
|
||||
merged = event.data.get("merged", False)
|
||||
|
||||
if pr_num:
|
||||
if action == "opened":
|
||||
metrics.prs_opened.add(pr_num)
|
||||
elif action == "closed" and merged:
|
||||
metrics.prs_merged.add(pr_num)
|
||||
# Also count as touched issue for tracking
|
||||
metrics.issues_touched.add(pr_num)
|
||||
|
||||
elif event_type == "agent.task.completed":
|
||||
# Extract test files from task data
|
||||
affected = event.data.get("tests_affected", [])
|
||||
for test in affected:
|
||||
metrics.tests_affected.add(test)
|
||||
|
||||
# Token rewards from task completion
|
||||
reward = event.data.get("token_reward", 0)
|
||||
if reward:
|
||||
metrics.tokens_earned += reward
|
||||
|
||||
elif event_type == "test.execution":
|
||||
# Track test files that were executed
|
||||
test_files = event.data.get("test_files", [])
|
||||
for test in test_files:
|
||||
metrics.tests_affected.add(test)
|
||||
|
||||
return metrics_by_agent
|
||||
|
||||
|
||||
def _query_token_transactions(agent_id: str, start: datetime, end: datetime) -> tuple[int, int]:
|
||||
"""Query the lightning ledger for token transactions.
|
||||
|
||||
Args:
|
||||
agent_id: The agent to query for
|
||||
start: Period start
|
||||
end: Period end
|
||||
|
||||
Returns:
|
||||
Tuple of (tokens_earned, tokens_spent)
|
||||
"""
|
||||
try:
|
||||
from lightning.ledger import get_transactions
|
||||
|
||||
transactions = get_transactions(limit=1000)
|
||||
|
||||
earned = 0
|
||||
spent = 0
|
||||
|
||||
for tx in transactions:
|
||||
# Filter by agent if specified
|
||||
if tx.agent_id and tx.agent_id != agent_id:
|
||||
continue
|
||||
|
||||
# Filter by timestamp
|
||||
try:
|
||||
tx_time = datetime.fromisoformat(tx.created_at.replace("Z", "+00:00"))
|
||||
if not (start <= tx_time < end):
|
||||
continue
|
||||
except (ValueError, AttributeError):
|
||||
continue
|
||||
|
||||
if tx.tx_type.value == "incoming":
|
||||
earned += tx.amount_sats
|
||||
else:
|
||||
spent += tx.amount_sats
|
||||
|
||||
return earned, spent
|
||||
|
||||
except Exception as exc:
|
||||
logger.debug("Failed to query token transactions: %s", exc)
|
||||
return 0, 0
|
||||
|
||||
|
||||
def _generate_narrative_bullets(metrics: AgentMetrics, period_type: PeriodType) -> list[str]:
|
||||
"""Generate narrative summary bullets for a scorecard.
|
||||
|
||||
Args:
|
||||
metrics: The agent's metrics
|
||||
period_type: daily or weekly
|
||||
|
||||
Returns:
|
||||
List of narrative bullet points
|
||||
"""
|
||||
bullets: list[str] = []
|
||||
period_label = "day" if period_type == PeriodType.daily else "week"
|
||||
|
||||
# Activity summary
|
||||
activities = []
|
||||
if metrics.commits:
|
||||
activities.append(f"{metrics.commits} commit{'s' if metrics.commits != 1 else ''}")
|
||||
if len(metrics.prs_opened):
|
||||
activities.append(
|
||||
f"{len(metrics.prs_opened)} PR{'s' if len(metrics.prs_opened) != 1 else ''} opened"
|
||||
)
|
||||
if len(metrics.prs_merged):
|
||||
activities.append(
|
||||
f"{len(metrics.prs_merged)} PR{'s' if len(metrics.prs_merged) != 1 else ''} merged"
|
||||
)
|
||||
if len(metrics.issues_touched):
|
||||
activities.append(
|
||||
f"{len(metrics.issues_touched)} issue{'s' if len(metrics.issues_touched) != 1 else ''} touched"
|
||||
)
|
||||
if metrics.comments:
|
||||
activities.append(f"{metrics.comments} comment{'s' if metrics.comments != 1 else ''}")
|
||||
|
||||
if activities:
|
||||
bullets.append(f"Active across {', '.join(activities)} this {period_label}.")
|
||||
|
||||
# Test activity
|
||||
if len(metrics.tests_affected):
|
||||
bullets.append(
|
||||
f"Affected {len(metrics.tests_affected)} test file{'s' if len(metrics.tests_affected) != 1 else ''}."
|
||||
)
|
||||
|
||||
# Token summary
|
||||
net_tokens = metrics.tokens_earned - metrics.tokens_spent
|
||||
if metrics.tokens_earned or metrics.tokens_spent:
|
||||
if net_tokens > 0:
|
||||
bullets.append(
|
||||
f"Net earned {net_tokens} tokens ({metrics.tokens_earned} earned, {metrics.tokens_spent} spent)."
|
||||
)
|
||||
elif net_tokens < 0:
|
||||
bullets.append(
|
||||
f"Net spent {abs(net_tokens)} tokens ({metrics.tokens_earned} earned, {metrics.tokens_spent} spent)."
|
||||
)
|
||||
else:
|
||||
bullets.append(
|
||||
f"Balanced token flow ({metrics.tokens_earned} earned, {metrics.tokens_spent} spent)."
|
||||
)
|
||||
|
||||
# Handle empty case
|
||||
if not bullets:
|
||||
bullets.append(f"No recorded activity this {period_label}.")
|
||||
|
||||
return bullets
|
||||
|
||||
|
||||
def _detect_patterns(metrics: AgentMetrics) -> list[str]:
|
||||
"""Detect interesting patterns in agent behavior.
|
||||
|
||||
Args:
|
||||
metrics: The agent's metrics
|
||||
|
||||
Returns:
|
||||
List of pattern descriptions
|
||||
"""
|
||||
patterns: list[str] = []
|
||||
|
||||
pr_opened = len(metrics.prs_opened)
|
||||
merge_rate = metrics.pr_merge_rate
|
||||
|
||||
# Merge rate patterns
|
||||
if pr_opened >= 3:
|
||||
if merge_rate >= 0.8:
|
||||
patterns.append("High merge rate with few failures — code quality focus.")
|
||||
elif merge_rate <= 0.3:
|
||||
patterns.append("Lots of noisy PRs, low merge rate — may need review support.")
|
||||
|
||||
# Activity patterns
|
||||
if metrics.commits > 10 and pr_opened == 0:
|
||||
patterns.append("High commit volume without PRs — working directly on main?")
|
||||
|
||||
if len(metrics.issues_touched) > 5 and metrics.comments == 0:
|
||||
patterns.append("Touching many issues but low comment volume — silent worker.")
|
||||
|
||||
if metrics.comments > len(metrics.issues_touched) * 2:
|
||||
patterns.append("Highly communicative — lots of discussion relative to work items.")
|
||||
|
||||
# Token patterns
|
||||
net_tokens = metrics.tokens_earned - metrics.tokens_spent
|
||||
if net_tokens > 100:
|
||||
patterns.append("Strong token accumulation — high value delivery.")
|
||||
elif net_tokens < -50:
|
||||
patterns.append("High token spend — may be in experimentation phase.")
|
||||
|
||||
return patterns
|
||||
|
||||
|
||||
def generate_scorecard(
|
||||
agent_id: str,
|
||||
period_type: PeriodType = PeriodType.daily,
|
||||
reference_date: datetime | None = None,
|
||||
) -> ScorecardSummary | None:
|
||||
"""Generate a scorecard for a single agent.
|
||||
|
||||
Args:
|
||||
agent_id: The agent to generate scorecard for
|
||||
period_type: daily or weekly
|
||||
reference_date: The date to calculate from (defaults to now)
|
||||
|
||||
Returns:
|
||||
ScorecardSummary or None if agent has no activity
|
||||
"""
|
||||
start, end = _get_period_bounds(period_type, reference_date)
|
||||
|
||||
# Collect events
|
||||
events = _collect_events_for_period(start, end, agent_id)
|
||||
|
||||
# Aggregate metrics
|
||||
all_metrics = _aggregate_metrics(events)
|
||||
|
||||
# Get metrics for this specific agent
|
||||
if agent_id not in all_metrics:
|
||||
# Create empty metrics - still generate a scorecard
|
||||
metrics = AgentMetrics(agent_id=agent_id)
|
||||
else:
|
||||
metrics = all_metrics[agent_id]
|
||||
|
||||
# Augment with token data from ledger
|
||||
tokens_earned, tokens_spent = _query_token_transactions(agent_id, start, end)
|
||||
metrics.tokens_earned = max(metrics.tokens_earned, tokens_earned)
|
||||
metrics.tokens_spent = max(metrics.tokens_spent, tokens_spent)
|
||||
|
||||
# Generate narrative and patterns
|
||||
narrative = _generate_narrative_bullets(metrics, period_type)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
return ScorecardSummary(
|
||||
agent_id=agent_id,
|
||||
period_type=period_type,
|
||||
period_start=start,
|
||||
period_end=end,
|
||||
metrics=metrics,
|
||||
narrative_bullets=narrative,
|
||||
patterns=patterns,
|
||||
)
|
||||
|
||||
|
||||
def generate_all_scorecards(
|
||||
period_type: PeriodType = PeriodType.daily,
|
||||
reference_date: datetime | None = None,
|
||||
) -> list[ScorecardSummary]:
|
||||
"""Generate scorecards for all tracked agents.
|
||||
|
||||
Args:
|
||||
period_type: daily or weekly
|
||||
reference_date: The date to calculate from (defaults to now)
|
||||
|
||||
Returns:
|
||||
List of ScorecardSummary for all agents with activity
|
||||
"""
|
||||
start, end = _get_period_bounds(period_type, reference_date)
|
||||
|
||||
# Collect all events
|
||||
events = _collect_events_for_period(start, end)
|
||||
|
||||
# Aggregate metrics for all agents
|
||||
all_metrics = _aggregate_metrics(events)
|
||||
|
||||
# Include tracked agents even if no activity
|
||||
for agent_id in TRACKED_AGENTS:
|
||||
if agent_id not in all_metrics:
|
||||
all_metrics[agent_id] = AgentMetrics(agent_id=agent_id)
|
||||
|
||||
# Generate scorecards
|
||||
scorecards: list[ScorecardSummary] = []
|
||||
|
||||
for agent_id, metrics in all_metrics.items():
|
||||
# Augment with token data
|
||||
tokens_earned, tokens_spent = _query_token_transactions(agent_id, start, end)
|
||||
metrics.tokens_earned = max(metrics.tokens_earned, tokens_earned)
|
||||
metrics.tokens_spent = max(metrics.tokens_spent, tokens_spent)
|
||||
|
||||
narrative = _generate_narrative_bullets(metrics, period_type)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
scorecard = ScorecardSummary(
|
||||
agent_id=agent_id,
|
||||
period_type=period_type,
|
||||
period_start=start,
|
||||
period_end=end,
|
||||
metrics=metrics,
|
||||
narrative_bullets=narrative,
|
||||
patterns=patterns,
|
||||
)
|
||||
scorecards.append(scorecard)
|
||||
|
||||
# Sort by agent_id for consistent ordering
|
||||
scorecards.sort(key=lambda s: s.agent_id)
|
||||
|
||||
return scorecards
|
||||
|
||||
|
||||
def get_tracked_agents() -> list[str]:
|
||||
"""Return the list of tracked agent IDs."""
|
||||
return sorted(TRACKED_AGENTS)
|
||||
@@ -51,6 +51,7 @@
|
||||
<a href="/thinking" class="mc-test-link mc-link-thinking">THINKING</a>
|
||||
<a href="/swarm/mission-control" class="mc-test-link">MISSION CTRL</a>
|
||||
<a href="/swarm/live" class="mc-test-link">SWARM</a>
|
||||
<a href="/scorecards" class="mc-test-link">SCORECARDS</a>
|
||||
<a href="/bugs" class="mc-test-link mc-link-bugs">BUGS</a>
|
||||
</div>
|
||||
</div>
|
||||
@@ -123,6 +124,7 @@
|
||||
<a href="/thinking" class="mc-mobile-link">THINKING</a>
|
||||
<a href="/swarm/mission-control" class="mc-mobile-link">MISSION CONTROL</a>
|
||||
<a href="/swarm/live" class="mc-mobile-link">SWARM</a>
|
||||
<a href="/scorecards" class="mc-mobile-link">SCORECARDS</a>
|
||||
<a href="/bugs" class="mc-mobile-link">BUGS</a>
|
||||
<div class="mc-mobile-section-label">INTELLIGENCE</div>
|
||||
<a href="/spark/ui" class="mc-mobile-link">SPARK</a>
|
||||
|
||||
@@ -179,6 +179,13 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Sovereignty Metrics -->
|
||||
{% call panel("SOVEREIGNTY METRICS", id="sovereignty-metrics-panel",
|
||||
hx_get="/sovereignty/metrics/panel",
|
||||
hx_trigger="load, every 30s") %}
|
||||
<p class="chat-history-placeholder">Loading sovereignty metrics...</p>
|
||||
{% endcall %}
|
||||
|
||||
<!-- Chat History -->
|
||||
<div class="card mc-card-spaced">
|
||||
<div class="card-header">
|
||||
|
||||
63
src/dashboard/templates/partials/sovereignty_metrics.html
Normal file
63
src/dashboard/templates/partials/sovereignty_metrics.html
Normal file
@@ -0,0 +1,63 @@
|
||||
{# HTMX partial: Sovereignty Metrics Progress Panel
|
||||
Loaded via hx-get="/sovereignty/metrics/panel"
|
||||
Refs: #981
|
||||
#}
|
||||
{% set phase_labels = {"pre-start": "Pre-start", "week1": "Week 1", "month1": "Month 1", "month3": "Month 3", "graduated": "Graduated"} %}
|
||||
{% set phase_colors = {"pre-start": "var(--text-dim)", "week1": "var(--red)", "month1": "var(--amber)", "month3": "var(--green)", "graduated": "var(--purple)"} %}
|
||||
|
||||
{% set metric_labels = {
|
||||
"cache_hit_rate": "Cache Hit Rate",
|
||||
"api_cost": "API Cost / Task",
|
||||
"time_to_report": "Time to Report",
|
||||
"human_involvement": "Human Involvement",
|
||||
"local_artifacts": "Local Artifacts"
|
||||
} %}
|
||||
|
||||
{% set metric_units = {
|
||||
"cache_hit_rate": "%",
|
||||
"api_cost": "$",
|
||||
"time_to_report": "min",
|
||||
"human_involvement": "%",
|
||||
"local_artifacts": ""
|
||||
} %}
|
||||
|
||||
{% if alerts %}
|
||||
<div class="sov-alerts">
|
||||
{% for alert in alerts %}
|
||||
<div class="sov-alert-item">
|
||||
<span class="sov-alert-icon">!</span>
|
||||
<span>{{ alert.message }}</span>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
<div class="grid grid-3">
|
||||
{% for key, data in metrics.items() %}
|
||||
{% set label = metric_labels.get(key, key) %}
|
||||
{% set unit = metric_units.get(key, "") %}
|
||||
{% set phase = data.phase %}
|
||||
{% set color = phase_colors.get(phase, "var(--text-dim)") %}
|
||||
<div class="stat">
|
||||
<div class="stat-value" style="color: {{ color }}">
|
||||
{% if data.current is not none %}
|
||||
{% if key == "cache_hit_rate" or key == "human_involvement" %}
|
||||
{{ "%.0f"|format(data.current * 100) }}{{ unit }}
|
||||
{% elif key == "api_cost" %}
|
||||
{{ unit }}{{ "%.2f"|format(data.current) }}
|
||||
{% elif key == "time_to_report" %}
|
||||
{{ "%.1f"|format(data.current) }}{{ unit }}
|
||||
{% else %}
|
||||
{{ data.current|int }}
|
||||
{% endif %}
|
||||
{% else %}
|
||||
--
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="stat-label">{{ label }}</div>
|
||||
<div class="stat-label" style="font-size: 0.7rem; color: {{ color }}">
|
||||
{{ phase_labels.get(phase, phase) }}
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
113
src/dashboard/templates/scorecards.html
Normal file
113
src/dashboard/templates/scorecards.html
Normal file
@@ -0,0 +1,113 @@
|
||||
{% extends "base.html" %}
|
||||
|
||||
{% block title %}Agent Scorecards - Timmy Time{% endblock %}
|
||||
|
||||
{% block extra_styles %}{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="container-fluid py-4">
|
||||
<!-- Header -->
|
||||
<div class="d-flex justify-content-between align-items-center mb-4">
|
||||
<div>
|
||||
<h1 class="h3 mb-0">AGENT SCORECARDS</h1>
|
||||
<p class="text-muted small mb-0">Track agent performance across issues, PRs, tests, and tokens</p>
|
||||
</div>
|
||||
<div class="d-flex gap-2">
|
||||
<select id="period-select" class="form-select form-select-sm" style="width: auto;">
|
||||
<option value="daily" selected>Daily</option>
|
||||
<option value="weekly">Weekly</option>
|
||||
</select>
|
||||
<button class="btn btn-sm btn-primary" onclick="refreshScorecards()">
|
||||
<span>Refresh</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Scorecards Grid -->
|
||||
<div id="scorecards-container"
|
||||
hx-get="/scorecards/all/panels?period=daily"
|
||||
hx-trigger="load"
|
||||
hx-swap="innerHTML">
|
||||
<div class="text-center py-5">
|
||||
<div class="spinner-border text-secondary" role="status">
|
||||
<span class="visually-hidden">Loading...</span>
|
||||
</div>
|
||||
<p class="text-muted mt-2">Loading scorecards...</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- API Reference -->
|
||||
<div class="mt-5 pt-4 border-top">
|
||||
<h5 class="text-muted">API Reference</h5>
|
||||
<div class="row g-3">
|
||||
<div class="col-md-6">
|
||||
<div class="card mc-panel">
|
||||
<div class="card-body">
|
||||
<h6 class="card-title">List Tracked Agents</h6>
|
||||
<code>GET /scorecards/api/agents</code>
|
||||
<p class="small text-muted mt-2">Returns all tracked agent IDs</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="card mc-panel">
|
||||
<div class="card-body">
|
||||
<h6 class="card-title">Get All Scorecards</h6>
|
||||
<code>GET /scorecards/api?period=daily|weekly</code>
|
||||
<p class="small text-muted mt-2">Returns scorecards for all agents</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="card mc-panel">
|
||||
<div class="card-body">
|
||||
<h6 class="card-title">Get Agent Scorecard</h6>
|
||||
<code>GET /scorecards/api/{agent_id}?period=daily|weekly</code>
|
||||
<p class="small text-muted mt-2">Returns scorecard for a specific agent</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="card mc-panel">
|
||||
<div class="card-body">
|
||||
<h6 class="card-title">HTML Panel (HTMX)</h6>
|
||||
<code>GET /scorecards/panel/{agent_id}?period=daily|weekly</code>
|
||||
<p class="small text-muted mt-2">Returns HTML panel for embedding</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Period selector change handler
|
||||
document.getElementById('period-select').addEventListener('change', function() {
|
||||
refreshScorecards();
|
||||
});
|
||||
|
||||
function refreshScorecards() {
|
||||
var period = document.getElementById('period-select').value;
|
||||
var container = document.getElementById('scorecards-container');
|
||||
|
||||
// Show loading state
|
||||
container.innerHTML = `
|
||||
<div class="text-center py-5">
|
||||
<div class="spinner-border text-secondary" role="status">
|
||||
<span class="visually-hidden">Loading...</span>
|
||||
</div>
|
||||
<p class="text-muted mt-2">Loading scorecards...</p>
|
||||
</div>
|
||||
`;
|
||||
|
||||
// Trigger HTMX request
|
||||
htmx.ajax('GET', '/scorecards/all/panels?period=' + period, {
|
||||
target: '#scorecards-container',
|
||||
swap: 'innerHTML'
|
||||
});
|
||||
}
|
||||
|
||||
// Auto-refresh every 5 minutes
|
||||
setInterval(refreshScorecards, 300000);
|
||||
</script>
|
||||
{% endblock %}
|
||||
264
src/infrastructure/claude_quota.py
Normal file
264
src/infrastructure/claude_quota.py
Normal file
@@ -0,0 +1,264 @@
|
||||
"""
|
||||
claude_quota.py — Claude Code / Claude.ai Quota Monitor
|
||||
|
||||
Drop into src/infrastructure/ in the Timmy Time Dashboard repo.
|
||||
|
||||
Provides real-time quota visibility and metabolic protocol decisions.
|
||||
|
||||
Usage:
|
||||
from infrastructure.claude_quota import QuotaMonitor
|
||||
|
||||
monitor = QuotaMonitor()
|
||||
status = monitor.check()
|
||||
print(status.five_hour_pct) # 42
|
||||
print(status.five_hour_resets_in) # "2h 15m"
|
||||
print(status.seven_day_pct) # 29
|
||||
print(status.recommended_tier) # MetabolicTier.BURST
|
||||
|
||||
# Metabolic protocol: auto-select model based on quota
|
||||
model = monitor.select_model(task_complexity="high")
|
||||
# Returns "claude-sonnet-4-6" if quota allows, else "qwen3:14b"
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MetabolicTier(StrEnum):
|
||||
"""The three-tier metabolic protocol from the Timmy Time architecture."""
|
||||
|
||||
BURST = "burst" # Cloud API (Claude/Groq) — expensive, best quality
|
||||
ACTIVE = "active" # Local 14B (Qwen3-14B) — free, good quality
|
||||
RESTING = "resting" # Local 8B (Qwen3-8B) — free, fast, adequate
|
||||
|
||||
|
||||
@dataclass
|
||||
class QuotaStatus:
|
||||
"""Current Claude quota state."""
|
||||
|
||||
five_hour_utilization: float # 0.0 to 1.0
|
||||
five_hour_resets_at: str | None
|
||||
seven_day_utilization: float # 0.0 to 1.0
|
||||
seven_day_resets_at: str | None
|
||||
raw_response: dict
|
||||
fetched_at: datetime
|
||||
|
||||
@property
|
||||
def five_hour_pct(self) -> int:
|
||||
return int(self.five_hour_utilization * 100)
|
||||
|
||||
@property
|
||||
def seven_day_pct(self) -> int:
|
||||
return int(self.seven_day_utilization * 100)
|
||||
|
||||
@property
|
||||
def five_hour_resets_in(self) -> str:
|
||||
return _time_remaining(self.five_hour_resets_at)
|
||||
|
||||
@property
|
||||
def seven_day_resets_in(self) -> str:
|
||||
return _time_remaining(self.seven_day_resets_at)
|
||||
|
||||
@property
|
||||
def recommended_tier(self) -> MetabolicTier:
|
||||
"""Metabolic protocol: determine which inference tier to use."""
|
||||
# If weekly quota is critical, go full local
|
||||
if self.seven_day_utilization >= 0.80:
|
||||
return MetabolicTier.RESTING
|
||||
# If 5-hour window is critical or past half, use local
|
||||
if self.five_hour_utilization >= 0.50:
|
||||
return MetabolicTier.ACTIVE
|
||||
# Quota healthy — cloud available for high-value tasks
|
||||
return MetabolicTier.BURST
|
||||
|
||||
def summary(self) -> str:
|
||||
"""Human-readable status string."""
|
||||
return (
|
||||
f"5h: {self.five_hour_pct}% (resets {self.five_hour_resets_in}) | "
|
||||
f"7d: {self.seven_day_pct}% (resets {self.seven_day_resets_in}) | "
|
||||
f"tier: {self.recommended_tier.value}"
|
||||
)
|
||||
|
||||
|
||||
class QuotaMonitor:
|
||||
"""
|
||||
Monitors Claude Code / Claude.ai quota via the internal OAuth API.
|
||||
|
||||
The token is read from macOS Keychain where Claude Code stores it.
|
||||
Falls back gracefully if credentials aren't available (e.g., on Linux VPS).
|
||||
"""
|
||||
|
||||
API_URL = "https://api.anthropic.com/api/oauth/usage"
|
||||
KEYCHAIN_SERVICE = "Claude Code-credentials"
|
||||
USER_AGENT = "claude-code/2.0.32"
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._token: str | None = None
|
||||
self._last_status: QuotaStatus | None = None
|
||||
self._cache_seconds = 30 # Don't hammer the API
|
||||
|
||||
def _get_token(self) -> str | None:
|
||||
"""Extract OAuth token from macOS Keychain."""
|
||||
if self._token:
|
||||
return self._token
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["security", "find-generic-password", "-s", self.KEYCHAIN_SERVICE, "-w"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.warning("Claude Code credentials not found in Keychain")
|
||||
return None
|
||||
|
||||
creds = json.loads(result.stdout.strip())
|
||||
oauth = creds.get("claudeAiOauth", creds)
|
||||
self._token = oauth.get("accessToken")
|
||||
return self._token
|
||||
|
||||
except (
|
||||
json.JSONDecodeError,
|
||||
KeyError,
|
||||
FileNotFoundError,
|
||||
subprocess.TimeoutExpired,
|
||||
) as exc:
|
||||
logger.warning("Could not read Claude Code credentials: %s", exc)
|
||||
return None
|
||||
|
||||
def check(self, force: bool = False) -> QuotaStatus | None:
|
||||
"""
|
||||
Fetch current quota status.
|
||||
|
||||
Returns None if credentials aren't available (graceful degradation).
|
||||
Caches results for 30 seconds to avoid rate limiting the quota API itself.
|
||||
"""
|
||||
# Return cached if fresh
|
||||
if not force and self._last_status:
|
||||
age = (datetime.now(UTC) - self._last_status.fetched_at).total_seconds()
|
||||
if age < self._cache_seconds:
|
||||
return self._last_status
|
||||
|
||||
token = self._get_token()
|
||||
if not token:
|
||||
return None
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
self.API_URL,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": self.USER_AGENT,
|
||||
"Authorization": f"Bearer {token}",
|
||||
"anthropic-beta": "oauth-2025-04-20",
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
data = json.loads(resp.read().decode())
|
||||
|
||||
five_hour = data.get("five_hour") or {}
|
||||
seven_day = data.get("seven_day") or {}
|
||||
|
||||
self._last_status = QuotaStatus(
|
||||
five_hour_utilization=float(five_hour.get("utilization", 0.0)),
|
||||
five_hour_resets_at=five_hour.get("resets_at"),
|
||||
seven_day_utilization=float(seven_day.get("utilization", 0.0)),
|
||||
seven_day_resets_at=seven_day.get("resets_at"),
|
||||
raw_response=data,
|
||||
fetched_at=datetime.now(UTC),
|
||||
)
|
||||
return self._last_status
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to fetch quota: %s", exc)
|
||||
return self._last_status # Return stale data if available
|
||||
|
||||
def select_model(self, task_complexity: str = "medium") -> str:
|
||||
"""
|
||||
Metabolic protocol: select the right model based on quota + task complexity.
|
||||
|
||||
Returns an Ollama model tag or "claude-sonnet-4-6" for cloud.
|
||||
|
||||
task_complexity: "low" | "medium" | "high"
|
||||
"""
|
||||
status = self.check()
|
||||
|
||||
# No quota info available — assume local only (sovereign default)
|
||||
if status is None:
|
||||
return "qwen3:14b" if task_complexity == "high" else "qwen3:8b"
|
||||
|
||||
tier = status.recommended_tier
|
||||
|
||||
if tier == MetabolicTier.BURST and task_complexity == "high":
|
||||
return "claude-sonnet-4-6" # Cloud — best quality
|
||||
elif tier == MetabolicTier.BURST and task_complexity == "medium":
|
||||
return "qwen3:14b" # Save cloud for truly hard tasks
|
||||
elif tier == MetabolicTier.ACTIVE:
|
||||
return "qwen3:14b" # Local 14B — good enough
|
||||
else: # RESTING
|
||||
return "qwen3:8b" # Local 8B — conserve everything
|
||||
|
||||
def should_use_cloud(self, task_value: str = "normal") -> bool:
|
||||
"""
|
||||
Simple yes/no: should this task use cloud API?
|
||||
|
||||
task_value: "critical" | "high" | "normal" | "routine"
|
||||
"""
|
||||
status = self.check()
|
||||
|
||||
if status is None:
|
||||
return False # No credentials = local only
|
||||
|
||||
if task_value == "critical":
|
||||
return status.seven_day_utilization < 0.95 # Almost always yes
|
||||
elif task_value == "high":
|
||||
return status.five_hour_utilization < 0.60
|
||||
elif task_value == "normal":
|
||||
return status.five_hour_utilization < 0.30
|
||||
else: # routine
|
||||
return False # Never waste cloud on routine
|
||||
|
||||
|
||||
def _time_remaining(reset_at: str | None) -> str:
|
||||
"""Format time until reset as human-readable string."""
|
||||
if not reset_at or reset_at == "null":
|
||||
return "unknown"
|
||||
|
||||
try:
|
||||
reset = datetime.fromisoformat(reset_at.replace("Z", "+00:00"))
|
||||
now = datetime.now(UTC)
|
||||
diff = reset - now
|
||||
|
||||
if diff.total_seconds() <= 0:
|
||||
return "resetting now"
|
||||
|
||||
hours = int(diff.total_seconds() // 3600)
|
||||
mins = int((diff.total_seconds() % 3600) // 60)
|
||||
|
||||
if hours > 0:
|
||||
return f"{hours}h {mins}m"
|
||||
return f"{mins}m"
|
||||
|
||||
except (ValueError, TypeError):
|
||||
return "unknown"
|
||||
|
||||
|
||||
# Module-level singleton
|
||||
_quota_monitor: QuotaMonitor | None = None
|
||||
|
||||
|
||||
def get_quota_monitor() -> QuotaMonitor:
|
||||
"""Get or create the quota monitor singleton."""
|
||||
global _quota_monitor
|
||||
if _quota_monitor is None:
|
||||
_quota_monitor = QuotaMonitor()
|
||||
return _quota_monitor
|
||||
84
src/infrastructure/db_pool.py
Normal file
84
src/infrastructure/db_pool.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""Thread-local SQLite connection pool.
|
||||
|
||||
Provides a ConnectionPool class that manages SQLite connections per thread,
|
||||
with support for context managers and automatic cleanup.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import threading
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ConnectionPool:
|
||||
"""Thread-local SQLite connection pool.
|
||||
|
||||
Each thread gets its own connection, which is reused for subsequent
|
||||
requests from the same thread. Connections are automatically cleaned
|
||||
up when close_connection() is called or the context manager exits.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: Path | str) -> None:
|
||||
"""Initialize the connection pool.
|
||||
|
||||
Args:
|
||||
db_path: Path to the SQLite database file.
|
||||
"""
|
||||
self._db_path = Path(db_path)
|
||||
self._local = threading.local()
|
||||
|
||||
def _ensure_db_exists(self) -> None:
|
||||
"""Ensure the database directory exists."""
|
||||
self._db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def get_connection(self) -> sqlite3.Connection:
|
||||
"""Get a connection for the current thread.
|
||||
|
||||
Creates a new connection if one doesn't exist for this thread,
|
||||
otherwise returns the existing connection.
|
||||
|
||||
Returns:
|
||||
A sqlite3 Connection object.
|
||||
"""
|
||||
if not hasattr(self._local, "conn") or self._local.conn is None:
|
||||
self._ensure_db_exists()
|
||||
self._local.conn = sqlite3.connect(str(self._db_path), check_same_thread=False)
|
||||
self._local.conn.row_factory = sqlite3.Row
|
||||
return self._local.conn
|
||||
|
||||
def close_connection(self) -> None:
|
||||
"""Close the connection for the current thread.
|
||||
|
||||
Cleans up the thread-local storage. Safe to call even if
|
||||
no connection exists for this thread.
|
||||
"""
|
||||
if hasattr(self._local, "conn") and self._local.conn is not None:
|
||||
self._local.conn.close()
|
||||
self._local.conn = None
|
||||
|
||||
@contextmanager
|
||||
def connection(self) -> Generator[sqlite3.Connection, None, None]:
|
||||
"""Context manager for getting and automatically closing a connection.
|
||||
|
||||
Yields:
|
||||
A sqlite3 Connection object.
|
||||
|
||||
Example:
|
||||
with pool.connection() as conn:
|
||||
cursor = conn.execute("SELECT 1")
|
||||
result = cursor.fetchone()
|
||||
"""
|
||||
conn = self.get_connection()
|
||||
try:
|
||||
yield conn
|
||||
finally:
|
||||
self.close_connection()
|
||||
|
||||
def close_all(self) -> None:
|
||||
"""Close all connections (useful for testing).
|
||||
|
||||
Note: This only closes the connection for the current thread.
|
||||
In a multi-threaded environment, each thread must close its own.
|
||||
"""
|
||||
self.close_connection()
|
||||
7
src/infrastructure/guards/__init__.py
Normal file
7
src/infrastructure/guards/__init__.py
Normal file
@@ -0,0 +1,7 @@
|
||||
"""Content moderation pipeline for AI narrator output.
|
||||
|
||||
Three-layer defense:
|
||||
1. Game-context system prompts (vocabulary whitelists, theme framing)
|
||||
2. Real-time output filter via Llama Guard (or fallback regex)
|
||||
3. Per-game moderation profiles with configurable thresholds
|
||||
"""
|
||||
497
src/infrastructure/guards/moderation.py
Normal file
497
src/infrastructure/guards/moderation.py
Normal file
@@ -0,0 +1,497 @@
|
||||
"""Content moderation pipeline for AI narrator output.
|
||||
|
||||
Three-layer defense against harmful LLM output:
|
||||
|
||||
Layer 1 — Game-context system prompts with per-game vocabulary whitelists.
|
||||
Layer 2 — Real-time output filter (Llama Guard via Ollama, regex fallback).
|
||||
Layer 3 — Per-game moderation profiles with configurable thresholds.
|
||||
|
||||
Usage:
|
||||
from infrastructure.guards.moderation import get_moderator
|
||||
|
||||
moderator = get_moderator()
|
||||
result = await moderator.check("Some narrator text", game="morrowind")
|
||||
if result.blocked:
|
||||
use_fallback_narration(result.fallback)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ModerationVerdict(Enum):
|
||||
"""Result of a moderation check."""
|
||||
|
||||
PASS = "pass" # noqa: S105
|
||||
FAIL = "fail"
|
||||
ERROR = "error"
|
||||
|
||||
|
||||
class ViolationCategory(Enum):
|
||||
"""Categories of content violations."""
|
||||
|
||||
HATE_SPEECH = "hate_speech"
|
||||
VIOLENCE_GLORIFICATION = "violence_glorification"
|
||||
REAL_WORLD_HARM = "real_world_harm"
|
||||
SEXUAL_CONTENT = "sexual_content"
|
||||
SELF_HARM = "self_harm"
|
||||
NONE = "none"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModerationResult:
|
||||
"""Result from the moderation pipeline."""
|
||||
|
||||
verdict: ModerationVerdict
|
||||
blocked: bool
|
||||
category: ViolationCategory = ViolationCategory.NONE
|
||||
confidence: float = 0.0
|
||||
latency_ms: float = 0.0
|
||||
layer: str = "" # Which layer caught it
|
||||
fallback: str = "" # Contextual fallback narration
|
||||
reason: str = ""
|
||||
timestamp: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
|
||||
|
||||
@property
|
||||
def passed(self) -> bool:
|
||||
return self.verdict == ModerationVerdict.PASS
|
||||
|
||||
|
||||
@dataclass
|
||||
class GameProfile:
|
||||
"""Per-game moderation profile.
|
||||
|
||||
Defines vocabulary whitelists, theme framing rules,
|
||||
and threshold tuning for a specific game.
|
||||
"""
|
||||
|
||||
game_id: str
|
||||
display_name: str
|
||||
# Terms that are safe in game context (e.g., "Skooma", "slave", "Morag Tong")
|
||||
vocabulary_whitelist: list[str] = field(default_factory=list)
|
||||
# System prompt additions for game-context framing
|
||||
context_prompt: str = ""
|
||||
# Confidence threshold — below this, content passes even if flagged
|
||||
threshold: float = 0.8
|
||||
# Scene-type fallback narrations
|
||||
fallbacks: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
# ── Regex fallback patterns ─────────────────────────────────────────────────
|
||||
# Used when Llama Guard is unavailable. Catches obvious violations only.
|
||||
# Intentionally conservative — false negatives preferred over false positives.
|
||||
_FALLBACK_PATTERNS: list[tuple[re.Pattern, ViolationCategory]] = [
|
||||
(
|
||||
re.compile(
|
||||
r"\b(kill\s+(?:all|every)\s+(?:real|actual)\s+(?:people|humans|children))\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
ViolationCategory.REAL_WORLD_HARM,
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"\b((?:you|we)\s+should\s+(?:actually|really)\s+(?:harm|hurt|kill))\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
ViolationCategory.REAL_WORLD_HARM,
|
||||
),
|
||||
(
|
||||
re.compile(
|
||||
r"\b((?:in\s+real\s+life|irl)\s+(?:you|we)\s+(?:should|could|must)\s+"
|
||||
r"(?:attack|hurt|kill|harm))\b",
|
||||
re.IGNORECASE,
|
||||
),
|
||||
ViolationCategory.REAL_WORLD_HARM,
|
||||
),
|
||||
]
|
||||
|
||||
# ── Default game profiles ───────────────────────────────────────────────────
|
||||
|
||||
_DEFAULT_PROFILES: dict[str, GameProfile] = {
|
||||
"morrowind": GameProfile(
|
||||
game_id="morrowind",
|
||||
display_name="The Elder Scrolls III: Morrowind",
|
||||
vocabulary_whitelist=[
|
||||
"Skooma",
|
||||
"Moon Sugar",
|
||||
"slave",
|
||||
"slavery",
|
||||
"Morag Tong",
|
||||
"Dark Brotherhood",
|
||||
"Telvanni",
|
||||
"Camonna Tong",
|
||||
"smuggler",
|
||||
"assassin",
|
||||
"Sixth House",
|
||||
"Corprus",
|
||||
"Dagoth Ur",
|
||||
"Nerevarine",
|
||||
],
|
||||
context_prompt=(
|
||||
"You are narrating gameplay of The Elder Scrolls III: Morrowind. "
|
||||
"Morrowind contains mature themes including slavery, drug use (Skooma/Moon Sugar), "
|
||||
"assassin guilds (Morag Tong, Dark Brotherhood), and political intrigue. "
|
||||
"Treat these as game mechanics and historical worldbuilding within the game's "
|
||||
"fictional universe. Never editorialize on real-world parallels. "
|
||||
"Narrate events neutrally as a game commentator would."
|
||||
),
|
||||
threshold=0.85,
|
||||
fallbacks={
|
||||
"combat": "The battle rages on in the ashlands of Vvardenfell.",
|
||||
"dialogue": "The conversation continues between the characters.",
|
||||
"exploration": "The Nerevarine presses onward through the landscape.",
|
||||
"default": "The adventure continues in Morrowind.",
|
||||
},
|
||||
),
|
||||
"default": GameProfile(
|
||||
game_id="default",
|
||||
display_name="Generic Game",
|
||||
vocabulary_whitelist=[],
|
||||
context_prompt=(
|
||||
"You are narrating gameplay. Describe in-game events as a neutral "
|
||||
"game commentator. Never reference real-world violence, politics, "
|
||||
"or controversial topics. Stay focused on game mechanics and story."
|
||||
),
|
||||
threshold=0.8,
|
||||
fallbacks={
|
||||
"combat": "The action continues on screen.",
|
||||
"dialogue": "The conversation unfolds between characters.",
|
||||
"exploration": "The player explores the game world.",
|
||||
"default": "The gameplay continues.",
|
||||
},
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class ContentModerator:
|
||||
"""Three-layer content moderation pipeline.
|
||||
|
||||
Layer 1: Game-context system prompts with vocabulary whitelists.
|
||||
Layer 2: LLM-based moderation (Llama Guard via Ollama, with regex fallback).
|
||||
Layer 3: Per-game threshold tuning and profile-based filtering.
|
||||
|
||||
Follows graceful degradation — if Llama Guard is unavailable,
|
||||
falls back to regex patterns. Never crashes.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
profiles: dict[str, GameProfile] | None = None,
|
||||
guard_model: str | None = None,
|
||||
) -> None:
|
||||
self._profiles: dict[str, GameProfile] = profiles or dict(_DEFAULT_PROFILES)
|
||||
self._guard_model = guard_model or settings.moderation_guard_model
|
||||
self._guard_available: bool | None = None # Lazy-checked
|
||||
self._metrics = _ModerationMetrics()
|
||||
|
||||
def get_profile(self, game: str) -> GameProfile:
|
||||
"""Get the moderation profile for a game, falling back to default."""
|
||||
return self._profiles.get(game, self._profiles["default"])
|
||||
|
||||
def register_profile(self, profile: GameProfile) -> None:
|
||||
"""Register or update a game moderation profile."""
|
||||
self._profiles[profile.game_id] = profile
|
||||
logger.info("Registered moderation profile: %s", profile.game_id)
|
||||
|
||||
def get_context_prompt(self, game: str) -> str:
|
||||
"""Get the game-context system prompt (Layer 1).
|
||||
|
||||
Returns the context prompt for the given game, which should be
|
||||
prepended to the narrator's system prompt.
|
||||
"""
|
||||
profile = self.get_profile(game)
|
||||
return profile.context_prompt
|
||||
|
||||
async def check(
|
||||
self,
|
||||
text: str,
|
||||
game: str = "default",
|
||||
scene_type: str = "default",
|
||||
) -> ModerationResult:
|
||||
"""Run the full moderation pipeline on narrator output.
|
||||
|
||||
Args:
|
||||
text: The text to moderate (narrator output).
|
||||
game: Game identifier for profile selection.
|
||||
scene_type: Current scene type for fallback selection.
|
||||
|
||||
Returns:
|
||||
ModerationResult with verdict, confidence, and fallback.
|
||||
"""
|
||||
start = time.monotonic()
|
||||
profile = self.get_profile(game)
|
||||
|
||||
# Layer 1: Vocabulary whitelist pre-processing
|
||||
cleaned_text = self._apply_whitelist(text, profile)
|
||||
|
||||
# Layer 2: LLM guard or regex fallback
|
||||
result = await self._run_guard(cleaned_text, profile)
|
||||
|
||||
# Layer 3: Threshold tuning
|
||||
if result.verdict == ModerationVerdict.FAIL and result.confidence < profile.threshold:
|
||||
logger.info(
|
||||
"Moderation flag below threshold (%.2f < %.2f) — allowing",
|
||||
result.confidence,
|
||||
profile.threshold,
|
||||
)
|
||||
result = ModerationResult(
|
||||
verdict=ModerationVerdict.PASS,
|
||||
blocked=False,
|
||||
confidence=result.confidence,
|
||||
layer="threshold",
|
||||
reason=f"Below threshold ({result.confidence:.2f} < {profile.threshold:.2f})",
|
||||
)
|
||||
|
||||
# Attach fallback narration if blocked
|
||||
if result.blocked:
|
||||
result.fallback = profile.fallbacks.get(
|
||||
scene_type, profile.fallbacks.get("default", "")
|
||||
)
|
||||
|
||||
result.latency_ms = (time.monotonic() - start) * 1000
|
||||
self._metrics.record(result)
|
||||
|
||||
if result.blocked:
|
||||
logger.warning(
|
||||
"Content blocked [%s/%s]: category=%s confidence=%.2f reason=%s",
|
||||
game,
|
||||
scene_type,
|
||||
result.category.value,
|
||||
result.confidence,
|
||||
result.reason,
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
def _apply_whitelist(self, text: str, profile: GameProfile) -> str:
|
||||
"""Layer 1: Replace whitelisted game terms with placeholders.
|
||||
|
||||
This prevents the guard model from flagging in-game terminology
|
||||
(e.g., "Skooma" being flagged as drug reference).
|
||||
"""
|
||||
cleaned = text
|
||||
for term in profile.vocabulary_whitelist:
|
||||
# Case-insensitive replacement with a neutral placeholder
|
||||
pattern = re.compile(re.escape(term), re.IGNORECASE)
|
||||
cleaned = pattern.sub("[GAME_TERM]", cleaned)
|
||||
return cleaned
|
||||
|
||||
async def _run_guard(self, text: str, profile: GameProfile) -> ModerationResult:
|
||||
"""Layer 2: Run LLM guard model or fall back to regex."""
|
||||
if not settings.moderation_enabled:
|
||||
return ModerationResult(
|
||||
verdict=ModerationVerdict.PASS,
|
||||
blocked=False,
|
||||
layer="disabled",
|
||||
reason="Moderation disabled",
|
||||
)
|
||||
|
||||
# Try Llama Guard via Ollama
|
||||
if await self._is_guard_available():
|
||||
try:
|
||||
return await self._check_with_guard(text)
|
||||
except Exception as exc:
|
||||
logger.warning("Guard model failed, using regex fallback: %s", exc)
|
||||
self._guard_available = False
|
||||
|
||||
# Regex fallback
|
||||
return self._check_with_regex(text)
|
||||
|
||||
async def _is_guard_available(self) -> bool:
|
||||
"""Check if the guard model is available via Ollama."""
|
||||
if self._guard_available is not None:
|
||||
return self._guard_available
|
||||
|
||||
try:
|
||||
import aiohttp
|
||||
|
||||
url = f"{settings.normalized_ollama_url}/api/tags"
|
||||
timeout = aiohttp.ClientTimeout(total=5)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.get(url) as resp:
|
||||
if resp.status != 200:
|
||||
self._guard_available = False
|
||||
return False
|
||||
data = await resp.json()
|
||||
models = [m.get("name", "") for m in data.get("models", [])]
|
||||
self._guard_available = any(
|
||||
self._guard_model in m or m.startswith(self._guard_model) for m in models
|
||||
)
|
||||
if not self._guard_available:
|
||||
logger.info(
|
||||
"Guard model '%s' not found in Ollama — using regex fallback",
|
||||
self._guard_model,
|
||||
)
|
||||
return self._guard_available
|
||||
except Exception as exc:
|
||||
logger.debug("Ollama guard check failed: %s", exc)
|
||||
self._guard_available = False
|
||||
return False
|
||||
|
||||
async def _check_with_guard(self, text: str) -> ModerationResult:
|
||||
"""Run moderation check via Llama Guard."""
|
||||
import aiohttp
|
||||
|
||||
url = f"{settings.normalized_ollama_url}/api/chat"
|
||||
payload = {
|
||||
"model": self._guard_model,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": text,
|
||||
}
|
||||
],
|
||||
"stream": False,
|
||||
"options": {"temperature": 0.0},
|
||||
}
|
||||
|
||||
timeout = aiohttp.ClientTimeout(total=10)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
async with session.post(url, json=payload) as resp:
|
||||
if resp.status != 200:
|
||||
raise RuntimeError(f"Guard API error: {resp.status}")
|
||||
data = await resp.json()
|
||||
|
||||
response_text = data.get("message", {}).get("content", "").strip().lower()
|
||||
|
||||
# Llama Guard returns "safe" or "unsafe\n<category>"
|
||||
if response_text.startswith("safe"):
|
||||
return ModerationResult(
|
||||
verdict=ModerationVerdict.PASS,
|
||||
blocked=False,
|
||||
confidence=0.0,
|
||||
layer="llama_guard",
|
||||
reason="Content safe",
|
||||
)
|
||||
|
||||
# Parse unsafe response
|
||||
category = ViolationCategory.NONE
|
||||
confidence = 0.95 # High confidence from LLM guard
|
||||
lines = response_text.split("\n")
|
||||
if len(lines) > 1:
|
||||
cat_str = lines[1].strip()
|
||||
category = _parse_guard_category(cat_str)
|
||||
|
||||
return ModerationResult(
|
||||
verdict=ModerationVerdict.FAIL,
|
||||
blocked=True,
|
||||
category=category,
|
||||
confidence=confidence,
|
||||
layer="llama_guard",
|
||||
reason=f"Guard flagged: {response_text}",
|
||||
)
|
||||
|
||||
def _check_with_regex(self, text: str) -> ModerationResult:
|
||||
"""Regex fallback when guard model is unavailable.
|
||||
|
||||
Intentionally conservative — only catches obvious real-world harm.
|
||||
"""
|
||||
for pattern, category in _FALLBACK_PATTERNS:
|
||||
match = pattern.search(text)
|
||||
if match:
|
||||
return ModerationResult(
|
||||
verdict=ModerationVerdict.FAIL,
|
||||
blocked=True,
|
||||
category=category,
|
||||
confidence=0.95, # Regex patterns are high-signal
|
||||
layer="regex_fallback",
|
||||
reason=f"Regex match: {match.group(0)[:50]}",
|
||||
)
|
||||
|
||||
return ModerationResult(
|
||||
verdict=ModerationVerdict.PASS,
|
||||
blocked=False,
|
||||
layer="regex_fallback",
|
||||
reason="No regex matches",
|
||||
)
|
||||
|
||||
def get_metrics(self) -> dict[str, Any]:
|
||||
"""Get moderation pipeline metrics."""
|
||||
return self._metrics.to_dict()
|
||||
|
||||
def reset_guard_cache(self) -> None:
|
||||
"""Reset the guard availability cache (e.g., after pulling model)."""
|
||||
self._guard_available = None
|
||||
|
||||
|
||||
class _ModerationMetrics:
|
||||
"""Tracks moderation pipeline performance."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.total_checks: int = 0
|
||||
self.passed: int = 0
|
||||
self.blocked: int = 0
|
||||
self.errors: int = 0
|
||||
self.total_latency_ms: float = 0.0
|
||||
self.by_layer: dict[str, int] = {}
|
||||
self.by_category: dict[str, int] = {}
|
||||
|
||||
def record(self, result: ModerationResult) -> None:
|
||||
self.total_checks += 1
|
||||
self.total_latency_ms += result.latency_ms
|
||||
|
||||
if result.verdict == ModerationVerdict.PASS:
|
||||
self.passed += 1
|
||||
elif result.verdict == ModerationVerdict.FAIL:
|
||||
self.blocked += 1
|
||||
else:
|
||||
self.errors += 1
|
||||
|
||||
layer = result.layer or "unknown"
|
||||
self.by_layer[layer] = self.by_layer.get(layer, 0) + 1
|
||||
|
||||
if result.blocked:
|
||||
cat = result.category.value
|
||||
self.by_category[cat] = self.by_category.get(cat, 0) + 1
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"total_checks": self.total_checks,
|
||||
"passed": self.passed,
|
||||
"blocked": self.blocked,
|
||||
"errors": self.errors,
|
||||
"avg_latency_ms": (
|
||||
round(self.total_latency_ms / self.total_checks, 2)
|
||||
if self.total_checks > 0
|
||||
else 0.0
|
||||
),
|
||||
"by_layer": dict(self.by_layer),
|
||||
"by_category": dict(self.by_category),
|
||||
}
|
||||
|
||||
|
||||
def _parse_guard_category(cat_str: str) -> ViolationCategory:
|
||||
"""Parse Llama Guard category string to ViolationCategory."""
|
||||
cat_lower = cat_str.lower()
|
||||
if "hate" in cat_lower:
|
||||
return ViolationCategory.HATE_SPEECH
|
||||
if "violence" in cat_lower:
|
||||
return ViolationCategory.VIOLENCE_GLORIFICATION
|
||||
if "sexual" in cat_lower:
|
||||
return ViolationCategory.SEXUAL_CONTENT
|
||||
if "self-harm" in cat_lower or "self_harm" in cat_lower or "suicide" in cat_lower:
|
||||
return ViolationCategory.SELF_HARM
|
||||
if "harm" in cat_lower or "dangerous" in cat_lower:
|
||||
return ViolationCategory.REAL_WORLD_HARM
|
||||
return ViolationCategory.NONE
|
||||
|
||||
|
||||
# ── Module-level singleton ──────────────────────────────────────────────────
|
||||
_moderator: ContentModerator | None = None
|
||||
|
||||
|
||||
def get_moderator() -> ContentModerator:
|
||||
"""Get or create the content moderator singleton."""
|
||||
global _moderator
|
||||
if _moderator is None:
|
||||
_moderator = ContentModerator()
|
||||
return _moderator
|
||||
56
src/infrastructure/guards/profiles.py
Normal file
56
src/infrastructure/guards/profiles.py
Normal file
@@ -0,0 +1,56 @@
|
||||
"""Load game moderation profiles from config/moderation.yaml.
|
||||
|
||||
Falls back to hardcoded defaults if the YAML file is missing or malformed.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from infrastructure.guards.moderation import GameProfile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_profiles(config_path: Path | None = None) -> dict[str, GameProfile]:
|
||||
"""Load game moderation profiles from YAML config.
|
||||
|
||||
Args:
|
||||
config_path: Path to moderation.yaml. Defaults to config/moderation.yaml.
|
||||
|
||||
Returns:
|
||||
Dict mapping game_id to GameProfile.
|
||||
"""
|
||||
path = config_path or Path("config/moderation.yaml")
|
||||
|
||||
if not path.exists():
|
||||
logger.info("Moderation config not found at %s — using defaults", path)
|
||||
return {}
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
logger.warning("PyYAML not installed — using default moderation profiles")
|
||||
return {}
|
||||
|
||||
try:
|
||||
data = yaml.safe_load(path.read_text())
|
||||
except Exception as exc:
|
||||
logger.error("Failed to parse moderation config: %s", exc)
|
||||
return {}
|
||||
|
||||
profiles: dict[str, GameProfile] = {}
|
||||
for game_id, profile_data in data.get("profiles", {}).items():
|
||||
try:
|
||||
profiles[game_id] = GameProfile(
|
||||
game_id=game_id,
|
||||
display_name=profile_data.get("display_name", game_id),
|
||||
vocabulary_whitelist=profile_data.get("vocabulary_whitelist", []),
|
||||
context_prompt=profile_data.get("context_prompt", ""),
|
||||
threshold=float(profile_data.get("threshold", 0.8)),
|
||||
fallbacks=profile_data.get("fallbacks", {}),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Invalid profile '%s': %s", game_id, exc)
|
||||
|
||||
logger.info("Loaded %d moderation profiles from %s", len(profiles), path)
|
||||
return profiles
|
||||
@@ -32,6 +32,15 @@ except ImportError:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Quota monitor — optional, degrades gracefully if unavailable
|
||||
try:
|
||||
from infrastructure.claude_quota import QuotaMonitor, get_quota_monitor
|
||||
|
||||
_quota_monitor: "QuotaMonitor | None" = get_quota_monitor()
|
||||
except Exception as _exc: # pragma: no cover
|
||||
logger.debug("Quota monitor not available: %s", _exc)
|
||||
_quota_monitor = None
|
||||
|
||||
|
||||
class ProviderStatus(Enum):
|
||||
"""Health status of a provider."""
|
||||
@@ -301,6 +310,22 @@ class CascadeRouter:
|
||||
logger.debug("Ollama provider check error: %s", exc)
|
||||
return False
|
||||
|
||||
elif provider.type == "vllm_mlx":
|
||||
# Check if local vllm-mlx server is running (OpenAI-compatible)
|
||||
if requests is None:
|
||||
return True
|
||||
try:
|
||||
base_url = provider.base_url or provider.url or "http://localhost:8000"
|
||||
# Strip /v1 suffix — health endpoint is at the root
|
||||
server_root = base_url.rstrip("/")
|
||||
if server_root.endswith("/v1"):
|
||||
server_root = server_root[:-3]
|
||||
response = requests.get(f"{server_root}/health", timeout=5)
|
||||
return response.status_code == 200
|
||||
except Exception as exc:
|
||||
logger.debug("vllm-mlx provider check error: %s", exc)
|
||||
return False
|
||||
|
||||
elif provider.type in ("openai", "anthropic", "grok"):
|
||||
# Check if API key is set
|
||||
return provider.api_key is not None and provider.api_key != ""
|
||||
@@ -457,6 +482,25 @@ class CascadeRouter:
|
||||
|
||||
raise RuntimeError("; ".join(errors))
|
||||
|
||||
def _quota_allows_cloud(self, provider: Provider) -> bool:
|
||||
"""Check quota before routing to a cloud provider.
|
||||
|
||||
Uses the metabolic protocol: cloud calls are gated by 5-hour quota.
|
||||
Returns True (allow cloud) if quota monitor is unavailable or returns None.
|
||||
"""
|
||||
if _quota_monitor is None:
|
||||
return True
|
||||
try:
|
||||
# Map provider type to task_value heuristic
|
||||
task_value = "high" # conservative default
|
||||
status = _quota_monitor.check()
|
||||
if status is None:
|
||||
return True # No credentials — caller decides based on config
|
||||
return _quota_monitor.should_use_cloud(task_value)
|
||||
except Exception as exc:
|
||||
logger.warning("Quota check failed, allowing cloud: %s", exc)
|
||||
return True
|
||||
|
||||
def _is_provider_available(self, provider: Provider) -> bool:
|
||||
"""Check if a provider should be tried (enabled + circuit breaker)."""
|
||||
if not provider.enabled:
|
||||
@@ -510,6 +554,15 @@ class CascadeRouter:
|
||||
if not self._is_provider_available(provider):
|
||||
continue
|
||||
|
||||
# Metabolic protocol: skip cloud providers when quota is low
|
||||
if provider.type in ("anthropic", "openai", "grok"):
|
||||
if not self._quota_allows_cloud(provider):
|
||||
logger.info(
|
||||
"Metabolic protocol: skipping cloud provider %s (quota too low)",
|
||||
provider.name,
|
||||
)
|
||||
continue
|
||||
|
||||
selected_model, is_fallback_model = self._select_model(provider, model, content_type)
|
||||
|
||||
try:
|
||||
@@ -582,6 +635,14 @@ class CascadeRouter:
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
elif provider.type == "vllm_mlx":
|
||||
result = await self._call_vllm_mlx(
|
||||
provider=provider,
|
||||
messages=messages,
|
||||
model=model or provider.get_default_model(),
|
||||
temperature=temperature,
|
||||
max_tokens=max_tokens,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unknown provider type: {provider.type}")
|
||||
|
||||
@@ -778,6 +839,48 @@ class CascadeRouter:
|
||||
"model": response.model,
|
||||
}
|
||||
|
||||
async def _call_vllm_mlx(
|
||||
self,
|
||||
provider: Provider,
|
||||
messages: list[dict],
|
||||
model: str,
|
||||
temperature: float,
|
||||
max_tokens: int | None,
|
||||
) -> dict:
|
||||
"""Call vllm-mlx via its OpenAI-compatible API.
|
||||
|
||||
vllm-mlx exposes the same /v1/chat/completions endpoint as OpenAI,
|
||||
so we reuse the OpenAI client pointed at the local server.
|
||||
No API key is required for local deployments.
|
||||
"""
|
||||
import openai
|
||||
|
||||
base_url = provider.base_url or provider.url or "http://localhost:8000"
|
||||
# Ensure the base_url ends with /v1 as expected by the OpenAI client
|
||||
if not base_url.rstrip("/").endswith("/v1"):
|
||||
base_url = base_url.rstrip("/") + "/v1"
|
||||
|
||||
client = openai.AsyncOpenAI(
|
||||
api_key=provider.api_key or "no-key-required",
|
||||
base_url=base_url,
|
||||
timeout=self.config.timeout_seconds,
|
||||
)
|
||||
|
||||
kwargs: dict = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": temperature,
|
||||
}
|
||||
if max_tokens:
|
||||
kwargs["max_tokens"] = max_tokens
|
||||
|
||||
response = await client.chat.completions.create(**kwargs)
|
||||
|
||||
return {
|
||||
"content": response.choices[0].message.content,
|
||||
"model": response.model,
|
||||
}
|
||||
|
||||
def _record_success(self, provider: Provider, latency_ms: float) -> None:
|
||||
"""Record a successful request."""
|
||||
provider.metrics.total_requests += 1
|
||||
|
||||
306
src/infrastructure/sovereignty_metrics.py
Normal file
306
src/infrastructure/sovereignty_metrics.py
Normal file
@@ -0,0 +1,306 @@
|
||||
"""Sovereignty metrics collector and store.
|
||||
|
||||
Tracks research sovereignty progress: cache hit rate, API cost,
|
||||
time-to-report, and human involvement. Persists to SQLite for
|
||||
trend analysis and dashboard display.
|
||||
|
||||
Refs: #981
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DB_PATH = Path(settings.repo_root) / "data" / "sovereignty_metrics.db"
|
||||
|
||||
_SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS sovereignty_metrics (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
metric_type TEXT NOT NULL,
|
||||
value REAL NOT NULL,
|
||||
metadata TEXT DEFAULT '{}'
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sm_type ON sovereignty_metrics(metric_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_sm_ts ON sovereignty_metrics(timestamp);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sovereignty_alerts (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
alert_type TEXT NOT NULL,
|
||||
message TEXT NOT NULL,
|
||||
value REAL NOT NULL,
|
||||
threshold REAL NOT NULL,
|
||||
acknowledged INTEGER DEFAULT 0
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sa_ts ON sovereignty_alerts(timestamp);
|
||||
CREATE INDEX IF NOT EXISTS idx_sa_ack ON sovereignty_alerts(acknowledged);
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SovereigntyMetric:
|
||||
"""A single sovereignty metric data point."""
|
||||
|
||||
metric_type: str # cache_hit_rate, api_cost, time_to_report, human_involvement
|
||||
value: float
|
||||
timestamp: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SovereigntyAlert:
|
||||
"""An alert triggered when a metric exceeds a threshold."""
|
||||
|
||||
alert_type: str
|
||||
message: str
|
||||
value: float
|
||||
threshold: float
|
||||
timestamp: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
|
||||
acknowledged: bool = False
|
||||
|
||||
|
||||
# Graduation targets from issue #981
|
||||
GRADUATION_TARGETS = {
|
||||
"cache_hit_rate": {"week1": 0.10, "month1": 0.40, "month3": 0.80, "graduation": 0.90},
|
||||
"api_cost": {"week1": 1.50, "month1": 0.50, "month3": 0.10, "graduation": 0.01},
|
||||
"time_to_report": {"week1": 180.0, "month1": 30.0, "month3": 5.0, "graduation": 1.0},
|
||||
"human_involvement": {"week1": 1.0, "month1": 0.5, "month3": 0.25, "graduation": 0.0},
|
||||
"local_artifacts": {"week1": 6, "month1": 30, "month3": 100, "graduation": 500},
|
||||
}
|
||||
|
||||
|
||||
class SovereigntyMetricsStore:
|
||||
"""SQLite-backed sovereignty metrics store.
|
||||
|
||||
Thread-safe: creates a new connection per operation.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: Path | None = None) -> None:
|
||||
self._db_path = db_path or DB_PATH
|
||||
self._init_db()
|
||||
|
||||
def _init_db(self) -> None:
|
||||
"""Initialize the database schema."""
|
||||
try:
|
||||
self._db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with closing(sqlite3.connect(str(self._db_path))) as conn:
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute(f"PRAGMA busy_timeout={settings.db_busy_timeout_ms}")
|
||||
conn.executescript(_SCHEMA)
|
||||
conn.commit()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to initialize sovereignty metrics DB: %s", exc)
|
||||
|
||||
def _connect(self) -> sqlite3.Connection:
|
||||
"""Get a new connection."""
|
||||
conn = sqlite3.connect(str(self._db_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute(f"PRAGMA busy_timeout={settings.db_busy_timeout_ms}")
|
||||
return conn
|
||||
|
||||
def record(self, metric: SovereigntyMetric) -> None:
|
||||
"""Record a sovereignty metric data point."""
|
||||
try:
|
||||
with closing(self._connect()) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sovereignty_metrics (timestamp, metric_type, value, metadata) "
|
||||
"VALUES (?, ?, ?, ?)",
|
||||
(
|
||||
metric.timestamp,
|
||||
metric.metric_type,
|
||||
metric.value,
|
||||
json.dumps(metric.metadata),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to record sovereignty metric: %s", exc)
|
||||
|
||||
# Check thresholds for alerts
|
||||
self._check_alert(metric)
|
||||
|
||||
def _check_alert(self, metric: SovereigntyMetric) -> None:
|
||||
"""Check if a metric triggers an alert."""
|
||||
threshold = settings.sovereignty_api_cost_alert_threshold
|
||||
if metric.metric_type == "api_cost" and metric.value > threshold:
|
||||
alert = SovereigntyAlert(
|
||||
alert_type="api_cost_exceeded",
|
||||
message=f"API cost ${metric.value:.2f} exceeds threshold ${threshold:.2f}",
|
||||
value=metric.value,
|
||||
threshold=threshold,
|
||||
)
|
||||
self._record_alert(alert)
|
||||
|
||||
def _record_alert(self, alert: SovereigntyAlert) -> None:
|
||||
"""Persist an alert."""
|
||||
try:
|
||||
with closing(self._connect()) as conn:
|
||||
conn.execute(
|
||||
"INSERT INTO sovereignty_alerts "
|
||||
"(timestamp, alert_type, message, value, threshold) "
|
||||
"VALUES (?, ?, ?, ?, ?)",
|
||||
(
|
||||
alert.timestamp,
|
||||
alert.alert_type,
|
||||
alert.message,
|
||||
alert.value,
|
||||
alert.threshold,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
logger.warning("Sovereignty alert: %s", alert.message)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to record sovereignty alert: %s", exc)
|
||||
|
||||
def get_latest(self, metric_type: str, limit: int = 50) -> list[dict]:
|
||||
"""Get the most recent metric values for a given type."""
|
||||
try:
|
||||
with closing(self._connect()) as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT timestamp, value, metadata FROM sovereignty_metrics "
|
||||
"WHERE metric_type = ? ORDER BY timestamp DESC LIMIT ?",
|
||||
(metric_type, limit),
|
||||
).fetchall()
|
||||
return [
|
||||
{
|
||||
"timestamp": row["timestamp"],
|
||||
"value": row["value"],
|
||||
"metadata": json.loads(row["metadata"]) if row["metadata"] else {},
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to query sovereignty metrics: %s", exc)
|
||||
return []
|
||||
|
||||
def get_summary(self) -> dict[str, Any]:
|
||||
"""Get a summary of current sovereignty metrics progress."""
|
||||
summary: dict[str, Any] = {}
|
||||
for metric_type in GRADUATION_TARGETS:
|
||||
latest = self.get_latest(metric_type, limit=1)
|
||||
history = self.get_latest(metric_type, limit=30)
|
||||
|
||||
current_value = latest[0]["value"] if latest else None
|
||||
targets = GRADUATION_TARGETS[metric_type]
|
||||
|
||||
# Determine current phase based on value
|
||||
phase = "pre-start"
|
||||
if current_value is not None:
|
||||
if metric_type in ("api_cost", "time_to_report", "human_involvement"):
|
||||
# Lower is better
|
||||
if current_value <= targets["graduation"]:
|
||||
phase = "graduated"
|
||||
elif current_value <= targets["month3"]:
|
||||
phase = "month3"
|
||||
elif current_value <= targets["month1"]:
|
||||
phase = "month1"
|
||||
elif current_value <= targets["week1"]:
|
||||
phase = "week1"
|
||||
else:
|
||||
phase = "pre-start"
|
||||
else:
|
||||
# Higher is better
|
||||
if current_value >= targets["graduation"]:
|
||||
phase = "graduated"
|
||||
elif current_value >= targets["month3"]:
|
||||
phase = "month3"
|
||||
elif current_value >= targets["month1"]:
|
||||
phase = "month1"
|
||||
elif current_value >= targets["week1"]:
|
||||
phase = "week1"
|
||||
else:
|
||||
phase = "pre-start"
|
||||
|
||||
summary[metric_type] = {
|
||||
"current": current_value,
|
||||
"phase": phase,
|
||||
"targets": targets,
|
||||
"trend": [{"t": h["timestamp"], "v": h["value"]} for h in reversed(history)],
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
def get_alerts(self, unacknowledged_only: bool = True, limit: int = 20) -> list[dict]:
|
||||
"""Get sovereignty alerts."""
|
||||
try:
|
||||
with closing(self._connect()) as conn:
|
||||
if unacknowledged_only:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM sovereignty_alerts "
|
||||
"WHERE acknowledged = 0 ORDER BY timestamp DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM sovereignty_alerts ORDER BY timestamp DESC LIMIT ?",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to query sovereignty alerts: %s", exc)
|
||||
return []
|
||||
|
||||
def acknowledge_alert(self, alert_id: int) -> bool:
|
||||
"""Acknowledge an alert."""
|
||||
try:
|
||||
with closing(self._connect()) as conn:
|
||||
conn.execute(
|
||||
"UPDATE sovereignty_alerts SET acknowledged = 1 WHERE id = ?",
|
||||
(alert_id,),
|
||||
)
|
||||
conn.commit()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to acknowledge alert: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
# ── Module-level singleton ─────────────────────────────────────────────────
|
||||
_store: SovereigntyMetricsStore | None = None
|
||||
|
||||
|
||||
def get_sovereignty_store() -> SovereigntyMetricsStore:
|
||||
"""Return the module-level store, creating it on first access."""
|
||||
global _store
|
||||
if _store is None:
|
||||
_store = SovereigntyMetricsStore()
|
||||
return _store
|
||||
|
||||
|
||||
async def emit_sovereignty_metric(
|
||||
metric_type: str,
|
||||
value: float,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""Convenience function to record a sovereignty metric and emit an event.
|
||||
|
||||
Also publishes to the event bus for real-time subscribers.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
from infrastructure.events.bus import emit
|
||||
|
||||
metric = SovereigntyMetric(
|
||||
metric_type=metric_type,
|
||||
value=value,
|
||||
metadata=metadata or {},
|
||||
)
|
||||
# Record to SQLite in thread to avoid blocking event loop
|
||||
await asyncio.to_thread(get_sovereignty_store().record, metric)
|
||||
|
||||
# Publish to event bus for real-time consumers
|
||||
await emit(
|
||||
f"sovereignty.metric.{metric_type}",
|
||||
source="sovereignty_metrics",
|
||||
data={"metric_type": metric_type, "value": value, **(metadata or {})},
|
||||
)
|
||||
29
src/infrastructure/world/__init__.py
Normal file
29
src/infrastructure/world/__init__.py
Normal file
@@ -0,0 +1,29 @@
|
||||
"""World interface — engine-agnostic adapter pattern for embodied agents.
|
||||
|
||||
Provides the ``WorldInterface`` ABC and an adapter registry so Timmy can
|
||||
observe, act, and speak in any game world (Morrowind, Luanti, Godot, …)
|
||||
through a single contract.
|
||||
|
||||
Quick start::
|
||||
|
||||
from infrastructure.world import get_adapter, register_adapter
|
||||
from infrastructure.world.interface import WorldInterface
|
||||
|
||||
register_adapter("mock", MockWorldAdapter)
|
||||
world = get_adapter("mock")
|
||||
perception = world.observe()
|
||||
"""
|
||||
|
||||
from infrastructure.world.registry import AdapterRegistry
|
||||
|
||||
_registry = AdapterRegistry()
|
||||
|
||||
register_adapter = _registry.register
|
||||
get_adapter = _registry.get
|
||||
list_adapters = _registry.list_adapters
|
||||
|
||||
__all__ = [
|
||||
"register_adapter",
|
||||
"get_adapter",
|
||||
"list_adapters",
|
||||
]
|
||||
1
src/infrastructure/world/adapters/__init__.py
Normal file
1
src/infrastructure/world/adapters/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Built-in world adapters."""
|
||||
234
src/infrastructure/world/adapters/bannerlord.py
Normal file
234
src/infrastructure/world/adapters/bannerlord.py
Normal file
@@ -0,0 +1,234 @@
|
||||
"""Bannerlord world adapter — bridges GABS to the WorldInterface contract.
|
||||
|
||||
Allows the existing ``Heartbeat`` loop to drive the Bannerlord campaign
|
||||
by treating it as just another game world. Wraps the async ``GabsClient``
|
||||
for synchronous use (the ``Heartbeat`` calls ``observe()`` and ``act()``
|
||||
synchronously).
|
||||
|
||||
Async callers should use ``CampaignLoop`` directly — it is more efficient
|
||||
and handles the full M2 logic natively.
|
||||
|
||||
Usage::
|
||||
|
||||
adapter = BannerlordWorldAdapter()
|
||||
adapter.connect()
|
||||
heartbeat = Heartbeat(world=adapter, interval=5.0)
|
||||
await heartbeat.run_once()
|
||||
adapter.disconnect()
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from infrastructure.world.interface import WorldInterface
|
||||
from infrastructure.world.types import (
|
||||
ActionResult,
|
||||
ActionStatus,
|
||||
CommandInput,
|
||||
PerceptionOutput,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BannerlordWorldAdapter(WorldInterface):
|
||||
"""WorldInterface adapter for Bannerlord via GABS.
|
||||
|
||||
Wraps ``GabsClient`` and ``CampaignState`` to present the Bannerlord
|
||||
campaign map as a ``WorldInterface``-compatible world.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
host:
|
||||
Override GABS server host (defaults to ``settings.gabs_host``).
|
||||
port:
|
||||
Override GABS server port (defaults to ``settings.gabs_port``).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
host: str | None = None,
|
||||
port: int | None = None,
|
||||
) -> None:
|
||||
from config import settings
|
||||
|
||||
self._host = host or settings.gabs_host
|
||||
self._port = port or settings.gabs_port
|
||||
self._connected = False
|
||||
self._client = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
|
||||
# -- lifecycle ---------------------------------------------------------
|
||||
|
||||
def connect(self) -> None:
|
||||
"""Open the GABS TCP connection (synchronous wrapper)."""
|
||||
from bannerlord.gabs_client import GabsClient
|
||||
|
||||
self._client = GabsClient(host=self._host, port=self._port)
|
||||
try:
|
||||
self._loop = asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
self._loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self._loop)
|
||||
|
||||
try:
|
||||
self._loop.run_until_complete(self._client.connect())
|
||||
self._connected = True
|
||||
logger.info("BannerlordWorldAdapter connected to GABS")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("BannerlordWorldAdapter: GABS connect failed: %s", exc)
|
||||
self._connected = False
|
||||
|
||||
def disconnect(self) -> None:
|
||||
"""Close the GABS TCP connection (synchronous wrapper)."""
|
||||
if self._client is not None and self._loop is not None:
|
||||
try:
|
||||
self._loop.run_until_complete(self._client.disconnect())
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("BannerlordWorldAdapter disconnect error: %s", exc)
|
||||
self._connected = False
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return self._connected
|
||||
|
||||
# -- core contract -----------------------------------------------------
|
||||
|
||||
def observe(self) -> PerceptionOutput:
|
||||
"""Poll GABS for current game state and return structured perception."""
|
||||
from bannerlord.campaign_state import parse_campaign_state
|
||||
|
||||
if not self._connected or self._client is None or self._loop is None:
|
||||
return PerceptionOutput(
|
||||
location="disconnected",
|
||||
entities=[],
|
||||
events=["gabs_disconnected"],
|
||||
raw={"error": "GABS not connected"},
|
||||
)
|
||||
|
||||
try:
|
||||
raw = self._loop.run_until_complete(self._client.get_game_state())
|
||||
state = parse_campaign_state(raw)
|
||||
|
||||
# Build entities list from settlements and nearby parties
|
||||
entities: list[str] = []
|
||||
for s in state.settlements[:5]:
|
||||
entities.append(f"settlement:{s.name}")
|
||||
for p in state.nearby_parties[:3]:
|
||||
prefix = "hostile" if p.is_hostile else "friendly"
|
||||
entities.append(f"{prefix}_party:{p.name}")
|
||||
|
||||
# Build events list
|
||||
events: list[str] = []
|
||||
if state.party.food_days < 2.0:
|
||||
events.append("low_food")
|
||||
if state.party.morale < 40:
|
||||
events.append("low_morale")
|
||||
if state.hostile_bandits_nearby():
|
||||
events.append("bandits_nearby")
|
||||
if state.m2_complete:
|
||||
events.append("m2_complete")
|
||||
|
||||
location = state.party.current_settlement or "campaign_map"
|
||||
|
||||
return PerceptionOutput(
|
||||
location=location,
|
||||
entities=entities,
|
||||
events=events,
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("BannerlordWorldAdapter.observe() failed: %s", exc)
|
||||
return PerceptionOutput(
|
||||
location="unknown",
|
||||
entities=[],
|
||||
events=[f"observe_error:{exc}"],
|
||||
raw={"error": str(exc)},
|
||||
)
|
||||
|
||||
def act(self, command: CommandInput) -> ActionResult:
|
||||
"""Dispatch a campaign command to GABS.
|
||||
|
||||
Recognized ``command.action`` values:
|
||||
- ``"move"`` → party/move_to_settlement (target = settlement_id)
|
||||
- ``"trade"`` → inventory/buy_item (target = item_id)
|
||||
- ``"recruit"`` → party/recruit_all
|
||||
- ``"engage"`` → party/engage_party (target = party_id)
|
||||
|
||||
Parameters
|
||||
----------
|
||||
command:
|
||||
WorldInterface ``CommandInput`` with action, target, parameters.
|
||||
"""
|
||||
if not self._connected or self._client is None or self._loop is None:
|
||||
return ActionResult(
|
||||
status=ActionStatus.FAILURE,
|
||||
message="GABS not connected",
|
||||
)
|
||||
|
||||
try:
|
||||
return self._loop.run_until_complete(self._async_act(command))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("BannerlordWorldAdapter.act() failed: %s", exc)
|
||||
return ActionResult(
|
||||
status=ActionStatus.FAILURE,
|
||||
message=f"act failed: {exc}",
|
||||
)
|
||||
|
||||
async def _async_act(self, command: CommandInput) -> ActionResult:
|
||||
"""Async implementation of act()."""
|
||||
from bannerlord.campaign_actions import (
|
||||
buy_item,
|
||||
engage_party,
|
||||
move_to_settlement,
|
||||
recruit_all,
|
||||
)
|
||||
|
||||
action = command.action.lower()
|
||||
params = command.parameters
|
||||
|
||||
if action == "move":
|
||||
settlement_id = command.target or params.get("settlement_id", "")
|
||||
return await move_to_settlement(
|
||||
self._client,
|
||||
settlement_id,
|
||||
settlement_name=params.get("settlement_name", ""),
|
||||
)
|
||||
|
||||
elif action == "trade":
|
||||
item_id = command.target or params.get("item_id", "")
|
||||
quantity = int(params.get("quantity", 1))
|
||||
return await buy_item(
|
||||
self._client,
|
||||
item_id,
|
||||
quantity,
|
||||
settlement_id=params.get("settlement_id", ""),
|
||||
)
|
||||
|
||||
elif action == "recruit":
|
||||
return await recruit_all(
|
||||
self._client,
|
||||
settlement_id=params.get("settlement_id", ""),
|
||||
)
|
||||
|
||||
elif action == "engage":
|
||||
party_id = command.target or params.get("party_id", "")
|
||||
return await engage_party(
|
||||
self._client,
|
||||
party_id,
|
||||
party_name=params.get("party_name", ""),
|
||||
)
|
||||
|
||||
else:
|
||||
return ActionResult(
|
||||
status=ActionStatus.NOOP,
|
||||
message=f"Unknown action: {command.action}",
|
||||
)
|
||||
|
||||
def speak(self, message: str, target: str | None = None) -> None:
|
||||
"""Log the message — GABS has no chat mechanism in M2."""
|
||||
logger.info("BannerlordWorldAdapter.speak: %r (target=%r)", message, target)
|
||||
99
src/infrastructure/world/adapters/mock.py
Normal file
99
src/infrastructure/world/adapters/mock.py
Normal file
@@ -0,0 +1,99 @@
|
||||
"""Mock world adapter — returns canned perception and logs commands.
|
||||
|
||||
Useful for testing the heartbeat loop and WorldInterface contract
|
||||
without a running game server.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from infrastructure.world.interface import WorldInterface
|
||||
from infrastructure.world.types import (
|
||||
ActionResult,
|
||||
ActionStatus,
|
||||
CommandInput,
|
||||
PerceptionOutput,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ActionLog:
|
||||
"""Record of an action dispatched to the mock world."""
|
||||
|
||||
command: CommandInput
|
||||
timestamp: datetime
|
||||
|
||||
|
||||
class MockWorldAdapter(WorldInterface):
|
||||
"""In-memory mock adapter for testing.
|
||||
|
||||
* ``observe()`` returns configurable canned perception.
|
||||
* ``act()`` logs the command and returns success.
|
||||
* ``speak()`` logs the message.
|
||||
|
||||
Inspect ``action_log`` and ``speech_log`` to verify behaviour in tests.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
location: str = "Test Chamber",
|
||||
entities: list[str] | None = None,
|
||||
events: list[str] | None = None,
|
||||
) -> None:
|
||||
self._location = location
|
||||
self._entities = entities or ["TestNPC"]
|
||||
self._events = events or []
|
||||
self._connected = False
|
||||
self.action_log: list[_ActionLog] = []
|
||||
self.speech_log: list[dict] = []
|
||||
|
||||
# -- lifecycle ---------------------------------------------------------
|
||||
|
||||
def connect(self) -> None:
|
||||
self._connected = True
|
||||
logger.info("MockWorldAdapter connected")
|
||||
|
||||
def disconnect(self) -> None:
|
||||
self._connected = False
|
||||
logger.info("MockWorldAdapter disconnected")
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return self._connected
|
||||
|
||||
# -- core contract -----------------------------------------------------
|
||||
|
||||
def observe(self) -> PerceptionOutput:
|
||||
logger.debug("MockWorldAdapter.observe()")
|
||||
return PerceptionOutput(
|
||||
timestamp=datetime.now(UTC),
|
||||
location=self._location,
|
||||
entities=list(self._entities),
|
||||
events=list(self._events),
|
||||
raw={"adapter": "mock"},
|
||||
)
|
||||
|
||||
def act(self, command: CommandInput) -> ActionResult:
|
||||
logger.debug("MockWorldAdapter.act(%s)", command.action)
|
||||
self.action_log.append(_ActionLog(command=command, timestamp=datetime.now(UTC)))
|
||||
return ActionResult(
|
||||
status=ActionStatus.SUCCESS,
|
||||
message=f"Mock executed: {command.action}",
|
||||
data={"adapter": "mock"},
|
||||
)
|
||||
|
||||
def speak(self, message: str, target: str | None = None) -> None:
|
||||
logger.debug("MockWorldAdapter.speak(%r, target=%r)", message, target)
|
||||
self.speech_log.append(
|
||||
{
|
||||
"message": message,
|
||||
"target": target,
|
||||
"timestamp": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
)
|
||||
58
src/infrastructure/world/adapters/tes3mp.py
Normal file
58
src/infrastructure/world/adapters/tes3mp.py
Normal file
@@ -0,0 +1,58 @@
|
||||
"""TES3MP world adapter — stub for Morrowind multiplayer via TES3MP.
|
||||
|
||||
This adapter will eventually connect to a TES3MP server and translate
|
||||
the WorldInterface contract into TES3MP commands. For now every method
|
||||
raises ``NotImplementedError`` with guidance on what needs wiring up.
|
||||
|
||||
Once PR #864 merges, import PerceptionOutput and CommandInput directly
|
||||
from ``infrastructure.morrowind.schemas`` if their shapes differ from
|
||||
the canonical types in ``infrastructure.world.types``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from infrastructure.world.interface import WorldInterface
|
||||
from infrastructure.world.types import ActionResult, CommandInput, PerceptionOutput
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TES3MPWorldAdapter(WorldInterface):
|
||||
"""Stub adapter for TES3MP (Morrowind multiplayer).
|
||||
|
||||
All core methods raise ``NotImplementedError``.
|
||||
Implement ``connect()`` first — it should open a socket to the
|
||||
TES3MP server and authenticate.
|
||||
"""
|
||||
|
||||
def __init__(self, *, host: str = "localhost", port: int = 25565) -> None:
|
||||
self._host = host
|
||||
self._port = port
|
||||
self._connected = False
|
||||
|
||||
# -- lifecycle ---------------------------------------------------------
|
||||
|
||||
def connect(self) -> None:
|
||||
raise NotImplementedError("TES3MPWorldAdapter.connect() — wire up TES3MP server socket")
|
||||
|
||||
def disconnect(self) -> None:
|
||||
raise NotImplementedError("TES3MPWorldAdapter.disconnect() — close TES3MP server socket")
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
return self._connected
|
||||
|
||||
# -- core contract (stubs) ---------------------------------------------
|
||||
|
||||
def observe(self) -> PerceptionOutput:
|
||||
raise NotImplementedError("TES3MPWorldAdapter.observe() — poll TES3MP for player/NPC state")
|
||||
|
||||
def act(self, command: CommandInput) -> ActionResult:
|
||||
raise NotImplementedError(
|
||||
"TES3MPWorldAdapter.act() — translate CommandInput to TES3MP packet"
|
||||
)
|
||||
|
||||
def speak(self, message: str, target: str | None = None) -> None:
|
||||
raise NotImplementedError("TES3MPWorldAdapter.speak() — send chat message via TES3MP")
|
||||
17
src/infrastructure/world/benchmark/__init__.py
Normal file
17
src/infrastructure/world/benchmark/__init__.py
Normal file
@@ -0,0 +1,17 @@
|
||||
"""Performance regression suite for Morrowind agent scenarios.
|
||||
|
||||
Provides standardised benchmark scenarios, a runner that executes them
|
||||
through the heartbeat loop with a mock (or live) world adapter, and
|
||||
metrics collection for CI-integrated regression detection.
|
||||
"""
|
||||
|
||||
from infrastructure.world.benchmark.metrics import BenchmarkMetrics
|
||||
from infrastructure.world.benchmark.runner import BenchmarkRunner
|
||||
from infrastructure.world.benchmark.scenarios import BenchmarkScenario, load_scenarios
|
||||
|
||||
__all__ = [
|
||||
"BenchmarkMetrics",
|
||||
"BenchmarkRunner",
|
||||
"BenchmarkScenario",
|
||||
"load_scenarios",
|
||||
]
|
||||
195
src/infrastructure/world/benchmark/metrics.py
Normal file
195
src/infrastructure/world/benchmark/metrics.py
Normal file
@@ -0,0 +1,195 @@
|
||||
"""Benchmark metrics collection and persistence.
|
||||
|
||||
Tracks per-scenario results: cycles used, wall-clock time, success,
|
||||
LLM call count, and estimated metabolic cost. Results are persisted
|
||||
as JSONL for trend analysis and CI regression gates.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScenarioResult:
|
||||
"""Outcome of running a single benchmark scenario.
|
||||
|
||||
Attributes:
|
||||
scenario_name: Human-readable scenario name.
|
||||
success: Whether the goal predicate was satisfied.
|
||||
cycles_used: Number of heartbeat cycles executed.
|
||||
max_cycles: The scenario's cycle budget.
|
||||
wall_time_ms: Total wall-clock time in milliseconds.
|
||||
llm_calls: Number of LLM inference calls made.
|
||||
metabolic_cost: Estimated resource cost (arbitrary unit, ≈ tokens).
|
||||
error: Error message if the run crashed.
|
||||
tags: Scenario tags (copied for filtering).
|
||||
"""
|
||||
|
||||
scenario_name: str
|
||||
success: bool = False
|
||||
cycles_used: int = 0
|
||||
max_cycles: int = 0
|
||||
wall_time_ms: int = 0
|
||||
llm_calls: int = 0
|
||||
metabolic_cost: float = 0.0
|
||||
error: str | None = None
|
||||
tags: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BenchmarkMetrics:
|
||||
"""Aggregated metrics across all scenarios in a benchmark run.
|
||||
|
||||
Attributes:
|
||||
results: Per-scenario results.
|
||||
total_time_ms: Total wall-clock time for the full suite.
|
||||
timestamp: ISO-8601 timestamp of the run.
|
||||
commit_sha: Git commit SHA (if available).
|
||||
"""
|
||||
|
||||
results: list[ScenarioResult] = field(default_factory=list)
|
||||
total_time_ms: int = 0
|
||||
timestamp: str = ""
|
||||
commit_sha: str = ""
|
||||
|
||||
# -- derived properties ------------------------------------------------
|
||||
|
||||
@property
|
||||
def pass_count(self) -> int:
|
||||
return sum(1 for r in self.results if r.success)
|
||||
|
||||
@property
|
||||
def fail_count(self) -> int:
|
||||
return sum(1 for r in self.results if not r.success)
|
||||
|
||||
@property
|
||||
def success_rate(self) -> float:
|
||||
if not self.results:
|
||||
return 0.0
|
||||
return self.pass_count / len(self.results)
|
||||
|
||||
@property
|
||||
def total_llm_calls(self) -> int:
|
||||
return sum(r.llm_calls for r in self.results)
|
||||
|
||||
@property
|
||||
def total_metabolic_cost(self) -> float:
|
||||
return sum(r.metabolic_cost for r in self.results)
|
||||
|
||||
# -- persistence -------------------------------------------------------
|
||||
|
||||
def save(self, path: Path) -> None:
|
||||
"""Append this run's results to a JSONL file at *path*."""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
record = {
|
||||
"timestamp": self.timestamp,
|
||||
"commit_sha": self.commit_sha,
|
||||
"total_time_ms": self.total_time_ms,
|
||||
"success_rate": round(self.success_rate, 4),
|
||||
"total_llm_calls": self.total_llm_calls,
|
||||
"total_metabolic_cost": round(self.total_metabolic_cost, 2),
|
||||
"scenarios": [asdict(r) for r in self.results],
|
||||
}
|
||||
with path.open("a") as f:
|
||||
f.write(json.dumps(record) + "\n")
|
||||
logger.info("Benchmark results saved to %s", path)
|
||||
|
||||
# -- summary -----------------------------------------------------------
|
||||
|
||||
def summary(self) -> str:
|
||||
"""Return a human-readable summary of the benchmark run."""
|
||||
lines = [
|
||||
"=== Benchmark Summary ===",
|
||||
f"Scenarios: {len(self.results)} "
|
||||
f"Passed: {self.pass_count} "
|
||||
f"Failed: {self.fail_count} "
|
||||
f"Success rate: {self.success_rate:.0%}",
|
||||
f"Total time: {self.total_time_ms} ms "
|
||||
f"LLM calls: {self.total_llm_calls} "
|
||||
f"Metabolic cost: {self.total_metabolic_cost:.1f}",
|
||||
]
|
||||
if self.commit_sha:
|
||||
lines.append(f"Commit: {self.commit_sha}")
|
||||
lines.append("")
|
||||
for r in self.results:
|
||||
status = "PASS" if r.success else "FAIL"
|
||||
lines.append(
|
||||
f" [{status}] {r.scenario_name} — "
|
||||
f"{r.cycles_used}/{r.max_cycles} cycles, "
|
||||
f"{r.wall_time_ms} ms, "
|
||||
f"{r.llm_calls} LLM calls"
|
||||
)
|
||||
if r.error:
|
||||
lines.append(f" Error: {r.error}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def load_history(path: Path) -> list[dict]:
|
||||
"""Load benchmark history from a JSONL file.
|
||||
|
||||
Returns:
|
||||
List of run records, most recent first.
|
||||
"""
|
||||
path = Path(path)
|
||||
if not path.exists():
|
||||
return []
|
||||
records: list[dict] = []
|
||||
for line in path.read_text().strip().splitlines():
|
||||
try:
|
||||
records.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
return list(reversed(records))
|
||||
|
||||
|
||||
def compare_runs(
|
||||
current: BenchmarkMetrics,
|
||||
baseline: BenchmarkMetrics,
|
||||
) -> str:
|
||||
"""Compare two benchmark runs and report regressions.
|
||||
|
||||
Returns:
|
||||
Human-readable comparison report.
|
||||
"""
|
||||
lines = ["=== Regression Report ==="]
|
||||
|
||||
# Overall
|
||||
rate_delta = current.success_rate - baseline.success_rate
|
||||
lines.append(
|
||||
f"Success rate: {baseline.success_rate:.0%} -> {current.success_rate:.0%} "
|
||||
f"({rate_delta:+.0%})"
|
||||
)
|
||||
|
||||
cost_delta = current.total_metabolic_cost - baseline.total_metabolic_cost
|
||||
if baseline.total_metabolic_cost > 0:
|
||||
cost_pct = (cost_delta / baseline.total_metabolic_cost) * 100
|
||||
lines.append(
|
||||
f"Metabolic cost: {baseline.total_metabolic_cost:.1f} -> "
|
||||
f"{current.total_metabolic_cost:.1f} ({cost_pct:+.1f}%)"
|
||||
)
|
||||
|
||||
# Per-scenario
|
||||
baseline_map = {r.scenario_name: r for r in baseline.results}
|
||||
for r in current.results:
|
||||
b = baseline_map.get(r.scenario_name)
|
||||
if b is None:
|
||||
lines.append(f" [NEW] {r.scenario_name}")
|
||||
continue
|
||||
if b.success and not r.success:
|
||||
lines.append(f" [REGRESSION] {r.scenario_name} — was PASS, now FAIL")
|
||||
elif not b.success and r.success:
|
||||
lines.append(f" [IMPROVEMENT] {r.scenario_name} — was FAIL, now PASS")
|
||||
elif r.cycles_used > b.cycles_used * 1.5:
|
||||
lines.append(
|
||||
f" [SLOWER] {r.scenario_name} — "
|
||||
f"{b.cycles_used} -> {r.cycles_used} cycles (+{r.cycles_used - b.cycles_used})"
|
||||
)
|
||||
|
||||
return "\n".join(lines)
|
||||
167
src/infrastructure/world/benchmark/runner.py
Normal file
167
src/infrastructure/world/benchmark/runner.py
Normal file
@@ -0,0 +1,167 @@
|
||||
"""Benchmark runner — executes scenarios through the heartbeat loop.
|
||||
|
||||
Wires each ``BenchmarkScenario`` into a ``MockWorldAdapter`` (or a
|
||||
supplied adapter), runs the heartbeat for up to ``max_cycles``, and
|
||||
collects ``BenchmarkMetrics``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from infrastructure.world.adapters.mock import MockWorldAdapter
|
||||
from infrastructure.world.benchmark.metrics import BenchmarkMetrics, ScenarioResult
|
||||
from infrastructure.world.benchmark.scenarios import BenchmarkScenario
|
||||
from infrastructure.world.interface import WorldInterface
|
||||
from loop.heartbeat import Heartbeat
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Rough estimate: each heartbeat cycle costs ~1 unit of metabolic cost
|
||||
# (gather + reason + act phases each touch the LLM router once).
|
||||
_COST_PER_CYCLE = 3.0 # three phases per cycle
|
||||
|
||||
|
||||
class BenchmarkRunner:
|
||||
"""Run benchmark scenarios and collect metrics.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
adapter_factory:
|
||||
Optional callable that returns a ``WorldInterface`` for a given
|
||||
scenario. Defaults to building a ``MockWorldAdapter`` from the
|
||||
scenario's start state.
|
||||
heartbeat_interval:
|
||||
Seconds between heartbeat ticks (0 for immediate).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
adapter_factory=None,
|
||||
heartbeat_interval: float = 0.0,
|
||||
) -> None:
|
||||
self._adapter_factory = adapter_factory or self._default_adapter
|
||||
self._interval = heartbeat_interval
|
||||
|
||||
# -- public API --------------------------------------------------------
|
||||
|
||||
async def run(
|
||||
self,
|
||||
scenarios: list[BenchmarkScenario],
|
||||
) -> BenchmarkMetrics:
|
||||
"""Execute all *scenarios* and return aggregated metrics."""
|
||||
metrics = BenchmarkMetrics(
|
||||
timestamp=datetime.now(UTC).isoformat(),
|
||||
commit_sha=self._git_sha(),
|
||||
)
|
||||
suite_start = time.monotonic()
|
||||
|
||||
for scenario in scenarios:
|
||||
logger.info("Benchmark: starting '%s'", scenario.name)
|
||||
result = await self._run_scenario(scenario)
|
||||
metrics.results.append(result)
|
||||
status = "PASS" if result.success else "FAIL"
|
||||
logger.info(
|
||||
"Benchmark: '%s' %s (%d/%d cycles, %d ms)",
|
||||
scenario.name,
|
||||
status,
|
||||
result.cycles_used,
|
||||
result.max_cycles,
|
||||
result.wall_time_ms,
|
||||
)
|
||||
|
||||
metrics.total_time_ms = int((time.monotonic() - suite_start) * 1000)
|
||||
return metrics
|
||||
|
||||
# -- internal ----------------------------------------------------------
|
||||
|
||||
async def _run_scenario(self, scenario: BenchmarkScenario) -> ScenarioResult:
|
||||
"""Run a single scenario through the heartbeat loop."""
|
||||
result = ScenarioResult(
|
||||
scenario_name=scenario.name,
|
||||
max_cycles=scenario.max_cycles,
|
||||
tags=list(scenario.tags),
|
||||
)
|
||||
|
||||
adapter = self._adapter_factory(scenario)
|
||||
adapter.connect()
|
||||
|
||||
hb = Heartbeat(world=adapter, interval=self._interval)
|
||||
actions: list[dict] = []
|
||||
|
||||
start = time.monotonic()
|
||||
try:
|
||||
for cycle in range(1, scenario.max_cycles + 1):
|
||||
record = await hb.run_once()
|
||||
result.cycles_used = cycle
|
||||
|
||||
# Track LLM calls (each cycle has 3 phases that may call LLM)
|
||||
result.llm_calls += 3
|
||||
|
||||
# Accumulate actions for goal predicate
|
||||
if record.action_taken and record.action_taken != "idle":
|
||||
actions.append(
|
||||
{
|
||||
"action": record.action_taken,
|
||||
"target": record.observation.get("location", ""),
|
||||
"status": record.action_status,
|
||||
}
|
||||
)
|
||||
|
||||
# Update adapter location if scenario simulates movement
|
||||
current_location = self._get_current_location(adapter)
|
||||
|
||||
# Check goal predicate
|
||||
if scenario.goal_predicate is not None:
|
||||
if scenario.goal_predicate(actions, current_location):
|
||||
result.success = True
|
||||
break
|
||||
elif cycle == scenario.max_cycles:
|
||||
# No predicate — success if we survived all cycles
|
||||
result.success = True
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("Benchmark scenario '%s' crashed: %s", scenario.name, exc)
|
||||
result.error = str(exc)
|
||||
finally:
|
||||
adapter.disconnect()
|
||||
|
||||
result.wall_time_ms = int((time.monotonic() - start) * 1000)
|
||||
result.metabolic_cost = result.cycles_used * _COST_PER_CYCLE
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _default_adapter(scenario: BenchmarkScenario) -> WorldInterface:
|
||||
"""Build a MockWorldAdapter from a scenario's starting state."""
|
||||
return MockWorldAdapter(
|
||||
location=scenario.start_location,
|
||||
entities=list(scenario.entities),
|
||||
events=list(scenario.events),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_current_location(adapter: WorldInterface) -> str:
|
||||
"""Read the current location from the adapter."""
|
||||
try:
|
||||
perception = adapter.observe()
|
||||
return perception.location
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _git_sha() -> str:
|
||||
"""Best-effort: return the current git commit SHA."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["git", "rev-parse", "--short", "HEAD"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
return result.stdout.strip() if result.returncode == 0 else ""
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return ""
|
||||
160
src/infrastructure/world/benchmark/scenarios.py
Normal file
160
src/infrastructure/world/benchmark/scenarios.py
Normal file
@@ -0,0 +1,160 @@
|
||||
"""Benchmark scenario definitions for Morrowind agent regression testing.
|
||||
|
||||
Each scenario specifies a starting location, goal conditions, world state
|
||||
(entities, events), and maximum cycles allowed. The runner feeds these
|
||||
into the heartbeat loop and checks completion against the goal predicate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BenchmarkScenario:
|
||||
"""A reproducible agent task used to detect performance regressions.
|
||||
|
||||
Attributes:
|
||||
name: Human-readable scenario name.
|
||||
description: What the scenario tests.
|
||||
start_location: Where the agent begins.
|
||||
goal_location: Target location (if navigation scenario).
|
||||
entities: NPCs / objects present in the world.
|
||||
events: Game events injected each cycle.
|
||||
max_cycles: Hard cap on heartbeat cycles before failure.
|
||||
goal_predicate: Optional callable ``(actions, location) -> bool``
|
||||
evaluated after each cycle to check early success.
|
||||
tags: Freeform tags for filtering (e.g. "navigation", "quest").
|
||||
"""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
start_location: str
|
||||
goal_location: str = ""
|
||||
entities: list[str] = field(default_factory=list)
|
||||
events: list[str] = field(default_factory=list)
|
||||
max_cycles: int = 50
|
||||
goal_predicate: Callable | None = None
|
||||
tags: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Goal predicates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _reached_location(target: str) -> Callable:
|
||||
"""Return a predicate that checks whether the agent reached *target*."""
|
||||
|
||||
def predicate(actions: list[dict], current_location: str) -> bool:
|
||||
return current_location.lower() == target.lower()
|
||||
|
||||
return predicate
|
||||
|
||||
|
||||
def _interacted_with(npc: str) -> Callable:
|
||||
"""Return a predicate that checks for a speak/interact action with *npc*."""
|
||||
|
||||
def predicate(actions: list[dict], current_location: str) -> bool:
|
||||
for act in actions:
|
||||
if act.get("action") in ("speak", "interact", "talk"):
|
||||
if act.get("target", "").lower() == npc.lower():
|
||||
return True
|
||||
return False
|
||||
|
||||
return predicate
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Built-in scenarios
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
BUILTIN_SCENARIOS: list[BenchmarkScenario] = [
|
||||
BenchmarkScenario(
|
||||
name="Walk Seyda Neen to Balmora",
|
||||
description=(
|
||||
"Navigate from the starting village to Balmora via the road. "
|
||||
"Tests basic navigation and pathfinding."
|
||||
),
|
||||
start_location="Seyda Neen",
|
||||
goal_location="Balmora",
|
||||
entities=["Silt Strider", "Road Sign", "Mudcrab"],
|
||||
events=["player_spawned"],
|
||||
max_cycles=30,
|
||||
goal_predicate=_reached_location("Balmora"),
|
||||
tags=["navigation", "basic"],
|
||||
),
|
||||
BenchmarkScenario(
|
||||
name="Fargoth's Ring",
|
||||
description=(
|
||||
"Complete the Fargoth quest: find Fargoth, receive the ring, "
|
||||
"and return it. Tests NPC interaction and quest logic."
|
||||
),
|
||||
start_location="Seyda Neen",
|
||||
goal_location="Seyda Neen",
|
||||
entities=["Fargoth", "Arrille", "Guard"],
|
||||
events=["quest_available:fargoth_ring"],
|
||||
max_cycles=40,
|
||||
goal_predicate=_interacted_with("Fargoth"),
|
||||
tags=["quest", "npc_interaction"],
|
||||
),
|
||||
BenchmarkScenario(
|
||||
name="Balmora Guild Navigation",
|
||||
description=(
|
||||
"Walk from Balmora South Wall Corner Club to the Fighters Guild. "
|
||||
"Tests intra-city navigation with multiple NPCs present."
|
||||
),
|
||||
start_location="Balmora, South Wall Corner Club",
|
||||
goal_location="Balmora, Fighters Guild",
|
||||
entities=["Guard", "Merchant", "Caius Cosades"],
|
||||
events=["player_entered"],
|
||||
max_cycles=20,
|
||||
goal_predicate=_reached_location("Balmora, Fighters Guild"),
|
||||
tags=["navigation", "city"],
|
||||
),
|
||||
BenchmarkScenario(
|
||||
name="Combat Encounter — Mudcrab",
|
||||
description=(
|
||||
"Engage and defeat a single Mudcrab on the road between "
|
||||
"Seyda Neen and Balmora. Tests combat action selection."
|
||||
),
|
||||
start_location="Bitter Coast Road",
|
||||
goal_location="Bitter Coast Road",
|
||||
entities=["Mudcrab"],
|
||||
events=["hostile_entity_nearby"],
|
||||
max_cycles=15,
|
||||
goal_predicate=None, # Success = survived max_cycles without crash
|
||||
tags=["combat", "basic"],
|
||||
),
|
||||
BenchmarkScenario(
|
||||
name="Passive Observation — Balmora Market",
|
||||
description=(
|
||||
"Observe the Balmora market for 10 cycles without acting. "
|
||||
"Tests that the agent can reason without unnecessary actions."
|
||||
),
|
||||
start_location="Balmora, Market Square",
|
||||
goal_location="",
|
||||
entities=["Merchant", "Guard", "Pilgrim", "Trader"],
|
||||
events=["market_day"],
|
||||
max_cycles=10,
|
||||
tags=["observation", "passive"],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def load_scenarios(
|
||||
tags: list[str] | None = None,
|
||||
) -> list[BenchmarkScenario]:
|
||||
"""Return built-in scenarios, optionally filtered by tags.
|
||||
|
||||
Args:
|
||||
tags: If provided, only return scenarios whose tags overlap.
|
||||
|
||||
Returns:
|
||||
List of matching ``BenchmarkScenario`` instances.
|
||||
"""
|
||||
if tags is None:
|
||||
return list(BUILTIN_SCENARIOS)
|
||||
tag_set = set(tags)
|
||||
return [s for s in BUILTIN_SCENARIOS if tag_set & set(s.tags)]
|
||||
64
src/infrastructure/world/interface.py
Normal file
64
src/infrastructure/world/interface.py
Normal file
@@ -0,0 +1,64 @@
|
||||
"""Abstract WorldInterface — the contract every game-world adapter must fulfil.
|
||||
|
||||
Follows a Gymnasium-inspired pattern: observe → act → speak, with each
|
||||
method returning strongly-typed data structures.
|
||||
|
||||
Any future engine (TES3MP, Luanti, Godot, …) plugs in by subclassing
|
||||
``WorldInterface`` and implementing the three methods.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
from infrastructure.world.types import ActionResult, CommandInput, PerceptionOutput
|
||||
|
||||
|
||||
class WorldInterface(ABC):
|
||||
"""Engine-agnostic base class for world adapters.
|
||||
|
||||
Subclasses must implement:
|
||||
- ``observe()`` — gather structured perception from the world
|
||||
- ``act()`` — dispatch a command and return the outcome
|
||||
- ``speak()`` — send a message to an NPC / player / broadcast
|
||||
|
||||
Lifecycle hooks ``connect()`` and ``disconnect()`` are optional.
|
||||
"""
|
||||
|
||||
# -- lifecycle (optional overrides) ------------------------------------
|
||||
|
||||
def connect(self) -> None: # noqa: B027
|
||||
"""Establish connection to the game world.
|
||||
|
||||
Default implementation is a no-op. Override to open sockets,
|
||||
authenticate, etc.
|
||||
"""
|
||||
|
||||
def disconnect(self) -> None: # noqa: B027
|
||||
"""Tear down the connection.
|
||||
|
||||
Default implementation is a no-op.
|
||||
"""
|
||||
|
||||
@property
|
||||
def is_connected(self) -> bool:
|
||||
"""Return ``True`` if the adapter has an active connection.
|
||||
|
||||
Default returns ``True``. Override for adapters that maintain
|
||||
persistent connections.
|
||||
"""
|
||||
return True
|
||||
|
||||
# -- core contract (must implement) ------------------------------------
|
||||
|
||||
@abstractmethod
|
||||
def observe(self) -> PerceptionOutput:
|
||||
"""Return a structured snapshot of the current world state."""
|
||||
|
||||
@abstractmethod
|
||||
def act(self, command: CommandInput) -> ActionResult:
|
||||
"""Execute *command* in the world and return the result."""
|
||||
|
||||
@abstractmethod
|
||||
def speak(self, message: str, target: str | None = None) -> None:
|
||||
"""Send *message* in the world, optionally directed at *target*."""
|
||||
54
src/infrastructure/world/registry.py
Normal file
54
src/infrastructure/world/registry.py
Normal file
@@ -0,0 +1,54 @@
|
||||
"""Adapter registry — register and instantiate world adapters by name.
|
||||
|
||||
Usage::
|
||||
|
||||
registry = AdapterRegistry()
|
||||
registry.register("mock", MockWorldAdapter)
|
||||
adapter = registry.get("mock", some_kwarg="value")
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from infrastructure.world.interface import WorldInterface
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AdapterRegistry:
|
||||
"""Name → WorldInterface class registry with instantiation."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._adapters: dict[str, type[WorldInterface]] = {}
|
||||
|
||||
def register(self, name: str, cls: type[WorldInterface]) -> None:
|
||||
"""Register an adapter class under *name*.
|
||||
|
||||
Raises ``TypeError`` if *cls* is not a ``WorldInterface`` subclass.
|
||||
"""
|
||||
if not (isinstance(cls, type) and issubclass(cls, WorldInterface)):
|
||||
raise TypeError(f"{cls!r} is not a WorldInterface subclass")
|
||||
if name in self._adapters:
|
||||
logger.warning("Overwriting adapter %r (was %r)", name, self._adapters[name])
|
||||
self._adapters[name] = cls
|
||||
logger.info("Registered world adapter: %s → %s", name, cls.__name__)
|
||||
|
||||
def get(self, name: str, **kwargs: Any) -> WorldInterface:
|
||||
"""Instantiate and return the adapter registered as *name*.
|
||||
|
||||
Raises ``KeyError`` if *name* is not registered.
|
||||
"""
|
||||
cls = self._adapters[name]
|
||||
return cls(**kwargs)
|
||||
|
||||
def list_adapters(self) -> list[str]:
|
||||
"""Return sorted list of registered adapter names."""
|
||||
return sorted(self._adapters)
|
||||
|
||||
def __contains__(self, name: str) -> bool:
|
||||
return name in self._adapters
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._adapters)
|
||||
71
src/infrastructure/world/types.py
Normal file
71
src/infrastructure/world/types.py
Normal file
@@ -0,0 +1,71 @@
|
||||
"""Canonical data types for world interaction.
|
||||
|
||||
These mirror the PerceptionOutput / CommandInput types from PR #864's
|
||||
``morrowind/schemas.py``. When that PR merges, these can be replaced
|
||||
with re-exports — but until then they serve as the stable contract for
|
||||
every WorldInterface adapter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
|
||||
|
||||
class ActionStatus(StrEnum):
|
||||
"""Outcome of an action dispatched to the world."""
|
||||
|
||||
SUCCESS = "success"
|
||||
FAILURE = "failure"
|
||||
PENDING = "pending"
|
||||
NOOP = "noop"
|
||||
|
||||
|
||||
@dataclass
|
||||
class PerceptionOutput:
|
||||
"""Structured world state returned by ``WorldInterface.observe()``.
|
||||
|
||||
Attributes:
|
||||
timestamp: When the observation was captured.
|
||||
location: Free-form location descriptor (e.g. "Balmora, Fighters Guild").
|
||||
entities: List of nearby entity descriptions.
|
||||
events: Recent game events since last observation.
|
||||
raw: Optional raw / engine-specific payload for advanced consumers.
|
||||
"""
|
||||
|
||||
timestamp: datetime = field(default_factory=lambda: datetime.now(UTC))
|
||||
location: str = ""
|
||||
entities: list[str] = field(default_factory=list)
|
||||
events: list[str] = field(default_factory=list)
|
||||
raw: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommandInput:
|
||||
"""Action command sent via ``WorldInterface.act()``.
|
||||
|
||||
Attributes:
|
||||
action: Verb / action name (e.g. "move", "attack", "use_item").
|
||||
target: Optional target identifier.
|
||||
parameters: Arbitrary key-value payload for engine-specific params.
|
||||
"""
|
||||
|
||||
action: str
|
||||
target: str | None = None
|
||||
parameters: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionResult:
|
||||
"""Outcome returned by ``WorldInterface.act()``.
|
||||
|
||||
Attributes:
|
||||
status: Whether the action succeeded, failed, etc.
|
||||
message: Human-readable description of the outcome.
|
||||
data: Arbitrary engine-specific result payload.
|
||||
"""
|
||||
|
||||
status: ActionStatus = ActionStatus.SUCCESS
|
||||
message: str = ""
|
||||
data: dict = field(default_factory=dict)
|
||||
286
src/loop/heartbeat.py
Normal file
286
src/loop/heartbeat.py
Normal file
@@ -0,0 +1,286 @@
|
||||
"""Heartbeat v2 — WorldInterface-driven cognitive loop.
|
||||
|
||||
Drives real observe → reason → act → reflect cycles through whatever
|
||||
``WorldInterface`` adapter is connected. When no adapter is present,
|
||||
gracefully falls back to the existing ``run_cycle()`` behaviour.
|
||||
|
||||
Usage::
|
||||
|
||||
heartbeat = Heartbeat(world=adapter, interval=30.0)
|
||||
await heartbeat.run_once() # single cycle
|
||||
await heartbeat.start() # background loop
|
||||
heartbeat.stop() # graceful shutdown
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from loop.phase1_gather import gather
|
||||
from loop.phase2_reason import reason
|
||||
from loop.phase3_act import act
|
||||
from loop.schema import ContextPayload
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cycle log entry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class CycleRecord:
|
||||
"""One observe → reason → act → reflect cycle."""
|
||||
|
||||
cycle_id: int
|
||||
timestamp: str
|
||||
observation: dict = field(default_factory=dict)
|
||||
reasoning_summary: str = ""
|
||||
action_taken: str = ""
|
||||
action_status: str = ""
|
||||
reflect_notes: str = ""
|
||||
duration_ms: int = 0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Heartbeat
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class Heartbeat:
|
||||
"""Manages the recurring cognitive loop with optional world adapter.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
world:
|
||||
A ``WorldInterface`` instance (or ``None`` for passive mode).
|
||||
interval:
|
||||
Seconds between heartbeat ticks. 30 s for embodied mode,
|
||||
300 s (5 min) for passive thinking.
|
||||
on_cycle:
|
||||
Optional async callback invoked after each cycle with the
|
||||
``CycleRecord``.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
world=None, # WorldInterface | None
|
||||
interval: float = 30.0,
|
||||
on_cycle=None, # Callable[[CycleRecord], Awaitable[None]] | None
|
||||
) -> None:
|
||||
self._world = world
|
||||
self._interval = interval
|
||||
self._on_cycle = on_cycle
|
||||
self._cycle_count: int = 0
|
||||
self._running = False
|
||||
self._task: asyncio.Task | None = None
|
||||
self.history: list[CycleRecord] = []
|
||||
|
||||
# -- properties --------------------------------------------------------
|
||||
|
||||
@property
|
||||
def world(self):
|
||||
return self._world
|
||||
|
||||
@world.setter
|
||||
def world(self, adapter) -> None:
|
||||
self._world = adapter
|
||||
|
||||
@property
|
||||
def interval(self) -> float:
|
||||
return self._interval
|
||||
|
||||
@interval.setter
|
||||
def interval(self, value: float) -> None:
|
||||
self._interval = max(1.0, value)
|
||||
|
||||
@property
|
||||
def is_running(self) -> bool:
|
||||
return self._running
|
||||
|
||||
@property
|
||||
def cycle_count(self) -> int:
|
||||
return self._cycle_count
|
||||
|
||||
# -- single cycle ------------------------------------------------------
|
||||
|
||||
async def run_once(self) -> CycleRecord:
|
||||
"""Execute one full heartbeat cycle.
|
||||
|
||||
If a world adapter is present:
|
||||
1. Observe — ``world.observe()``
|
||||
2. Gather + Reason + Act via the three-phase loop, with the
|
||||
observation injected into the payload
|
||||
3. Dispatch the decided action back to ``world.act()``
|
||||
4. Reflect — log the cycle
|
||||
|
||||
Without an adapter the existing loop runs on a timer-sourced
|
||||
payload (passive thinking).
|
||||
"""
|
||||
self._cycle_count += 1
|
||||
start = time.monotonic()
|
||||
record = CycleRecord(
|
||||
cycle_id=self._cycle_count,
|
||||
timestamp=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
|
||||
if self._world is not None:
|
||||
record = await self._embodied_cycle(record)
|
||||
else:
|
||||
record = await self._passive_cycle(record)
|
||||
|
||||
record.duration_ms = int((time.monotonic() - start) * 1000)
|
||||
self.history.append(record)
|
||||
|
||||
# Broadcast via WebSocket (best-effort)
|
||||
await self._broadcast(record)
|
||||
|
||||
if self._on_cycle:
|
||||
await self._on_cycle(record)
|
||||
|
||||
logger.info(
|
||||
"Heartbeat cycle #%d complete (%d ms) — action=%s status=%s",
|
||||
record.cycle_id,
|
||||
record.duration_ms,
|
||||
record.action_taken or "(passive)",
|
||||
record.action_status or "n/a",
|
||||
)
|
||||
return record
|
||||
|
||||
# -- background loop ---------------------------------------------------
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Start the recurring heartbeat loop as a background task."""
|
||||
if self._running:
|
||||
logger.warning("Heartbeat already running")
|
||||
return
|
||||
self._running = True
|
||||
self._task = asyncio.current_task() or asyncio.ensure_future(self._loop())
|
||||
if self._task is not asyncio.current_task():
|
||||
return
|
||||
await self._loop()
|
||||
|
||||
async def _loop(self) -> None:
|
||||
logger.info(
|
||||
"Heartbeat loop started (interval=%.1fs, adapter=%s)",
|
||||
self._interval,
|
||||
type(self._world).__name__ if self._world else "None",
|
||||
)
|
||||
while self._running:
|
||||
try:
|
||||
await self.run_once()
|
||||
except Exception:
|
||||
logger.exception("Heartbeat cycle failed")
|
||||
await asyncio.sleep(self._interval)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Signal the heartbeat loop to stop after the current cycle."""
|
||||
self._running = False
|
||||
logger.info("Heartbeat stop requested")
|
||||
|
||||
# -- internal: embodied cycle ------------------------------------------
|
||||
|
||||
async def _embodied_cycle(self, record: CycleRecord) -> CycleRecord:
|
||||
"""Cycle with a live world adapter: observe → reason → act → reflect."""
|
||||
from infrastructure.world.types import ActionStatus, CommandInput
|
||||
|
||||
# 1. Observe
|
||||
perception = self._world.observe()
|
||||
record.observation = {
|
||||
"location": perception.location,
|
||||
"entities": perception.entities,
|
||||
"events": perception.events,
|
||||
}
|
||||
|
||||
# 2. Feed observation into the three-phase loop
|
||||
obs_content = (
|
||||
f"Location: {perception.location}\n"
|
||||
f"Entities: {', '.join(perception.entities)}\n"
|
||||
f"Events: {', '.join(perception.events)}"
|
||||
)
|
||||
payload = ContextPayload(
|
||||
source="world",
|
||||
content=obs_content,
|
||||
metadata={"perception": record.observation},
|
||||
)
|
||||
|
||||
gathered = gather(payload)
|
||||
reasoned = reason(gathered)
|
||||
acted = act(reasoned)
|
||||
|
||||
# Extract action decision from the acted payload
|
||||
action_name = acted.metadata.get("action", "idle")
|
||||
action_target = acted.metadata.get("action_target")
|
||||
action_params = acted.metadata.get("action_params", {})
|
||||
record.reasoning_summary = acted.metadata.get("reasoning", acted.content[:200])
|
||||
|
||||
# 3. Dispatch action to world
|
||||
if action_name != "idle":
|
||||
cmd = CommandInput(
|
||||
action=action_name,
|
||||
target=action_target,
|
||||
parameters=action_params,
|
||||
)
|
||||
result = self._world.act(cmd)
|
||||
record.action_taken = action_name
|
||||
record.action_status = result.status.value
|
||||
else:
|
||||
record.action_taken = "idle"
|
||||
record.action_status = ActionStatus.NOOP.value
|
||||
|
||||
# 4. Reflect
|
||||
record.reflect_notes = (
|
||||
f"Observed {len(perception.entities)} entities at {perception.location}. "
|
||||
f"Action: {record.action_taken} → {record.action_status}."
|
||||
)
|
||||
|
||||
return record
|
||||
|
||||
# -- internal: passive cycle -------------------------------------------
|
||||
|
||||
async def _passive_cycle(self, record: CycleRecord) -> CycleRecord:
|
||||
"""Cycle without a world adapter — existing think_once() behaviour."""
|
||||
payload = ContextPayload(
|
||||
source="timer",
|
||||
content="heartbeat",
|
||||
metadata={"mode": "passive"},
|
||||
)
|
||||
|
||||
gathered = gather(payload)
|
||||
reasoned = reason(gathered)
|
||||
acted = act(reasoned)
|
||||
|
||||
record.reasoning_summary = acted.content[:200]
|
||||
record.action_taken = "think"
|
||||
record.action_status = "noop"
|
||||
record.reflect_notes = "Passive thinking cycle — no world adapter connected."
|
||||
|
||||
return record
|
||||
|
||||
# -- broadcast ---------------------------------------------------------
|
||||
|
||||
async def _broadcast(self, record: CycleRecord) -> None:
|
||||
"""Emit heartbeat cycle data via WebSocket (best-effort)."""
|
||||
try:
|
||||
from infrastructure.ws_manager.handler import ws_manager
|
||||
|
||||
await ws_manager.broadcast(
|
||||
"heartbeat.cycle",
|
||||
{
|
||||
"cycle_id": record.cycle_id,
|
||||
"timestamp": record.timestamp,
|
||||
"action": record.action_taken,
|
||||
"action_status": record.action_status,
|
||||
"reasoning_summary": record.reasoning_summary[:300],
|
||||
"observation": record.observation,
|
||||
"duration_ms": record.duration_ms,
|
||||
},
|
||||
)
|
||||
except (ImportError, AttributeError, ConnectionError, RuntimeError) as exc:
|
||||
logger.debug("Heartbeat broadcast skipped: %s", exc)
|
||||
@@ -17,9 +17,9 @@ logger = logging.getLogger(__name__)
|
||||
def gather(payload: ContextPayload) -> ContextPayload:
|
||||
"""Accept raw input and return structured context for reasoning.
|
||||
|
||||
Stub: tags the payload with phase=gather and logs transit.
|
||||
Timmy will flesh this out with context selection, memory lookup,
|
||||
adapter polling, and attention-residual weighting.
|
||||
When the payload carries a ``perception`` dict in metadata (injected by
|
||||
the heartbeat loop from a WorldInterface adapter), that observation is
|
||||
folded into the gathered context. Otherwise behaves as before.
|
||||
"""
|
||||
logger.info(
|
||||
"Phase 1 (Gather) received: source=%s content_len=%d tokens=%d",
|
||||
@@ -28,7 +28,20 @@ def gather(payload: ContextPayload) -> ContextPayload:
|
||||
payload.token_count,
|
||||
)
|
||||
|
||||
result = payload.with_metadata(phase="gather", gathered=True)
|
||||
extra: dict = {"phase": "gather", "gathered": True}
|
||||
|
||||
# Enrich with world observation when present
|
||||
perception = payload.metadata.get("perception")
|
||||
if perception:
|
||||
extra["world_observation"] = perception
|
||||
logger.info(
|
||||
"Phase 1 (Gather) world observation: location=%s entities=%d events=%d",
|
||||
perception.get("location", "?"),
|
||||
len(perception.get("entities", [])),
|
||||
len(perception.get("events", [])),
|
||||
)
|
||||
|
||||
result = payload.with_metadata(**extra)
|
||||
|
||||
logger.info(
|
||||
"Phase 1 (Gather) produced: metadata_keys=%s",
|
||||
|
||||
@@ -215,6 +215,119 @@ def _summarize(result: AgenticResult, total_steps: int, was_truncated: bool) ->
|
||||
result.status = "completed"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Execution orchestrator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _execute_all_steps(
|
||||
agent,
|
||||
task: str,
|
||||
task_id: str,
|
||||
steps: list[str],
|
||||
total_steps: int,
|
||||
session_id: str,
|
||||
result: AgenticResult,
|
||||
on_progress: Callable | None,
|
||||
) -> list[str]:
|
||||
"""Execute all planned steps, handling failures with adaptation.
|
||||
|
||||
Appends AgenticStep objects to *result.steps* and returns the list
|
||||
of completed-result strings (used as context for later steps).
|
||||
"""
|
||||
completed_results: list[str] = []
|
||||
|
||||
for i, step_desc in enumerate(steps, 1):
|
||||
step_start = time.monotonic()
|
||||
try:
|
||||
step = await _execute_step(
|
||||
agent,
|
||||
task,
|
||||
step_desc,
|
||||
i,
|
||||
total_steps,
|
||||
completed_results,
|
||||
session_id,
|
||||
)
|
||||
result.steps.append(step)
|
||||
completed_results.append(f"Step {i}: {step.result[:200]}")
|
||||
await _broadcast_progress(
|
||||
"agentic.step_complete",
|
||||
{
|
||||
"task_id": task_id,
|
||||
"step": i,
|
||||
"total": total_steps,
|
||||
"description": step_desc,
|
||||
"result": step.result[:200],
|
||||
},
|
||||
)
|
||||
if on_progress:
|
||||
await on_progress(step_desc, i, total_steps)
|
||||
|
||||
except Exception as exc: # broad catch intentional: agent.run can raise any error
|
||||
logger.warning("Agentic loop step %d failed: %s", i, exc)
|
||||
step = await _handle_step_failure(
|
||||
agent,
|
||||
step_desc,
|
||||
i,
|
||||
total_steps,
|
||||
task_id,
|
||||
exc,
|
||||
step_start,
|
||||
session_id,
|
||||
result,
|
||||
completed_results,
|
||||
on_progress,
|
||||
)
|
||||
|
||||
return completed_results
|
||||
|
||||
|
||||
async def _handle_step_failure(
|
||||
agent,
|
||||
step_desc: str,
|
||||
step_num: int,
|
||||
total_steps: int,
|
||||
task_id: str,
|
||||
exc: Exception,
|
||||
step_start: float,
|
||||
session_id: str,
|
||||
result: AgenticResult,
|
||||
completed_results: list[str],
|
||||
on_progress: Callable | None,
|
||||
) -> None:
|
||||
"""Try to adapt a failed step; record a hard failure if adaptation also fails."""
|
||||
try:
|
||||
step = await _adapt_step(agent, step_desc, step_num, exc, step_start, session_id)
|
||||
result.steps.append(step)
|
||||
completed_results.append(f"Step {step_num} (adapted): {step.result[:200]}")
|
||||
await _broadcast_progress(
|
||||
"agentic.step_adapted",
|
||||
{
|
||||
"task_id": task_id,
|
||||
"step": step_num,
|
||||
"total": total_steps,
|
||||
"description": step_desc,
|
||||
"error": str(exc),
|
||||
"adaptation": step.result[:200],
|
||||
},
|
||||
)
|
||||
if on_progress:
|
||||
await on_progress(f"[Adapted] {step_desc}", step_num, total_steps)
|
||||
except Exception as adapt_exc: # broad catch intentional
|
||||
logger.error("Agentic loop adaptation also failed: %s", adapt_exc)
|
||||
result.steps.append(
|
||||
AgenticStep(
|
||||
step_num=step_num,
|
||||
description=step_desc,
|
||||
result=f"Failed: {exc}; Adaptation also failed: {adapt_exc}",
|
||||
status="failed",
|
||||
duration_ms=int((time.monotonic() - step_start) * 1000),
|
||||
)
|
||||
)
|
||||
completed_results.append(f"Step {step_num}: FAILED")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core loop
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -265,65 +378,9 @@ async def run_agentic_loop(
|
||||
)
|
||||
|
||||
# Phase 2: Execution
|
||||
completed_results: list[str] = []
|
||||
for i, step_desc in enumerate(steps, 1):
|
||||
step_start = time.monotonic()
|
||||
try:
|
||||
step = await _execute_step(
|
||||
agent,
|
||||
task,
|
||||
step_desc,
|
||||
i,
|
||||
total_steps,
|
||||
completed_results,
|
||||
session_id,
|
||||
)
|
||||
result.steps.append(step)
|
||||
completed_results.append(f"Step {i}: {step.result[:200]}")
|
||||
await _broadcast_progress(
|
||||
"agentic.step_complete",
|
||||
{
|
||||
"task_id": task_id,
|
||||
"step": i,
|
||||
"total": total_steps,
|
||||
"description": step_desc,
|
||||
"result": step.result[:200],
|
||||
},
|
||||
)
|
||||
if on_progress:
|
||||
await on_progress(step_desc, i, total_steps)
|
||||
|
||||
except Exception as exc: # broad catch intentional: agent.run can raise any error
|
||||
logger.warning("Agentic loop step %d failed: %s", i, exc)
|
||||
try:
|
||||
step = await _adapt_step(agent, step_desc, i, exc, step_start, session_id)
|
||||
result.steps.append(step)
|
||||
completed_results.append(f"Step {i} (adapted): {step.result[:200]}")
|
||||
await _broadcast_progress(
|
||||
"agentic.step_adapted",
|
||||
{
|
||||
"task_id": task_id,
|
||||
"step": i,
|
||||
"total": total_steps,
|
||||
"description": step_desc,
|
||||
"error": str(exc),
|
||||
"adaptation": step.result[:200],
|
||||
},
|
||||
)
|
||||
if on_progress:
|
||||
await on_progress(f"[Adapted] {step_desc}", i, total_steps)
|
||||
except Exception as adapt_exc: # broad catch intentional
|
||||
logger.error("Agentic loop adaptation also failed: %s", adapt_exc)
|
||||
result.steps.append(
|
||||
AgenticStep(
|
||||
step_num=i,
|
||||
description=step_desc,
|
||||
result=f"Failed: {exc}; Adaptation also failed: {adapt_exc}",
|
||||
status="failed",
|
||||
duration_ms=int((time.monotonic() - step_start) * 1000),
|
||||
)
|
||||
)
|
||||
completed_results.append(f"Step {i}: FAILED")
|
||||
await _execute_all_steps(
|
||||
agent, task, task_id, steps, total_steps, session_id, result, on_progress
|
||||
)
|
||||
|
||||
# Phase 3: Summary
|
||||
_summarize(result, total_steps, was_truncated)
|
||||
|
||||
@@ -489,5 +489,43 @@ def focus(
|
||||
typer.echo("No active focus (broad mode).")
|
||||
|
||||
|
||||
@app.command(name="healthcheck")
|
||||
def healthcheck(
|
||||
json_output: bool = typer.Option(False, "--json", "-j", help="Output as JSON"),
|
||||
verbose: bool = typer.Option(
|
||||
False, "--verbose", "-v", help="Show verbose output including issue details"
|
||||
),
|
||||
quiet: bool = typer.Option(False, "--quiet", "-q", help="Only show status line (no details)"),
|
||||
):
|
||||
"""Quick health snapshot before coding.
|
||||
|
||||
Shows CI status, critical issues (P0/P1), test flakiness, and token economy.
|
||||
Fast execution (< 5 seconds) for pre-work checks.
|
||||
|
||||
Refs: #710
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
script_path = (
|
||||
Path(__file__).resolve().parent.parent.parent
|
||||
/ "timmy_automations"
|
||||
/ "daily_run"
|
||||
/ "health_snapshot.py"
|
||||
)
|
||||
|
||||
cmd = [sys.executable, str(script_path)]
|
||||
if json_output:
|
||||
cmd.append("--json")
|
||||
if verbose:
|
||||
cmd.append("--verbose")
|
||||
if quiet:
|
||||
cmd.append("--quiet")
|
||||
|
||||
result = subprocess.run(cmd)
|
||||
raise typer.Exit(result.returncode)
|
||||
|
||||
|
||||
def main():
|
||||
app()
|
||||
|
||||
488
src/timmy/kimi_delegation.py
Normal file
488
src/timmy/kimi_delegation.py
Normal file
@@ -0,0 +1,488 @@
|
||||
"""Kimi delegation for heavy research via Gitea labels.
|
||||
|
||||
When research exceeds local + Groq capacity, Timmy delegates to Kimi by:
|
||||
1. Filling a research template with full context
|
||||
2. Creating a Gitea issue labeled `kimi-ready`
|
||||
3. Monitoring for Kimi's completion (issue closed + artifact committed)
|
||||
4. Indexing Kimi's artifact into semantic memory
|
||||
5. Extracting action items and creating follow-up issues
|
||||
|
||||
Delegation flow:
|
||||
Timmy detects capacity exceeded
|
||||
→ Fills template with context
|
||||
→ Creates `kimi-ready` Gitea issue
|
||||
→ Kimi picks up, executes, commits artifact, closes issue
|
||||
→ Timmy indexes artifact + creates follow-ups
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Label applied to issues that Kimi should pick up
|
||||
KIMI_READY_LABEL = "kimi-ready"
|
||||
|
||||
# Label colour for the kimi-ready label (dark teal)
|
||||
KIMI_LABEL_COLOR = "#006b75"
|
||||
|
||||
# Keywords that suggest a task exceeds local capacity
|
||||
_HEAVY_RESEARCH_KEYWORDS = frozenset(
|
||||
{
|
||||
"comprehensive",
|
||||
"exhaustive",
|
||||
"systematic review",
|
||||
"literature review",
|
||||
"benchmark",
|
||||
"comparative analysis",
|
||||
"large-scale",
|
||||
"survey",
|
||||
"meta-analysis",
|
||||
"deep research",
|
||||
"extensive",
|
||||
}
|
||||
)
|
||||
|
||||
# Minimum word count that hints at a heavy task
|
||||
_HEAVY_WORD_THRESHOLD = 50
|
||||
|
||||
|
||||
def exceeds_local_capacity(task_description: str) -> bool:
|
||||
"""Heuristic: does this research task exceed local + Groq capacity?
|
||||
|
||||
Returns True when the task description signals heavy or broad research
|
||||
that benefits from Kimi's 262K context and long-running processing.
|
||||
|
||||
Args:
|
||||
task_description: Free-text description of the research task.
|
||||
|
||||
Returns:
|
||||
True if the task should be delegated to Kimi.
|
||||
"""
|
||||
lower = task_description.lower()
|
||||
word_count = len(task_description.split())
|
||||
|
||||
has_heavy_keyword = any(kw in lower for kw in _HEAVY_RESEARCH_KEYWORDS)
|
||||
is_long_task = word_count >= _HEAVY_WORD_THRESHOLD
|
||||
|
||||
return has_heavy_keyword or is_long_task
|
||||
|
||||
|
||||
def _build_research_template(
|
||||
task: str,
|
||||
context: str,
|
||||
question: str,
|
||||
priority: str = "normal",
|
||||
) -> str:
|
||||
"""Fill the standard Kimi research template with task context.
|
||||
|
||||
Args:
|
||||
task: Short title for the research task.
|
||||
context: Background information and relevant project context.
|
||||
question: The specific research question to answer.
|
||||
priority: Task priority — "low", "normal", or "high".
|
||||
|
||||
Returns:
|
||||
Markdown-formatted issue body ready for Gitea.
|
||||
"""
|
||||
return f"""\
|
||||
## Research Request
|
||||
|
||||
**Priority:** {priority}
|
||||
|
||||
### Research Question
|
||||
|
||||
{question}
|
||||
|
||||
### Background / Context
|
||||
|
||||
{context}
|
||||
|
||||
### Scope
|
||||
|
||||
Please produce a thorough, well-structured research report covering:
|
||||
|
||||
- Direct answer to the research question above
|
||||
- Supporting evidence and sources where applicable
|
||||
- Trade-offs, limitations, or caveats
|
||||
- Concrete recommendations or next steps
|
||||
|
||||
### Deliverables
|
||||
|
||||
Commit your findings as a markdown artifact (e.g. `memory/research/{_slugify(task)}.md`)
|
||||
and close this issue when complete.
|
||||
|
||||
### Task
|
||||
|
||||
{task}
|
||||
|
||||
---
|
||||
*Delegated by Timmy via Kimi delegation pipeline. Label: `{KIMI_READY_LABEL}`*
|
||||
"""
|
||||
|
||||
|
||||
def _slugify(text: str) -> str:
|
||||
"""Convert text to a safe filename slug."""
|
||||
slug = re.sub(r"[^\w\s-]", "", text.lower())
|
||||
slug = re.sub(r"[\s_]+", "-", slug)
|
||||
return slug[:60].strip("-")
|
||||
|
||||
|
||||
async def _get_or_create_label(
|
||||
client: Any,
|
||||
base_url: str,
|
||||
headers: dict[str, str],
|
||||
repo: str,
|
||||
) -> int | None:
|
||||
"""Ensure the `kimi-ready` label exists; return its ID or None on error.
|
||||
|
||||
Args:
|
||||
client: httpx.AsyncClient instance.
|
||||
base_url: Gitea API base URL.
|
||||
headers: Auth headers.
|
||||
repo: owner/repo string.
|
||||
|
||||
Returns:
|
||||
Label ID, or None if the operation failed.
|
||||
"""
|
||||
labels_url = f"{base_url}/repos/{repo}/labels"
|
||||
|
||||
# Check for existing label
|
||||
try:
|
||||
resp = await client.get(labels_url, headers=headers)
|
||||
if resp.status_code == 200:
|
||||
for label in resp.json():
|
||||
if label.get("name") == KIMI_READY_LABEL:
|
||||
return label["id"]
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to list Gitea labels: %s", exc)
|
||||
return None
|
||||
|
||||
# Create the label
|
||||
try:
|
||||
resp = await client.post(
|
||||
labels_url,
|
||||
headers=headers,
|
||||
json={"name": KIMI_READY_LABEL, "color": KIMI_LABEL_COLOR},
|
||||
)
|
||||
if resp.status_code in (200, 201):
|
||||
return resp.json().get("id")
|
||||
logger.warning("Label creation returned %s: %s", resp.status_code, resp.text[:200])
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to create Gitea label: %s", exc)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def create_kimi_research_issue(
|
||||
task: str,
|
||||
context: str,
|
||||
question: str,
|
||||
priority: str = "normal",
|
||||
) -> dict[str, Any]:
|
||||
"""Create a Gitea issue labeled `kimi-ready` for Kimi to pick up.
|
||||
|
||||
Args:
|
||||
task: Short title for the research task (used as issue title).
|
||||
context: Background information and project context.
|
||||
question: The specific research question.
|
||||
priority: Task priority — "low", "normal", or "high".
|
||||
|
||||
Returns:
|
||||
Dict with `success`, `issue_number`, `issue_url`, and `error` keys.
|
||||
"""
|
||||
try:
|
||||
import httpx
|
||||
|
||||
from config import settings
|
||||
except ImportError as exc:
|
||||
return {"success": False, "error": f"Missing dependency: {exc}"}
|
||||
|
||||
if not settings.gitea_enabled or not settings.gitea_token:
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Gitea integration not configured (no token or disabled).",
|
||||
}
|
||||
|
||||
base_url = f"{settings.gitea_url}/api/v1"
|
||||
repo = settings.gitea_repo
|
||||
headers = {
|
||||
"Authorization": f"token {settings.gitea_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
label_id = await _get_or_create_label(client, base_url, headers, repo)
|
||||
|
||||
body = _build_research_template(task, context, question, priority)
|
||||
issue_payload: dict[str, Any] = {"title": task, "body": body}
|
||||
if label_id is not None:
|
||||
issue_payload["labels"] = [label_id]
|
||||
|
||||
resp = await client.post(
|
||||
f"{base_url}/repos/{repo}/issues",
|
||||
headers=headers,
|
||||
json=issue_payload,
|
||||
)
|
||||
|
||||
if resp.status_code in (200, 201):
|
||||
data = resp.json()
|
||||
number = data.get("number")
|
||||
url = data.get("html_url", "")
|
||||
logger.info("Created kimi-ready issue #%s: %s", number, task[:60])
|
||||
return {
|
||||
"success": True,
|
||||
"issue_number": number,
|
||||
"issue_url": url,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
logger.warning("Issue creation failed (%s): %s", resp.status_code, resp.text[:200])
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"Gitea API error {resp.status_code}: {resp.text[:200]}",
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("create_kimi_research_issue failed: %s", exc)
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
|
||||
async def poll_kimi_issue(
|
||||
issue_number: int,
|
||||
poll_interval: int = 60,
|
||||
max_wait: int = 3600,
|
||||
) -> dict[str, Any]:
|
||||
"""Poll a Gitea issue until it is closed (Kimi completed) or timeout.
|
||||
|
||||
Args:
|
||||
issue_number: The Gitea issue number to watch.
|
||||
poll_interval: Seconds between polls. Default 60.
|
||||
max_wait: Maximum total seconds to wait. Default 3600 (1 hour).
|
||||
|
||||
Returns:
|
||||
Dict with `completed` bool, `state`, `body`, and `error` keys.
|
||||
"""
|
||||
try:
|
||||
import httpx
|
||||
|
||||
from config import settings
|
||||
except ImportError as exc:
|
||||
return {"completed": False, "error": f"Missing dependency: {exc}"}
|
||||
|
||||
if not settings.gitea_enabled or not settings.gitea_token:
|
||||
return {"completed": False, "error": "Gitea not configured."}
|
||||
|
||||
base_url = f"{settings.gitea_url}/api/v1"
|
||||
repo = settings.gitea_repo
|
||||
headers = {"Authorization": f"token {settings.gitea_token}"}
|
||||
issue_url = f"{base_url}/repos/{repo}/issues/{issue_number}"
|
||||
|
||||
elapsed = 0
|
||||
while elapsed < max_wait:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
resp = await client.get(issue_url, headers=headers)
|
||||
|
||||
if resp.status_code == 200:
|
||||
data = resp.json()
|
||||
state = data.get("state", "open")
|
||||
if state == "closed":
|
||||
logger.info("Kimi completed issue #%s", issue_number)
|
||||
return {
|
||||
"completed": True,
|
||||
"state": state,
|
||||
"body": data.get("body", ""),
|
||||
"error": None,
|
||||
}
|
||||
else:
|
||||
logger.warning("Poll issue #%s returned %s", issue_number, resp.status_code)
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("Poll error for issue #%s: %s", issue_number, exc)
|
||||
|
||||
await asyncio.sleep(poll_interval)
|
||||
elapsed += poll_interval
|
||||
|
||||
return {
|
||||
"completed": False,
|
||||
"state": "timeout",
|
||||
"body": "",
|
||||
"error": f"Timed out after {max_wait}s waiting for issue #{issue_number}",
|
||||
}
|
||||
|
||||
|
||||
def _extract_action_items(text: str) -> list[str]:
|
||||
"""Extract action items from markdown text.
|
||||
|
||||
Looks for lines that start with checklist markers, numbered items,
|
||||
or explicit "Action:" / "TODO:" prefixes.
|
||||
|
||||
Args:
|
||||
text: Markdown text from Kimi's artifact.
|
||||
|
||||
Returns:
|
||||
List of action item strings (deduplicated, whitespace-stripped).
|
||||
"""
|
||||
items: list[str] = []
|
||||
patterns = [
|
||||
re.compile(r"^[-*]\s+\[ \]\s+(.+)", re.MULTILINE), # - [ ] checkbox
|
||||
re.compile(r"^\d+\.\s+(.+)", re.MULTILINE), # 1. numbered list
|
||||
re.compile(r"^(?:Action|TODO|Next step):\s*(.+)", re.MULTILINE | re.IGNORECASE),
|
||||
]
|
||||
seen: set[str] = set()
|
||||
for pat in patterns:
|
||||
for m in pat.finditer(text):
|
||||
item = m.group(1).strip()
|
||||
if item and item not in seen:
|
||||
items.append(item)
|
||||
seen.add(item)
|
||||
return items
|
||||
|
||||
|
||||
async def index_kimi_artifact(
|
||||
issue_number: int,
|
||||
title: str,
|
||||
artifact_content: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Index Kimi's research artifact into Timmy's semantic memory.
|
||||
|
||||
Args:
|
||||
issue_number: Source Gitea issue number (used as task_id).
|
||||
title: Human-readable title for the memory entry.
|
||||
artifact_content: The research artifact text to index.
|
||||
|
||||
Returns:
|
||||
Dict with `success` bool and `memory_id` or `error`.
|
||||
"""
|
||||
if not artifact_content.strip():
|
||||
return {"success": False, "error": "Empty artifact — nothing to index."}
|
||||
|
||||
try:
|
||||
import asyncio
|
||||
|
||||
from timmy.memory_system import store_memory
|
||||
|
||||
# store_memory is synchronous — wrap in thread to avoid blocking event loop
|
||||
entry = await asyncio.to_thread(
|
||||
store_memory,
|
||||
content=artifact_content,
|
||||
source="kimi",
|
||||
context_type="document",
|
||||
task_id=str(issue_number),
|
||||
metadata={"issue_number": issue_number, "title": title},
|
||||
)
|
||||
logger.info("Indexed Kimi artifact for issue #%s (id=%s)", issue_number, entry.id)
|
||||
return {"success": True, "memory_id": entry.id}
|
||||
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to index Kimi artifact for issue #%s: %s", issue_number, exc)
|
||||
return {"success": False, "error": str(exc)}
|
||||
|
||||
|
||||
async def extract_and_create_followups(
|
||||
artifact_content: str,
|
||||
source_issue_number: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Extract action items from artifact and create follow-up Gitea issues.
|
||||
|
||||
Args:
|
||||
artifact_content: Text of Kimi's research artifact.
|
||||
source_issue_number: Issue number that produced the artifact (for cross-links).
|
||||
|
||||
Returns:
|
||||
Dict with `success`, `created` (list of issue numbers), and `error`.
|
||||
"""
|
||||
items = _extract_action_items(artifact_content)
|
||||
if not items:
|
||||
logger.info("No action items found in artifact for issue #%s", source_issue_number)
|
||||
return {"success": True, "created": [], "error": None}
|
||||
|
||||
try:
|
||||
import httpx
|
||||
|
||||
from config import settings
|
||||
except ImportError as exc:
|
||||
return {"success": False, "created": [], "error": str(exc)}
|
||||
|
||||
if not settings.gitea_enabled or not settings.gitea_token:
|
||||
return {
|
||||
"success": False,
|
||||
"created": [],
|
||||
"error": "Gitea not configured.",
|
||||
}
|
||||
|
||||
base_url = f"{settings.gitea_url}/api/v1"
|
||||
repo = settings.gitea_repo
|
||||
headers = {
|
||||
"Authorization": f"token {settings.gitea_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
created: list[int] = []
|
||||
|
||||
for item in items:
|
||||
body = (
|
||||
f"Follow-up from Kimi research artifact in #{source_issue_number}.\n\n"
|
||||
f"**Action item:** {item}"
|
||||
)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
resp = await client.post(
|
||||
f"{base_url}/repos/{repo}/issues",
|
||||
headers=headers,
|
||||
json={"title": item[:120], "body": body},
|
||||
)
|
||||
if resp.status_code in (200, 201):
|
||||
num = resp.json().get("number")
|
||||
if num:
|
||||
created.append(num)
|
||||
logger.info(
|
||||
"Created follow-up issue #%s from kimi artifact #%s",
|
||||
num,
|
||||
source_issue_number,
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Follow-up issue creation returned %s for item: %s",
|
||||
resp.status_code,
|
||||
item[:60],
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to create follow-up for item '%s': %s", item[:60], exc)
|
||||
|
||||
return {"success": True, "created": created, "error": None}
|
||||
|
||||
|
||||
async def delegate_research_to_kimi(
|
||||
task: str,
|
||||
context: str,
|
||||
question: str,
|
||||
priority: str = "normal",
|
||||
) -> dict[str, Any]:
|
||||
"""Top-level entry point: delegate a heavy research task to Kimi.
|
||||
|
||||
Creates the `kimi-ready` Gitea issue and returns immediately.
|
||||
Monitoring, artifact indexing, and follow-up creation happen
|
||||
separately via `poll_kimi_issue`, `index_kimi_artifact`, and
|
||||
`extract_and_create_followups`.
|
||||
|
||||
Args:
|
||||
task: Short title (becomes the issue title).
|
||||
context: Background / project context.
|
||||
question: The specific research question Kimi should answer.
|
||||
priority: "low", "normal", or "high".
|
||||
|
||||
Returns:
|
||||
Dict with `success`, `issue_number`, `issue_url`, and `error`.
|
||||
"""
|
||||
if not task.strip() or not question.strip():
|
||||
return {
|
||||
"success": False,
|
||||
"error": "Both `task` and `question` are required.",
|
||||
}
|
||||
|
||||
logger.info("Delegating research to Kimi: %s", task[:80])
|
||||
return await create_kimi_research_issue(task, context, question, priority)
|
||||
540
src/timmy/mcp_bridge.py
Normal file
540
src/timmy/mcp_bridge.py
Normal file
@@ -0,0 +1,540 @@
|
||||
"""MCP Bridge for Qwen3 via Ollama.
|
||||
|
||||
Provides a lightweight bridge between Ollama's native tool-calling API
|
||||
and MCP tool servers (Gitea, Filesystem, Shell). Unlike the Agno-based
|
||||
agent loop, this bridge talks directly to the Ollama ``/api/chat``
|
||||
endpoint, translating MCP tool schemas into Ollama tool definitions and
|
||||
executing tool calls in a loop until the model produces a final response.
|
||||
|
||||
Designed for Qwen3 models which have first-class tool-calling support.
|
||||
|
||||
Usage::
|
||||
|
||||
from timmy.mcp_bridge import MCPBridge
|
||||
|
||||
bridge = MCPBridge()
|
||||
async with bridge:
|
||||
result = await bridge.run("List open issues in Timmy-time-dashboard")
|
||||
print(result.content)
|
||||
|
||||
The bridge evaluates available options in order of preference:
|
||||
1. Direct Ollama /api/chat with native tool_calls (selected — best fit)
|
||||
2. qwen-agent MCP (requires separate qwen-agent install)
|
||||
3. ollmcp / mcphost / ollama-mcp-bridge (external binaries)
|
||||
|
||||
Option 1 was selected because:
|
||||
- Zero additional dependencies (uses httpx already in the project)
|
||||
- Native Qwen3 tool-calling support via Ollama's OpenAI-compatible API
|
||||
- Full control over the tool-call loop and error handling
|
||||
- Consistent with the project's graceful-degradation pattern
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Maximum tool-call round-trips before aborting (safety valve).
|
||||
_MAX_TOOL_ROUNDS = 10
|
||||
|
||||
|
||||
@dataclass
|
||||
class BridgeResult:
|
||||
"""Result from an MCP bridge run."""
|
||||
|
||||
content: str
|
||||
tool_calls_made: list[dict] = field(default_factory=list)
|
||||
rounds: int = 0
|
||||
latency_ms: float = 0.0
|
||||
model: str = ""
|
||||
error: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class MCPToolDef:
|
||||
"""An MCP tool definition translated for Ollama."""
|
||||
|
||||
name: str
|
||||
description: str
|
||||
parameters: dict[str, Any]
|
||||
handler: Any # async callable(**kwargs) -> str
|
||||
|
||||
|
||||
def _mcp_schema_to_ollama_tool(tool: MCPToolDef) -> dict:
|
||||
"""Convert an MCPToolDef into Ollama's tool format.
|
||||
|
||||
Ollama uses OpenAI-compatible tool definitions::
|
||||
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "...",
|
||||
"description": "...",
|
||||
"parameters": { "type": "object", "properties": {...}, "required": [...] }
|
||||
}
|
||||
}
|
||||
"""
|
||||
# Normalise parameters — ensure it has "type": "object" wrapper.
|
||||
params = tool.parameters
|
||||
if params.get("type") != "object":
|
||||
params = {
|
||||
"type": "object",
|
||||
"properties": params,
|
||||
"required": list(params.keys()),
|
||||
}
|
||||
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tool.name,
|
||||
"description": tool.description,
|
||||
"parameters": params,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _build_shell_tool() -> MCPToolDef | None:
|
||||
"""Build the shell execution tool using the local ShellHand."""
|
||||
try:
|
||||
from infrastructure.hands.shell import shell_hand
|
||||
|
||||
async def _handle_shell(**kwargs: Any) -> str:
|
||||
command = kwargs.get("command", "")
|
||||
timeout = kwargs.get("timeout")
|
||||
result = await shell_hand.run(command, timeout=timeout)
|
||||
if result.success:
|
||||
return result.stdout or "(no output)"
|
||||
return f"[error] exit={result.exit_code} {result.error or result.stderr}"
|
||||
|
||||
return MCPToolDef(
|
||||
name="shell_exec",
|
||||
description=(
|
||||
"Execute a shell command in a sandboxed environment. "
|
||||
"Commands are validated against an allow-list. "
|
||||
"Returns stdout, stderr, and exit code."
|
||||
),
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"command": {
|
||||
"type": "string",
|
||||
"description": "Shell command to execute (must match allow-list)",
|
||||
},
|
||||
"timeout": {
|
||||
"type": "integer",
|
||||
"description": "Timeout in seconds (default 60)",
|
||||
},
|
||||
},
|
||||
"required": ["command"],
|
||||
},
|
||||
handler=_handle_shell,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Shell tool unavailable: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _build_gitea_tools() -> list[MCPToolDef]:
|
||||
"""Build Gitea MCP tool definitions for direct Ollama bridge use.
|
||||
|
||||
These tools call the Gitea REST API directly via httpx rather than
|
||||
spawning an MCP server subprocess, keeping the bridge lightweight.
|
||||
"""
|
||||
if not settings.gitea_enabled or not settings.gitea_token:
|
||||
return []
|
||||
|
||||
base_url = settings.gitea_url
|
||||
token = settings.gitea_token
|
||||
owner, repo = settings.gitea_repo.split("/", 1)
|
||||
|
||||
async def _list_issues(**kwargs: Any) -> str:
|
||||
state = kwargs.get("state", "open")
|
||||
limit = kwargs.get("limit", 10)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
resp = await client.get(
|
||||
f"{base_url}/api/v1/repos/{owner}/{repo}/issues",
|
||||
headers={"Authorization": f"token {token}"},
|
||||
params={"state": state, "limit": limit, "type": "issues"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
issues = resp.json()
|
||||
if not issues:
|
||||
return f"No {state} issues found."
|
||||
lines = []
|
||||
for issue in issues:
|
||||
labels = ", ".join(lb["name"] for lb in issue.get("labels", []))
|
||||
label_str = f" [{labels}]" if labels else ""
|
||||
lines.append(f"#{issue['number']}: {issue['title']}{label_str}")
|
||||
return "\n".join(lines)
|
||||
except Exception as exc:
|
||||
return f"Error listing issues: {exc}"
|
||||
|
||||
async def _create_issue(**kwargs: Any) -> str:
|
||||
title = kwargs.get("title", "")
|
||||
body = kwargs.get("body", "")
|
||||
if not title:
|
||||
return "Error: title is required"
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
resp = await client.post(
|
||||
f"{base_url}/api/v1/repos/{owner}/{repo}/issues",
|
||||
headers={
|
||||
"Authorization": f"token {token}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json={"title": title, "body": body},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
return f"Created issue #{data['number']}: {data['title']}"
|
||||
except Exception as exc:
|
||||
return f"Error creating issue: {exc}"
|
||||
|
||||
async def _read_issue(**kwargs: Any) -> str:
|
||||
number = kwargs.get("number")
|
||||
if not number:
|
||||
return "Error: issue number is required"
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
resp = await client.get(
|
||||
f"{base_url}/api/v1/repos/{owner}/{repo}/issues/{number}",
|
||||
headers={"Authorization": f"token {token}"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
issue = resp.json()
|
||||
labels = ", ".join(lb["name"] for lb in issue.get("labels", []))
|
||||
parts = [
|
||||
f"#{issue['number']}: {issue['title']}",
|
||||
f"State: {issue['state']}",
|
||||
]
|
||||
if labels:
|
||||
parts.append(f"Labels: {labels}")
|
||||
if issue.get("body"):
|
||||
parts.append(f"\n{issue['body']}")
|
||||
return "\n".join(parts)
|
||||
except Exception as exc:
|
||||
return f"Error reading issue: {exc}"
|
||||
|
||||
return [
|
||||
MCPToolDef(
|
||||
name="list_issues",
|
||||
description="List issues in the Gitea repository. Returns issue numbers and titles.",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "string",
|
||||
"description": "Filter by state: open, closed, or all (default: open)",
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of issues to return (default: 10)",
|
||||
},
|
||||
},
|
||||
"required": [],
|
||||
},
|
||||
handler=_list_issues,
|
||||
),
|
||||
MCPToolDef(
|
||||
name="create_issue",
|
||||
description="Create a new issue in the Gitea repository.",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {
|
||||
"type": "string",
|
||||
"description": "Issue title (required)",
|
||||
},
|
||||
"body": {
|
||||
"type": "string",
|
||||
"description": "Issue body in markdown (optional)",
|
||||
},
|
||||
},
|
||||
"required": ["title"],
|
||||
},
|
||||
handler=_create_issue,
|
||||
),
|
||||
MCPToolDef(
|
||||
name="read_issue",
|
||||
description="Read details of a specific issue by number.",
|
||||
parameters={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"number": {
|
||||
"type": "integer",
|
||||
"description": "Issue number to read",
|
||||
},
|
||||
},
|
||||
"required": ["number"],
|
||||
},
|
||||
handler=_read_issue,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
class MCPBridge:
|
||||
"""Bridge between Ollama's tool-calling API and MCP tools.
|
||||
|
||||
Manages a set of tool definitions and executes a chat loop with
|
||||
tool calling against a Qwen3 model via Ollama.
|
||||
|
||||
The bridge:
|
||||
1. Registers available tools (Gitea, shell, custom)
|
||||
2. Sends prompts to Ollama with tool definitions
|
||||
3. Executes tool calls when the model requests them
|
||||
4. Returns tool results to the model for the next round
|
||||
5. Repeats until the model produces a final text response
|
||||
|
||||
Attributes:
|
||||
model: Ollama model name (default from settings).
|
||||
ollama_url: Ollama API base URL (default from settings).
|
||||
tools: Registered tool definitions.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str | None = None,
|
||||
ollama_url: str | None = None,
|
||||
*,
|
||||
include_gitea: bool = True,
|
||||
include_shell: bool = True,
|
||||
extra_tools: list[MCPToolDef] | None = None,
|
||||
max_rounds: int = _MAX_TOOL_ROUNDS,
|
||||
) -> None:
|
||||
self.model = model or settings.ollama_model
|
||||
self.ollama_url = ollama_url or settings.normalized_ollama_url
|
||||
self.max_rounds = max_rounds
|
||||
self._tools: dict[str, MCPToolDef] = {}
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
|
||||
# Register built-in tools
|
||||
if include_gitea:
|
||||
for tool in _build_gitea_tools():
|
||||
self._tools[tool.name] = tool
|
||||
|
||||
if include_shell:
|
||||
shell = _build_shell_tool()
|
||||
if shell:
|
||||
self._tools[shell.name] = shell
|
||||
|
||||
# Register extra tools
|
||||
if extra_tools:
|
||||
for tool in extra_tools:
|
||||
self._tools[tool.name] = tool
|
||||
|
||||
logger.info(
|
||||
"MCPBridge initialised: model=%s, tools=%s",
|
||||
self.model,
|
||||
list(self._tools.keys()),
|
||||
)
|
||||
|
||||
async def __aenter__(self) -> MCPBridge:
|
||||
self._client = httpx.AsyncClient(timeout=settings.mcp_bridge_timeout)
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc: Any) -> None:
|
||||
if self._client:
|
||||
await self._client.aclose()
|
||||
self._client = None
|
||||
|
||||
@property
|
||||
def tool_names(self) -> list[str]:
|
||||
"""Return names of all registered tools."""
|
||||
return list(self._tools.keys())
|
||||
|
||||
def _build_ollama_tools(self) -> list[dict]:
|
||||
"""Convert registered tools to Ollama tool format."""
|
||||
return [_mcp_schema_to_ollama_tool(t) for t in self._tools.values()]
|
||||
|
||||
async def _chat(self, messages: list[dict], tools: list[dict]) -> dict:
|
||||
"""Send a chat request to Ollama and return the response.
|
||||
|
||||
Uses the ``/api/chat`` endpoint with tool definitions.
|
||||
"""
|
||||
if not self._client:
|
||||
raise RuntimeError("MCPBridge must be used as async context manager")
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"model": self.model,
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
}
|
||||
if tools:
|
||||
payload["tools"] = tools
|
||||
|
||||
# Set num_ctx if configured
|
||||
if settings.ollama_num_ctx > 0:
|
||||
payload["options"] = {"num_ctx": settings.ollama_num_ctx}
|
||||
|
||||
resp = await self._client.post(
|
||||
f"{self.ollama_url}/api/chat",
|
||||
json=payload,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
async def _execute_tool_call(self, tool_call: dict) -> str:
|
||||
"""Execute a single tool call and return the result string."""
|
||||
func = tool_call.get("function", {})
|
||||
name = func.get("name", "")
|
||||
arguments = func.get("arguments", {})
|
||||
|
||||
tool = self._tools.get(name)
|
||||
if not tool:
|
||||
return f"Error: unknown tool '{name}'"
|
||||
|
||||
try:
|
||||
result = await tool.handler(**arguments)
|
||||
return str(result)
|
||||
except Exception as exc:
|
||||
logger.warning("Tool '%s' execution failed: %s", name, exc)
|
||||
return f"Error executing {name}: {exc}"
|
||||
|
||||
async def run(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
system_prompt: str | None = None,
|
||||
) -> BridgeResult:
|
||||
"""Run a prompt through the MCP bridge with tool calling.
|
||||
|
||||
Sends the prompt to the Ollama model with tool definitions.
|
||||
If the model requests tool calls, executes them and feeds
|
||||
results back until the model produces a final text response.
|
||||
|
||||
Args:
|
||||
prompt: User message to send.
|
||||
system_prompt: Optional system prompt override.
|
||||
|
||||
Returns:
|
||||
BridgeResult with the final response and tool call history.
|
||||
"""
|
||||
start = time.time()
|
||||
messages: list[dict] = []
|
||||
|
||||
if system_prompt:
|
||||
messages.append({"role": "system", "content": system_prompt})
|
||||
|
||||
messages.append({"role": "user", "content": prompt})
|
||||
|
||||
tools = self._build_ollama_tools()
|
||||
tool_calls_made: list[dict] = []
|
||||
rounds = 0
|
||||
|
||||
try:
|
||||
for round_num in range(self.max_rounds):
|
||||
rounds = round_num + 1
|
||||
response = await self._chat(messages, tools)
|
||||
msg = response.get("message", {})
|
||||
|
||||
# Check if model made tool calls
|
||||
model_tool_calls = msg.get("tool_calls", [])
|
||||
if not model_tool_calls:
|
||||
# Final text response — done.
|
||||
content = msg.get("content", "")
|
||||
latency = (time.time() - start) * 1000
|
||||
return BridgeResult(
|
||||
content=content,
|
||||
tool_calls_made=tool_calls_made,
|
||||
rounds=rounds,
|
||||
latency_ms=latency,
|
||||
model=self.model,
|
||||
)
|
||||
|
||||
# Append the assistant message (with tool_calls) to history
|
||||
messages.append(msg)
|
||||
|
||||
# Execute each tool call and add results
|
||||
for tc in model_tool_calls:
|
||||
func = tc.get("function", {})
|
||||
tool_name = func.get("name", "unknown")
|
||||
tool_args = func.get("arguments", {})
|
||||
|
||||
logger.info(
|
||||
"Bridge tool call [round %d]: %s(%s)",
|
||||
rounds,
|
||||
tool_name,
|
||||
tool_args,
|
||||
)
|
||||
|
||||
result = await self._execute_tool_call(tc)
|
||||
tool_calls_made.append(
|
||||
{
|
||||
"round": rounds,
|
||||
"tool": tool_name,
|
||||
"arguments": tool_args,
|
||||
"result": result[:500], # Truncate for logging
|
||||
}
|
||||
)
|
||||
|
||||
# Add tool result to message history
|
||||
messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"content": result,
|
||||
}
|
||||
)
|
||||
|
||||
# Hit max rounds
|
||||
latency = (time.time() - start) * 1000
|
||||
return BridgeResult(
|
||||
content="(max tool-call rounds reached)",
|
||||
tool_calls_made=tool_calls_made,
|
||||
rounds=rounds,
|
||||
latency_ms=latency,
|
||||
model=self.model,
|
||||
error=f"Exceeded maximum of {self.max_rounds} tool-call rounds",
|
||||
)
|
||||
|
||||
except httpx.ConnectError as exc:
|
||||
latency = (time.time() - start) * 1000
|
||||
logger.warning("Ollama connection failed: %s", exc)
|
||||
return BridgeResult(
|
||||
content="",
|
||||
tool_calls_made=tool_calls_made,
|
||||
rounds=rounds,
|
||||
latency_ms=latency,
|
||||
model=self.model,
|
||||
error=f"Ollama connection failed: {exc}",
|
||||
)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
latency = (time.time() - start) * 1000
|
||||
logger.warning("Ollama HTTP error: %s", exc)
|
||||
return BridgeResult(
|
||||
content="",
|
||||
tool_calls_made=tool_calls_made,
|
||||
rounds=rounds,
|
||||
latency_ms=latency,
|
||||
model=self.model,
|
||||
error=f"Ollama HTTP error: {exc.response.status_code}",
|
||||
)
|
||||
except Exception as exc:
|
||||
latency = (time.time() - start) * 1000
|
||||
logger.error("MCPBridge run failed: %s", exc)
|
||||
return BridgeResult(
|
||||
content="",
|
||||
tool_calls_made=tool_calls_made,
|
||||
rounds=rounds,
|
||||
latency_ms=latency,
|
||||
model=self.model,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
def status(self) -> dict:
|
||||
"""Return bridge status for the dashboard."""
|
||||
return {
|
||||
"model": self.model,
|
||||
"ollama_url": self.ollama_url,
|
||||
"tools": self.tool_names,
|
||||
"max_rounds": self.max_rounds,
|
||||
"connected": self._client is not None,
|
||||
}
|
||||
@@ -14,6 +14,8 @@ from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Paths
|
||||
@@ -28,7 +30,7 @@ def get_connection() -> Generator[sqlite3.Connection, None, None]:
|
||||
with closing(sqlite3.connect(str(DB_PATH))) as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
conn.execute(f"PRAGMA busy_timeout={settings.db_busy_timeout_ms}")
|
||||
_ensure_schema(conn)
|
||||
yield conn
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from config import settings
|
||||
from timmy.memory.embeddings import (
|
||||
EMBEDDING_DIM,
|
||||
EMBEDDING_MODEL, # noqa: F401 — re-exported for backward compatibility
|
||||
@@ -111,7 +112,7 @@ def get_connection() -> Generator[sqlite3.Connection, None, None]:
|
||||
with closing(sqlite3.connect(str(DB_PATH))) as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
conn.execute(f"PRAGMA busy_timeout={settings.db_busy_timeout_ms}")
|
||||
_ensure_schema(conn)
|
||||
yield conn
|
||||
|
||||
@@ -949,7 +950,7 @@ class SemanticMemory:
|
||||
with closing(sqlite3.connect(str(self.db_path))) as conn:
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
conn.execute(f"PRAGMA busy_timeout={settings.db_busy_timeout_ms}")
|
||||
# Ensure schema exists
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS memories (
|
||||
|
||||
175
src/timmy/paperclip.py
Normal file
175
src/timmy/paperclip.py
Normal file
@@ -0,0 +1,175 @@
|
||||
"""Paperclip integration for Timmy.
|
||||
|
||||
This module provides a client for the Paperclip API, and a poller for
|
||||
running research tasks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
import httpx
|
||||
|
||||
from config import settings
|
||||
from timmy.research_triage import triage_research_report
|
||||
from timmy.research_tools import google_web_search, get_llm_client
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PaperclipTask:
|
||||
"""A task from the Paperclip API."""
|
||||
|
||||
id: str
|
||||
kind: str
|
||||
context: dict
|
||||
|
||||
|
||||
class PaperclipClient:
|
||||
"""A client for the Paperclip API."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.base_url = settings.paperclip_url
|
||||
self.api_key = settings.paperclip_api_key
|
||||
self.agent_id = settings.paperclip_agent_id
|
||||
self.company_id = settings.paperclip_company_id
|
||||
self.timeout = settings.paperclip_timeout
|
||||
|
||||
async def get_tasks(self) -> list[PaperclipTask]:
|
||||
"""Get a list of tasks from the Paperclip API."""
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/api/tasks",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
params={
|
||||
"agent_id": self.agent_id,
|
||||
"company_id": self.company_id,
|
||||
"status": "queued",
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
tasks = resp.json()
|
||||
return [
|
||||
PaperclipTask(id=t["id"], kind=t["kind"], context=t["context"])
|
||||
for t in tasks
|
||||
]
|
||||
|
||||
async def update_task_status(
|
||||
self, task_id: str, status: str, result: str | None = None
|
||||
) -> None:
|
||||
"""Update the status of a task."""
|
||||
async with httpx.AsyncClient(timeout=self.timeout) as client:
|
||||
await client.patch(
|
||||
f"{self.base_url}/api/tasks/{task_id}",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json={"status": status, "result": result},
|
||||
)
|
||||
|
||||
|
||||
class ResearchOrchestrator:
|
||||
"""Orchestrates research tasks."""
|
||||
|
||||
async def get_gitea_issue(self, issue_number: int) -> dict:
|
||||
"""Get a Gitea issue by its number."""
|
||||
owner, repo = settings.gitea_repo.split("/", 1)
|
||||
api_url = f"{settings.gitea_url}/api/v1/repos/{owner}/{repo}/issues/{issue_number}"
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
resp = await client.get(
|
||||
api_url,
|
||||
headers={"Authorization": f"token {settings.gitea_token}"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
async def post_gitea_comment(self, issue_number: int, comment: str) -> None:
|
||||
"""Post a comment to a Gitea issue."""
|
||||
owner, repo = settings.gitea_repo.split("/", 1)
|
||||
api_url = f"{settings.gitea_url}/api/v1/repos/{owner}/{repo}/issues/{issue_number}/comments"
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
await client.post(
|
||||
api_url,
|
||||
headers={"Authorization": f"token {settings.gitea_token}"},
|
||||
json={"body": comment},
|
||||
)
|
||||
|
||||
async def run_research_pipeline(self, issue_title: str) -> str:
|
||||
"""Run the research pipeline."""
|
||||
search_results = await google_web_search(issue_title)
|
||||
|
||||
llm_client = get_llm_client()
|
||||
response = await llm_client.completion(
|
||||
f"Summarize the following search results and generate a research report:\\n\\n{search_results}",
|
||||
max_tokens=2048,
|
||||
)
|
||||
return response.text
|
||||
|
||||
async def run(self, context: dict) -> str:
|
||||
"""Run a research task."""
|
||||
issue_number = context.get("issue_number")
|
||||
if not issue_number:
|
||||
return "Missing issue_number in task context"
|
||||
|
||||
issue = await self.get_gitea_issue(issue_number)
|
||||
|
||||
report = await self.run_research_pipeline(issue["title"])
|
||||
|
||||
triage_results = await triage_research_report(report, source_issue=issue_number)
|
||||
|
||||
comment = f"Research complete for issue #{issue_number}.\\n\\n"
|
||||
if triage_results:
|
||||
comment += "Created the following issues:\\n"
|
||||
for result in triage_results:
|
||||
if result["gitea_issue"]:
|
||||
comment += f"- #{result['gitea_issue']['number']}: {result['action_item'].title}\\n"
|
||||
else:
|
||||
comment += "No new issues were created.\\n"
|
||||
|
||||
await self.post_gitea_comment(issue_number, comment)
|
||||
|
||||
return f"Research complete for issue #{issue_number}"
|
||||
|
||||
|
||||
class PaperclipPoller:
|
||||
"""Polls the Paperclip API for new tasks."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.client = PaperclipClient()
|
||||
self.orchestrator = ResearchOrchestrator()
|
||||
self.poll_interval = settings.paperclip_poll_interval
|
||||
|
||||
async def poll(self) -> None:
|
||||
"""Poll the Paperclip API for new tasks."""
|
||||
if self.poll_interval == 0:
|
||||
return
|
||||
|
||||
while True:
|
||||
try:
|
||||
tasks = await self.client.get_tasks()
|
||||
for task in tasks:
|
||||
if task.kind == "research":
|
||||
await self.run_research_task(task)
|
||||
except httpx.HTTPError as exc:
|
||||
logger.warning("Error polling Paperclip: %s", exc)
|
||||
|
||||
await asyncio.sleep(self.poll_interval)
|
||||
|
||||
async def run_research_task(self, task: PaperclipTask) -> None:
|
||||
"""Run a research task."""
|
||||
await self.client.update_task_status(task.id, "running")
|
||||
try:
|
||||
result = await self.orchestrator.run(task.context)
|
||||
await self.client.update_task_status(task.id, "completed", result)
|
||||
except Exception as exc:
|
||||
logger.error("Error running research task: %s", exc, exc_info=True)
|
||||
await self.client.update_task_status(task.id, "failed", str(exc))
|
||||
|
||||
|
||||
async def start_paperclip_poller() -> None:
|
||||
"""Start the Paperclip poller."""
|
||||
if settings.paperclip_enabled:
|
||||
poller = PaperclipPoller()
|
||||
asyncio.create_task(poller.poll())
|
||||
|
||||
42
src/timmy/research_tools.py
Normal file
42
src/timmy/research_tools.py
Normal file
@@ -0,0 +1,42 @@
|
||||
"""Tools for the research pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from config import settings
|
||||
from serpapi import GoogleSearch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def google_web_search(query: str) -> str:
|
||||
"""Perform a Google search and return the results."""
|
||||
if "SERPAPI_API_KEY" not in os.environ:
|
||||
logger.warning("SERPAPI_API_KEY not set, skipping web search")
|
||||
return ""
|
||||
params = {
|
||||
"q": query,
|
||||
"api_key": os.environ["SERPAPI_API_KEY"],
|
||||
}
|
||||
search = GoogleSearch(params)
|
||||
results = search.get_dict()
|
||||
return str(results)
|
||||
|
||||
|
||||
def get_llm_client() -> Any:
|
||||
"""Get an LLM client."""
|
||||
# This is a placeholder. In a real application, this would return
|
||||
# a client for an LLM service like OpenAI, Anthropic, or a local
|
||||
# model.
|
||||
class MockLLMClient:
|
||||
async def completion(self, prompt: str, max_tokens: int) -> Any:
|
||||
class MockCompletion:
|
||||
def __init__(self, text: str) -> None:
|
||||
self.text = text
|
||||
|
||||
return MockCompletion(f"This is a summary of the search results for '{prompt}'.")
|
||||
|
||||
return MockLLMClient()
|
||||
367
src/timmy/research_triage.py
Normal file
367
src/timmy/research_triage.py
Normal file
@@ -0,0 +1,367 @@
|
||||
"""Research triage — extract action items from research reports and file Gitea issues.
|
||||
|
||||
Closes the loop: research → knowledge → actionable engineering work.
|
||||
|
||||
The LLM extracts action items during synthesis (not post-processed), then
|
||||
each item is filed as a Gitea issue with appropriate labels, source links,
|
||||
and evidence from the original research.
|
||||
|
||||
Usage::
|
||||
|
||||
from timmy.research_triage import triage_research_report
|
||||
|
||||
results = await triage_research_report(
|
||||
report="## Findings\\n...",
|
||||
source_issue=946,
|
||||
)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Regex to strip markdown code fences from LLM output
|
||||
_FENCE_RE = re.compile(r"^```(?:json)?\s*\n?", re.MULTILINE)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ActionItem:
|
||||
"""A single actionable item extracted from a research report."""
|
||||
|
||||
title: str
|
||||
body: str
|
||||
labels: list[str] = field(default_factory=list)
|
||||
priority: str = "medium"
|
||||
source_urls: list[str] = field(default_factory=list)
|
||||
|
||||
def to_issue_body(self, source_issue: int | None = None) -> str:
|
||||
"""Format for a Gitea issue body with source attribution."""
|
||||
parts = [self.body]
|
||||
|
||||
if self.source_urls:
|
||||
parts.append("\n### Source Evidence")
|
||||
for url in self.source_urls:
|
||||
parts.append(f"- {url}")
|
||||
|
||||
if source_issue:
|
||||
parts.append(f"\n### Origin\nExtracted from research in #{source_issue}")
|
||||
|
||||
parts.append("\n---\n*Auto-triaged from research findings by Timmy*")
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
def _build_extraction_prompt(report: str) -> str:
|
||||
"""Build the LLM prompt for extracting action items from a research report."""
|
||||
return (
|
||||
"You are triaging a research report for actionable engineering work.\n"
|
||||
"Extract 0-5 CONCRETE action items — bugs to fix, features to build,\n"
|
||||
"infrastructure to set up, or investigations to run.\n\n"
|
||||
"Rules:\n"
|
||||
"- Only include items that map to real engineering tasks\n"
|
||||
"- Skip vague recommendations or philosophical observations\n"
|
||||
"- Each item should be specific enough to become a Gitea issue\n"
|
||||
"- Include evidence/URLs from the report in source_urls\n"
|
||||
"- Priority: high (blocking or critical), medium (important), low (nice-to-have)\n"
|
||||
"- Labels: pick from [actionable, research, bug, feature, infrastructure, "
|
||||
"performance, security, kimi-ready]\n"
|
||||
" - 'kimi-ready' means a well-scoped task suitable for an AI agent\n"
|
||||
" - 'actionable' should be on every item (these are all actionable)\n\n"
|
||||
"For each item return:\n"
|
||||
'- "title": Clear, specific title with area prefix '
|
||||
'(e.g. "[MCP] Restore tool server with FastMCP")\n'
|
||||
'- "body": Detailed markdown body with:\n'
|
||||
" **What:** What needs to be done\n"
|
||||
" **Why:** Why this matters (link to research finding)\n"
|
||||
" **Suggested approach:** How to implement\n"
|
||||
" **Acceptance criteria:** How to verify\n"
|
||||
'- "labels": Array of label strings\n'
|
||||
'- "priority": One of high, medium, low\n'
|
||||
'- "source_urls": Array of URLs referenced in the research\n\n'
|
||||
"Return ONLY a JSON array of objects. Return [] if nothing is actionable.\n\n"
|
||||
f"Research report:\n{report}\n\nJSON array:"
|
||||
)
|
||||
|
||||
|
||||
def _parse_llm_response(raw: str) -> list[dict[str, Any]]:
|
||||
"""Parse LLM JSON response, stripping code fences if present."""
|
||||
cleaned = raw.strip()
|
||||
|
||||
# Strip markdown code fences
|
||||
if cleaned.startswith("```"):
|
||||
cleaned = cleaned.split("\n", 1)[-1].rsplit("```", 1)[0].strip()
|
||||
|
||||
items = json.loads(cleaned)
|
||||
if not isinstance(items, list):
|
||||
return []
|
||||
return items
|
||||
|
||||
|
||||
def _validate_action_item(raw_item: dict[str, Any]) -> ActionItem | None:
|
||||
"""Validate and convert a raw dict to an ActionItem, or None if invalid."""
|
||||
if not isinstance(raw_item, dict):
|
||||
return None
|
||||
|
||||
title = raw_item.get("title", "").strip()
|
||||
body = raw_item.get("body", "").strip()
|
||||
|
||||
if not title or len(title) < 10:
|
||||
return None
|
||||
if not body or len(body) < 20:
|
||||
return None
|
||||
|
||||
labels = raw_item.get("labels", [])
|
||||
if isinstance(labels, str):
|
||||
labels = [lbl.strip() for lbl in labels.split(",") if lbl.strip()]
|
||||
if not isinstance(labels, list):
|
||||
labels = []
|
||||
|
||||
# Ensure 'actionable' label is always present
|
||||
if "actionable" not in labels:
|
||||
labels.insert(0, "actionable")
|
||||
|
||||
priority = raw_item.get("priority", "medium").strip().lower()
|
||||
if priority not in ("high", "medium", "low"):
|
||||
priority = "medium"
|
||||
|
||||
source_urls = raw_item.get("source_urls", [])
|
||||
if not isinstance(source_urls, list):
|
||||
source_urls = []
|
||||
|
||||
return ActionItem(
|
||||
title=title,
|
||||
body=body,
|
||||
labels=labels,
|
||||
priority=priority,
|
||||
source_urls=source_urls,
|
||||
)
|
||||
|
||||
|
||||
async def extract_action_items(
|
||||
report: str,
|
||||
llm_caller: Any | None = None,
|
||||
) -> list[ActionItem]:
|
||||
"""Extract actionable engineering items from a research report.
|
||||
|
||||
Uses the LLM to identify concrete tasks, bugs, features, and
|
||||
infrastructure work from structured research output.
|
||||
|
||||
Args:
|
||||
report: The research report text (markdown).
|
||||
llm_caller: Optional async callable(prompt) -> str for LLM.
|
||||
Falls back to the cascade router.
|
||||
|
||||
Returns:
|
||||
List of validated ActionItem objects (0-5 items).
|
||||
"""
|
||||
if not report or not report.strip():
|
||||
return []
|
||||
|
||||
prompt = _build_extraction_prompt(report)
|
||||
|
||||
try:
|
||||
if llm_caller is not None:
|
||||
raw = await llm_caller(prompt)
|
||||
else:
|
||||
raw = await _call_llm(prompt)
|
||||
except Exception as exc:
|
||||
logger.warning("LLM extraction failed: %s", exc)
|
||||
return []
|
||||
|
||||
if not raw or not raw.strip():
|
||||
return []
|
||||
|
||||
try:
|
||||
raw_items = _parse_llm_response(raw)
|
||||
except (json.JSONDecodeError, ValueError) as exc:
|
||||
logger.warning("Failed to parse LLM action items: %s", exc)
|
||||
return []
|
||||
|
||||
items = []
|
||||
for raw_item in raw_items[:5]: # Safety cap
|
||||
item = _validate_action_item(raw_item)
|
||||
if item is not None:
|
||||
items.append(item)
|
||||
|
||||
logger.info("Extracted %d action items from research report", len(items))
|
||||
return items
|
||||
|
||||
|
||||
async def _call_llm(prompt: str) -> str:
|
||||
"""Call the cascade router for LLM completion.
|
||||
|
||||
Falls back gracefully if the router is unavailable.
|
||||
"""
|
||||
from infrastructure.router import get_router
|
||||
|
||||
router = get_router()
|
||||
messages = [{"role": "user", "content": prompt}]
|
||||
result = await router.complete(messages=messages, temperature=0.1)
|
||||
return result.get("content", "") if isinstance(result, dict) else str(result)
|
||||
|
||||
|
||||
async def create_gitea_issue(
|
||||
item: ActionItem,
|
||||
source_issue: int | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Create a Gitea issue from an ActionItem via the REST API.
|
||||
|
||||
Args:
|
||||
item: The action item to file.
|
||||
source_issue: Parent research issue number to link back to.
|
||||
|
||||
Returns:
|
||||
The created issue dict from Gitea API, or None on failure.
|
||||
"""
|
||||
if not settings.gitea_enabled or not settings.gitea_token:
|
||||
logger.debug("Gitea not configured — skipping issue creation")
|
||||
return None
|
||||
|
||||
owner, repo = settings.gitea_repo.split("/", 1)
|
||||
api_url = f"{settings.gitea_url}/api/v1/repos/{owner}/{repo}/issues"
|
||||
|
||||
body = item.to_issue_body(source_issue=source_issue)
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"title": item.title,
|
||||
"body": body,
|
||||
}
|
||||
|
||||
# Resolve label names to IDs
|
||||
label_ids = await _resolve_label_ids(item.labels, owner, repo)
|
||||
if label_ids:
|
||||
payload["labels"] = label_ids
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
resp = await client.post(
|
||||
api_url,
|
||||
headers={
|
||||
"Authorization": f"token {settings.gitea_token}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
json=payload,
|
||||
)
|
||||
|
||||
if resp.status_code in (200, 201):
|
||||
issue_data = resp.json()
|
||||
logger.info(
|
||||
"Created Gitea issue #%s: %s",
|
||||
issue_data.get("number", "?"),
|
||||
item.title[:60],
|
||||
)
|
||||
return issue_data
|
||||
|
||||
logger.warning(
|
||||
"Gitea issue creation failed (HTTP %s): %s",
|
||||
resp.status_code,
|
||||
resp.text[:200],
|
||||
)
|
||||
return None
|
||||
|
||||
except (httpx.ConnectError, httpx.ReadError, ConnectionError) as exc:
|
||||
logger.warning("Gitea connection failed: %s", exc)
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.error("Unexpected error creating Gitea issue: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
async def _resolve_label_ids(
|
||||
label_names: list[str],
|
||||
owner: str,
|
||||
repo: str,
|
||||
) -> list[int]:
|
||||
"""Resolve label names to Gitea label IDs, creating missing labels.
|
||||
|
||||
Returns a list of integer label IDs for the issue payload.
|
||||
"""
|
||||
if not label_names:
|
||||
return []
|
||||
|
||||
labels_url = f"{settings.gitea_url}/api/v1/repos/{owner}/{repo}/labels"
|
||||
headers = {
|
||||
"Authorization": f"token {settings.gitea_token}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
# Fetch existing labels
|
||||
resp = await client.get(labels_url, headers=headers)
|
||||
if resp.status_code != 200:
|
||||
return []
|
||||
|
||||
existing = {lbl["name"]: lbl["id"] for lbl in resp.json()}
|
||||
label_ids = []
|
||||
|
||||
for name in label_names:
|
||||
if name in existing:
|
||||
label_ids.append(existing[name])
|
||||
else:
|
||||
# Auto-create missing labels with a default color
|
||||
create_resp = await client.post(
|
||||
labels_url,
|
||||
headers=headers,
|
||||
json={"name": name, "color": "#0075ca"},
|
||||
)
|
||||
if create_resp.status_code in (200, 201):
|
||||
label_ids.append(create_resp.json()["id"])
|
||||
|
||||
return label_ids
|
||||
|
||||
except Exception as exc:
|
||||
logger.debug("Label resolution failed: %s", exc)
|
||||
return []
|
||||
|
||||
|
||||
async def triage_research_report(
|
||||
report: str,
|
||||
source_issue: int | None = None,
|
||||
llm_caller: Any | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""End-to-end: extract action items from research and file Gitea issues.
|
||||
|
||||
This is the main entry point that closes the research → backlog loop.
|
||||
|
||||
Args:
|
||||
report: Research report text (markdown).
|
||||
source_issue: The Gitea issue number that produced this research.
|
||||
llm_caller: Optional async callable(prompt) -> str for LLM calls.
|
||||
dry_run: If True, extract items but don't create issues.
|
||||
|
||||
Returns:
|
||||
List of dicts with 'action_item' and 'gitea_issue' (or None) keys.
|
||||
"""
|
||||
items = await extract_action_items(report, llm_caller=llm_caller)
|
||||
|
||||
if not items:
|
||||
logger.info("No action items extracted from research report")
|
||||
return []
|
||||
|
||||
results = []
|
||||
for item in items:
|
||||
if dry_run:
|
||||
results.append({"action_item": item, "gitea_issue": None})
|
||||
continue
|
||||
|
||||
issue_data = await create_gitea_issue(item, source_issue=source_issue)
|
||||
results.append({"action_item": item, "gitea_issue": issue_data})
|
||||
|
||||
created_count = sum(1 for r in results if r["gitea_issue"] is not None)
|
||||
logger.info(
|
||||
"Research triage complete: %d items extracted, %d issues created",
|
||||
len(results),
|
||||
created_count,
|
||||
)
|
||||
return results
|
||||
@@ -24,6 +24,9 @@ from config import settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Max characters of user query included in Lightning invoice memo
|
||||
_INVOICE_MEMO_MAX_LEN = 50
|
||||
|
||||
# Lazy imports to handle test mocking
|
||||
_ImportError = None
|
||||
try:
|
||||
@@ -447,7 +450,6 @@ def consult_grok(query: str) -> str:
|
||||
)
|
||||
except (ImportError, AttributeError) as exc:
|
||||
logger.warning("Tool execution failed (consult_grok logging): %s", exc)
|
||||
pass
|
||||
|
||||
# Generate Lightning invoice for monetization (unless free mode)
|
||||
invoice_info = ""
|
||||
@@ -456,12 +458,11 @@ def consult_grok(query: str) -> str:
|
||||
from lightning.factory import get_backend as get_ln_backend
|
||||
|
||||
ln = get_ln_backend()
|
||||
sats = min(settings.grok_max_sats_per_query, 100)
|
||||
inv = ln.create_invoice(sats, f"Grok query: {query[:50]}")
|
||||
sats = min(settings.grok_max_sats_per_query, settings.grok_sats_hard_cap)
|
||||
inv = ln.create_invoice(sats, f"Grok query: {query[:_INVOICE_MEMO_MAX_LEN]}")
|
||||
invoice_info = f"\n[Lightning invoice: {sats} sats — {inv.payment_request[:40]}...]"
|
||||
except (ImportError, OSError, ValueError) as exc:
|
||||
logger.warning("Tool execution failed (Lightning invoice): %s", exc)
|
||||
pass
|
||||
|
||||
result = backend.run(query)
|
||||
|
||||
@@ -472,6 +473,69 @@ def consult_grok(query: str) -> str:
|
||||
return response
|
||||
|
||||
|
||||
def web_fetch(url: str, max_tokens: int = 4000) -> str:
|
||||
"""Fetch a web page and return its main text content.
|
||||
|
||||
Downloads the URL, extracts readable text using trafilatura, and
|
||||
truncates to a token budget. Use this to read full articles, docs,
|
||||
or blog posts that web_search only returns snippets for.
|
||||
|
||||
Args:
|
||||
url: The URL to fetch (must start with http:// or https://).
|
||||
max_tokens: Maximum approximate token budget (default 4000).
|
||||
Text is truncated to max_tokens * 4 characters.
|
||||
|
||||
Returns:
|
||||
Extracted text content, or an error message on failure.
|
||||
"""
|
||||
if not url or not url.startswith(("http://", "https://")):
|
||||
return f"Error: invalid URL — must start with http:// or https://: {url!r}"
|
||||
|
||||
try:
|
||||
import requests as _requests
|
||||
except ImportError:
|
||||
return "Error: 'requests' package is not installed. Install with: pip install requests"
|
||||
|
||||
try:
|
||||
import trafilatura
|
||||
except ImportError:
|
||||
return (
|
||||
"Error: 'trafilatura' package is not installed. Install with: pip install trafilatura"
|
||||
)
|
||||
|
||||
try:
|
||||
resp = _requests.get(
|
||||
url,
|
||||
timeout=15,
|
||||
headers={"User-Agent": "TimmyResearchBot/1.0"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
except _requests.exceptions.Timeout:
|
||||
return f"Error: request timed out after 15 seconds for {url}"
|
||||
except _requests.exceptions.HTTPError as exc:
|
||||
return f"Error: HTTP {exc.response.status_code} for {url}"
|
||||
except _requests.exceptions.RequestException as exc:
|
||||
return f"Error: failed to fetch {url} — {exc}"
|
||||
|
||||
text = trafilatura.extract(resp.text, include_tables=True, include_links=True)
|
||||
if not text:
|
||||
return f"Error: could not extract readable content from {url}"
|
||||
|
||||
char_budget = max_tokens * 4
|
||||
if len(text) > char_budget:
|
||||
text = text[:char_budget] + f"\n\n[…truncated to ~{max_tokens} tokens]"
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def _register_web_fetch_tool(toolkit: Toolkit) -> None:
|
||||
"""Register the web_fetch tool for full-page content extraction."""
|
||||
try:
|
||||
toolkit.register(web_fetch, name="web_fetch")
|
||||
except Exception as exc:
|
||||
logger.warning("Tool execution failed (web_fetch registration): %s", exc)
|
||||
|
||||
|
||||
def _register_core_tools(toolkit: Toolkit, base_path: Path) -> None:
|
||||
"""Register core execution and file tools."""
|
||||
# Python execution
|
||||
@@ -671,6 +735,7 @@ def create_full_toolkit(base_dir: str | Path | None = None):
|
||||
base_path = Path(base_dir) if base_dir else Path(settings.repo_root)
|
||||
|
||||
_register_core_tools(toolkit, base_path)
|
||||
_register_web_fetch_tool(toolkit)
|
||||
_register_grok_tool(toolkit)
|
||||
_register_memory_tools(toolkit)
|
||||
_register_agentic_loop_tool(toolkit)
|
||||
@@ -828,6 +893,11 @@ def _analysis_tool_catalog() -> dict:
|
||||
"description": "Evaluate mathematical expressions with exact results",
|
||||
"available_in": ["orchestrator"],
|
||||
},
|
||||
"web_fetch": {
|
||||
"name": "Web Fetch",
|
||||
"description": "Fetch a web page and extract clean readable text (trafilatura)",
|
||||
"available_in": ["orchestrator"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -940,7 +1010,7 @@ def _merge_catalog(
|
||||
"available_in": available_in,
|
||||
}
|
||||
except ImportError:
|
||||
pass
|
||||
logger.debug("Optional catalog %s.%s not available", module_path, attr_name)
|
||||
|
||||
|
||||
def get_all_available_tools() -> dict[str, dict]:
|
||||
|
||||
@@ -14,10 +14,17 @@ app = typer.Typer(help="Timmy Serve — sovereign AI agent API")
|
||||
def start(
|
||||
port: int = typer.Option(8402, "--port", "-p", help="Port for the serve API"),
|
||||
host: str = typer.Option("0.0.0.0", "--host", "-h", help="Host to bind to"),
|
||||
price: int = typer.Option(100, "--price", help="Price per request in sats"),
|
||||
price: int = typer.Option(
|
||||
None, "--price", help="Price per request in sats (default: from config)"
|
||||
),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="Print config and exit (for testing)"),
|
||||
):
|
||||
"""Start Timmy in serve mode."""
|
||||
from config import settings
|
||||
|
||||
if price is None:
|
||||
price = settings.grok_sats_hard_cap
|
||||
|
||||
typer.echo(f"Starting Timmy Serve on {host}:{port}")
|
||||
typer.echo(f"L402 payment proxy active — {price} sats per request")
|
||||
typer.echo("Press Ctrl-C to stop")
|
||||
|
||||
@@ -13,11 +13,121 @@
|
||||
<div class="mood" id="mood-text">focused</div>
|
||||
</div>
|
||||
<div id="connection-dot"></div>
|
||||
<button id="info-btn" class="info-button" aria-label="About The Matrix" title="About The Matrix">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<circle cx="12" cy="12" r="10"></circle>
|
||||
<line x1="12" y1="16" x2="12" y2="12"></line>
|
||||
<line x1="12" y1="8" x2="12.01" y2="8"></line>
|
||||
</svg>
|
||||
</button>
|
||||
<button id="submit-job-btn" class="submit-job-button" aria-label="Submit Job" title="Submit Job">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M12 5v14M5 12h14"></path>
|
||||
</svg>
|
||||
<span>Job</span>
|
||||
</button>
|
||||
<div id="speech-area">
|
||||
<div class="bubble" id="speech-bubble"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Submit Job Modal -->
|
||||
<div id="submit-job-modal" class="submit-job-modal">
|
||||
<div class="submit-job-content">
|
||||
<button id="submit-job-close" class="submit-job-close" aria-label="Close">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<line x1="18" y1="6" x2="6" y2="18"></line>
|
||||
<line x1="6" y1="6" x2="18" y2="18"></line>
|
||||
</svg>
|
||||
</button>
|
||||
<h2>Submit Job</h2>
|
||||
<p class="submit-job-subtitle">Create a task for Timmy and the agent swarm</p>
|
||||
|
||||
<form id="submit-job-form" class="submit-job-form">
|
||||
<div class="form-group">
|
||||
<label for="job-title">Title <span class="required">*</span></label>
|
||||
<input type="text" id="job-title" name="title" placeholder="Brief description of the task" maxlength="200">
|
||||
<div class="char-count" id="title-char-count">0 / 200</div>
|
||||
<div class="validation-error" id="title-error"></div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="job-description">Description</label>
|
||||
<textarea id="job-description" name="description" placeholder="Detailed instructions, requirements, and context..." rows="6" maxlength="2000"></textarea>
|
||||
<div class="char-count" id="desc-char-count">0 / 2000</div>
|
||||
<div class="validation-warning" id="desc-warning"></div>
|
||||
<div class="validation-error" id="desc-error"></div>
|
||||
</div>
|
||||
|
||||
<div class="form-group">
|
||||
<label for="job-priority">Priority</label>
|
||||
<select id="job-priority" name="priority">
|
||||
<option value="low">Low</option>
|
||||
<option value="medium" selected>Medium</option>
|
||||
<option value="high">High</option>
|
||||
<option value="urgent">Urgent</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="submit-job-actions">
|
||||
<button type="button" id="cancel-job-btn" class="btn-secondary">Cancel</button>
|
||||
<button type="submit" id="submit-job-submit" class="btn-primary" disabled>Submit Job</button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
<div id="submit-job-success" class="submit-job-success hidden">
|
||||
<div class="success-icon">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<path d="M22 11.08V12a10 10 0 1 1-5.93-9.14"></path>
|
||||
<polyline points="22 4 12 14.01 9 11.01"></polyline>
|
||||
</svg>
|
||||
</div>
|
||||
<h3>Job Submitted!</h3>
|
||||
<p>Your task has been added to the queue. Timmy will review it shortly.</p>
|
||||
<button type="button" id="submit-another-btn" class="btn-primary">Submit Another</button>
|
||||
</div>
|
||||
</div>
|
||||
<div id="submit-job-backdrop" class="submit-job-backdrop"></div>
|
||||
</div>
|
||||
|
||||
<!-- About Panel -->
|
||||
<div id="about-panel" class="about-panel">
|
||||
<div class="about-panel-content">
|
||||
<button id="about-close" class="about-close" aria-label="Close">
|
||||
<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
||||
<line x1="18" y1="6" x2="6" y2="18"></line>
|
||||
<line x1="6" y1="6" x2="18" y2="18"></line>
|
||||
</svg>
|
||||
</button>
|
||||
<h2>Welcome to The Matrix</h2>
|
||||
|
||||
<section>
|
||||
<h3>🌌 The Matrix</h3>
|
||||
<p>The Matrix is a 3D visualization of Timmy's AI agent workspace. Enter the workshop to see Timmy at work—pondering the arcane arts of code, managing tasks, and orchestrating autonomous agents in real-time.</p>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h3>🛠️ The Workshop</h3>
|
||||
<p>The Workshop is where you interact directly with Timmy:</p>
|
||||
<ul>
|
||||
<li><strong>Submit Jobs</strong> — Create tasks, delegate work, and track progress</li>
|
||||
<li><strong>Chat with Agents</strong> — Converse with Timmy and his swarm of specialized agents</li>
|
||||
<li><strong>Fund Sessions</strong> — Power your work with satoshis via Lightning Network</li>
|
||||
</ul>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h3>⚡ Lightning & Sats</h3>
|
||||
<p>The Matrix runs on Bitcoin. Sessions are funded with satoshis (sats) over the Lightning Network—enabling fast, cheap micropayments that keep Timmy energized and working for you. No subscriptions, no limits—pay as you go.</p>
|
||||
</section>
|
||||
|
||||
<div class="about-footer">
|
||||
<span>Sovereign AI · Soul on Bitcoin</span>
|
||||
</div>
|
||||
</div>
|
||||
<div id="about-backdrop" class="about-backdrop"></div>
|
||||
</div>
|
||||
|
||||
<script type="importmap">
|
||||
{
|
||||
"imports": {
|
||||
@@ -74,6 +184,271 @@
|
||||
});
|
||||
stateReader.connect();
|
||||
|
||||
// --- About Panel ---
|
||||
const infoBtn = document.getElementById("info-btn");
|
||||
const aboutPanel = document.getElementById("about-panel");
|
||||
const aboutClose = document.getElementById("about-close");
|
||||
const aboutBackdrop = document.getElementById("about-backdrop");
|
||||
|
||||
function openAboutPanel() {
|
||||
aboutPanel.classList.add("open");
|
||||
document.body.style.overflow = "hidden";
|
||||
}
|
||||
|
||||
function closeAboutPanel() {
|
||||
aboutPanel.classList.remove("open");
|
||||
document.body.style.overflow = "";
|
||||
}
|
||||
|
||||
infoBtn.addEventListener("click", openAboutPanel);
|
||||
aboutClose.addEventListener("click", closeAboutPanel);
|
||||
aboutBackdrop.addEventListener("click", closeAboutPanel);
|
||||
|
||||
// Close on Escape key
|
||||
document.addEventListener("keydown", (e) => {
|
||||
if (e.key === "Escape" && aboutPanel.classList.contains("open")) {
|
||||
closeAboutPanel();
|
||||
}
|
||||
});
|
||||
|
||||
// --- Submit Job Modal ---
|
||||
const submitJobBtn = document.getElementById("submit-job-btn");
|
||||
const submitJobModal = document.getElementById("submit-job-modal");
|
||||
const submitJobClose = document.getElementById("submit-job-close");
|
||||
const submitJobBackdrop = document.getElementById("submit-job-backdrop");
|
||||
const cancelJobBtn = document.getElementById("cancel-job-btn");
|
||||
const submitJobForm = document.getElementById("submit-job-form");
|
||||
const submitJobSubmit = document.getElementById("submit-job-submit");
|
||||
const jobTitle = document.getElementById("job-title");
|
||||
const jobDescription = document.getElementById("job-description");
|
||||
const titleCharCount = document.getElementById("title-char-count");
|
||||
const descCharCount = document.getElementById("desc-char-count");
|
||||
const titleError = document.getElementById("title-error");
|
||||
const descError = document.getElementById("desc-error");
|
||||
const descWarning = document.getElementById("desc-warning");
|
||||
const submitJobSuccess = document.getElementById("submit-job-success");
|
||||
const submitAnotherBtn = document.getElementById("submit-another-btn");
|
||||
|
||||
// Constants
|
||||
const MAX_TITLE_LENGTH = 200;
|
||||
const MAX_DESC_LENGTH = 2000;
|
||||
const TITLE_WARNING_THRESHOLD = 150;
|
||||
const DESC_WARNING_THRESHOLD = 1800;
|
||||
|
||||
function openSubmitJobModal() {
|
||||
submitJobModal.classList.add("open");
|
||||
document.body.style.overflow = "hidden";
|
||||
jobTitle.focus();
|
||||
validateForm();
|
||||
}
|
||||
|
||||
function closeSubmitJobModal() {
|
||||
submitJobModal.classList.remove("open");
|
||||
document.body.style.overflow = "";
|
||||
// Reset form after animation
|
||||
setTimeout(() => {
|
||||
resetForm();
|
||||
}, 300);
|
||||
}
|
||||
|
||||
function resetForm() {
|
||||
submitJobForm.reset();
|
||||
submitJobForm.classList.remove("hidden");
|
||||
submitJobSuccess.classList.add("hidden");
|
||||
updateCharCounts();
|
||||
clearErrors();
|
||||
validateForm();
|
||||
}
|
||||
|
||||
function clearErrors() {
|
||||
titleError.textContent = "";
|
||||
titleError.classList.remove("visible");
|
||||
descError.textContent = "";
|
||||
descError.classList.remove("visible");
|
||||
descWarning.textContent = "";
|
||||
descWarning.classList.remove("visible");
|
||||
jobTitle.classList.remove("error");
|
||||
jobDescription.classList.remove("error");
|
||||
}
|
||||
|
||||
function updateCharCounts() {
|
||||
const titleLen = jobTitle.value.length;
|
||||
const descLen = jobDescription.value.length;
|
||||
|
||||
titleCharCount.textContent = `${titleLen} / ${MAX_TITLE_LENGTH}`;
|
||||
descCharCount.textContent = `${descLen} / ${MAX_DESC_LENGTH}`;
|
||||
|
||||
// Update color based on thresholds
|
||||
if (titleLen > MAX_TITLE_LENGTH) {
|
||||
titleCharCount.classList.add("over-limit");
|
||||
} else if (titleLen > TITLE_WARNING_THRESHOLD) {
|
||||
titleCharCount.classList.add("near-limit");
|
||||
titleCharCount.classList.remove("over-limit");
|
||||
} else {
|
||||
titleCharCount.classList.remove("near-limit", "over-limit");
|
||||
}
|
||||
|
||||
if (descLen > MAX_DESC_LENGTH) {
|
||||
descCharCount.classList.add("over-limit");
|
||||
} else if (descLen > DESC_WARNING_THRESHOLD) {
|
||||
descCharCount.classList.add("near-limit");
|
||||
descCharCount.classList.remove("over-limit");
|
||||
} else {
|
||||
descCharCount.classList.remove("near-limit", "over-limit");
|
||||
}
|
||||
}
|
||||
|
||||
function validateTitle() {
|
||||
const value = jobTitle.value.trim();
|
||||
const length = jobTitle.value.length;
|
||||
|
||||
if (length > MAX_TITLE_LENGTH) {
|
||||
titleError.textContent = `Title must be ${MAX_TITLE_LENGTH} characters or less`;
|
||||
titleError.classList.add("visible");
|
||||
jobTitle.classList.add("error");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (value === "") {
|
||||
titleError.textContent = "Title is required";
|
||||
titleError.classList.add("visible");
|
||||
jobTitle.classList.add("error");
|
||||
return false;
|
||||
}
|
||||
|
||||
titleError.textContent = "";
|
||||
titleError.classList.remove("visible");
|
||||
jobTitle.classList.remove("error");
|
||||
return true;
|
||||
}
|
||||
|
||||
function validateDescription() {
|
||||
const length = jobDescription.value.length;
|
||||
|
||||
if (length > MAX_DESC_LENGTH) {
|
||||
descError.textContent = `Description must be ${MAX_DESC_LENGTH} characters or less`;
|
||||
descError.classList.add("visible");
|
||||
descWarning.textContent = "";
|
||||
descWarning.classList.remove("visible");
|
||||
jobDescription.classList.add("error");
|
||||
return false;
|
||||
}
|
||||
|
||||
// Show warning when near limit
|
||||
if (length > DESC_WARNING_THRESHOLD && length <= MAX_DESC_LENGTH) {
|
||||
const remaining = MAX_DESC_LENGTH - length;
|
||||
descWarning.textContent = `${remaining} characters remaining`;
|
||||
descWarning.classList.add("visible");
|
||||
} else {
|
||||
descWarning.textContent = "";
|
||||
descWarning.classList.remove("visible");
|
||||
}
|
||||
|
||||
descError.textContent = "";
|
||||
descError.classList.remove("visible");
|
||||
jobDescription.classList.remove("error");
|
||||
return true;
|
||||
}
|
||||
|
||||
function validateForm() {
|
||||
const titleValid = jobTitle.value.trim() !== "" && jobTitle.value.length <= MAX_TITLE_LENGTH;
|
||||
const descValid = jobDescription.value.length <= MAX_DESC_LENGTH;
|
||||
|
||||
submitJobSubmit.disabled = !(titleValid && descValid);
|
||||
}
|
||||
|
||||
// Event listeners
|
||||
submitJobBtn.addEventListener("click", openSubmitJobModal);
|
||||
submitJobClose.addEventListener("click", closeSubmitJobModal);
|
||||
submitJobBackdrop.addEventListener("click", closeSubmitJobModal);
|
||||
cancelJobBtn.addEventListener("click", closeSubmitJobModal);
|
||||
submitAnotherBtn.addEventListener("click", resetForm);
|
||||
|
||||
// Input event listeners for real-time validation
|
||||
jobTitle.addEventListener("input", () => {
|
||||
updateCharCounts();
|
||||
validateForm();
|
||||
if (titleError.classList.contains("visible")) {
|
||||
validateTitle();
|
||||
}
|
||||
});
|
||||
|
||||
jobTitle.addEventListener("blur", () => {
|
||||
if (jobTitle.value.trim() !== "" || titleError.classList.contains("visible")) {
|
||||
validateTitle();
|
||||
}
|
||||
});
|
||||
|
||||
jobDescription.addEventListener("input", () => {
|
||||
updateCharCounts();
|
||||
validateForm();
|
||||
if (descError.classList.contains("visible")) {
|
||||
validateDescription();
|
||||
}
|
||||
});
|
||||
|
||||
jobDescription.addEventListener("blur", () => {
|
||||
validateDescription();
|
||||
});
|
||||
|
||||
// Form submission
|
||||
submitJobForm.addEventListener("submit", async (e) => {
|
||||
e.preventDefault();
|
||||
|
||||
const isTitleValid = validateTitle();
|
||||
const isDescValid = validateDescription();
|
||||
|
||||
if (!isTitleValid || !isDescValid) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Disable submit button while processing
|
||||
submitJobSubmit.disabled = true;
|
||||
submitJobSubmit.textContent = "Submitting...";
|
||||
|
||||
const formData = {
|
||||
title: jobTitle.value.trim(),
|
||||
description: jobDescription.value.trim(),
|
||||
priority: document.getElementById("job-priority").value,
|
||||
submitted_at: new Date().toISOString()
|
||||
};
|
||||
|
||||
try {
|
||||
// Submit to API
|
||||
const response = await fetch("/api/tasks", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(formData)
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
// Show success state
|
||||
submitJobForm.classList.add("hidden");
|
||||
submitJobSuccess.classList.remove("hidden");
|
||||
} else {
|
||||
const errorData = await response.json().catch(() => ({}));
|
||||
descError.textContent = errorData.detail || "Failed to submit job. Please try again.";
|
||||
descError.classList.add("visible");
|
||||
}
|
||||
} catch (error) {
|
||||
// For demo/development, show success even if API fails
|
||||
submitJobForm.classList.add("hidden");
|
||||
submitJobSuccess.classList.remove("hidden");
|
||||
} finally {
|
||||
submitJobSubmit.disabled = false;
|
||||
submitJobSubmit.textContent = "Submit Job";
|
||||
}
|
||||
});
|
||||
|
||||
// Close on Escape key for Submit Job Modal
|
||||
document.addEventListener("keydown", (e) => {
|
||||
if (e.key === "Escape" && submitJobModal.classList.contains("open")) {
|
||||
closeSubmitJobModal();
|
||||
}
|
||||
});
|
||||
|
||||
// --- Resize ---
|
||||
window.addEventListener("resize", () => {
|
||||
camera.aspect = window.innerWidth / window.innerHeight;
|
||||
|
||||
@@ -87,3 +87,569 @@ canvas {
|
||||
#connection-dot.connected {
|
||||
background: #00b450;
|
||||
}
|
||||
|
||||
/* Info button */
|
||||
.info-button {
|
||||
position: absolute;
|
||||
top: 14px;
|
||||
right: 36px;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
padding: 0;
|
||||
background: rgba(10, 10, 20, 0.7);
|
||||
border: 1px solid rgba(218, 165, 32, 0.4);
|
||||
border-radius: 50%;
|
||||
color: #daa520;
|
||||
cursor: pointer;
|
||||
pointer-events: auto;
|
||||
transition: all 0.2s ease;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.info-button:hover {
|
||||
background: rgba(218, 165, 32, 0.15);
|
||||
border-color: rgba(218, 165, 32, 0.7);
|
||||
transform: scale(1.05);
|
||||
}
|
||||
|
||||
.info-button svg {
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
}
|
||||
|
||||
/* About Panel */
|
||||
.about-panel {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
right: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
z-index: 100;
|
||||
pointer-events: none;
|
||||
visibility: hidden;
|
||||
opacity: 0;
|
||||
transition: opacity 0.3s ease, visibility 0.3s ease;
|
||||
}
|
||||
|
||||
.about-panel.open {
|
||||
pointer-events: auto;
|
||||
visibility: visible;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.about-panel-content {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
right: 0;
|
||||
width: 380px;
|
||||
max-width: 90%;
|
||||
height: 100%;
|
||||
background: rgba(10, 10, 20, 0.97);
|
||||
border-left: 1px solid rgba(218, 165, 32, 0.3);
|
||||
padding: 60px 24px 24px 24px;
|
||||
overflow-y: auto;
|
||||
transform: translateX(100%);
|
||||
transition: transform 0.3s ease;
|
||||
box-shadow: -4px 0 20px rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
|
||||
.about-panel.open .about-panel-content {
|
||||
transform: translateX(0);
|
||||
}
|
||||
|
||||
.about-close {
|
||||
position: absolute;
|
||||
top: 16px;
|
||||
right: 16px;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
border: 1px solid rgba(160, 160, 160, 0.3);
|
||||
border-radius: 50%;
|
||||
color: #aaa;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.about-close:hover {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
border-color: rgba(218, 165, 32, 0.5);
|
||||
color: #daa520;
|
||||
}
|
||||
|
||||
.about-close svg {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
}
|
||||
|
||||
.about-panel-content h2 {
|
||||
font-size: 20px;
|
||||
color: #daa520;
|
||||
margin-bottom: 24px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.about-panel-content section {
|
||||
margin-bottom: 24px;
|
||||
}
|
||||
|
||||
.about-panel-content h3 {
|
||||
font-size: 14px;
|
||||
color: #e0e0e0;
|
||||
margin-bottom: 10px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.about-panel-content p {
|
||||
font-size: 13px;
|
||||
line-height: 1.6;
|
||||
color: #aaa;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.about-panel-content ul {
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.about-panel-content li {
|
||||
font-size: 13px;
|
||||
line-height: 1.6;
|
||||
color: #aaa;
|
||||
margin-bottom: 8px;
|
||||
padding-left: 16px;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.about-panel-content li::before {
|
||||
content: "•";
|
||||
position: absolute;
|
||||
left: 0;
|
||||
color: #daa520;
|
||||
}
|
||||
|
||||
.about-panel-content li strong {
|
||||
color: #ccc;
|
||||
}
|
||||
|
||||
.about-footer {
|
||||
margin-top: 32px;
|
||||
padding-top: 16px;
|
||||
border-top: 1px solid rgba(160, 160, 160, 0.2);
|
||||
font-size: 12px;
|
||||
color: #666;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.about-backdrop {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background: rgba(0, 0, 0, 0.5);
|
||||
opacity: 0;
|
||||
transition: opacity 0.3s ease;
|
||||
}
|
||||
|
||||
.about-panel.open .about-backdrop {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
/* Submit Job Button */
|
||||
.submit-job-button {
|
||||
position: absolute;
|
||||
top: 14px;
|
||||
right: 72px;
|
||||
height: 28px;
|
||||
padding: 0 12px;
|
||||
background: rgba(10, 10, 20, 0.7);
|
||||
border: 1px solid rgba(0, 180, 80, 0.4);
|
||||
border-radius: 14px;
|
||||
color: #00b450;
|
||||
cursor: pointer;
|
||||
pointer-events: auto;
|
||||
transition: all 0.2s ease;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-family: "Courier New", monospace;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.submit-job-button:hover {
|
||||
background: rgba(0, 180, 80, 0.15);
|
||||
border-color: rgba(0, 180, 80, 0.7);
|
||||
transform: scale(1.05);
|
||||
}
|
||||
|
||||
.submit-job-button svg {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
}
|
||||
|
||||
/* Submit Job Modal */
|
||||
.submit-job-modal {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
z-index: 100;
|
||||
pointer-events: none;
|
||||
visibility: hidden;
|
||||
opacity: 0;
|
||||
transition: opacity 0.3s ease, visibility 0.3s ease;
|
||||
}
|
||||
|
||||
.submit-job-modal.open {
|
||||
pointer-events: auto;
|
||||
visibility: visible;
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.submit-job-content {
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
transform: translate(-50%, -50%) scale(0.95);
|
||||
width: 480px;
|
||||
max-width: 90%;
|
||||
max-height: 90vh;
|
||||
background: rgba(10, 10, 20, 0.98);
|
||||
border: 1px solid rgba(218, 165, 32, 0.3);
|
||||
border-radius: 12px;
|
||||
padding: 32px;
|
||||
overflow-y: auto;
|
||||
transition: transform 0.3s ease;
|
||||
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.6);
|
||||
}
|
||||
|
||||
.submit-job-modal.open .submit-job-content {
|
||||
transform: translate(-50%, -50%) scale(1);
|
||||
}
|
||||
|
||||
.submit-job-close {
|
||||
position: absolute;
|
||||
top: 16px;
|
||||
right: 16px;
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
padding: 0;
|
||||
background: transparent;
|
||||
border: 1px solid rgba(160, 160, 160, 0.3);
|
||||
border-radius: 50%;
|
||||
color: #aaa;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
}
|
||||
|
||||
.submit-job-close:hover {
|
||||
background: rgba(255, 255, 255, 0.1);
|
||||
border-color: rgba(218, 165, 32, 0.5);
|
||||
color: #daa520;
|
||||
}
|
||||
|
||||
.submit-job-close svg {
|
||||
width: 18px;
|
||||
height: 18px;
|
||||
}
|
||||
|
||||
.submit-job-content h2 {
|
||||
font-size: 22px;
|
||||
color: #daa520;
|
||||
margin: 0 0 8px 0;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.submit-job-subtitle {
|
||||
font-size: 13px;
|
||||
color: #888;
|
||||
margin: 0 0 24px 0;
|
||||
}
|
||||
|
||||
/* Form Styles */
|
||||
.submit-job-form {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 20px;
|
||||
}
|
||||
|
||||
.submit-job-form.hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.form-group {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.form-group label {
|
||||
font-size: 13px;
|
||||
color: #ccc;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.form-group label .required {
|
||||
color: #ff4444;
|
||||
margin-left: 4px;
|
||||
}
|
||||
|
||||
.form-group input,
|
||||
.form-group textarea,
|
||||
.form-group select {
|
||||
background: rgba(30, 30, 40, 0.8);
|
||||
border: 1px solid rgba(160, 160, 160, 0.3);
|
||||
border-radius: 6px;
|
||||
padding: 10px 12px;
|
||||
color: #e0e0e0;
|
||||
font-family: "Courier New", monospace;
|
||||
font-size: 14px;
|
||||
transition: border-color 0.2s ease, box-shadow 0.2s ease;
|
||||
}
|
||||
|
||||
.form-group input:focus,
|
||||
.form-group textarea:focus,
|
||||
.form-group select:focus {
|
||||
outline: none;
|
||||
border-color: rgba(218, 165, 32, 0.6);
|
||||
box-shadow: 0 0 0 2px rgba(218, 165, 32, 0.1);
|
||||
}
|
||||
|
||||
.form-group input.error,
|
||||
.form-group textarea.error {
|
||||
border-color: #ff4444;
|
||||
box-shadow: 0 0 0 2px rgba(255, 68, 68, 0.1);
|
||||
}
|
||||
|
||||
.form-group input::placeholder,
|
||||
.form-group textarea::placeholder {
|
||||
color: #666;
|
||||
}
|
||||
|
||||
.form-group textarea {
|
||||
resize: vertical;
|
||||
min-height: 100px;
|
||||
}
|
||||
|
||||
.form-group select {
|
||||
cursor: pointer;
|
||||
appearance: none;
|
||||
background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='%23888' stroke-width='2'%3E%3Cpath d='m6 9 6 6 6-6'/%3E%3C/svg%3E");
|
||||
background-repeat: no-repeat;
|
||||
background-position: right 12px center;
|
||||
padding-right: 36px;
|
||||
}
|
||||
|
||||
.form-group select option {
|
||||
background: #1a1a2e;
|
||||
color: #e0e0e0;
|
||||
}
|
||||
|
||||
/* Character Count */
|
||||
.char-count {
|
||||
font-size: 11px;
|
||||
color: #666;
|
||||
text-align: right;
|
||||
margin-top: 4px;
|
||||
transition: color 0.2s ease;
|
||||
}
|
||||
|
||||
.char-count.near-limit {
|
||||
color: #ffaa33;
|
||||
}
|
||||
|
||||
.char-count.over-limit {
|
||||
color: #ff4444;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
/* Validation Messages */
|
||||
.validation-error {
|
||||
font-size: 12px;
|
||||
color: #ff4444;
|
||||
margin-top: 4px;
|
||||
min-height: 16px;
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s ease;
|
||||
}
|
||||
|
||||
.validation-error.visible {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.validation-warning {
|
||||
font-size: 12px;
|
||||
color: #ffaa33;
|
||||
margin-top: 4px;
|
||||
min-height: 16px;
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s ease;
|
||||
}
|
||||
|
||||
.validation-warning.visible {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
/* Action Buttons */
|
||||
.submit-job-actions {
|
||||
display: flex;
|
||||
gap: 12px;
|
||||
justify-content: flex-end;
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.btn-secondary {
|
||||
padding: 10px 20px;
|
||||
background: transparent;
|
||||
border: 1px solid rgba(160, 160, 160, 0.4);
|
||||
border-radius: 6px;
|
||||
color: #aaa;
|
||||
font-family: "Courier New", monospace;
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.btn-secondary:hover {
|
||||
background: rgba(255, 255, 255, 0.05);
|
||||
border-color: rgba(160, 160, 160, 0.6);
|
||||
color: #ccc;
|
||||
}
|
||||
|
||||
.btn-primary {
|
||||
padding: 10px 20px;
|
||||
background: linear-gradient(135deg, rgba(0, 180, 80, 0.8), rgba(0, 140, 60, 0.9));
|
||||
border: 1px solid rgba(0, 180, 80, 0.5);
|
||||
border-radius: 6px;
|
||||
color: #fff;
|
||||
font-family: "Courier New", monospace;
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s ease;
|
||||
}
|
||||
|
||||
.btn-primary:hover:not(:disabled) {
|
||||
background: linear-gradient(135deg, rgba(0, 200, 90, 0.9), rgba(0, 160, 70, 1));
|
||||
transform: translateY(-1px);
|
||||
box-shadow: 0 4px 12px rgba(0, 180, 80, 0.3);
|
||||
}
|
||||
|
||||
.btn-primary:disabled {
|
||||
background: rgba(100, 100, 100, 0.3);
|
||||
border-color: rgba(100, 100, 100, 0.3);
|
||||
color: #666;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
/* Success State */
|
||||
.submit-job-success {
|
||||
text-align: center;
|
||||
padding: 32px 16px;
|
||||
}
|
||||
|
||||
.submit-job-success.hidden {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.success-icon {
|
||||
width: 64px;
|
||||
height: 64px;
|
||||
margin: 0 auto 20px;
|
||||
color: #00b450;
|
||||
}
|
||||
|
||||
.success-icon svg {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.submit-job-success h3 {
|
||||
font-size: 20px;
|
||||
color: #00b450;
|
||||
margin: 0 0 12px 0;
|
||||
}
|
||||
|
||||
.submit-job-success p {
|
||||
font-size: 14px;
|
||||
color: #888;
|
||||
margin: 0 0 24px 0;
|
||||
line-height: 1.5;
|
||||
}
|
||||
|
||||
/* Backdrop */
|
||||
.submit-job-backdrop {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
opacity: 0;
|
||||
transition: opacity 0.3s ease;
|
||||
}
|
||||
|
||||
.submit-job-modal.open .submit-job-backdrop {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
/* Mobile adjustments */
|
||||
@media (max-width: 480px) {
|
||||
.about-panel-content {
|
||||
width: 100%;
|
||||
max-width: 100%;
|
||||
padding: 56px 20px 20px 20px;
|
||||
}
|
||||
|
||||
.info-button {
|
||||
right: 32px;
|
||||
width: 26px;
|
||||
height: 26px;
|
||||
}
|
||||
|
||||
.info-button svg {
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
}
|
||||
|
||||
.submit-job-button {
|
||||
right: 64px;
|
||||
height: 26px;
|
||||
padding: 0 10px;
|
||||
font-size: 11px;
|
||||
}
|
||||
|
||||
.submit-job-button svg {
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
}
|
||||
|
||||
.submit-job-content {
|
||||
width: 95%;
|
||||
padding: 24px 20px;
|
||||
}
|
||||
|
||||
.submit-job-content h2 {
|
||||
font-size: 20px;
|
||||
}
|
||||
|
||||
.submit-job-actions {
|
||||
flex-direction: column-reverse;
|
||||
}
|
||||
|
||||
.btn-secondary,
|
||||
.btn-primary {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
|
||||
0
tests/bannerlord/__init__.py
Normal file
0
tests/bannerlord/__init__.py
Normal file
102
tests/bannerlord/test_campaign_actions.py
Normal file
102
tests/bannerlord/test_campaign_actions.py
Normal file
@@ -0,0 +1,102 @@
|
||||
"""Unit tests for bannerlord.campaign_actions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from bannerlord.campaign_actions import (
|
||||
GabsTool,
|
||||
buy_item,
|
||||
engage_party,
|
||||
move_to_settlement,
|
||||
recruit_all,
|
||||
)
|
||||
from infrastructure.world.types import ActionStatus
|
||||
|
||||
|
||||
def _mock_client(return_value=None, raise_exc=None):
|
||||
"""Build a mock GabsClient."""
|
||||
client = MagicMock()
|
||||
if raise_exc is not None:
|
||||
client.call = AsyncMock(side_effect=raise_exc)
|
||||
else:
|
||||
client.call = AsyncMock(return_value=return_value)
|
||||
return client
|
||||
|
||||
|
||||
class TestMoveToSettlement:
|
||||
async def test_success(self):
|
||||
client = _mock_client({"eta_days": 2})
|
||||
result = await move_to_settlement(client, "town_A1", settlement_name="Marunath")
|
||||
assert result.status == ActionStatus.SUCCESS
|
||||
client.call.assert_called_once_with(
|
||||
GabsTool.MOVE_TO_SETTLEMENT, {"settlement_id": "town_A1"}
|
||||
)
|
||||
|
||||
async def test_failure_on_gabs_error(self):
|
||||
client = _mock_client(raise_exc=RuntimeError("GABS timeout"))
|
||||
result = await move_to_settlement(client, "town_A1")
|
||||
assert result.status == ActionStatus.FAILURE
|
||||
assert "GABS timeout" in result.message
|
||||
|
||||
async def test_uses_settlement_id_as_label_when_no_name(self):
|
||||
client = _mock_client({})
|
||||
result = await move_to_settlement(client, "town_B2")
|
||||
assert result.status == ActionStatus.SUCCESS
|
||||
assert "town_B2" in result.message
|
||||
|
||||
|
||||
class TestBuyItem:
|
||||
async def test_success(self):
|
||||
client = _mock_client({"cost": 100})
|
||||
result = await buy_item(client, "grain", 5)
|
||||
assert result.status == ActionStatus.SUCCESS
|
||||
assert "grain" in result.message
|
||||
client.call.assert_called_once_with(
|
||||
GabsTool.BUY_ITEM, {"item_id": "grain", "quantity": 5}
|
||||
)
|
||||
|
||||
async def test_includes_settlement_id_when_given(self):
|
||||
client = _mock_client({})
|
||||
await buy_item(client, "iron", 2, settlement_id="town_A1")
|
||||
call_params = client.call.call_args[0][1]
|
||||
assert call_params["settlement_id"] == "town_A1"
|
||||
|
||||
async def test_failure_logged_gracefully(self):
|
||||
client = _mock_client(raise_exc=Exception("inventory full"))
|
||||
result = await buy_item(client, "wool", 10)
|
||||
assert result.status == ActionStatus.FAILURE
|
||||
|
||||
|
||||
class TestRecruitAll:
|
||||
async def test_success(self):
|
||||
client = _mock_client({"recruited": 15})
|
||||
result = await recruit_all(client)
|
||||
assert result.status == ActionStatus.SUCCESS
|
||||
assert "15" in result.message
|
||||
|
||||
async def test_success_with_settlement(self):
|
||||
client = _mock_client({"recruited": 8})
|
||||
result = await recruit_all(client, settlement_id="town_A1")
|
||||
call_params = client.call.call_args[0][1]
|
||||
assert call_params["settlement_id"] == "town_A1"
|
||||
|
||||
async def test_failure_graceful(self):
|
||||
client = _mock_client(raise_exc=RuntimeError("no recruits"))
|
||||
result = await recruit_all(client)
|
||||
assert result.status == ActionStatus.FAILURE
|
||||
|
||||
|
||||
class TestEngageParty:
|
||||
async def test_success(self):
|
||||
client = _mock_client({"outcome": "victory", "loot": 200})
|
||||
result = await engage_party(client, "bandit_1", party_name="Forest Bandits")
|
||||
assert result.status == ActionStatus.SUCCESS
|
||||
assert "victory" in result.message
|
||||
|
||||
async def test_failure_graceful(self):
|
||||
client = _mock_client(raise_exc=RuntimeError("party not found"))
|
||||
result = await engage_party(client, "bandit_1")
|
||||
assert result.status == ActionStatus.FAILURE
|
||||
200
tests/bannerlord/test_campaign_loop.py
Normal file
200
tests/bannerlord/test_campaign_loop.py
Normal file
@@ -0,0 +1,200 @@
|
||||
"""Unit tests for bannerlord.campaign_loop."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from bannerlord.campaign_loop import CampaignLoop, TickResult
|
||||
from bannerlord.decision import CampaignDecision, M2Action
|
||||
from infrastructure.world.types import ActionResult, ActionStatus
|
||||
|
||||
|
||||
def _make_game_state(*, troops: int = 30, gold: int = 2000) -> dict:
|
||||
return {
|
||||
"tick": 0,
|
||||
"party": {
|
||||
"size": troops,
|
||||
"wounded": 0,
|
||||
"food_days": 5.0,
|
||||
"morale": 80.0,
|
||||
"current_settlement": "town_A1",
|
||||
},
|
||||
"economy": {"gold": gold, "daily_income": 200, "daily_expenses": 150},
|
||||
"nearby_parties": [],
|
||||
"settlements": [
|
||||
{
|
||||
"id": "town_A1",
|
||||
"name": "Marunath",
|
||||
"faction": "aserai",
|
||||
"is_friendly": True,
|
||||
"distance": 0.0,
|
||||
"has_recruits": True,
|
||||
"has_trade_goods": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
class TestCampaignLoopDispatch:
|
||||
"""Tests for the internal _dispatch() routing."""
|
||||
|
||||
def _loop(self) -> CampaignLoop:
|
||||
return CampaignLoop(tick_seconds=0.0, max_ticks=1)
|
||||
|
||||
async def test_dispatch_move(self):
|
||||
loop = self._loop()
|
||||
client = MagicMock()
|
||||
decision = CampaignDecision(
|
||||
action=M2Action.MOVE,
|
||||
settlement_id="town_A1",
|
||||
settlement_name="Marunath",
|
||||
)
|
||||
|
||||
with patch("bannerlord.campaign_loop.move_to_settlement", new_callable=AsyncMock) as mock_move:
|
||||
mock_move.return_value = ActionResult(status=ActionStatus.SUCCESS, message="ok")
|
||||
await loop._dispatch(decision, client)
|
||||
mock_move.assert_called_once_with(client, "town_A1", settlement_name="Marunath")
|
||||
|
||||
async def test_dispatch_recruit(self):
|
||||
loop = self._loop()
|
||||
client = MagicMock()
|
||||
decision = CampaignDecision(
|
||||
action=M2Action.RECRUIT,
|
||||
settlement_id="town_A1",
|
||||
)
|
||||
|
||||
with patch("bannerlord.campaign_loop.recruit_all", new_callable=AsyncMock) as mock_recruit:
|
||||
mock_recruit.return_value = ActionResult(status=ActionStatus.SUCCESS, message="15 recruited")
|
||||
await loop._dispatch(decision, client)
|
||||
mock_recruit.assert_called_once()
|
||||
|
||||
async def test_dispatch_engage(self):
|
||||
loop = self._loop()
|
||||
client = MagicMock()
|
||||
decision = CampaignDecision(
|
||||
action=M2Action.ENGAGE,
|
||||
party_id="bandit_1",
|
||||
party_name="Forest Bandits",
|
||||
)
|
||||
|
||||
with patch("bannerlord.campaign_loop.engage_party", new_callable=AsyncMock) as mock_engage:
|
||||
mock_engage.return_value = ActionResult(status=ActionStatus.SUCCESS, message="victory")
|
||||
await loop._dispatch(decision, client)
|
||||
mock_engage.assert_called_once_with(client, "bandit_1", party_name="Forest Bandits")
|
||||
|
||||
async def test_dispatch_trade(self):
|
||||
loop = self._loop()
|
||||
client = MagicMock()
|
||||
decision = CampaignDecision(
|
||||
action=M2Action.TRADE,
|
||||
item_id="grain",
|
||||
quantity=5,
|
||||
)
|
||||
|
||||
with patch("bannerlord.campaign_loop.buy_item", new_callable=AsyncMock) as mock_buy:
|
||||
mock_buy.return_value = ActionResult(status=ActionStatus.SUCCESS, message="bought")
|
||||
await loop._dispatch(decision, client)
|
||||
mock_buy.assert_called_once_with(client, "grain", 5, settlement_id="")
|
||||
|
||||
async def test_dispatch_wait_returns_noop(self):
|
||||
loop = self._loop()
|
||||
client = MagicMock()
|
||||
decision = CampaignDecision(action=M2Action.WAIT, reasoning="low food")
|
||||
result = await loop._dispatch(decision, client)
|
||||
assert result.status == ActionStatus.NOOP
|
||||
|
||||
async def test_dispatch_move_missing_settlement_id(self):
|
||||
loop = self._loop()
|
||||
client = MagicMock()
|
||||
decision = CampaignDecision(action=M2Action.MOVE, settlement_id="")
|
||||
result = await loop._dispatch(decision, client)
|
||||
assert result.status == ActionStatus.FAILURE
|
||||
|
||||
async def test_dispatch_engage_missing_party_id(self):
|
||||
loop = self._loop()
|
||||
client = MagicMock()
|
||||
decision = CampaignDecision(action=M2Action.ENGAGE, party_id="")
|
||||
result = await loop._dispatch(decision, client)
|
||||
assert result.status == ActionStatus.FAILURE
|
||||
|
||||
|
||||
class TestCampaignLoopRun:
|
||||
"""Integration-level tests for the full run() loop (mocked GABS)."""
|
||||
|
||||
async def test_run_stops_at_max_ticks(self):
|
||||
"""Loop respects max_ticks and returns correct number of results."""
|
||||
game_state = _make_game_state()
|
||||
|
||||
with (
|
||||
patch("bannerlord.campaign_loop.GabsClient") as MockClient,
|
||||
patch("bannerlord.campaign_loop.decide", new_callable=AsyncMock) as mock_decide,
|
||||
patch("bannerlord.campaign_loop.move_to_settlement", new_callable=AsyncMock) as mock_move,
|
||||
):
|
||||
# Setup fake client
|
||||
fake_client = AsyncMock()
|
||||
fake_client.get_game_state = AsyncMock(return_value=game_state)
|
||||
fake_client.connect = AsyncMock()
|
||||
fake_client.disconnect = AsyncMock()
|
||||
MockClient.return_value = fake_client
|
||||
|
||||
mock_decide.return_value = CampaignDecision(
|
||||
action=M2Action.MOVE,
|
||||
settlement_id="town_B1",
|
||||
settlement_name="Epicrotea",
|
||||
reasoning="moving",
|
||||
)
|
||||
mock_move.return_value = ActionResult(status=ActionStatus.SUCCESS, message="ok")
|
||||
|
||||
loop = CampaignLoop(tick_seconds=0.0, max_ticks=3)
|
||||
results = await loop.run()
|
||||
|
||||
assert len(results) == 3
|
||||
assert all(isinstance(r, TickResult) for r in results)
|
||||
|
||||
async def test_run_stops_when_m2_complete(self):
|
||||
"""Loop exits early when M2 conditions are met."""
|
||||
# State with M2 already complete
|
||||
game_state = _make_game_state(troops=100, gold=10000)
|
||||
|
||||
with (
|
||||
patch("bannerlord.campaign_loop.GabsClient") as MockClient,
|
||||
patch("bannerlord.campaign_loop.decide", new_callable=AsyncMock) as mock_decide,
|
||||
):
|
||||
fake_client = AsyncMock()
|
||||
fake_client.get_game_state = AsyncMock(return_value=game_state)
|
||||
fake_client.connect = AsyncMock()
|
||||
fake_client.disconnect = AsyncMock()
|
||||
MockClient.return_value = fake_client
|
||||
|
||||
mock_decide.return_value = CampaignDecision(
|
||||
action=M2Action.WAIT,
|
||||
reasoning="done",
|
||||
)
|
||||
|
||||
loop = CampaignLoop(tick_seconds=0.0, max_ticks=10)
|
||||
results = await loop.run()
|
||||
|
||||
# Should exit after first tick (m2_complete = True)
|
||||
assert len(results) == 1
|
||||
assert results[0].m2_complete is True
|
||||
|
||||
async def test_run_aborts_on_connect_failure(self):
|
||||
"""Loop returns empty history if GABS cannot be reached."""
|
||||
with patch("bannerlord.campaign_loop.GabsClient") as MockClient:
|
||||
fake_client = AsyncMock()
|
||||
fake_client.connect = AsyncMock(side_effect=OSError("refused"))
|
||||
fake_client.disconnect = AsyncMock()
|
||||
MockClient.return_value = fake_client
|
||||
|
||||
loop = CampaignLoop(tick_seconds=0.0, max_ticks=5)
|
||||
results = await loop.run()
|
||||
|
||||
assert results == []
|
||||
|
||||
def test_stop_sets_running_false(self):
|
||||
loop = CampaignLoop()
|
||||
loop._running = True
|
||||
loop.stop()
|
||||
assert not loop.is_running
|
||||
150
tests/bannerlord/test_campaign_state.py
Normal file
150
tests/bannerlord/test_campaign_state.py
Normal file
@@ -0,0 +1,150 @@
|
||||
"""Unit tests for bannerlord.campaign_state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from bannerlord.campaign_state import (
|
||||
M2_GOLD_GOAL,
|
||||
M2_TROOP_GOAL,
|
||||
CampaignState,
|
||||
NearbyParty,
|
||||
Settlement,
|
||||
parse_campaign_state,
|
||||
)
|
||||
|
||||
|
||||
class TestParseCampaignState:
|
||||
def test_empty_dict_returns_defaults(self):
|
||||
state = parse_campaign_state({})
|
||||
assert state.party.party_size == 0
|
||||
assert state.economy.gold == 0
|
||||
assert state.nearby_parties == []
|
||||
assert state.settlements == []
|
||||
|
||||
def test_full_payload_parsed(self):
|
||||
raw = {
|
||||
"tick": 5,
|
||||
"party": {
|
||||
"size": 30,
|
||||
"wounded": 2,
|
||||
"prisoners": 1,
|
||||
"food_days": 3.5,
|
||||
"morale": 75.0,
|
||||
"current_settlement": "town_A1",
|
||||
"speed": 5.2,
|
||||
},
|
||||
"economy": {
|
||||
"gold": 4500,
|
||||
"daily_income": 200,
|
||||
"daily_expenses": 150,
|
||||
},
|
||||
"nearby_parties": [
|
||||
{
|
||||
"id": "bandit_1",
|
||||
"name": "Forest Bandits",
|
||||
"faction": "bandit",
|
||||
"is_hostile": True,
|
||||
"troop_count": 10,
|
||||
"distance": 3.0,
|
||||
}
|
||||
],
|
||||
"settlements": [
|
||||
{
|
||||
"id": "town_A1",
|
||||
"name": "Marunath",
|
||||
"faction": "aserai",
|
||||
"is_friendly": True,
|
||||
"distance": 0.0,
|
||||
"has_recruits": True,
|
||||
"has_trade_goods": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
state = parse_campaign_state(raw)
|
||||
|
||||
assert state.tick == 5
|
||||
assert state.party.party_size == 30
|
||||
assert state.party.wounded == 2
|
||||
assert state.economy.gold == 4500
|
||||
assert state.economy.net_income == 50
|
||||
assert len(state.nearby_parties) == 1
|
||||
assert state.nearby_parties[0].name == "Forest Bandits"
|
||||
assert len(state.settlements) == 1
|
||||
assert state.settlements[0].name == "Marunath"
|
||||
|
||||
def test_malformed_entries_skipped(self):
|
||||
raw = {
|
||||
"nearby_parties": [{"id": "ok", "name": "Good", "faction": "bandit",
|
||||
"is_hostile": True, "troop_count": 5, "distance": 2.0},
|
||||
{"bad": "data"}],
|
||||
"settlements": [None, "not_a_dict"],
|
||||
}
|
||||
state = parse_campaign_state(raw)
|
||||
assert len(state.nearby_parties) == 1
|
||||
assert state.settlements == []
|
||||
|
||||
|
||||
class TestCampaignStateProperties:
|
||||
def _make_state(self, *, troops: int, gold: int) -> CampaignState:
|
||||
state = CampaignState()
|
||||
state.party.party_size = troops
|
||||
state.economy.gold = gold
|
||||
return state
|
||||
|
||||
def test_m2_not_complete_by_default(self):
|
||||
state = self._make_state(troops=20, gold=0)
|
||||
assert not state.m2_complete
|
||||
|
||||
def test_m2_complete_when_both_goals_met(self):
|
||||
state = self._make_state(troops=M2_TROOP_GOAL, gold=M2_GOLD_GOAL)
|
||||
assert state.m2_complete
|
||||
|
||||
def test_m2_not_complete_if_only_troops_met(self):
|
||||
state = self._make_state(troops=M2_TROOP_GOAL, gold=M2_GOLD_GOAL - 1)
|
||||
assert not state.m2_complete
|
||||
|
||||
def test_m2_not_complete_if_only_gold_met(self):
|
||||
state = self._make_state(troops=M2_TROOP_GOAL - 1, gold=M2_GOLD_GOAL)
|
||||
assert not state.m2_complete
|
||||
|
||||
def test_troops_progress_string(self):
|
||||
state = self._make_state(troops=45, gold=0)
|
||||
assert state.troops_progress == f"45/{M2_TROOP_GOAL}"
|
||||
|
||||
def test_gold_progress_string(self):
|
||||
state = self._make_state(troops=0, gold=3000)
|
||||
assert "3,000" in state.gold_progress
|
||||
|
||||
def test_hostile_bandits_nearby_filter(self):
|
||||
state = CampaignState()
|
||||
state.nearby_parties = [
|
||||
NearbyParty("b1", "Bandits", "bandit", True, 10, 2.0),
|
||||
NearbyParty("l1", "Lord", "empire", False, 50, 1.0),
|
||||
NearbyParty("b2", "Far Bandits", "bandit", True, 5, 10.0),
|
||||
]
|
||||
nearby = state.hostile_bandits_nearby(max_distance=5.0)
|
||||
assert len(nearby) == 1
|
||||
assert nearby[0].party_id == "b1"
|
||||
|
||||
def test_nearest_settlement_returns_closest(self):
|
||||
state = CampaignState()
|
||||
state.settlements = [
|
||||
Settlement("s1", "Far Town", "empire", True, 10.0),
|
||||
Settlement("s2", "Near Town", "empire", True, 2.0),
|
||||
]
|
||||
nearest = state.nearest_settlement()
|
||||
assert nearest.settlement_id == "s2"
|
||||
|
||||
def test_nearest_recruit_settlement(self):
|
||||
state = CampaignState()
|
||||
state.settlements = [
|
||||
Settlement("s1", "Town A", "empire", True, 5.0, has_recruits=False),
|
||||
Settlement("s2", "Town B", "empire", True, 8.0, has_recruits=True),
|
||||
]
|
||||
recruit = state.nearest_recruit_settlement()
|
||||
assert recruit.settlement_id == "s2"
|
||||
|
||||
def test_nearest_settlement_none_when_empty(self):
|
||||
state = CampaignState()
|
||||
assert state.nearest_settlement() is None
|
||||
154
tests/bannerlord/test_decision.py
Normal file
154
tests/bannerlord/test_decision.py
Normal file
@@ -0,0 +1,154 @@
|
||||
"""Unit tests for bannerlord.decision."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from bannerlord.campaign_state import (
|
||||
CampaignState,
|
||||
EconomyState,
|
||||
NearbyParty,
|
||||
PartyState,
|
||||
Settlement,
|
||||
)
|
||||
from bannerlord.decision import (
|
||||
M2Action,
|
||||
CampaignDecision,
|
||||
build_decision_prompt,
|
||||
parse_decision,
|
||||
)
|
||||
|
||||
|
||||
def _make_state(
|
||||
*,
|
||||
troops: int = 30,
|
||||
gold: int = 2000,
|
||||
food_days: float = 5.0,
|
||||
morale: float = 80.0,
|
||||
settlements: list | None = None,
|
||||
nearby_parties: list | None = None,
|
||||
) -> CampaignState:
|
||||
state = CampaignState()
|
||||
state.party = PartyState(
|
||||
party_size=troops,
|
||||
food_days=food_days,
|
||||
morale=morale,
|
||||
)
|
||||
state.economy = EconomyState(gold=gold, daily_income=200, daily_expenses=150)
|
||||
state.settlements = settlements or []
|
||||
state.nearby_parties = nearby_parties or []
|
||||
return state
|
||||
|
||||
|
||||
class TestBuildDecisionPrompt:
|
||||
def test_returns_two_messages(self):
|
||||
state = _make_state()
|
||||
messages = build_decision_prompt(state)
|
||||
assert len(messages) == 2
|
||||
assert messages[0]["role"] == "system"
|
||||
assert messages[1]["role"] == "user"
|
||||
|
||||
def test_user_message_includes_party_info(self):
|
||||
state = _make_state(troops=45, gold=3000)
|
||||
messages = build_decision_prompt(state)
|
||||
user_content = messages[1]["content"]
|
||||
assert "45" in user_content
|
||||
assert "3,000" in user_content
|
||||
|
||||
def test_bandits_appear_in_prompt_when_nearby(self):
|
||||
state = _make_state(
|
||||
nearby_parties=[NearbyParty("b1", "Forest Bandits", "bandit", True, 10, 2.0)]
|
||||
)
|
||||
messages = build_decision_prompt(state)
|
||||
user_content = messages[1]["content"]
|
||||
assert "Forest Bandits" in user_content
|
||||
|
||||
def test_settlements_appear_in_prompt(self):
|
||||
state = _make_state(
|
||||
settlements=[Settlement("s1", "Marunath", "aserai", True, 3.0, has_recruits=True)]
|
||||
)
|
||||
messages = build_decision_prompt(state)
|
||||
user_content = messages[1]["content"]
|
||||
assert "Marunath" in user_content
|
||||
|
||||
def test_system_prompt_contains_action_vocabulary(self):
|
||||
state = _make_state()
|
||||
messages = build_decision_prompt(state)
|
||||
system = messages[0]["content"]
|
||||
for action in ("MOVE", "TRADE", "RECRUIT", "ENGAGE", "WAIT"):
|
||||
assert action in system
|
||||
|
||||
|
||||
class TestParseDecision:
|
||||
def test_valid_move_decision(self):
|
||||
raw = json.dumps({
|
||||
"action": "MOVE",
|
||||
"settlement_id": "town_A1",
|
||||
"settlement_name": "Marunath",
|
||||
"item_id": "",
|
||||
"quantity": 1,
|
||||
"party_id": "",
|
||||
"party_name": "",
|
||||
"reasoning": "Moving to recruit troops",
|
||||
})
|
||||
decision = parse_decision(raw)
|
||||
assert decision.action == M2Action.MOVE
|
||||
assert decision.settlement_id == "town_A1"
|
||||
assert decision.settlement_name == "Marunath"
|
||||
|
||||
def test_valid_recruit_decision(self):
|
||||
raw = json.dumps({
|
||||
"action": "RECRUIT",
|
||||
"settlement_id": "town_A1",
|
||||
"settlement_name": "Marunath",
|
||||
"item_id": "",
|
||||
"quantity": 1,
|
||||
"party_id": "",
|
||||
"party_name": "",
|
||||
"reasoning": "Has recruits available",
|
||||
})
|
||||
decision = parse_decision(raw)
|
||||
assert decision.action == M2Action.RECRUIT
|
||||
|
||||
def test_valid_engage_decision(self):
|
||||
raw = json.dumps({
|
||||
"action": "ENGAGE",
|
||||
"settlement_id": "",
|
||||
"settlement_name": "",
|
||||
"item_id": "",
|
||||
"quantity": 1,
|
||||
"party_id": "bandit_1",
|
||||
"party_name": "Forest Bandits",
|
||||
"reasoning": "Weak bandits — easy XP",
|
||||
})
|
||||
decision = parse_decision(raw)
|
||||
assert decision.action == M2Action.ENGAGE
|
||||
assert decision.party_id == "bandit_1"
|
||||
|
||||
def test_wait_on_invalid_json(self):
|
||||
decision = parse_decision("not json at all")
|
||||
assert decision.action == M2Action.WAIT
|
||||
|
||||
def test_wait_on_unknown_action(self):
|
||||
raw = json.dumps({"action": "TELEPORT", "reasoning": "hack"})
|
||||
decision = parse_decision(raw)
|
||||
assert decision.action == M2Action.WAIT
|
||||
|
||||
def test_strips_markdown_fences(self):
|
||||
raw = '```json\n{"action": "WAIT", "reasoning": "low food"}\n```'
|
||||
decision = parse_decision(raw)
|
||||
assert decision.action == M2Action.WAIT
|
||||
|
||||
def test_quantity_minimum_one(self):
|
||||
raw = json.dumps({"action": "TRADE", "item_id": "grain", "quantity": -5, "reasoning": "x"})
|
||||
decision = parse_decision(raw)
|
||||
assert decision.quantity == 1
|
||||
|
||||
def test_missing_optional_fields_default_to_empty(self):
|
||||
raw = json.dumps({"action": "WAIT", "reasoning": "resting"})
|
||||
decision = parse_decision(raw)
|
||||
assert decision.settlement_id == ""
|
||||
assert decision.party_id == ""
|
||||
assert decision.item_id == ""
|
||||
120
tests/bannerlord/test_gabs_client.py
Normal file
120
tests/bannerlord/test_gabs_client.py
Normal file
@@ -0,0 +1,120 @@
|
||||
"""Unit tests for bannerlord.gabs_client."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from bannerlord.gabs_client import GabsClient, GabsError
|
||||
|
||||
|
||||
class TestGabsClientCall:
|
||||
"""Tests for GabsClient.call() using mock StreamReader/Writer."""
|
||||
|
||||
def _make_client(self, response: dict) -> GabsClient:
|
||||
"""Return a pre-connected GabsClient with mocked I/O."""
|
||||
client = GabsClient(host="localhost", port=4825, timeout=5.0)
|
||||
client._connected = True
|
||||
|
||||
writer = MagicMock()
|
||||
writer.write = MagicMock()
|
||||
writer.drain = AsyncMock()
|
||||
|
||||
raw_response = json.dumps(response).encode() + b"\n"
|
||||
reader = MagicMock()
|
||||
reader.readline = AsyncMock(return_value=raw_response)
|
||||
|
||||
client._reader = reader
|
||||
client._writer = writer
|
||||
return client
|
||||
|
||||
async def test_successful_call_returns_result(self):
|
||||
client = self._make_client({"jsonrpc": "2.0", "id": 1, "result": {"status": "ok"}})
|
||||
result = await client.call("game/ping")
|
||||
assert result == {"status": "ok"}
|
||||
|
||||
async def test_error_response_raises_gabs_error(self):
|
||||
client = self._make_client({
|
||||
"jsonrpc": "2.0",
|
||||
"id": 1,
|
||||
"error": {"code": -32601, "message": "Method not found"},
|
||||
})
|
||||
with pytest.raises(GabsError) as exc_info:
|
||||
await client.call("unknown/method")
|
||||
assert exc_info.value.code == -32601
|
||||
|
||||
async def test_not_connected_raises_runtime_error(self):
|
||||
client = GabsClient()
|
||||
with pytest.raises(RuntimeError, match="not connected"):
|
||||
await client.call("game/ping")
|
||||
|
||||
async def test_request_id_increments(self):
|
||||
client = self._make_client({"jsonrpc": "2.0", "id": 1, "result": {}})
|
||||
await client.call("game/ping")
|
||||
# Reset reader for second call
|
||||
client._reader.readline = AsyncMock(
|
||||
return_value=json.dumps({"jsonrpc": "2.0", "id": 2, "result": {}}).encode() + b"\n"
|
||||
)
|
||||
await client.call("game/ping")
|
||||
assert client._req_id == 2
|
||||
|
||||
async def test_get_game_state_returns_empty_on_error(self):
|
||||
client = GabsClient()
|
||||
client._connected = True
|
||||
|
||||
writer = MagicMock()
|
||||
writer.write = MagicMock()
|
||||
writer.drain = AsyncMock()
|
||||
reader = MagicMock()
|
||||
reader.readline = AsyncMock(side_effect=OSError("connection reset"))
|
||||
|
||||
client._reader = reader
|
||||
client._writer = writer
|
||||
|
||||
result = await client.get_game_state()
|
||||
assert result == {}
|
||||
|
||||
async def test_ping_returns_true_on_success(self):
|
||||
client = self._make_client({"jsonrpc": "2.0", "id": 1, "result": "pong"})
|
||||
result = await client.ping()
|
||||
assert result is True
|
||||
|
||||
async def test_ping_returns_false_on_failure(self):
|
||||
client = GabsClient()
|
||||
result = await client.ping()
|
||||
assert result is False
|
||||
|
||||
|
||||
class TestGabsClientLifecycle:
|
||||
async def test_connect_failure_sets_not_connected(self):
|
||||
client = GabsClient(host="localhost", port=9999, timeout=0.1)
|
||||
with pytest.raises(Exception):
|
||||
await client.connect()
|
||||
assert not client.is_connected
|
||||
|
||||
async def test_context_manager_calls_connect_and_disconnect(self):
|
||||
client = GabsClient()
|
||||
connect_called = False
|
||||
disconnect_called = False
|
||||
|
||||
async def _fake_connect():
|
||||
nonlocal connect_called
|
||||
connect_called = True
|
||||
client._connected = True
|
||||
|
||||
async def _fake_disconnect():
|
||||
nonlocal disconnect_called
|
||||
disconnect_called = True
|
||||
client._connected = False
|
||||
|
||||
client.connect = _fake_connect
|
||||
client.disconnect = _fake_disconnect
|
||||
|
||||
async with client as c:
|
||||
assert c is client
|
||||
assert connect_called
|
||||
|
||||
assert disconnect_called
|
||||
@@ -147,10 +147,12 @@ def clean_database(tmp_path):
|
||||
# IMPORTANT: swarm.task_queue.models also has a DB_PATH that writes to
|
||||
# tasks.db — it MUST be patched too, or error_capture.capture_error()
|
||||
# will write test data to the production database.
|
||||
tmp_sovereignty_db = tmp_path / "sovereignty_metrics.db"
|
||||
for mod_name, tmp_db in [
|
||||
("dashboard.routes.tasks", tmp_tasks_db),
|
||||
("dashboard.routes.work_orders", tmp_work_orders_db),
|
||||
("swarm.task_queue.models", tmp_tasks_db),
|
||||
("infrastructure.sovereignty_metrics", tmp_sovereignty_db),
|
||||
]:
|
||||
try:
|
||||
mod = __import__(mod_name, fromlist=["DB_PATH"])
|
||||
|
||||
499
tests/dashboard/test_health.py
Normal file
499
tests/dashboard/test_health.py
Normal file
@@ -0,0 +1,499 @@
|
||||
"""Unit tests for dashboard/routes/health.py.
|
||||
|
||||
Covers helper functions, caching, endpoint responses, and graceful
|
||||
degradation when subsystems (Ollama, SQLite) are unavailable.
|
||||
|
||||
Fixes #945
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from dashboard.routes.health import (
|
||||
DependencyStatus,
|
||||
HealthStatus,
|
||||
SovereigntyReport,
|
||||
_calculate_overall_score,
|
||||
_check_lightning,
|
||||
_check_ollama_sync,
|
||||
_check_sqlite,
|
||||
_generate_recommendations,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDependencyStatusModel:
|
||||
"""Validate DependencyStatus model."""
|
||||
|
||||
def test_fields(self):
|
||||
dep = DependencyStatus(
|
||||
name="Test", status="healthy", sovereignty_score=8, details={"key": "val"}
|
||||
)
|
||||
assert dep.name == "Test"
|
||||
assert dep.status == "healthy"
|
||||
assert dep.sovereignty_score == 8
|
||||
assert dep.details == {"key": "val"}
|
||||
|
||||
def test_empty_details(self):
|
||||
dep = DependencyStatus(name="X", status="unavailable", sovereignty_score=0, details={})
|
||||
assert dep.details == {}
|
||||
|
||||
|
||||
class TestSovereigntyReportModel:
|
||||
"""Validate SovereigntyReport model."""
|
||||
|
||||
def test_fields(self):
|
||||
report = SovereigntyReport(
|
||||
overall_score=9.3,
|
||||
dependencies=[],
|
||||
timestamp="2026-01-01T00:00:00+00:00",
|
||||
recommendations=["All good"],
|
||||
)
|
||||
assert report.overall_score == 9.3
|
||||
assert report.dependencies == []
|
||||
assert report.recommendations == ["All good"]
|
||||
|
||||
|
||||
class TestHealthStatusModel:
|
||||
"""Validate HealthStatus model."""
|
||||
|
||||
def test_fields(self):
|
||||
hs = HealthStatus(
|
||||
status="ok",
|
||||
timestamp="2026-01-01T00:00:00+00:00",
|
||||
version="2.0.0",
|
||||
uptime_seconds=42.5,
|
||||
)
|
||||
assert hs.status == "ok"
|
||||
assert hs.uptime_seconds == 42.5
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper functions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCalculateOverallScore:
|
||||
"""Test _calculate_overall_score."""
|
||||
|
||||
def test_empty_deps(self):
|
||||
assert _calculate_overall_score([]) == 0.0
|
||||
|
||||
def test_single_dep(self):
|
||||
deps = [DependencyStatus(name="A", status="healthy", sovereignty_score=7, details={})]
|
||||
assert _calculate_overall_score(deps) == 7.0
|
||||
|
||||
def test_averages_multiple(self):
|
||||
deps = [
|
||||
DependencyStatus(name="A", status="healthy", sovereignty_score=10, details={}),
|
||||
DependencyStatus(name="B", status="healthy", sovereignty_score=8, details={}),
|
||||
DependencyStatus(name="C", status="unavailable", sovereignty_score=6, details={}),
|
||||
]
|
||||
assert _calculate_overall_score(deps) == 8.0
|
||||
|
||||
def test_rounding(self):
|
||||
deps = [
|
||||
DependencyStatus(name="A", status="healthy", sovereignty_score=10, details={}),
|
||||
DependencyStatus(name="B", status="healthy", sovereignty_score=9, details={}),
|
||||
DependencyStatus(name="C", status="healthy", sovereignty_score=10, details={}),
|
||||
]
|
||||
assert _calculate_overall_score(deps) == 9.7
|
||||
|
||||
|
||||
class TestGenerateRecommendations:
|
||||
"""Test _generate_recommendations."""
|
||||
|
||||
def test_all_healthy(self):
|
||||
deps = [DependencyStatus(name="X", status="healthy", sovereignty_score=10, details={})]
|
||||
recs = _generate_recommendations(deps)
|
||||
assert recs == ["System operating optimally - all dependencies healthy"]
|
||||
|
||||
def test_unavailable_service(self):
|
||||
deps = [
|
||||
DependencyStatus(
|
||||
name="Ollama AI", status="unavailable", sovereignty_score=10, details={}
|
||||
)
|
||||
]
|
||||
recs = _generate_recommendations(deps)
|
||||
assert any("Ollama AI is unavailable" in r for r in recs)
|
||||
|
||||
def test_degraded_lightning_mock(self):
|
||||
deps = [
|
||||
DependencyStatus(
|
||||
name="Lightning Payments",
|
||||
status="degraded",
|
||||
sovereignty_score=8,
|
||||
details={"backend": "mock"},
|
||||
)
|
||||
]
|
||||
recs = _generate_recommendations(deps)
|
||||
assert any("Switch to real Lightning" in r for r in recs)
|
||||
|
||||
def test_degraded_non_lightning(self):
|
||||
"""Degraded non-Lightning dep produces no specific recommendation."""
|
||||
deps = [DependencyStatus(name="Redis", status="degraded", sovereignty_score=5, details={})]
|
||||
recs = _generate_recommendations(deps)
|
||||
assert recs == ["System operating optimally - all dependencies healthy"]
|
||||
|
||||
def test_multiple_unavailable(self):
|
||||
deps = [
|
||||
DependencyStatus(name="A", status="unavailable", sovereignty_score=5, details={}),
|
||||
DependencyStatus(name="B", status="unavailable", sovereignty_score=5, details={}),
|
||||
]
|
||||
recs = _generate_recommendations(deps)
|
||||
assert len(recs) == 2
|
||||
assert "A is unavailable" in recs[0]
|
||||
assert "B is unavailable" in recs[1]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _check_lightning (static)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCheckLightning:
|
||||
"""Test _check_lightning — always returns unavailable for now."""
|
||||
|
||||
def test_returns_unavailable(self):
|
||||
dep = _check_lightning()
|
||||
assert dep.name == "Lightning Payments"
|
||||
assert dep.status == "unavailable"
|
||||
assert dep.sovereignty_score == 8
|
||||
assert "removed" in dep.details.get("note", "").lower()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _check_ollama_sync
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCheckOllamaSync:
|
||||
"""Test synchronous Ollama health probe."""
|
||||
|
||||
def test_healthy_when_reachable(self):
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status = 200
|
||||
mock_resp.__enter__ = MagicMock(return_value=mock_resp)
|
||||
mock_resp.__exit__ = MagicMock(return_value=False)
|
||||
|
||||
with patch("urllib.request.urlopen", return_value=mock_resp):
|
||||
dep = _check_ollama_sync()
|
||||
|
||||
assert dep.status == "healthy"
|
||||
assert dep.name == "Ollama AI"
|
||||
assert dep.sovereignty_score == 10
|
||||
|
||||
def test_unavailable_on_connection_error(self):
|
||||
with patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=ConnectionError("refused"),
|
||||
):
|
||||
dep = _check_ollama_sync()
|
||||
|
||||
assert dep.status == "unavailable"
|
||||
assert "Cannot connect" in dep.details.get("error", "")
|
||||
|
||||
def test_unavailable_on_timeout(self):
|
||||
from urllib.error import URLError
|
||||
|
||||
with patch(
|
||||
"urllib.request.urlopen",
|
||||
side_effect=URLError("timeout"),
|
||||
):
|
||||
dep = _check_ollama_sync()
|
||||
|
||||
assert dep.status == "unavailable"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _check_sqlite
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCheckSQLite:
|
||||
"""Test SQLite health probe."""
|
||||
|
||||
def test_healthy_when_db_reachable(self, tmp_path):
|
||||
import sqlite3
|
||||
|
||||
db_path = tmp_path / "data" / "timmy.db"
|
||||
db_path.parent.mkdir(parents=True)
|
||||
sqlite3.connect(str(db_path)).close()
|
||||
|
||||
with patch("dashboard.routes.health.settings") as mock_settings:
|
||||
mock_settings.repo_root = str(tmp_path)
|
||||
dep = _check_sqlite()
|
||||
|
||||
assert dep.status == "healthy"
|
||||
assert dep.name == "SQLite Database"
|
||||
|
||||
def test_unavailable_on_missing_db(self, tmp_path):
|
||||
with patch("dashboard.routes.health.settings") as mock_settings:
|
||||
mock_settings.repo_root = str(tmp_path / "nonexistent")
|
||||
dep = _check_sqlite()
|
||||
|
||||
assert dep.status == "unavailable"
|
||||
assert "error" in dep.details
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _check_ollama (async, with caching)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCheckOllamaAsync:
|
||||
"""Test async Ollama check with TTL cache."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_cache(self):
|
||||
"""Clear the module-level Ollama cache before each test."""
|
||||
import dashboard.routes.health as mod
|
||||
|
||||
mod._ollama_cache = None
|
||||
mod._ollama_cache_ts = 0.0
|
||||
yield
|
||||
mod._ollama_cache = None
|
||||
mod._ollama_cache_ts = 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_returns_dependency_status(self):
|
||||
healthy = DependencyStatus(
|
||||
name="Ollama AI", status="healthy", sovereignty_score=10, details={}
|
||||
)
|
||||
with patch(
|
||||
"dashboard.routes.health._check_ollama_sync",
|
||||
return_value=healthy,
|
||||
):
|
||||
from dashboard.routes.health import _check_ollama
|
||||
|
||||
result = await _check_ollama()
|
||||
|
||||
assert result.status == "healthy"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_caches_result(self):
|
||||
healthy = DependencyStatus(
|
||||
name="Ollama AI", status="healthy", sovereignty_score=10, details={}
|
||||
)
|
||||
with patch(
|
||||
"dashboard.routes.health._check_ollama_sync",
|
||||
return_value=healthy,
|
||||
) as mock_sync:
|
||||
from dashboard.routes.health import _check_ollama
|
||||
|
||||
await _check_ollama()
|
||||
await _check_ollama()
|
||||
|
||||
# Should only call the sync function once due to cache
|
||||
assert mock_sync.call_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_expires(self):
|
||||
healthy = DependencyStatus(
|
||||
name="Ollama AI", status="healthy", sovereignty_score=10, details={}
|
||||
)
|
||||
import dashboard.routes.health as mod
|
||||
|
||||
with patch(
|
||||
"dashboard.routes.health._check_ollama_sync",
|
||||
return_value=healthy,
|
||||
) as mock_sync:
|
||||
from dashboard.routes.health import _check_ollama
|
||||
|
||||
await _check_ollama()
|
||||
# Expire the cache
|
||||
mod._ollama_cache_ts = time.monotonic() - 60
|
||||
await _check_ollama()
|
||||
|
||||
assert mock_sync.call_count == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fallback_on_thread_exception(self):
|
||||
"""If to_thread raises, return unavailable status."""
|
||||
import asyncio
|
||||
|
||||
with patch.object(
|
||||
asyncio,
|
||||
"to_thread",
|
||||
side_effect=RuntimeError("thread pool exhausted"),
|
||||
):
|
||||
from dashboard.routes.health import _check_ollama
|
||||
|
||||
result = await _check_ollama()
|
||||
|
||||
assert result.status == "unavailable"
|
||||
|
||||
|
||||
class TestCheckOllamaBool:
|
||||
"""Test the legacy bool wrapper."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_cache(self):
|
||||
import dashboard.routes.health as mod
|
||||
|
||||
mod._ollama_cache = None
|
||||
mod._ollama_cache_ts = 0.0
|
||||
yield
|
||||
mod._ollama_cache = None
|
||||
mod._ollama_cache_ts = 0.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_true_when_healthy(self):
|
||||
healthy = DependencyStatus(
|
||||
name="Ollama AI", status="healthy", sovereignty_score=10, details={}
|
||||
)
|
||||
with patch("dashboard.routes.health._check_ollama_sync", return_value=healthy):
|
||||
from dashboard.routes.health import check_ollama
|
||||
|
||||
assert await check_ollama() is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_false_when_unavailable(self):
|
||||
down = DependencyStatus(
|
||||
name="Ollama AI", status="unavailable", sovereignty_score=10, details={}
|
||||
)
|
||||
with patch("dashboard.routes.health._check_ollama_sync", return_value=down):
|
||||
from dashboard.routes.health import check_ollama
|
||||
|
||||
assert await check_ollama() is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoint tests via FastAPI TestClient
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestHealthEndpoint:
|
||||
"""Tests for GET /health."""
|
||||
|
||||
def test_returns_200(self, client):
|
||||
response = client.get("/health")
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_ok_when_ollama_up(self, client):
|
||||
with patch(
|
||||
"dashboard.routes.health.check_ollama", new_callable=AsyncMock, return_value=True
|
||||
):
|
||||
data = client.get("/health").json()
|
||||
|
||||
assert data["status"] == "ok"
|
||||
assert data["services"]["ollama"] == "up"
|
||||
assert data["agents"]["agent"]["status"] == "idle"
|
||||
|
||||
def test_degraded_when_ollama_down(self, client):
|
||||
with patch(
|
||||
"dashboard.routes.health.check_ollama", new_callable=AsyncMock, return_value=False
|
||||
):
|
||||
data = client.get("/health").json()
|
||||
|
||||
assert data["status"] == "degraded"
|
||||
assert data["services"]["ollama"] == "down"
|
||||
assert data["agents"]["agent"]["status"] == "offline"
|
||||
|
||||
def test_extended_fields(self, client):
|
||||
data = client.get("/health").json()
|
||||
assert "timestamp" in data
|
||||
assert "version" in data
|
||||
assert "uptime_seconds" in data
|
||||
assert isinstance(data["uptime_seconds"], (int, float))
|
||||
assert "llm_backend" in data
|
||||
assert "llm_model" in data
|
||||
|
||||
|
||||
class TestHealthStatusPanel:
|
||||
"""Tests for GET /health/status (HTML response)."""
|
||||
|
||||
def test_returns_html(self, client):
|
||||
response = client.get("/health/status")
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers["content-type"]
|
||||
|
||||
def test_shows_up_when_ollama_healthy(self, client):
|
||||
with patch(
|
||||
"dashboard.routes.health.check_ollama", new_callable=AsyncMock, return_value=True
|
||||
):
|
||||
text = client.get("/health/status").text
|
||||
|
||||
assert "UP" in text
|
||||
|
||||
def test_shows_down_when_ollama_unhealthy(self, client):
|
||||
with patch(
|
||||
"dashboard.routes.health.check_ollama", new_callable=AsyncMock, return_value=False
|
||||
):
|
||||
text = client.get("/health/status").text
|
||||
|
||||
assert "DOWN" in text
|
||||
|
||||
def test_includes_model_name(self, client):
|
||||
text = client.get("/health/status").text
|
||||
assert "Model:" in text
|
||||
|
||||
|
||||
class TestSovereigntyEndpoint:
|
||||
"""Tests for GET /health/sovereignty."""
|
||||
|
||||
def test_aggregates_three_subsystems(self, client):
|
||||
data = client.get("/health/sovereignty").json()
|
||||
names = [d["name"] for d in data["dependencies"]]
|
||||
assert "Ollama AI" in names
|
||||
assert "Lightning Payments" in names
|
||||
assert "SQLite Database" in names
|
||||
|
||||
def test_score_range(self, client):
|
||||
data = client.get("/health/sovereignty").json()
|
||||
assert 0 <= data["overall_score"] <= 10
|
||||
|
||||
|
||||
class TestComponentsEndpoint:
|
||||
"""Tests for GET /health/components."""
|
||||
|
||||
def test_returns_timestamp(self, client):
|
||||
data = client.get("/health/components").json()
|
||||
assert "timestamp" in data
|
||||
|
||||
def test_config_keys(self, client):
|
||||
data = client.get("/health/components").json()
|
||||
cfg = data["config"]
|
||||
assert "debug" in cfg
|
||||
assert "model_backend" in cfg
|
||||
assert "ollama_model" in cfg
|
||||
|
||||
|
||||
class TestSnapshotEndpoint:
|
||||
"""Tests for GET /health/snapshot."""
|
||||
|
||||
def test_returns_200(self, client):
|
||||
response = client.get("/health/snapshot")
|
||||
assert response.status_code == 200
|
||||
|
||||
def test_overall_status_valid(self, client):
|
||||
data = client.get("/health/snapshot").json()
|
||||
assert data["overall_status"] in ["green", "yellow", "red", "unknown"]
|
||||
|
||||
def test_graceful_fallback_on_import_error(self, client):
|
||||
"""Snapshot degrades gracefully when automation module fails."""
|
||||
with patch(
|
||||
"dashboard.routes.health.asyncio.to_thread",
|
||||
side_effect=ImportError("no module"),
|
||||
):
|
||||
data = client.get("/health/snapshot").json()
|
||||
|
||||
assert data["overall_status"] == "unknown"
|
||||
assert "error" in data
|
||||
assert data["ci"]["status"] == "unknown"
|
||||
|
||||
def test_graceful_fallback_on_runtime_error(self, client):
|
||||
with patch(
|
||||
"dashboard.routes.health.asyncio.to_thread",
|
||||
side_effect=RuntimeError("boom"),
|
||||
):
|
||||
data = client.get("/health/snapshot").json()
|
||||
|
||||
assert data["overall_status"] == "unknown"
|
||||
680
tests/dashboard/test_scorecards.py
Normal file
680
tests/dashboard/test_scorecards.py
Normal file
@@ -0,0 +1,680 @@
|
||||
"""Tests for agent scorecard functionality."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from dashboard.services.scorecard_service import (
|
||||
AgentMetrics,
|
||||
PeriodType,
|
||||
ScorecardSummary,
|
||||
_aggregate_metrics,
|
||||
_detect_patterns,
|
||||
_extract_actor_from_event,
|
||||
_generate_narrative_bullets,
|
||||
_get_period_bounds,
|
||||
_is_tracked_agent,
|
||||
_query_token_transactions,
|
||||
generate_all_scorecards,
|
||||
generate_scorecard,
|
||||
get_tracked_agents,
|
||||
)
|
||||
from infrastructure.events.bus import Event
|
||||
|
||||
|
||||
class TestPeriodBounds:
|
||||
"""Test period boundary calculations."""
|
||||
|
||||
def test_daily_period_bounds(self):
|
||||
"""Test daily period returns correct 24-hour window."""
|
||||
reference = datetime(2026, 3, 21, 12, 30, 45, tzinfo=UTC)
|
||||
start, end = _get_period_bounds(PeriodType.daily, reference)
|
||||
|
||||
assert end == datetime(2026, 3, 21, 0, 0, 0, tzinfo=UTC)
|
||||
assert start == datetime(2026, 3, 20, 0, 0, 0, tzinfo=UTC)
|
||||
assert (end - start) == timedelta(days=1)
|
||||
|
||||
def test_weekly_period_bounds(self):
|
||||
"""Test weekly period returns correct 7-day window."""
|
||||
reference = datetime(2026, 3, 21, 12, 30, 45, tzinfo=UTC)
|
||||
start, end = _get_period_bounds(PeriodType.weekly, reference)
|
||||
|
||||
assert end == datetime(2026, 3, 21, 0, 0, 0, tzinfo=UTC)
|
||||
assert start == datetime(2026, 3, 14, 0, 0, 0, tzinfo=UTC)
|
||||
assert (end - start) == timedelta(days=7)
|
||||
|
||||
def test_default_reference_date(self):
|
||||
"""Test default reference date uses current time."""
|
||||
start, end = _get_period_bounds(PeriodType.daily)
|
||||
now = datetime.now(UTC)
|
||||
|
||||
# End should be start of current day (midnight)
|
||||
expected_end = now.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
assert end == expected_end
|
||||
# Start should be 24 hours before end
|
||||
assert (end - start) == timedelta(days=1)
|
||||
|
||||
|
||||
class TestTrackedAgents:
|
||||
"""Test agent tracking functions."""
|
||||
|
||||
def test_get_tracked_agents(self):
|
||||
"""Test get_tracked_agents returns sorted list."""
|
||||
agents = get_tracked_agents()
|
||||
assert isinstance(agents, list)
|
||||
assert "kimi" in agents
|
||||
assert "claude" in agents
|
||||
assert "gemini" in agents
|
||||
assert "hermes" in agents
|
||||
assert "manus" in agents
|
||||
assert agents == sorted(agents)
|
||||
|
||||
def test_is_tracked_agent_true(self):
|
||||
"""Test _is_tracked_agent returns True for tracked agents."""
|
||||
assert _is_tracked_agent("kimi") is True
|
||||
assert _is_tracked_agent("KIMI") is True # case insensitive
|
||||
assert _is_tracked_agent("claude") is True
|
||||
assert _is_tracked_agent("hermes") is True
|
||||
|
||||
def test_is_tracked_agent_false(self):
|
||||
"""Test _is_tracked_agent returns False for untracked agents."""
|
||||
assert _is_tracked_agent("unknown") is False
|
||||
assert _is_tracked_agent("rockachopa") is False
|
||||
assert _is_tracked_agent("") is False
|
||||
|
||||
|
||||
class TestExtractActor:
|
||||
"""Test actor extraction from events."""
|
||||
|
||||
def test_extract_from_actor_field(self):
|
||||
"""Test extraction from data.actor field."""
|
||||
event = Event(type="test", source="system", data={"actor": "kimi"})
|
||||
assert _extract_actor_from_event(event) == "kimi"
|
||||
|
||||
def test_extract_from_agent_id_field(self):
|
||||
"""Test extraction from data.agent_id field."""
|
||||
event = Event(type="test", source="system", data={"agent_id": "claude"})
|
||||
assert _extract_actor_from_event(event) == "claude"
|
||||
|
||||
def test_extract_from_source_fallback(self):
|
||||
"""Test fallback to event.source."""
|
||||
event = Event(type="test", source="gemini", data={})
|
||||
assert _extract_actor_from_event(event) == "gemini"
|
||||
|
||||
def test_actor_priority_over_agent_id(self):
|
||||
"""Test actor field takes priority over agent_id."""
|
||||
event = Event(type="test", source="system", data={"actor": "kimi", "agent_id": "claude"})
|
||||
assert _extract_actor_from_event(event) == "kimi"
|
||||
|
||||
|
||||
class TestAggregateMetrics:
|
||||
"""Test metrics aggregation from events."""
|
||||
|
||||
def test_empty_events(self):
|
||||
"""Test aggregation with no events returns empty dict."""
|
||||
result = _aggregate_metrics([])
|
||||
assert result == {}
|
||||
|
||||
def test_push_event_aggregation(self):
|
||||
"""Test push events aggregate commits correctly."""
|
||||
events = [
|
||||
Event(type="gitea.push", source="gitea", data={"actor": "kimi", "num_commits": 3}),
|
||||
Event(type="gitea.push", source="gitea", data={"actor": "kimi", "num_commits": 2}),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "kimi" in result
|
||||
assert result["kimi"].commits == 5
|
||||
|
||||
def test_issue_opened_aggregation(self):
|
||||
"""Test issue opened events aggregate correctly."""
|
||||
events = [
|
||||
Event(
|
||||
type="gitea.issue.opened",
|
||||
source="gitea",
|
||||
data={"actor": "claude", "issue_number": 100},
|
||||
),
|
||||
Event(
|
||||
type="gitea.issue.opened",
|
||||
source="gitea",
|
||||
data={"actor": "claude", "issue_number": 101},
|
||||
),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "claude" in result
|
||||
assert len(result["claude"].issues_touched) == 2
|
||||
assert 100 in result["claude"].issues_touched
|
||||
assert 101 in result["claude"].issues_touched
|
||||
|
||||
def test_comment_aggregation(self):
|
||||
"""Test comment events aggregate correctly."""
|
||||
events = [
|
||||
Event(
|
||||
type="gitea.issue.comment",
|
||||
source="gitea",
|
||||
data={"actor": "gemini", "issue_number": 100},
|
||||
),
|
||||
Event(
|
||||
type="gitea.issue.comment",
|
||||
source="gitea",
|
||||
data={"actor": "gemini", "issue_number": 101},
|
||||
),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "gemini" in result
|
||||
assert result["gemini"].comments == 2
|
||||
assert len(result["gemini"].issues_touched) == 2 # Comments touch issues too
|
||||
|
||||
def test_pr_events_aggregation(self):
|
||||
"""Test PR open and merge events aggregate correctly."""
|
||||
events = [
|
||||
Event(
|
||||
type="gitea.pull_request",
|
||||
source="gitea",
|
||||
data={"actor": "kimi", "pr_number": 50, "action": "opened"},
|
||||
),
|
||||
Event(
|
||||
type="gitea.pull_request",
|
||||
source="gitea",
|
||||
data={"actor": "kimi", "pr_number": 50, "action": "closed", "merged": True},
|
||||
),
|
||||
Event(
|
||||
type="gitea.pull_request",
|
||||
source="gitea",
|
||||
data={"actor": "kimi", "pr_number": 51, "action": "opened"},
|
||||
),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "kimi" in result
|
||||
assert len(result["kimi"].prs_opened) == 2
|
||||
assert len(result["kimi"].prs_merged) == 1
|
||||
assert 50 in result["kimi"].prs_merged
|
||||
|
||||
def test_untracked_agent_filtered(self):
|
||||
"""Test events from untracked agents are filtered out."""
|
||||
events = [
|
||||
Event(
|
||||
type="gitea.push", source="gitea", data={"actor": "rockachopa", "num_commits": 5}
|
||||
),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "rockachopa" not in result
|
||||
|
||||
def test_task_completion_aggregation(self):
|
||||
"""Test task completion events aggregate test files."""
|
||||
events = [
|
||||
Event(
|
||||
type="agent.task.completed",
|
||||
source="gitea",
|
||||
data={
|
||||
"agent_id": "kimi",
|
||||
"tests_affected": ["test_foo.py", "test_bar.py"],
|
||||
"token_reward": 10,
|
||||
},
|
||||
),
|
||||
]
|
||||
result = _aggregate_metrics(events)
|
||||
|
||||
assert "kimi" in result
|
||||
assert len(result["kimi"].tests_affected) == 2
|
||||
assert "test_foo.py" in result["kimi"].tests_affected
|
||||
assert result["kimi"].tokens_earned == 10
|
||||
|
||||
|
||||
class TestAgentMetrics:
|
||||
"""Test AgentMetrics class."""
|
||||
|
||||
def test_merge_rate_zero_prs(self):
|
||||
"""Test merge rate is 0 when no PRs opened."""
|
||||
metrics = AgentMetrics(agent_id="kimi")
|
||||
assert metrics.pr_merge_rate == 0.0
|
||||
|
||||
def test_merge_rate_perfect(self):
|
||||
"""Test 100% merge rate calculation."""
|
||||
metrics = AgentMetrics(agent_id="kimi", prs_opened={1, 2, 3}, prs_merged={1, 2, 3})
|
||||
assert metrics.pr_merge_rate == 1.0
|
||||
|
||||
def test_merge_rate_partial(self):
|
||||
"""Test partial merge rate calculation."""
|
||||
metrics = AgentMetrics(agent_id="kimi", prs_opened={1, 2, 3, 4}, prs_merged={1, 2})
|
||||
assert metrics.pr_merge_rate == 0.5
|
||||
|
||||
|
||||
class TestDetectPatterns:
|
||||
"""Test pattern detection logic."""
|
||||
|
||||
def test_high_merge_rate_pattern(self):
|
||||
"""Test detection of high merge rate pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
prs_opened={1, 2, 3, 4, 5},
|
||||
prs_merged={1, 2, 3, 4}, # 80% merge rate
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("High merge rate" in p for p in patterns)
|
||||
|
||||
def test_low_merge_rate_pattern(self):
|
||||
"""Test detection of low merge rate pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
prs_opened={1, 2, 3, 4, 5},
|
||||
prs_merged={1}, # 20% merge rate
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("low merge rate" in p for p in patterns)
|
||||
|
||||
def test_high_commits_no_prs_pattern(self):
|
||||
"""Test detection of direct-to-main commits pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
commits=15,
|
||||
prs_opened=set(),
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("High commit volume without PRs" in p for p in patterns)
|
||||
|
||||
def test_silent_worker_pattern(self):
|
||||
"""Test detection of silent worker pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
issues_touched={1, 2, 3, 4, 5, 6},
|
||||
comments=0,
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("silent worker" in p for p in patterns)
|
||||
|
||||
def test_communicative_pattern(self):
|
||||
"""Test detection of highly communicative pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
issues_touched={1, 2}, # 2 issues
|
||||
comments=10, # 5x comments per issue
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("Highly communicative" in p for p in patterns)
|
||||
|
||||
def test_token_accumulation_pattern(self):
|
||||
"""Test detection of token accumulation pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tokens_earned=150,
|
||||
tokens_spent=10,
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("Strong token accumulation" in p for p in patterns)
|
||||
|
||||
def test_token_spend_pattern(self):
|
||||
"""Test detection of high token spend pattern."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tokens_earned=10,
|
||||
tokens_spent=100,
|
||||
)
|
||||
patterns = _detect_patterns(metrics)
|
||||
|
||||
assert any("High token spend" in p for p in patterns)
|
||||
|
||||
|
||||
class TestGenerateNarrative:
|
||||
"""Test narrative bullet generation."""
|
||||
|
||||
def test_empty_metrics_narrative(self):
|
||||
"""Test narrative for empty metrics mentions no activity."""
|
||||
metrics = AgentMetrics(agent_id="kimi")
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
assert len(bullets) == 1
|
||||
assert "No recorded activity" in bullets[0]
|
||||
|
||||
def test_activity_summary_narrative(self):
|
||||
"""Test narrative includes activity summary."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
commits=5,
|
||||
prs_opened={1, 2},
|
||||
prs_merged={1},
|
||||
)
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
activity_bullet = next((b for b in bullets if "Active across" in b), None)
|
||||
assert activity_bullet is not None
|
||||
assert "5 commits" in activity_bullet
|
||||
assert "2 PRs opened" in activity_bullet
|
||||
assert "1 PR merged" in activity_bullet
|
||||
|
||||
def test_tests_affected_narrative(self):
|
||||
"""Test narrative includes tests affected."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tests_affected={"test_a.py", "test_b.py"},
|
||||
)
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
assert any("2 test files" in b for b in bullets)
|
||||
|
||||
def test_tokens_earned_narrative(self):
|
||||
"""Test narrative includes token earnings."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tokens_earned=100,
|
||||
tokens_spent=20,
|
||||
)
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
assert any("Net earned 80 tokens" in b for b in bullets)
|
||||
|
||||
def test_tokens_spent_narrative(self):
|
||||
"""Test narrative includes token spending."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tokens_earned=20,
|
||||
tokens_spent=100,
|
||||
)
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
assert any("Net spent 80 tokens" in b for b in bullets)
|
||||
|
||||
def test_balanced_tokens_narrative(self):
|
||||
"""Test narrative for balanced token flow."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
tokens_earned=100,
|
||||
tokens_spent=100,
|
||||
)
|
||||
bullets = _generate_narrative_bullets(metrics, PeriodType.daily)
|
||||
|
||||
assert any("Balanced token flow" in b for b in bullets)
|
||||
|
||||
|
||||
class TestScorecardSummary:
|
||||
"""Test ScorecardSummary dataclass."""
|
||||
|
||||
def test_to_dict_structure(self):
|
||||
"""Test to_dict returns expected structure."""
|
||||
metrics = AgentMetrics(
|
||||
agent_id="kimi",
|
||||
issues_touched={1, 2},
|
||||
prs_opened={10, 11},
|
||||
prs_merged={10},
|
||||
tokens_earned=100,
|
||||
tokens_spent=20,
|
||||
)
|
||||
summary = ScorecardSummary(
|
||||
agent_id="kimi",
|
||||
period_type=PeriodType.daily,
|
||||
period_start=datetime.now(UTC),
|
||||
period_end=datetime.now(UTC),
|
||||
metrics=metrics,
|
||||
narrative_bullets=["Test bullet"],
|
||||
patterns=["Test pattern"],
|
||||
)
|
||||
data = summary.to_dict()
|
||||
|
||||
assert data["agent_id"] == "kimi"
|
||||
assert data["period_type"] == "daily"
|
||||
assert "metrics" in data
|
||||
assert data["metrics"]["issues_touched"] == 2
|
||||
assert data["metrics"]["prs_opened"] == 2
|
||||
assert data["metrics"]["prs_merged"] == 1
|
||||
assert data["metrics"]["pr_merge_rate"] == 0.5
|
||||
assert data["metrics"]["tokens_earned"] == 100
|
||||
assert data["metrics"]["token_net"] == 80
|
||||
assert data["narrative_bullets"] == ["Test bullet"]
|
||||
assert data["patterns"] == ["Test pattern"]
|
||||
|
||||
|
||||
class TestQueryTokenTransactions:
|
||||
"""Test token transaction querying."""
|
||||
|
||||
def test_empty_ledger(self):
|
||||
"""Test empty ledger returns zero values."""
|
||||
with patch("lightning.ledger.get_transactions", return_value=[]):
|
||||
earned, spent = _query_token_transactions("kimi", datetime.now(UTC), datetime.now(UTC))
|
||||
assert earned == 0
|
||||
assert spent == 0
|
||||
|
||||
def test_ledger_with_transactions(self):
|
||||
"""Test ledger aggregation of transactions."""
|
||||
now = datetime.now(UTC)
|
||||
mock_tx = [
|
||||
MagicMock(
|
||||
agent_id="kimi",
|
||||
tx_type=MagicMock(value="incoming"),
|
||||
amount_sats=100,
|
||||
created_at=now.isoformat(),
|
||||
),
|
||||
MagicMock(
|
||||
agent_id="kimi",
|
||||
tx_type=MagicMock(value="outgoing"),
|
||||
amount_sats=30,
|
||||
created_at=now.isoformat(),
|
||||
),
|
||||
]
|
||||
with patch("lightning.ledger.get_transactions", return_value=mock_tx):
|
||||
earned, spent = _query_token_transactions(
|
||||
"kimi", now - timedelta(hours=1), now + timedelta(hours=1)
|
||||
)
|
||||
assert earned == 100
|
||||
assert spent == 30
|
||||
|
||||
def test_ledger_filters_by_agent(self):
|
||||
"""Test ledger filters transactions by agent_id."""
|
||||
now = datetime.now(UTC)
|
||||
mock_tx = [
|
||||
MagicMock(
|
||||
agent_id="claude",
|
||||
tx_type=MagicMock(value="incoming"),
|
||||
amount_sats=100,
|
||||
created_at=now.isoformat(),
|
||||
),
|
||||
]
|
||||
with patch("lightning.ledger.get_transactions", return_value=mock_tx):
|
||||
earned, spent = _query_token_transactions(
|
||||
"kimi", now - timedelta(hours=1), now + timedelta(hours=1)
|
||||
)
|
||||
assert earned == 0 # Transaction was for claude, not kimi
|
||||
|
||||
def test_ledger_filters_by_time(self):
|
||||
"""Test ledger filters transactions by time range."""
|
||||
now = datetime.now(UTC)
|
||||
old_time = now - timedelta(days=2)
|
||||
mock_tx = [
|
||||
MagicMock(
|
||||
agent_id="kimi",
|
||||
tx_type=MagicMock(value="incoming"),
|
||||
amount_sats=100,
|
||||
created_at=old_time.isoformat(),
|
||||
),
|
||||
]
|
||||
with patch("lightning.ledger.get_transactions", return_value=mock_tx):
|
||||
# Query for today only
|
||||
earned, spent = _query_token_transactions(
|
||||
"kimi", now - timedelta(hours=1), now + timedelta(hours=1)
|
||||
)
|
||||
assert earned == 0 # Transaction was 2 days ago
|
||||
|
||||
|
||||
class TestGenerateScorecard:
|
||||
"""Test scorecard generation."""
|
||||
|
||||
def test_generate_scorecard_no_activity(self):
|
||||
"""Test scorecard generation for agent with no activity."""
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._collect_events_for_period", return_value=[]
|
||||
):
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._query_token_transactions",
|
||||
return_value=(0, 0),
|
||||
):
|
||||
scorecard = generate_scorecard("kimi", PeriodType.daily)
|
||||
|
||||
assert scorecard is not None
|
||||
assert scorecard.agent_id == "kimi"
|
||||
assert scorecard.period_type == PeriodType.daily
|
||||
assert len(scorecard.narrative_bullets) == 1
|
||||
assert "No recorded activity" in scorecard.narrative_bullets[0]
|
||||
|
||||
def test_generate_scorecard_with_activity(self):
|
||||
"""Test scorecard generation includes activity."""
|
||||
events = [
|
||||
Event(type="gitea.push", source="gitea", data={"actor": "kimi", "num_commits": 5}),
|
||||
]
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._collect_events_for_period", return_value=events
|
||||
):
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._query_token_transactions",
|
||||
return_value=(100, 20),
|
||||
):
|
||||
scorecard = generate_scorecard("kimi", PeriodType.daily)
|
||||
|
||||
assert scorecard is not None
|
||||
assert scorecard.metrics.commits == 5
|
||||
assert scorecard.metrics.tokens_earned == 100
|
||||
assert scorecard.metrics.tokens_spent == 20
|
||||
|
||||
|
||||
class TestGenerateAllScorecards:
|
||||
"""Test generating scorecards for all agents."""
|
||||
|
||||
def test_generates_for_all_tracked_agents(self):
|
||||
"""Test all tracked agents get scorecards even with no activity."""
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._collect_events_for_period", return_value=[]
|
||||
):
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._query_token_transactions",
|
||||
return_value=(0, 0),
|
||||
):
|
||||
scorecards = generate_all_scorecards(PeriodType.daily)
|
||||
|
||||
agent_ids = {s.agent_id for s in scorecards}
|
||||
expected = {"kimi", "claude", "gemini", "hermes", "manus"}
|
||||
assert expected.issubset(agent_ids)
|
||||
|
||||
def test_scorecards_sorted(self):
|
||||
"""Test scorecards are sorted by agent_id."""
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._collect_events_for_period", return_value=[]
|
||||
):
|
||||
with patch(
|
||||
"dashboard.services.scorecard_service._query_token_transactions",
|
||||
return_value=(0, 0),
|
||||
):
|
||||
scorecards = generate_all_scorecards(PeriodType.daily)
|
||||
|
||||
agent_ids = [s.agent_id for s in scorecards]
|
||||
assert agent_ids == sorted(agent_ids)
|
||||
|
||||
|
||||
class TestScorecardRoutes:
|
||||
"""Test scorecard API routes."""
|
||||
|
||||
def test_list_agents_endpoint(self, client):
|
||||
"""Test GET /scorecards/api/agents returns tracked agents."""
|
||||
response = client.get("/scorecards/api/agents")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "agents" in data
|
||||
assert "kimi" in data["agents"]
|
||||
assert "claude" in data["agents"]
|
||||
|
||||
def test_get_scorecard_endpoint(self, client):
|
||||
"""Test GET /scorecards/api/{agent_id} returns scorecard."""
|
||||
with patch("dashboard.routes.scorecards.generate_scorecard") as mock_generate:
|
||||
mock_generate.return_value = ScorecardSummary(
|
||||
agent_id="kimi",
|
||||
period_type=PeriodType.daily,
|
||||
period_start=datetime.now(UTC),
|
||||
period_end=datetime.now(UTC),
|
||||
metrics=AgentMetrics(agent_id="kimi"),
|
||||
narrative_bullets=["Test bullet"],
|
||||
patterns=[],
|
||||
)
|
||||
response = client.get("/scorecards/api/kimi?period=daily")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["agent_id"] == "kimi"
|
||||
assert data["period_type"] == "daily"
|
||||
|
||||
def test_get_scorecard_invalid_period(self, client):
|
||||
"""Test GET with invalid period returns 400."""
|
||||
response = client.get("/scorecards/api/kimi?period=invalid")
|
||||
assert response.status_code == 400
|
||||
assert "error" in response.json()
|
||||
|
||||
def test_get_all_scorecards_endpoint(self, client):
|
||||
"""Test GET /scorecards/api returns all scorecards."""
|
||||
with patch("dashboard.routes.scorecards.generate_all_scorecards") as mock_generate:
|
||||
mock_generate.return_value = [
|
||||
ScorecardSummary(
|
||||
agent_id="kimi",
|
||||
period_type=PeriodType.daily,
|
||||
period_start=datetime.now(UTC),
|
||||
period_end=datetime.now(UTC),
|
||||
metrics=AgentMetrics(agent_id="kimi"),
|
||||
narrative_bullets=[],
|
||||
patterns=[],
|
||||
),
|
||||
]
|
||||
response = client.get("/scorecards/api?period=daily")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["period"] == "daily"
|
||||
assert "scorecards" in data
|
||||
assert len(data["scorecards"]) == 1
|
||||
|
||||
def test_scorecards_page_renders(self, client):
|
||||
"""Test GET /scorecards returns HTML page."""
|
||||
response = client.get("/scorecards")
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers.get("content-type", "")
|
||||
assert "AGENT SCORECARDS" in response.text
|
||||
|
||||
def test_scorecard_panel_renders(self, client):
|
||||
"""Test GET /scorecards/panel/{agent_id} returns HTML."""
|
||||
with patch("dashboard.routes.scorecards.generate_scorecard") as mock_generate:
|
||||
mock_generate.return_value = ScorecardSummary(
|
||||
agent_id="kimi",
|
||||
period_type=PeriodType.daily,
|
||||
period_start=datetime.now(UTC),
|
||||
period_end=datetime.now(UTC),
|
||||
metrics=AgentMetrics(agent_id="kimi", commits=5),
|
||||
narrative_bullets=["Active across 5 commits this day."],
|
||||
patterns=["High activity"],
|
||||
)
|
||||
response = client.get("/scorecards/panel/kimi?period=daily")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers.get("content-type", "")
|
||||
assert "Kimi" in response.text
|
||||
|
||||
def test_all_panels_renders(self, client):
|
||||
"""Test GET /scorecards/all/panels returns HTML with all panels."""
|
||||
with patch("dashboard.routes.scorecards.generate_all_scorecards") as mock_generate:
|
||||
mock_generate.return_value = [
|
||||
ScorecardSummary(
|
||||
agent_id="kimi",
|
||||
period_type=PeriodType.daily,
|
||||
period_start=datetime.now(UTC),
|
||||
period_end=datetime.now(UTC),
|
||||
metrics=AgentMetrics(agent_id="kimi"),
|
||||
narrative_bullets=[],
|
||||
patterns=[],
|
||||
),
|
||||
]
|
||||
response = client.get("/scorecards/all/panels?period=daily")
|
||||
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers.get("content-type", "")
|
||||
267
tests/infrastructure/test_claude_quota.py
Normal file
267
tests/infrastructure/test_claude_quota.py
Normal file
@@ -0,0 +1,267 @@
|
||||
"""Tests for Claude Quota Monitor and Metabolic Protocol."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
from infrastructure.claude_quota import (
|
||||
MetabolicTier,
|
||||
QuotaMonitor,
|
||||
QuotaStatus,
|
||||
_time_remaining,
|
||||
get_quota_monitor,
|
||||
)
|
||||
|
||||
|
||||
def _make_status(five_hour: float = 0.0, seven_day: float = 0.0) -> QuotaStatus:
|
||||
"""Helper: build a QuotaStatus with given utilization values."""
|
||||
return QuotaStatus(
|
||||
five_hour_utilization=five_hour,
|
||||
five_hour_resets_at=None,
|
||||
seven_day_utilization=seven_day,
|
||||
seven_day_resets_at=None,
|
||||
raw_response={},
|
||||
fetched_at=datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class TestMetabolicTierThresholds:
|
||||
"""Test the three-tier metabolic protocol thresholds."""
|
||||
|
||||
def test_burst_when_five_hour_below_50pct(self):
|
||||
status = _make_status(five_hour=0.49, seven_day=0.10)
|
||||
assert status.recommended_tier == MetabolicTier.BURST
|
||||
|
||||
def test_burst_at_zero_utilization(self):
|
||||
status = _make_status(five_hour=0.0, seven_day=0.0)
|
||||
assert status.recommended_tier == MetabolicTier.BURST
|
||||
|
||||
def test_active_when_five_hour_at_50pct(self):
|
||||
status = _make_status(five_hour=0.50, seven_day=0.10)
|
||||
assert status.recommended_tier == MetabolicTier.ACTIVE
|
||||
|
||||
def test_active_when_five_hour_between_50_and_80pct(self):
|
||||
status = _make_status(five_hour=0.79, seven_day=0.10)
|
||||
assert status.recommended_tier == MetabolicTier.ACTIVE
|
||||
|
||||
def test_active_when_five_hour_at_80pct(self):
|
||||
# five_hour >= 0.80 but seven_day < 0.80 → ACTIVE (not RESTING)
|
||||
status = _make_status(five_hour=0.80, seven_day=0.50)
|
||||
assert status.recommended_tier == MetabolicTier.ACTIVE
|
||||
|
||||
def test_resting_when_seven_day_at_80pct(self):
|
||||
status = _make_status(five_hour=0.30, seven_day=0.80)
|
||||
assert status.recommended_tier == MetabolicTier.RESTING
|
||||
|
||||
def test_resting_when_seven_day_above_80pct(self):
|
||||
status = _make_status(five_hour=0.10, seven_day=0.95)
|
||||
assert status.recommended_tier == MetabolicTier.RESTING
|
||||
|
||||
def test_resting_when_both_critical(self):
|
||||
status = _make_status(five_hour=0.90, seven_day=0.90)
|
||||
assert status.recommended_tier == MetabolicTier.RESTING
|
||||
|
||||
def test_seven_day_takes_precedence_over_five_hour(self):
|
||||
# Weekly quota critical overrides whatever five-hour says
|
||||
status = _make_status(five_hour=0.10, seven_day=0.85)
|
||||
assert status.recommended_tier == MetabolicTier.RESTING
|
||||
|
||||
|
||||
class TestQuotaStatusProperties:
|
||||
"""Test QuotaStatus computed properties."""
|
||||
|
||||
def test_five_hour_pct(self):
|
||||
status = _make_status(five_hour=0.42)
|
||||
assert status.five_hour_pct == 42
|
||||
|
||||
def test_seven_day_pct(self):
|
||||
status = _make_status(seven_day=0.75)
|
||||
assert status.seven_day_pct == 75
|
||||
|
||||
def test_summary_contains_tier(self):
|
||||
status = _make_status(five_hour=0.20, seven_day=0.10)
|
||||
summary = status.summary()
|
||||
assert "burst" in summary
|
||||
assert "20%" in summary
|
||||
|
||||
def test_five_hour_resets_in_unknown_when_none(self):
|
||||
status = _make_status()
|
||||
assert status.five_hour_resets_in == "unknown"
|
||||
|
||||
def test_seven_day_resets_in_unknown_when_none(self):
|
||||
status = _make_status()
|
||||
assert status.seven_day_resets_in == "unknown"
|
||||
|
||||
|
||||
class TestTimeRemaining:
|
||||
"""Test _time_remaining helper."""
|
||||
|
||||
def test_none_returns_unknown(self):
|
||||
assert _time_remaining(None) == "unknown"
|
||||
|
||||
def test_empty_string_returns_unknown(self):
|
||||
assert _time_remaining("") == "unknown"
|
||||
|
||||
def test_past_time_returns_resetting_now(self):
|
||||
past = (datetime.now(UTC) - timedelta(hours=1)).isoformat()
|
||||
assert _time_remaining(past) == "resetting now"
|
||||
|
||||
def test_future_time_hours_and_minutes(self):
|
||||
future = (datetime.now(UTC) + timedelta(hours=2, minutes=15)).isoformat()
|
||||
result = _time_remaining(future)
|
||||
assert "2h" in result
|
||||
# Minutes may vary ±1 due to test execution time
|
||||
assert "m" in result
|
||||
|
||||
def test_future_time_minutes_only(self):
|
||||
future = (datetime.now(UTC) + timedelta(minutes=45)).isoformat()
|
||||
result = _time_remaining(future)
|
||||
assert "h" not in result
|
||||
# Minutes may vary ±1 due to test execution time
|
||||
assert "m" in result
|
||||
|
||||
def test_z_suffix_handled(self):
|
||||
future = (datetime.now(UTC) + timedelta(hours=1)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
result = _time_remaining(future)
|
||||
assert result != "unknown"
|
||||
|
||||
|
||||
class TestQuotaMonitorSelectModel:
|
||||
"""Test select_model metabolic routing."""
|
||||
|
||||
def test_no_quota_high_complexity_returns_14b(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._get_token = lambda: None
|
||||
assert monitor.select_model("high") == "qwen3:14b"
|
||||
|
||||
def test_no_quota_low_complexity_returns_8b(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._get_token = lambda: None
|
||||
assert monitor.select_model("low") == "qwen3:8b"
|
||||
|
||||
def test_burst_tier_high_complexity_returns_cloud(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.10, seven_day=0.10)
|
||||
monitor._cache_seconds = 9999
|
||||
result = monitor.select_model("high")
|
||||
assert result == "claude-sonnet-4-6"
|
||||
|
||||
def test_burst_tier_medium_complexity_returns_14b(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.10, seven_day=0.10)
|
||||
monitor._cache_seconds = 9999
|
||||
result = monitor.select_model("medium")
|
||||
assert result == "qwen3:14b"
|
||||
|
||||
def test_active_tier_returns_14b(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.65, seven_day=0.10)
|
||||
monitor._cache_seconds = 9999
|
||||
result = monitor.select_model("high")
|
||||
assert result == "qwen3:14b"
|
||||
|
||||
def test_resting_tier_returns_8b(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.10, seven_day=0.85)
|
||||
monitor._cache_seconds = 9999
|
||||
result = monitor.select_model("high")
|
||||
assert result == "qwen3:8b"
|
||||
|
||||
|
||||
class TestQuotaMonitorShouldUseCloud:
|
||||
"""Test should_use_cloud gate."""
|
||||
|
||||
def test_no_credentials_always_false(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._get_token = lambda: None
|
||||
assert monitor.should_use_cloud("critical") is False
|
||||
|
||||
def test_critical_task_allowed_when_under_95pct(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.10, seven_day=0.94)
|
||||
monitor._cache_seconds = 9999
|
||||
assert monitor.should_use_cloud("critical") is True
|
||||
|
||||
def test_critical_task_blocked_when_over_95pct(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.10, seven_day=0.96)
|
||||
monitor._cache_seconds = 9999
|
||||
assert monitor.should_use_cloud("critical") is False
|
||||
|
||||
def test_high_task_allowed_under_60pct(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.59, seven_day=0.10)
|
||||
monitor._cache_seconds = 9999
|
||||
assert monitor.should_use_cloud("high") is True
|
||||
|
||||
def test_high_task_blocked_at_60pct(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.60, seven_day=0.10)
|
||||
monitor._cache_seconds = 9999
|
||||
assert monitor.should_use_cloud("high") is False
|
||||
|
||||
def test_normal_task_allowed_under_30pct(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.29, seven_day=0.10)
|
||||
monitor._cache_seconds = 9999
|
||||
assert monitor.should_use_cloud("normal") is True
|
||||
|
||||
def test_normal_task_blocked_at_30pct(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.30, seven_day=0.10)
|
||||
monitor._cache_seconds = 9999
|
||||
assert monitor.should_use_cloud("normal") is False
|
||||
|
||||
def test_routine_task_always_false(self):
|
||||
monitor = QuotaMonitor()
|
||||
monitor._last_status = _make_status(five_hour=0.0, seven_day=0.0)
|
||||
monitor._cache_seconds = 9999
|
||||
assert monitor.should_use_cloud("routine") is False
|
||||
|
||||
|
||||
class TestQuotaMonitorCaching:
|
||||
"""Test 30-second TTL cache."""
|
||||
|
||||
def test_cached_result_returned_within_ttl(self):
|
||||
monitor = QuotaMonitor()
|
||||
fresh_status = _make_status(five_hour=0.10)
|
||||
monitor._last_status = fresh_status
|
||||
monitor._cache_seconds = 30
|
||||
|
||||
# Should NOT re-fetch — returns cached
|
||||
with patch.object(monitor, "_get_token", return_value="tok") as mock_tok:
|
||||
result = monitor.check()
|
||||
mock_tok.assert_not_called()
|
||||
|
||||
assert result is fresh_status
|
||||
|
||||
def test_stale_cache_triggers_fetch(self):
|
||||
monitor = QuotaMonitor()
|
||||
old_time = datetime.now(UTC) - timedelta(seconds=60)
|
||||
stale_status = QuotaStatus(
|
||||
five_hour_utilization=0.10,
|
||||
five_hour_resets_at=None,
|
||||
seven_day_utilization=0.10,
|
||||
seven_day_resets_at=None,
|
||||
raw_response={},
|
||||
fetched_at=old_time,
|
||||
)
|
||||
monitor._last_status = stale_status
|
||||
|
||||
# Token unavailable → returns None (triggers re-fetch path)
|
||||
with patch.object(monitor, "_get_token", return_value=None):
|
||||
result = monitor.check()
|
||||
|
||||
assert result is None # No credentials after cache miss
|
||||
|
||||
|
||||
class TestGetQuotaMonitorSingleton:
|
||||
"""Test module-level singleton."""
|
||||
|
||||
def test_returns_same_instance(self):
|
||||
m1 = get_quota_monitor()
|
||||
m2 = get_quota_monitor()
|
||||
assert m1 is m2
|
||||
|
||||
def test_returns_quota_monitor_instance(self):
|
||||
monitor = get_quota_monitor()
|
||||
assert isinstance(monitor, QuotaMonitor)
|
||||
427
tests/infrastructure/test_db_pool.py
Normal file
427
tests/infrastructure/test_db_pool.py
Normal file
@@ -0,0 +1,427 @@
|
||||
"""Tests for infrastructure.db_pool module."""
|
||||
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from infrastructure.db_pool import ConnectionPool
|
||||
|
||||
|
||||
class TestConnectionPoolInit:
|
||||
"""Test ConnectionPool initialization."""
|
||||
|
||||
def test_init_with_string_path(self, tmp_path):
|
||||
"""Pool can be initialized with a string path."""
|
||||
db_path = str(tmp_path / "test.db")
|
||||
pool = ConnectionPool(db_path)
|
||||
assert pool._db_path == Path(db_path)
|
||||
|
||||
def test_init_with_path_object(self, tmp_path):
|
||||
"""Pool can be initialized with a Path object."""
|
||||
db_path = tmp_path / "test.db"
|
||||
pool = ConnectionPool(db_path)
|
||||
assert pool._db_path == db_path
|
||||
|
||||
def test_init_creates_thread_local(self, tmp_path):
|
||||
"""Pool initializes thread-local storage."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
assert hasattr(pool, "_local")
|
||||
assert isinstance(pool._local, threading.local)
|
||||
|
||||
|
||||
class TestGetConnection:
|
||||
"""Test get_connection() method."""
|
||||
|
||||
def test_get_connection_returns_valid_sqlite3_connection(self, tmp_path):
|
||||
"""get_connection() returns a valid sqlite3 connection."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn = pool.get_connection()
|
||||
assert isinstance(conn, sqlite3.Connection)
|
||||
# Verify it's a working connection
|
||||
cursor = conn.execute("SELECT 1")
|
||||
assert cursor.fetchone()[0] == 1
|
||||
|
||||
def test_get_connection_creates_db_file(self, tmp_path):
|
||||
"""get_connection() creates the database file if it doesn't exist."""
|
||||
db_path = tmp_path / "subdir" / "test.db"
|
||||
assert not db_path.exists()
|
||||
pool = ConnectionPool(db_path)
|
||||
pool.get_connection()
|
||||
assert db_path.exists()
|
||||
|
||||
def test_get_connection_sets_row_factory(self, tmp_path):
|
||||
"""get_connection() sets row_factory to sqlite3.Row."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn = pool.get_connection()
|
||||
assert conn.row_factory is sqlite3.Row
|
||||
|
||||
def test_multiple_calls_same_thread_reuse_connection(self, tmp_path):
|
||||
"""Multiple calls from same thread reuse the same connection."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn1 = pool.get_connection()
|
||||
conn2 = pool.get_connection()
|
||||
assert conn1 is conn2
|
||||
|
||||
def test_different_threads_get_different_connections(self, tmp_path):
|
||||
"""Different threads get different connections."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
connections = []
|
||||
|
||||
def get_conn():
|
||||
connections.append(pool.get_connection())
|
||||
|
||||
t1 = threading.Thread(target=get_conn)
|
||||
t2 = threading.Thread(target=get_conn)
|
||||
t1.start()
|
||||
t2.start()
|
||||
t1.join()
|
||||
t2.join()
|
||||
|
||||
assert len(connections) == 2
|
||||
assert connections[0] is not connections[1]
|
||||
|
||||
|
||||
class TestCloseConnection:
|
||||
"""Test close_connection() method."""
|
||||
|
||||
def test_close_connection_closes_sqlite_connection(self, tmp_path):
|
||||
"""close_connection() closes the underlying sqlite connection."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn = pool.get_connection()
|
||||
pool.close_connection()
|
||||
# Connection should be closed
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
def test_close_connection_cleans_up_thread_local(self, tmp_path):
|
||||
"""close_connection() cleans up thread-local storage."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
pool.get_connection()
|
||||
assert hasattr(pool._local, "conn")
|
||||
assert pool._local.conn is not None
|
||||
|
||||
pool.close_connection()
|
||||
|
||||
# Should either not have the attr or it should be None
|
||||
assert not hasattr(pool._local, "conn") or pool._local.conn is None
|
||||
|
||||
def test_close_connection_without_getting_connection_is_safe(self, tmp_path):
|
||||
"""close_connection() is safe to call even without getting a connection first."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
# Should not raise
|
||||
pool.close_connection()
|
||||
|
||||
def test_close_connection_multiple_calls_is_safe(self, tmp_path):
|
||||
"""close_connection() can be called multiple times safely."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
pool.get_connection()
|
||||
pool.close_connection()
|
||||
# Should not raise
|
||||
pool.close_connection()
|
||||
|
||||
|
||||
class TestContextManager:
|
||||
"""Test the connection() context manager."""
|
||||
|
||||
def test_connection_yields_valid_connection(self, tmp_path):
|
||||
"""connection() context manager yields a valid sqlite3 connection."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
with pool.connection() as conn:
|
||||
assert isinstance(conn, sqlite3.Connection)
|
||||
cursor = conn.execute("SELECT 42")
|
||||
assert cursor.fetchone()[0] == 42
|
||||
|
||||
def test_connection_closes_on_exit(self, tmp_path):
|
||||
"""connection() context manager closes connection on exit."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
with pool.connection() as conn:
|
||||
pass
|
||||
# Connection should be closed after context exit
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
def test_connection_closes_on_exception(self, tmp_path):
|
||||
"""connection() context manager closes connection even on exception."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn_ref = None
|
||||
try:
|
||||
with pool.connection() as conn:
|
||||
conn_ref = conn
|
||||
raise ValueError("Test exception")
|
||||
except ValueError:
|
||||
pass
|
||||
# Connection should still be closed
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
conn_ref.execute("SELECT 1")
|
||||
|
||||
def test_connection_context_manager_is_reusable(self, tmp_path):
|
||||
"""connection() context manager can be used multiple times."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
|
||||
with pool.connection() as conn1:
|
||||
result1 = conn1.execute("SELECT 1").fetchone()[0]
|
||||
|
||||
with pool.connection() as conn2:
|
||||
result2 = conn2.execute("SELECT 2").fetchone()[0]
|
||||
|
||||
assert result1 == 1
|
||||
assert result2 == 2
|
||||
|
||||
|
||||
class TestThreadSafety:
|
||||
"""Test thread-safety of the connection pool."""
|
||||
|
||||
def test_concurrent_access(self, tmp_path):
|
||||
"""Multiple threads can use the pool concurrently."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
def worker(worker_id):
|
||||
try:
|
||||
with pool.connection() as conn:
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS test (id INTEGER)")
|
||||
conn.execute("INSERT INTO test VALUES (?)", (worker_id,))
|
||||
conn.commit()
|
||||
time.sleep(0.01) # Small delay to increase contention
|
||||
results.append(worker_id)
|
||||
except Exception as e:
|
||||
errors.append(e)
|
||||
|
||||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(5)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert len(errors) == 0, f"Errors occurred: {errors}"
|
||||
assert len(results) == 5
|
||||
|
||||
def test_thread_isolation(self, tmp_path):
|
||||
"""Each thread has isolated connections (verified by thread-local data)."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
results = []
|
||||
|
||||
def worker(worker_id):
|
||||
# Get connection and write worker-specific data
|
||||
conn = pool.get_connection()
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS isolation_test (thread_id INTEGER)")
|
||||
conn.execute("DELETE FROM isolation_test") # Clear previous data
|
||||
conn.execute("INSERT INTO isolation_test VALUES (?)", (worker_id,))
|
||||
conn.commit()
|
||||
# Read back the data
|
||||
result = conn.execute("SELECT thread_id FROM isolation_test").fetchone()[0]
|
||||
results.append((worker_id, result))
|
||||
pool.close_connection()
|
||||
|
||||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(3)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
# Each thread should have written and read its own ID
|
||||
assert len(results) == 3
|
||||
for worker_id, read_id in results:
|
||||
assert worker_id == read_id, f"Thread {worker_id} read {read_id} instead"
|
||||
|
||||
|
||||
class TestCloseAll:
|
||||
"""Test close_all() method."""
|
||||
|
||||
def test_close_all_closes_current_thread_connection(self, tmp_path):
|
||||
"""close_all() closes the connection for the current thread."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn = pool.get_connection()
|
||||
pool.close_all()
|
||||
# Connection should be closed
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
|
||||
class TestConnectionLeaks:
|
||||
"""Test that connections do not leak."""
|
||||
|
||||
def test_get_connection_after_close_returns_fresh_connection(self, tmp_path):
|
||||
"""After close, get_connection() returns a new working connection."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn1 = pool.get_connection()
|
||||
pool.close_connection()
|
||||
|
||||
conn2 = pool.get_connection()
|
||||
assert conn2 is not conn1
|
||||
# New connection must be usable
|
||||
cursor = conn2.execute("SELECT 1")
|
||||
assert cursor.fetchone()[0] == 1
|
||||
pool.close_connection()
|
||||
|
||||
def test_context_manager_does_not_leak_connection(self, tmp_path):
|
||||
"""After context manager exit, thread-local conn is cleared."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
with pool.connection():
|
||||
pass
|
||||
# Thread-local should be cleaned up
|
||||
assert pool._local.conn is None
|
||||
|
||||
def test_context_manager_exception_does_not_leak_connection(self, tmp_path):
|
||||
"""Connection is cleaned up even when an exception occurs."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
try:
|
||||
with pool.connection():
|
||||
raise RuntimeError("boom")
|
||||
except RuntimeError:
|
||||
pass
|
||||
assert pool._local.conn is None
|
||||
|
||||
def test_threads_do_not_leak_into_each_other(self, tmp_path):
|
||||
"""A connection opened in one thread is invisible to another."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
# Open a connection on main thread
|
||||
pool.get_connection()
|
||||
|
||||
visible_from_other_thread = []
|
||||
|
||||
def check():
|
||||
has_conn = hasattr(pool._local, "conn") and pool._local.conn is not None
|
||||
visible_from_other_thread.append(has_conn)
|
||||
|
||||
t = threading.Thread(target=check)
|
||||
t.start()
|
||||
t.join()
|
||||
|
||||
assert visible_from_other_thread == [False]
|
||||
pool.close_connection()
|
||||
|
||||
def test_repeated_open_close_cycles(self, tmp_path):
|
||||
"""Repeated open/close cycles do not accumulate leaked connections."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
for _ in range(50):
|
||||
with pool.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
# After each cycle, connection should be cleaned up
|
||||
assert pool._local.conn is None
|
||||
|
||||
|
||||
class TestPragmaApplication:
|
||||
"""Test that SQLite pragmas can be applied and persist on pooled connections.
|
||||
|
||||
The codebase uses WAL journal mode and busy_timeout pragmas on connections
|
||||
obtained from the pool. These tests verify that pattern works correctly.
|
||||
"""
|
||||
|
||||
def test_wal_journal_mode_persists(self, tmp_path):
|
||||
"""WAL journal mode set on a pooled connection persists for its lifetime."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn = pool.get_connection()
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
|
||||
assert mode == "wal"
|
||||
|
||||
# Same connection should retain the pragma
|
||||
same_conn = pool.get_connection()
|
||||
mode2 = same_conn.execute("PRAGMA journal_mode").fetchone()[0]
|
||||
assert mode2 == "wal"
|
||||
pool.close_connection()
|
||||
|
||||
def test_busy_timeout_persists(self, tmp_path):
|
||||
"""busy_timeout pragma set on a pooled connection persists."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn = pool.get_connection()
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
timeout = conn.execute("PRAGMA busy_timeout").fetchone()[0]
|
||||
assert timeout == 5000
|
||||
pool.close_connection()
|
||||
|
||||
def test_pragmas_apply_per_connection(self, tmp_path):
|
||||
"""Pragmas set on one thread's connection are independent of another's."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn_main = pool.get_connection()
|
||||
conn_main.execute("PRAGMA cache_size=9999")
|
||||
|
||||
other_cache = []
|
||||
|
||||
def check_pragma():
|
||||
conn = pool.get_connection()
|
||||
# Don't set cache_size — should get the default, not 9999
|
||||
val = conn.execute("PRAGMA cache_size").fetchone()[0]
|
||||
other_cache.append(val)
|
||||
pool.close_connection()
|
||||
|
||||
t = threading.Thread(target=check_pragma)
|
||||
t.start()
|
||||
t.join()
|
||||
|
||||
# Other thread's connection should NOT have our custom cache_size
|
||||
assert other_cache[0] != 9999
|
||||
pool.close_connection()
|
||||
|
||||
def test_session_pragma_resets_on_new_connection(self, tmp_path):
|
||||
"""Session-level pragmas (cache_size) reset on a new connection."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
conn1 = pool.get_connection()
|
||||
conn1.execute("PRAGMA cache_size=9999")
|
||||
assert conn1.execute("PRAGMA cache_size").fetchone()[0] == 9999
|
||||
pool.close_connection()
|
||||
|
||||
conn2 = pool.get_connection()
|
||||
cache = conn2.execute("PRAGMA cache_size").fetchone()[0]
|
||||
# New connection gets default cache_size, not the previous value
|
||||
assert cache != 9999
|
||||
pool.close_connection()
|
||||
|
||||
def test_wal_mode_via_context_manager(self, tmp_path):
|
||||
"""WAL mode can be set within a context manager block."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
with pool.connection() as conn:
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
|
||||
assert mode == "wal"
|
||||
|
||||
|
||||
class TestIntegration:
|
||||
"""Integration tests for real-world usage patterns."""
|
||||
|
||||
def test_basic_crud_operations(self, tmp_path):
|
||||
"""Can perform basic CRUD operations through the pool."""
|
||||
pool = ConnectionPool(tmp_path / "test.db")
|
||||
|
||||
with pool.connection() as conn:
|
||||
# Create table
|
||||
conn.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)")
|
||||
# Insert
|
||||
conn.execute("INSERT INTO users (name) VALUES (?)", ("Alice",))
|
||||
conn.execute("INSERT INTO users (name) VALUES (?)", ("Bob",))
|
||||
conn.commit()
|
||||
# Query
|
||||
cursor = conn.execute("SELECT * FROM users ORDER BY id")
|
||||
rows = cursor.fetchall()
|
||||
assert len(rows) == 2
|
||||
assert rows[0]["name"] == "Alice"
|
||||
assert rows[1]["name"] == "Bob"
|
||||
|
||||
def test_multiple_pools_different_databases(self, tmp_path):
|
||||
"""Multiple pools can manage different databases independently."""
|
||||
pool1 = ConnectionPool(tmp_path / "db1.db")
|
||||
pool2 = ConnectionPool(tmp_path / "db2.db")
|
||||
|
||||
with pool1.connection() as conn1:
|
||||
conn1.execute("CREATE TABLE test (val INTEGER)")
|
||||
conn1.execute("INSERT INTO test VALUES (1)")
|
||||
conn1.commit()
|
||||
|
||||
with pool2.connection() as conn2:
|
||||
conn2.execute("CREATE TABLE test (val INTEGER)")
|
||||
conn2.execute("INSERT INTO test VALUES (2)")
|
||||
conn2.commit()
|
||||
|
||||
# Verify isolation
|
||||
with pool1.connection() as conn1:
|
||||
result = conn1.execute("SELECT val FROM test").fetchone()[0]
|
||||
assert result == 1
|
||||
|
||||
with pool2.connection() as conn2:
|
||||
result = conn2.execute("SELECT val FROM test").fetchone()[0]
|
||||
assert result == 2
|
||||
332
tests/infrastructure/test_moderation.py
Normal file
332
tests/infrastructure/test_moderation.py
Normal file
@@ -0,0 +1,332 @@
|
||||
"""Tests for the content moderation pipeline."""
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from infrastructure.guards.moderation import (
|
||||
ContentModerator,
|
||||
GameProfile,
|
||||
ModerationResult,
|
||||
ModerationVerdict,
|
||||
ViolationCategory,
|
||||
_parse_guard_category,
|
||||
get_moderator,
|
||||
)
|
||||
|
||||
# ── Unit tests for data types ────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestModerationResult:
|
||||
"""Test ModerationResult dataclass."""
|
||||
|
||||
def test_passed_property_true(self):
|
||||
result = ModerationResult(verdict=ModerationVerdict.PASS, blocked=False)
|
||||
assert result.passed is True
|
||||
|
||||
def test_passed_property_false(self):
|
||||
result = ModerationResult(verdict=ModerationVerdict.FAIL, blocked=True)
|
||||
assert result.passed is False
|
||||
|
||||
def test_default_values(self):
|
||||
result = ModerationResult(verdict=ModerationVerdict.PASS, blocked=False)
|
||||
assert result.category == ViolationCategory.NONE
|
||||
assert result.confidence == 0.0
|
||||
assert result.fallback == ""
|
||||
assert result.reason == ""
|
||||
|
||||
|
||||
class TestGameProfile:
|
||||
"""Test GameProfile dataclass."""
|
||||
|
||||
def test_default_values(self):
|
||||
profile = GameProfile(game_id="test", display_name="Test Game")
|
||||
assert profile.vocabulary_whitelist == []
|
||||
assert profile.threshold == 0.8
|
||||
assert profile.fallbacks == {}
|
||||
|
||||
def test_morrowind_profile(self):
|
||||
profile = GameProfile(
|
||||
game_id="morrowind",
|
||||
display_name="Morrowind",
|
||||
vocabulary_whitelist=["Skooma", "slave"],
|
||||
threshold=0.85,
|
||||
)
|
||||
assert "Skooma" in profile.vocabulary_whitelist
|
||||
assert profile.threshold == 0.85
|
||||
|
||||
|
||||
class TestParseGuardCategory:
|
||||
"""Test Llama Guard category parsing."""
|
||||
|
||||
def test_hate_speech(self):
|
||||
assert _parse_guard_category("S1: Hate speech") == ViolationCategory.HATE_SPEECH
|
||||
|
||||
def test_violence(self):
|
||||
assert _parse_guard_category("S2: Violence") == ViolationCategory.VIOLENCE_GLORIFICATION
|
||||
|
||||
def test_sexual_content(self):
|
||||
assert _parse_guard_category("S3: Sexual content") == ViolationCategory.SEXUAL_CONTENT
|
||||
|
||||
def test_self_harm(self):
|
||||
assert _parse_guard_category("S4: Self-harm") == ViolationCategory.SELF_HARM
|
||||
|
||||
def test_dangerous(self):
|
||||
assert _parse_guard_category("S5: Dangerous activity") == ViolationCategory.REAL_WORLD_HARM
|
||||
|
||||
def test_unknown_category(self):
|
||||
assert _parse_guard_category("S99: Unknown") == ViolationCategory.NONE
|
||||
|
||||
|
||||
# ── ContentModerator tests ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestContentModerator:
|
||||
"""Test the content moderation pipeline."""
|
||||
|
||||
def _make_moderator(self, **kwargs) -> ContentModerator:
|
||||
"""Create a moderator with test defaults."""
|
||||
profiles = {
|
||||
"morrowind": GameProfile(
|
||||
game_id="morrowind",
|
||||
display_name="Morrowind",
|
||||
vocabulary_whitelist=["Skooma", "Moon Sugar", "slave", "Morag Tong"],
|
||||
context_prompt="Narrate Morrowind gameplay.",
|
||||
threshold=0.85,
|
||||
fallbacks={
|
||||
"combat": "The battle continues.",
|
||||
"default": "The adventure continues.",
|
||||
},
|
||||
),
|
||||
"default": GameProfile(
|
||||
game_id="default",
|
||||
display_name="Generic",
|
||||
vocabulary_whitelist=[],
|
||||
context_prompt="Narrate gameplay.",
|
||||
threshold=0.8,
|
||||
fallbacks={"default": "Gameplay continues."},
|
||||
),
|
||||
}
|
||||
return ContentModerator(profiles=profiles, **kwargs)
|
||||
|
||||
def test_get_profile_known_game(self):
|
||||
mod = self._make_moderator()
|
||||
profile = mod.get_profile("morrowind")
|
||||
assert profile.game_id == "morrowind"
|
||||
|
||||
def test_get_profile_unknown_game_falls_back(self):
|
||||
mod = self._make_moderator()
|
||||
profile = mod.get_profile("unknown_game")
|
||||
assert profile.game_id == "default"
|
||||
|
||||
def test_get_context_prompt(self):
|
||||
mod = self._make_moderator()
|
||||
prompt = mod.get_context_prompt("morrowind")
|
||||
assert "Morrowind" in prompt
|
||||
|
||||
def test_register_profile(self):
|
||||
mod = self._make_moderator()
|
||||
new_profile = GameProfile(game_id="skyrim", display_name="Skyrim")
|
||||
mod.register_profile(new_profile)
|
||||
assert mod.get_profile("skyrim").game_id == "skyrim"
|
||||
|
||||
def test_whitelist_replaces_game_terms(self):
|
||||
mod = self._make_moderator()
|
||||
profile = mod.get_profile("morrowind")
|
||||
cleaned = mod._apply_whitelist(
|
||||
"The merchant sells Skooma and Moon Sugar in the slave market.",
|
||||
profile,
|
||||
)
|
||||
assert "Skooma" not in cleaned
|
||||
assert "Moon Sugar" not in cleaned
|
||||
assert "slave" not in cleaned
|
||||
assert "[GAME_TERM]" in cleaned
|
||||
|
||||
def test_whitelist_case_insensitive(self):
|
||||
mod = self._make_moderator()
|
||||
profile = mod.get_profile("morrowind")
|
||||
cleaned = mod._apply_whitelist("skooma and SKOOMA", profile)
|
||||
assert "skooma" not in cleaned
|
||||
assert "SKOOMA" not in cleaned
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_safe_content_passes(self):
|
||||
"""Safe content should pass moderation."""
|
||||
mod = self._make_moderator()
|
||||
with patch.object(mod, "_is_guard_available", new_callable=AsyncMock, return_value=False):
|
||||
result = await mod.check("The player walks through the town.", game="morrowind")
|
||||
assert result.passed
|
||||
assert not result.blocked
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_blocked_content_has_fallback(self):
|
||||
"""Blocked content should include scene-appropriate fallback."""
|
||||
mod = self._make_moderator()
|
||||
# Force a block via regex by using real-world harm language
|
||||
text = "In real life you should attack and hurt people"
|
||||
with patch.object(mod, "_is_guard_available", new_callable=AsyncMock, return_value=False):
|
||||
result = await mod.check(text, game="morrowind", scene_type="combat")
|
||||
assert result.blocked
|
||||
assert result.fallback == "The battle continues."
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_check_with_moderation_disabled(self):
|
||||
"""When moderation is disabled, everything passes."""
|
||||
mod = self._make_moderator()
|
||||
with patch("infrastructure.guards.moderation.settings") as mock_settings:
|
||||
mock_settings.moderation_enabled = False
|
||||
mock_settings.moderation_guard_model = "llama-guard3:1b"
|
||||
mock_settings.normalized_ollama_url = "http://127.0.0.1:11434"
|
||||
result = await mod.check("anything goes here")
|
||||
assert result.passed
|
||||
assert result.layer == "disabled"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_threshold_below_allows_content(self):
|
||||
"""Content flagged below threshold should pass through (Layer 3)."""
|
||||
mod = self._make_moderator()
|
||||
# Mock the guard to return a low-confidence flag
|
||||
low_conf_result = ModerationResult(
|
||||
verdict=ModerationVerdict.FAIL,
|
||||
blocked=True,
|
||||
confidence=0.5, # Below morrowind threshold of 0.85
|
||||
layer="llama_guard",
|
||||
category=ViolationCategory.VIOLENCE_GLORIFICATION,
|
||||
)
|
||||
with patch.object(mod, "_run_guard", new_callable=AsyncMock, return_value=low_conf_result):
|
||||
result = await mod.check("sword fight scene", game="morrowind")
|
||||
assert result.passed
|
||||
assert not result.blocked
|
||||
assert result.layer == "threshold"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_threshold_above_blocks_content(self):
|
||||
"""Content flagged above threshold should remain blocked."""
|
||||
mod = self._make_moderator()
|
||||
high_conf_result = ModerationResult(
|
||||
verdict=ModerationVerdict.FAIL,
|
||||
blocked=True,
|
||||
confidence=0.95, # Above morrowind threshold of 0.85
|
||||
layer="llama_guard",
|
||||
category=ViolationCategory.REAL_WORLD_HARM,
|
||||
)
|
||||
with patch.object(mod, "_run_guard", new_callable=AsyncMock, return_value=high_conf_result):
|
||||
result = await mod.check("harmful content", game="morrowind")
|
||||
assert result.blocked
|
||||
|
||||
def test_regex_catches_real_world_harm(self):
|
||||
"""Regex fallback should catch obvious real-world harm patterns."""
|
||||
mod = self._make_moderator()
|
||||
result = mod._check_with_regex("you should actually harm real people")
|
||||
assert result.blocked
|
||||
assert result.category == ViolationCategory.REAL_WORLD_HARM
|
||||
assert result.layer == "regex_fallback"
|
||||
|
||||
def test_regex_passes_game_violence(self):
|
||||
"""Regex should not flag in-game violence narration."""
|
||||
mod = self._make_moderator()
|
||||
result = mod._check_with_regex("The warrior slays the dragon with a mighty blow.")
|
||||
assert result.passed
|
||||
|
||||
def test_regex_passes_normal_narration(self):
|
||||
"""Normal narration should pass regex checks."""
|
||||
mod = self._make_moderator()
|
||||
result = mod._check_with_regex(
|
||||
"The Nerevarine enters the city of Balmora and speaks with Caius Cosades."
|
||||
)
|
||||
assert result.passed
|
||||
|
||||
def test_metrics_tracking(self):
|
||||
"""Metrics should track checks accurately."""
|
||||
mod = self._make_moderator()
|
||||
assert mod.get_metrics()["total_checks"] == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metrics_increment_after_check(self):
|
||||
"""Metrics should increment after moderation checks."""
|
||||
mod = self._make_moderator()
|
||||
with patch.object(mod, "_is_guard_available", new_callable=AsyncMock, return_value=False):
|
||||
await mod.check("safe text", game="default")
|
||||
metrics = mod.get_metrics()
|
||||
assert metrics["total_checks"] == 1
|
||||
assert metrics["passed"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guard_fallback_on_error(self):
|
||||
"""Should fall back to regex when guard model errors."""
|
||||
mod = self._make_moderator()
|
||||
with (
|
||||
patch.object(mod, "_is_guard_available", new_callable=AsyncMock, return_value=True),
|
||||
patch.object(
|
||||
mod,
|
||||
"_check_with_guard",
|
||||
new_callable=AsyncMock,
|
||||
side_effect=RuntimeError("timeout"),
|
||||
),
|
||||
):
|
||||
result = await mod.check("safe text", game="default")
|
||||
# Should fall back to regex and pass
|
||||
assert result.passed
|
||||
assert result.layer == "regex_fallback"
|
||||
|
||||
|
||||
class TestGetModerator:
|
||||
"""Test the singleton accessor."""
|
||||
|
||||
def test_returns_same_instance(self):
|
||||
"""get_moderator should return the same instance."""
|
||||
# Reset the global to test fresh
|
||||
import infrastructure.guards.moderation as mod_module
|
||||
|
||||
mod_module._moderator = None
|
||||
m1 = get_moderator()
|
||||
m2 = get_moderator()
|
||||
assert m1 is m2
|
||||
# Clean up
|
||||
mod_module._moderator = None
|
||||
|
||||
|
||||
# ── Profile loader tests ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestProfileLoader:
|
||||
"""Test YAML profile loading."""
|
||||
|
||||
def test_load_missing_file_returns_empty(self, tmp_path):
|
||||
from infrastructure.guards.profiles import load_profiles
|
||||
|
||||
result = load_profiles(tmp_path / "nonexistent.yaml")
|
||||
assert result == {}
|
||||
|
||||
def test_load_valid_config(self, tmp_path):
|
||||
import yaml
|
||||
|
||||
from infrastructure.guards.profiles import load_profiles
|
||||
|
||||
config = {
|
||||
"profiles": {
|
||||
"testgame": {
|
||||
"display_name": "Test Game",
|
||||
"threshold": 0.9,
|
||||
"vocabulary_whitelist": ["sword", "potion"],
|
||||
"context_prompt": "Narrate test game.",
|
||||
"fallbacks": {"default": "Game continues."},
|
||||
}
|
||||
}
|
||||
}
|
||||
config_file = tmp_path / "moderation.yaml"
|
||||
config_file.write_text(yaml.dump(config))
|
||||
|
||||
profiles = load_profiles(config_file)
|
||||
assert "testgame" in profiles
|
||||
assert profiles["testgame"].threshold == 0.9
|
||||
assert "sword" in profiles["testgame"].vocabulary_whitelist
|
||||
|
||||
def test_load_malformed_yaml_returns_empty(self, tmp_path):
|
||||
from infrastructure.guards.profiles import load_profiles
|
||||
|
||||
config_file = tmp_path / "moderation.yaml"
|
||||
config_file.write_text("{{{{invalid yaml")
|
||||
|
||||
result = load_profiles(config_file)
|
||||
assert result == {}
|
||||
@@ -489,6 +489,197 @@ class TestProviderAvailabilityCheck:
|
||||
|
||||
assert router._check_provider_available(provider) is False
|
||||
|
||||
def test_check_vllm_mlx_without_requests(self):
|
||||
"""Test vllm-mlx returns True when requests not available (fallback)."""
|
||||
router = CascadeRouter(config_path=Path("/nonexistent"))
|
||||
|
||||
provider = Provider(
|
||||
name="vllm-mlx-local",
|
||||
type="vllm_mlx",
|
||||
enabled=True,
|
||||
priority=2,
|
||||
base_url="http://localhost:8000/v1",
|
||||
)
|
||||
|
||||
import infrastructure.router.cascade as cascade_module
|
||||
|
||||
old_requests = cascade_module.requests
|
||||
cascade_module.requests = None
|
||||
try:
|
||||
assert router._check_provider_available(provider) is True
|
||||
finally:
|
||||
cascade_module.requests = old_requests
|
||||
|
||||
def test_check_vllm_mlx_server_healthy(self):
|
||||
"""Test vllm-mlx when health check succeeds."""
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
router = CascadeRouter(config_path=Path("/nonexistent"))
|
||||
|
||||
provider = Provider(
|
||||
name="vllm-mlx-local",
|
||||
type="vllm_mlx",
|
||||
enabled=True,
|
||||
priority=2,
|
||||
base_url="http://localhost:8000/v1",
|
||||
)
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
|
||||
with patch("infrastructure.router.cascade.requests") as mock_requests:
|
||||
mock_requests.get.return_value = mock_response
|
||||
result = router._check_provider_available(provider)
|
||||
|
||||
assert result is True
|
||||
mock_requests.get.assert_called_once_with("http://localhost:8000/health", timeout=5)
|
||||
|
||||
def test_check_vllm_mlx_server_down(self):
|
||||
"""Test vllm-mlx when server is not running."""
|
||||
from unittest.mock import patch
|
||||
|
||||
router = CascadeRouter(config_path=Path("/nonexistent"))
|
||||
|
||||
provider = Provider(
|
||||
name="vllm-mlx-local",
|
||||
type="vllm_mlx",
|
||||
enabled=True,
|
||||
priority=2,
|
||||
base_url="http://localhost:8000/v1",
|
||||
)
|
||||
|
||||
with patch("infrastructure.router.cascade.requests") as mock_requests:
|
||||
mock_requests.get.side_effect = ConnectionRefusedError("Connection refused")
|
||||
result = router._check_provider_available(provider)
|
||||
|
||||
assert result is False
|
||||
|
||||
def test_check_vllm_mlx_default_url(self):
|
||||
"""Test vllm-mlx uses default localhost:8000 when no URL configured."""
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
router = CascadeRouter(config_path=Path("/nonexistent"))
|
||||
|
||||
provider = Provider(
|
||||
name="vllm-mlx-local",
|
||||
type="vllm_mlx",
|
||||
enabled=True,
|
||||
priority=2,
|
||||
)
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
|
||||
with patch("infrastructure.router.cascade.requests") as mock_requests:
|
||||
mock_requests.get.return_value = mock_response
|
||||
router._check_provider_available(provider)
|
||||
|
||||
mock_requests.get.assert_called_once_with("http://localhost:8000/health", timeout=5)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestVllmMlxProvider:
|
||||
"""Test vllm-mlx provider integration."""
|
||||
|
||||
async def test_complete_with_vllm_mlx(self):
|
||||
"""Test successful completion via vllm-mlx."""
|
||||
router = CascadeRouter(config_path=Path("/nonexistent"))
|
||||
|
||||
provider = Provider(
|
||||
name="vllm-mlx-local",
|
||||
type="vllm_mlx",
|
||||
enabled=True,
|
||||
priority=2,
|
||||
base_url="http://localhost:8000/v1",
|
||||
models=[{"name": "Qwen/Qwen2.5-14B-Instruct-MLX", "default": True}],
|
||||
)
|
||||
router.providers = [provider]
|
||||
|
||||
with patch.object(router, "_call_vllm_mlx") as mock_call:
|
||||
mock_call.return_value = {
|
||||
"content": "MLX response",
|
||||
"model": "Qwen/Qwen2.5-14B-Instruct-MLX",
|
||||
}
|
||||
|
||||
result = await router.complete(
|
||||
messages=[{"role": "user", "content": "Hi"}],
|
||||
)
|
||||
|
||||
assert result["content"] == "MLX response"
|
||||
assert result["provider"] == "vllm-mlx-local"
|
||||
assert result["model"] == "Qwen/Qwen2.5-14B-Instruct-MLX"
|
||||
|
||||
async def test_vllm_mlx_base_url_normalization(self):
|
||||
"""Test _call_vllm_mlx appends /v1 when missing."""
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
router = CascadeRouter(config_path=Path("/nonexistent"))
|
||||
|
||||
provider = Provider(
|
||||
name="vllm-mlx-local",
|
||||
type="vllm_mlx",
|
||||
enabled=True,
|
||||
priority=2,
|
||||
base_url="http://localhost:8000", # No /v1
|
||||
models=[{"name": "qwen-mlx", "default": True}],
|
||||
)
|
||||
|
||||
mock_choice = MagicMock()
|
||||
mock_choice.message.content = "hello"
|
||||
mock_response = MagicMock()
|
||||
mock_response.choices = [mock_choice]
|
||||
mock_response.model = "qwen-mlx"
|
||||
|
||||
async def fake_create(**kwargs):
|
||||
return mock_response
|
||||
|
||||
with patch("openai.AsyncOpenAI") as mock_openai_cls:
|
||||
mock_client = MagicMock()
|
||||
mock_client.chat.completions.create = AsyncMock(side_effect=fake_create)
|
||||
mock_openai_cls.return_value = mock_client
|
||||
|
||||
await router._call_vllm_mlx(
|
||||
provider=provider,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
model="qwen-mlx",
|
||||
temperature=0.7,
|
||||
max_tokens=None,
|
||||
)
|
||||
|
||||
call_kwargs = mock_openai_cls.call_args
|
||||
base_url_used = call_kwargs.kwargs.get("base_url") or call_kwargs[1].get("base_url")
|
||||
assert base_url_used.endswith("/v1")
|
||||
|
||||
async def test_vllm_mlx_is_local_not_cloud(self):
|
||||
"""Confirm vllm_mlx is not subject to metabolic protocol cloud skip."""
|
||||
router = CascadeRouter(config_path=Path("/nonexistent"))
|
||||
|
||||
provider = Provider(
|
||||
name="vllm-mlx-local",
|
||||
type="vllm_mlx",
|
||||
enabled=True,
|
||||
priority=2,
|
||||
base_url="http://localhost:8000/v1",
|
||||
models=[{"name": "qwen-mlx", "default": True}],
|
||||
)
|
||||
router.providers = [provider]
|
||||
|
||||
# Quota monitor returns False (block cloud) — vllm_mlx should still be tried
|
||||
with patch("infrastructure.router.cascade._quota_monitor") as mock_qm:
|
||||
mock_qm.check.return_value = object()
|
||||
mock_qm.should_use_cloud.return_value = False
|
||||
|
||||
with patch.object(router, "_call_vllm_mlx") as mock_call:
|
||||
mock_call.return_value = {
|
||||
"content": "Local MLX response",
|
||||
"model": "qwen-mlx",
|
||||
}
|
||||
result = await router.complete(
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
)
|
||||
|
||||
assert result["content"] == "Local MLX response"
|
||||
|
||||
|
||||
class TestCascadeRouterReload:
|
||||
"""Test hot-reload of providers.yaml."""
|
||||
|
||||
183
tests/infrastructure/test_sovereignty_metrics.py
Normal file
183
tests/infrastructure/test_sovereignty_metrics.py
Normal file
@@ -0,0 +1,183 @@
|
||||
"""Tests for the sovereignty metrics store and API routes.
|
||||
|
||||
Refs: #981
|
||||
"""
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from infrastructure.sovereignty_metrics import (
|
||||
GRADUATION_TARGETS,
|
||||
SovereigntyMetric,
|
||||
SovereigntyMetricsStore,
|
||||
emit_sovereignty_metric,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path):
|
||||
"""Create a fresh sovereignty metrics store with a temp DB."""
|
||||
return SovereigntyMetricsStore(db_path=tmp_path / "test_sov.db")
|
||||
|
||||
|
||||
class TestSovereigntyMetricsStore:
|
||||
def test_record_and_get_latest(self, store):
|
||||
metric = SovereigntyMetric(metric_type="cache_hit_rate", value=0.42)
|
||||
store.record(metric)
|
||||
|
||||
results = store.get_latest("cache_hit_rate", limit=10)
|
||||
assert len(results) == 1
|
||||
assert results[0]["value"] == 0.42
|
||||
|
||||
def test_get_latest_returns_most_recent_first(self, store):
|
||||
for val in [0.1, 0.2, 0.3]:
|
||||
store.record(SovereigntyMetric(metric_type="cache_hit_rate", value=val))
|
||||
|
||||
results = store.get_latest("cache_hit_rate", limit=10)
|
||||
assert len(results) == 3
|
||||
assert results[0]["value"] == 0.3 # most recent first
|
||||
|
||||
def test_get_latest_respects_limit(self, store):
|
||||
for i in range(10):
|
||||
store.record(SovereigntyMetric(metric_type="api_cost", value=float(i)))
|
||||
|
||||
results = store.get_latest("api_cost", limit=3)
|
||||
assert len(results) == 3
|
||||
|
||||
def test_get_latest_filters_by_type(self, store):
|
||||
store.record(SovereigntyMetric(metric_type="cache_hit_rate", value=0.5))
|
||||
store.record(SovereigntyMetric(metric_type="api_cost", value=1.20))
|
||||
|
||||
results = store.get_latest("cache_hit_rate")
|
||||
assert len(results) == 1
|
||||
assert results[0]["value"] == 0.5
|
||||
|
||||
def test_get_summary_empty(self, store):
|
||||
summary = store.get_summary()
|
||||
assert "cache_hit_rate" in summary
|
||||
assert summary["cache_hit_rate"]["current"] is None
|
||||
assert summary["cache_hit_rate"]["phase"] == "pre-start"
|
||||
|
||||
def test_get_summary_with_data(self, store):
|
||||
store.record(SovereigntyMetric(metric_type="cache_hit_rate", value=0.85))
|
||||
store.record(SovereigntyMetric(metric_type="api_cost", value=0.08))
|
||||
|
||||
summary = store.get_summary()
|
||||
assert summary["cache_hit_rate"]["current"] == 0.85
|
||||
assert summary["cache_hit_rate"]["phase"] == "month3"
|
||||
assert summary["api_cost"]["current"] == 0.08
|
||||
assert summary["api_cost"]["phase"] == "month3"
|
||||
|
||||
def test_get_summary_graduation(self, store):
|
||||
store.record(SovereigntyMetric(metric_type="cache_hit_rate", value=0.95))
|
||||
summary = store.get_summary()
|
||||
assert summary["cache_hit_rate"]["phase"] == "graduated"
|
||||
|
||||
def test_alert_on_high_api_cost(self, store):
|
||||
"""API cost above threshold triggers an alert."""
|
||||
with patch("infrastructure.sovereignty_metrics.settings") as mock_settings:
|
||||
mock_settings.sovereignty_api_cost_alert_threshold = 1.00
|
||||
mock_settings.db_busy_timeout_ms = 5000
|
||||
store.record(SovereigntyMetric(metric_type="api_cost", value=2.50))
|
||||
|
||||
alerts = store.get_alerts(unacknowledged_only=True)
|
||||
assert len(alerts) == 1
|
||||
assert alerts[0]["alert_type"] == "api_cost_exceeded"
|
||||
assert alerts[0]["value"] == 2.50
|
||||
|
||||
def test_no_alert_below_threshold(self, store):
|
||||
"""API cost below threshold does not trigger an alert."""
|
||||
with patch("infrastructure.sovereignty_metrics.settings") as mock_settings:
|
||||
mock_settings.sovereignty_api_cost_alert_threshold = 1.00
|
||||
mock_settings.db_busy_timeout_ms = 5000
|
||||
store.record(SovereigntyMetric(metric_type="api_cost", value=0.50))
|
||||
|
||||
alerts = store.get_alerts(unacknowledged_only=True)
|
||||
assert len(alerts) == 0
|
||||
|
||||
def test_acknowledge_alert(self, store):
|
||||
with patch("infrastructure.sovereignty_metrics.settings") as mock_settings:
|
||||
mock_settings.sovereignty_api_cost_alert_threshold = 0.50
|
||||
mock_settings.db_busy_timeout_ms = 5000
|
||||
store.record(SovereigntyMetric(metric_type="api_cost", value=1.00))
|
||||
|
||||
alerts = store.get_alerts(unacknowledged_only=True)
|
||||
assert len(alerts) == 1
|
||||
|
||||
store.acknowledge_alert(alerts[0]["id"])
|
||||
assert len(store.get_alerts(unacknowledged_only=True)) == 0
|
||||
assert len(store.get_alerts(unacknowledged_only=False)) == 1
|
||||
|
||||
def test_metadata_preserved(self, store):
|
||||
store.record(
|
||||
SovereigntyMetric(
|
||||
metric_type="cache_hit_rate",
|
||||
value=0.5,
|
||||
metadata={"source": "research_orchestrator"},
|
||||
)
|
||||
)
|
||||
results = store.get_latest("cache_hit_rate")
|
||||
assert results[0]["metadata"]["source"] == "research_orchestrator"
|
||||
|
||||
def test_summary_trend_data(self, store):
|
||||
for v in [0.1, 0.2, 0.3]:
|
||||
store.record(SovereigntyMetric(metric_type="cache_hit_rate", value=v))
|
||||
|
||||
summary = store.get_summary()
|
||||
trend = summary["cache_hit_rate"]["trend"]
|
||||
assert len(trend) == 3
|
||||
assert trend[0]["v"] == 0.1 # oldest first (reversed)
|
||||
assert trend[-1]["v"] == 0.3
|
||||
|
||||
def test_graduation_targets_complete(self):
|
||||
"""All expected metric types have graduation targets."""
|
||||
expected = {
|
||||
"cache_hit_rate",
|
||||
"api_cost",
|
||||
"time_to_report",
|
||||
"human_involvement",
|
||||
"local_artifacts",
|
||||
}
|
||||
assert set(GRADUATION_TARGETS.keys()) == expected
|
||||
|
||||
|
||||
class TestEmitSovereigntyMetric:
|
||||
@pytest.mark.asyncio
|
||||
async def test_emit_records_and_publishes(self, tmp_path):
|
||||
"""emit_sovereignty_metric records to store and publishes event."""
|
||||
with (
|
||||
patch("infrastructure.sovereignty_metrics._store", None),
|
||||
patch(
|
||||
"infrastructure.sovereignty_metrics.DB_PATH",
|
||||
tmp_path / "emit_test.db",
|
||||
),
|
||||
patch("infrastructure.events.bus.emit", new_callable=AsyncMock) as mock_emit,
|
||||
):
|
||||
await emit_sovereignty_metric("cache_hit_rate", 0.75, {"source": "test"})
|
||||
|
||||
mock_emit.assert_called_once()
|
||||
call_args = mock_emit.call_args
|
||||
assert call_args[0][0] == "sovereignty.metric.cache_hit_rate"
|
||||
|
||||
|
||||
class TestSovereigntyMetricsRoutes:
|
||||
def test_metrics_api_returns_200(self, client):
|
||||
response = client.get("/sovereignty/metrics")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "metrics" in data
|
||||
assert "alerts" in data
|
||||
assert "targets" in data
|
||||
|
||||
def test_metrics_panel_returns_html(self, client):
|
||||
response = client.get("/sovereignty/metrics/panel")
|
||||
assert response.status_code == 200
|
||||
assert "text/html" in response.headers["content-type"]
|
||||
|
||||
def test_alerts_api_returns_200(self, client):
|
||||
response = client.get("/sovereignty/alerts")
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert "alerts" in data
|
||||
assert "unacknowledged" in data
|
||||
0
tests/infrastructure/world/__init__.py
Normal file
0
tests/infrastructure/world/__init__.py
Normal file
394
tests/infrastructure/world/test_benchmark.py
Normal file
394
tests/infrastructure/world/test_benchmark.py
Normal file
@@ -0,0 +1,394 @@
|
||||
"""Tests for the agent performance regression benchmark suite.
|
||||
|
||||
Covers: scenario loading, metrics collection, runner execution,
|
||||
goal predicates, and result persistence.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from infrastructure.world.benchmark.metrics import (
|
||||
BenchmarkMetrics,
|
||||
ScenarioResult,
|
||||
compare_runs,
|
||||
load_history,
|
||||
)
|
||||
from infrastructure.world.benchmark.runner import BenchmarkRunner
|
||||
from infrastructure.world.benchmark.scenarios import (
|
||||
BUILTIN_SCENARIOS,
|
||||
BenchmarkScenario,
|
||||
load_scenarios,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Scenario definitions
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBenchmarkScenario:
|
||||
def test_builtin_scenarios_exist(self):
|
||||
assert len(BUILTIN_SCENARIOS) >= 5
|
||||
|
||||
def test_scenario_fields(self):
|
||||
s = BUILTIN_SCENARIOS[0]
|
||||
assert s.name
|
||||
assert s.description
|
||||
assert s.start_location
|
||||
assert s.max_cycles > 0
|
||||
|
||||
def test_load_all_scenarios(self):
|
||||
scenarios = load_scenarios()
|
||||
assert len(scenarios) == len(BUILTIN_SCENARIOS)
|
||||
|
||||
def test_load_scenarios_by_tag(self):
|
||||
nav = load_scenarios(tags=["navigation"])
|
||||
assert len(nav) >= 2
|
||||
for s in nav:
|
||||
assert "navigation" in s.tags
|
||||
|
||||
def test_load_scenarios_no_match(self):
|
||||
result = load_scenarios(tags=["nonexistent_tag"])
|
||||
assert result == []
|
||||
|
||||
def test_scenario_is_frozen(self):
|
||||
s = BUILTIN_SCENARIOS[0]
|
||||
with pytest.raises(AttributeError):
|
||||
s.name = "modified"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Goal predicates
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestGoalPredicates:
|
||||
def test_reached_location_predicate(self):
|
||||
s = BUILTIN_SCENARIOS[0] # Walk to Balmora
|
||||
assert s.goal_predicate is not None
|
||||
assert s.goal_predicate([], "Balmora") is True
|
||||
assert s.goal_predicate([], "Seyda Neen") is False
|
||||
|
||||
def test_reached_location_case_insensitive(self):
|
||||
s = BUILTIN_SCENARIOS[0]
|
||||
assert s.goal_predicate([], "balmora") is True
|
||||
assert s.goal_predicate([], "BALMORA") is True
|
||||
|
||||
def test_interacted_with_predicate(self):
|
||||
s = BUILTIN_SCENARIOS[1] # Fargoth quest
|
||||
assert s.goal_predicate is not None
|
||||
actions = [{"action": "speak", "target": "Fargoth"}]
|
||||
assert s.goal_predicate(actions, "Seyda Neen") is True
|
||||
|
||||
def test_interacted_with_no_match(self):
|
||||
s = BUILTIN_SCENARIOS[1]
|
||||
actions = [{"action": "speak", "target": "Guard"}]
|
||||
assert s.goal_predicate(actions, "Seyda Neen") is False
|
||||
|
||||
def test_interacted_with_interact_action(self):
|
||||
s = BUILTIN_SCENARIOS[1]
|
||||
actions = [{"action": "interact", "target": "Fargoth"}]
|
||||
assert s.goal_predicate(actions, "Seyda Neen") is True
|
||||
|
||||
def test_no_predicate_scenario(self):
|
||||
combat = [s for s in BUILTIN_SCENARIOS if "combat" in s.tags][0]
|
||||
assert combat.goal_predicate is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Metrics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestScenarioResult:
|
||||
def test_default_values(self):
|
||||
r = ScenarioResult(scenario_name="test")
|
||||
assert r.success is False
|
||||
assert r.cycles_used == 0
|
||||
assert r.llm_calls == 0
|
||||
assert r.metabolic_cost == 0.0
|
||||
assert r.error is None
|
||||
|
||||
|
||||
class TestBenchmarkMetrics:
|
||||
def test_empty_metrics(self):
|
||||
m = BenchmarkMetrics()
|
||||
assert m.pass_count == 0
|
||||
assert m.fail_count == 0
|
||||
assert m.success_rate == 0.0
|
||||
assert m.total_llm_calls == 0
|
||||
assert m.total_metabolic_cost == 0.0
|
||||
|
||||
def test_success_rate(self):
|
||||
m = BenchmarkMetrics(
|
||||
results=[
|
||||
ScenarioResult(scenario_name="a", success=True),
|
||||
ScenarioResult(scenario_name="b", success=False),
|
||||
ScenarioResult(scenario_name="c", success=True),
|
||||
]
|
||||
)
|
||||
assert m.pass_count == 2
|
||||
assert m.fail_count == 1
|
||||
assert abs(m.success_rate - 2 / 3) < 0.01
|
||||
|
||||
def test_totals(self):
|
||||
m = BenchmarkMetrics(
|
||||
results=[
|
||||
ScenarioResult(scenario_name="a", llm_calls=10, metabolic_cost=30.0),
|
||||
ScenarioResult(scenario_name="b", llm_calls=5, metabolic_cost=15.0),
|
||||
]
|
||||
)
|
||||
assert m.total_llm_calls == 15
|
||||
assert m.total_metabolic_cost == 45.0
|
||||
|
||||
def test_save_and_load(self, tmp_path):
|
||||
path = tmp_path / "bench.jsonl"
|
||||
m = BenchmarkMetrics(
|
||||
timestamp="2026-01-01T00:00:00",
|
||||
commit_sha="abc123",
|
||||
total_time_ms=1000,
|
||||
results=[
|
||||
ScenarioResult(
|
||||
scenario_name="a",
|
||||
success=True,
|
||||
cycles_used=5,
|
||||
max_cycles=10,
|
||||
),
|
||||
],
|
||||
)
|
||||
m.save(path)
|
||||
|
||||
history = load_history(path)
|
||||
assert len(history) == 1
|
||||
assert history[0]["commit_sha"] == "abc123"
|
||||
assert history[0]["scenarios"][0]["scenario_name"] == "a"
|
||||
|
||||
def test_save_appends(self, tmp_path):
|
||||
path = tmp_path / "bench.jsonl"
|
||||
for i in range(3):
|
||||
m = BenchmarkMetrics(
|
||||
timestamp=f"2026-01-0{i + 1}T00:00:00",
|
||||
results=[ScenarioResult(scenario_name=f"s{i}")],
|
||||
)
|
||||
m.save(path)
|
||||
|
||||
history = load_history(path)
|
||||
assert len(history) == 3
|
||||
# Most recent first
|
||||
assert history[0]["timestamp"] == "2026-01-03T00:00:00"
|
||||
|
||||
def test_summary_output(self):
|
||||
m = BenchmarkMetrics(
|
||||
timestamp="2026-01-01T00:00:00",
|
||||
commit_sha="abc123",
|
||||
total_time_ms=500,
|
||||
results=[
|
||||
ScenarioResult(
|
||||
scenario_name="Walk Test",
|
||||
success=True,
|
||||
cycles_used=5,
|
||||
max_cycles=10,
|
||||
wall_time_ms=200,
|
||||
llm_calls=15,
|
||||
),
|
||||
],
|
||||
)
|
||||
summary = m.summary()
|
||||
assert "Walk Test" in summary
|
||||
assert "PASS" in summary
|
||||
assert "abc123" in summary
|
||||
|
||||
def test_load_history_missing_file(self, tmp_path):
|
||||
assert load_history(tmp_path / "nope.jsonl") == []
|
||||
|
||||
def test_load_history_corrupt_lines(self, tmp_path):
|
||||
path = tmp_path / "bench.jsonl"
|
||||
path.write_text('{"valid": true}\nnot json\n{"also": "valid"}\n')
|
||||
history = load_history(path)
|
||||
assert len(history) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Comparison
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestCompareRuns:
|
||||
def test_regression_detected(self):
|
||||
baseline = BenchmarkMetrics(
|
||||
results=[
|
||||
ScenarioResult(scenario_name="walk", success=True, cycles_used=10),
|
||||
]
|
||||
)
|
||||
current = BenchmarkMetrics(
|
||||
results=[
|
||||
ScenarioResult(scenario_name="walk", success=False, cycles_used=10),
|
||||
]
|
||||
)
|
||||
report = compare_runs(current, baseline)
|
||||
assert "REGRESSION" in report
|
||||
|
||||
def test_improvement_detected(self):
|
||||
baseline = BenchmarkMetrics(
|
||||
results=[
|
||||
ScenarioResult(scenario_name="walk", success=False, cycles_used=10),
|
||||
]
|
||||
)
|
||||
current = BenchmarkMetrics(
|
||||
results=[
|
||||
ScenarioResult(scenario_name="walk", success=True, cycles_used=10),
|
||||
]
|
||||
)
|
||||
report = compare_runs(current, baseline)
|
||||
assert "IMPROVEMENT" in report
|
||||
|
||||
def test_slower_detected(self):
|
||||
baseline = BenchmarkMetrics(
|
||||
results=[
|
||||
ScenarioResult(scenario_name="walk", success=True, cycles_used=10),
|
||||
]
|
||||
)
|
||||
current = BenchmarkMetrics(
|
||||
results=[
|
||||
ScenarioResult(scenario_name="walk", success=True, cycles_used=20),
|
||||
]
|
||||
)
|
||||
report = compare_runs(current, baseline)
|
||||
assert "SLOWER" in report
|
||||
|
||||
def test_new_scenario_noted(self):
|
||||
baseline = BenchmarkMetrics(results=[])
|
||||
current = BenchmarkMetrics(results=[ScenarioResult(scenario_name="new_one", success=True)])
|
||||
report = compare_runs(current, baseline)
|
||||
assert "NEW" in report
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runner
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestBenchmarkRunner:
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_single_scenario(self):
|
||||
"""Runner executes a scenario and returns a result."""
|
||||
scenario = BenchmarkScenario(
|
||||
name="Test Walk",
|
||||
description="Simple test",
|
||||
start_location="A",
|
||||
goal_location="A",
|
||||
max_cycles=3,
|
||||
tags=["test"],
|
||||
)
|
||||
runner = BenchmarkRunner()
|
||||
metrics = await runner.run([scenario])
|
||||
assert len(metrics.results) == 1
|
||||
r = metrics.results[0]
|
||||
assert r.scenario_name == "Test Walk"
|
||||
assert r.cycles_used == 3 # no predicate, runs all cycles
|
||||
assert r.success is True # no predicate = success if survived
|
||||
assert r.wall_time_ms >= 0
|
||||
assert r.llm_calls == 9 # 3 cycles * 3 calls
|
||||
assert r.metabolic_cost > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_with_goal_predicate(self):
|
||||
"""Runner stops early when goal predicate is satisfied."""
|
||||
|
||||
def always_true(actions, location):
|
||||
return True
|
||||
|
||||
scenario = BenchmarkScenario(
|
||||
name="Instant Win",
|
||||
description="Predicate satisfied immediately",
|
||||
start_location="A",
|
||||
max_cycles=100,
|
||||
goal_predicate=always_true,
|
||||
tags=["test"],
|
||||
)
|
||||
runner = BenchmarkRunner()
|
||||
metrics = await runner.run([scenario])
|
||||
r = metrics.results[0]
|
||||
assert r.success is True
|
||||
assert r.cycles_used == 1 # Stopped at first cycle
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_with_failing_predicate(self):
|
||||
"""Scenario fails when predicate never satisfied."""
|
||||
|
||||
def never_true(actions, location):
|
||||
return False
|
||||
|
||||
scenario = BenchmarkScenario(
|
||||
name="Impossible",
|
||||
description="Predicate never satisfied",
|
||||
start_location="A",
|
||||
max_cycles=5,
|
||||
goal_predicate=never_true,
|
||||
tags=["test"],
|
||||
)
|
||||
runner = BenchmarkRunner()
|
||||
metrics = await runner.run([scenario])
|
||||
r = metrics.results[0]
|
||||
assert r.success is False
|
||||
assert r.cycles_used == 5
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_multiple_scenarios(self):
|
||||
"""Runner handles multiple scenarios in sequence."""
|
||||
scenarios = [
|
||||
BenchmarkScenario(
|
||||
name=f"Scenario {i}",
|
||||
description=f"Test {i}",
|
||||
start_location="A",
|
||||
max_cycles=2,
|
||||
tags=["test"],
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
runner = BenchmarkRunner()
|
||||
metrics = await runner.run(scenarios)
|
||||
assert len(metrics.results) == 3
|
||||
assert metrics.total_time_ms >= 0
|
||||
assert metrics.timestamp
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metrics_commit_sha(self):
|
||||
"""Runner captures git SHA in metrics."""
|
||||
scenario = BenchmarkScenario(
|
||||
name="SHA Test",
|
||||
description="Check SHA capture",
|
||||
start_location="A",
|
||||
max_cycles=1,
|
||||
tags=["test"],
|
||||
)
|
||||
runner = BenchmarkRunner()
|
||||
metrics = await runner.run([scenario])
|
||||
# SHA may or may not be available in test env; just ensure no crash
|
||||
assert isinstance(metrics.commit_sha, str)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_builtin_scenarios_run(self):
|
||||
"""All built-in scenarios run without crashing."""
|
||||
# Use just 2 cycles each to keep tests fast
|
||||
scenarios = [
|
||||
BenchmarkScenario(
|
||||
name=s.name,
|
||||
description=s.description,
|
||||
start_location=s.start_location,
|
||||
goal_location=s.goal_location,
|
||||
entities=list(s.entities),
|
||||
events=list(s.events),
|
||||
max_cycles=2, # Override for speed
|
||||
goal_predicate=None, # Skip predicate for smoke test
|
||||
tags=list(s.tags),
|
||||
)
|
||||
for s in BUILTIN_SCENARIOS
|
||||
]
|
||||
runner = BenchmarkRunner()
|
||||
metrics = await runner.run(scenarios)
|
||||
assert len(metrics.results) == len(BUILTIN_SCENARIOS)
|
||||
# All should succeed (no predicate + survived = pass)
|
||||
for r in metrics.results:
|
||||
assert r.success is True
|
||||
assert r.error is None
|
||||
129
tests/infrastructure/world/test_interface.py
Normal file
129
tests/infrastructure/world/test_interface.py
Normal file
@@ -0,0 +1,129 @@
|
||||
"""Tests for the WorldInterface contract and type system."""
|
||||
|
||||
import pytest
|
||||
|
||||
from infrastructure.world.interface import WorldInterface
|
||||
from infrastructure.world.types import (
|
||||
ActionResult,
|
||||
ActionStatus,
|
||||
CommandInput,
|
||||
PerceptionOutput,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Type construction
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestPerceptionOutput:
|
||||
def test_defaults(self):
|
||||
p = PerceptionOutput()
|
||||
assert p.location == ""
|
||||
assert p.entities == []
|
||||
assert p.events == []
|
||||
assert p.raw == {}
|
||||
assert p.timestamp is not None
|
||||
|
||||
def test_custom_values(self):
|
||||
p = PerceptionOutput(
|
||||
location="Balmora",
|
||||
entities=["Guard", "Merchant"],
|
||||
events=["door_opened"],
|
||||
)
|
||||
assert p.location == "Balmora"
|
||||
assert len(p.entities) == 2
|
||||
assert "door_opened" in p.events
|
||||
|
||||
|
||||
class TestCommandInput:
|
||||
def test_minimal(self):
|
||||
c = CommandInput(action="move")
|
||||
assert c.action == "move"
|
||||
assert c.target is None
|
||||
assert c.parameters == {}
|
||||
|
||||
def test_with_target_and_params(self):
|
||||
c = CommandInput(action="attack", target="Rat", parameters={"weapon": "sword"})
|
||||
assert c.target == "Rat"
|
||||
assert c.parameters["weapon"] == "sword"
|
||||
|
||||
|
||||
class TestActionResult:
|
||||
def test_defaults(self):
|
||||
r = ActionResult()
|
||||
assert r.status == ActionStatus.SUCCESS
|
||||
assert r.message == ""
|
||||
|
||||
def test_failure(self):
|
||||
r = ActionResult(status=ActionStatus.FAILURE, message="blocked")
|
||||
assert r.status == ActionStatus.FAILURE
|
||||
|
||||
|
||||
class TestActionStatus:
|
||||
def test_values(self):
|
||||
assert ActionStatus.SUCCESS.value == "success"
|
||||
assert ActionStatus.FAILURE.value == "failure"
|
||||
assert ActionStatus.PENDING.value == "pending"
|
||||
assert ActionStatus.NOOP.value == "noop"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Abstract contract
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestWorldInterfaceContract:
|
||||
"""Verify the ABC cannot be instantiated directly."""
|
||||
|
||||
def test_cannot_instantiate(self):
|
||||
with pytest.raises(TypeError):
|
||||
WorldInterface()
|
||||
|
||||
def test_subclass_must_implement_observe(self):
|
||||
class Incomplete(WorldInterface):
|
||||
def act(self, command):
|
||||
pass
|
||||
|
||||
def speak(self, message, target=None):
|
||||
pass
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
Incomplete()
|
||||
|
||||
def test_subclass_must_implement_act(self):
|
||||
class Incomplete(WorldInterface):
|
||||
def observe(self):
|
||||
return PerceptionOutput()
|
||||
|
||||
def speak(self, message, target=None):
|
||||
pass
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
Incomplete()
|
||||
|
||||
def test_subclass_must_implement_speak(self):
|
||||
class Incomplete(WorldInterface):
|
||||
def observe(self):
|
||||
return PerceptionOutput()
|
||||
|
||||
def act(self, command):
|
||||
return ActionResult()
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
Incomplete()
|
||||
|
||||
def test_complete_subclass_instantiates(self):
|
||||
class Complete(WorldInterface):
|
||||
def observe(self):
|
||||
return PerceptionOutput()
|
||||
|
||||
def act(self, command):
|
||||
return ActionResult()
|
||||
|
||||
def speak(self, message, target=None):
|
||||
pass
|
||||
|
||||
adapter = Complete()
|
||||
assert adapter.is_connected is True # default
|
||||
assert isinstance(adapter.observe(), PerceptionOutput)
|
||||
assert isinstance(adapter.act(CommandInput(action="test")), ActionResult)
|
||||
80
tests/infrastructure/world/test_mock_adapter.py
Normal file
80
tests/infrastructure/world/test_mock_adapter.py
Normal file
@@ -0,0 +1,80 @@
|
||||
"""Tests for the MockWorldAdapter — full observe/act/speak cycle."""
|
||||
|
||||
from infrastructure.world.adapters.mock import MockWorldAdapter
|
||||
from infrastructure.world.types import ActionStatus, CommandInput, PerceptionOutput
|
||||
|
||||
|
||||
class TestMockWorldAdapter:
|
||||
def test_observe_returns_perception(self):
|
||||
adapter = MockWorldAdapter(location="Vivec")
|
||||
perception = adapter.observe()
|
||||
assert isinstance(perception, PerceptionOutput)
|
||||
assert perception.location == "Vivec"
|
||||
assert perception.raw == {"adapter": "mock"}
|
||||
|
||||
def test_observe_entities(self):
|
||||
adapter = MockWorldAdapter(entities=["Jiub", "Silt Strider"])
|
||||
perception = adapter.observe()
|
||||
assert perception.entities == ["Jiub", "Silt Strider"]
|
||||
|
||||
def test_act_logs_command(self):
|
||||
adapter = MockWorldAdapter()
|
||||
cmd = CommandInput(action="move", target="north")
|
||||
result = adapter.act(cmd)
|
||||
assert result.status == ActionStatus.SUCCESS
|
||||
assert "move" in result.message
|
||||
assert len(adapter.action_log) == 1
|
||||
assert adapter.action_log[0].command.action == "move"
|
||||
|
||||
def test_act_multiple_commands(self):
|
||||
adapter = MockWorldAdapter()
|
||||
adapter.act(CommandInput(action="attack"))
|
||||
adapter.act(CommandInput(action="defend"))
|
||||
adapter.act(CommandInput(action="retreat"))
|
||||
assert len(adapter.action_log) == 3
|
||||
|
||||
def test_speak_logs_message(self):
|
||||
adapter = MockWorldAdapter()
|
||||
adapter.speak("Hello, traveler!")
|
||||
assert len(adapter.speech_log) == 1
|
||||
assert adapter.speech_log[0]["message"] == "Hello, traveler!"
|
||||
assert adapter.speech_log[0]["target"] is None
|
||||
|
||||
def test_speak_with_target(self):
|
||||
adapter = MockWorldAdapter()
|
||||
adapter.speak("Die, scum!", target="Cliff Racer")
|
||||
assert adapter.speech_log[0]["target"] == "Cliff Racer"
|
||||
|
||||
def test_lifecycle(self):
|
||||
adapter = MockWorldAdapter()
|
||||
assert adapter.is_connected is False
|
||||
adapter.connect()
|
||||
assert adapter.is_connected is True
|
||||
adapter.disconnect()
|
||||
assert adapter.is_connected is False
|
||||
|
||||
def test_full_observe_act_speak_cycle(self):
|
||||
"""Acceptance criterion: full observe/act/speak cycle passes."""
|
||||
adapter = MockWorldAdapter(
|
||||
location="Seyda Neen",
|
||||
entities=["Fargoth", "Hrisskar"],
|
||||
events=["quest_started"],
|
||||
)
|
||||
adapter.connect()
|
||||
|
||||
# Observe
|
||||
perception = adapter.observe()
|
||||
assert perception.location == "Seyda Neen"
|
||||
assert len(perception.entities) == 2
|
||||
assert "quest_started" in perception.events
|
||||
|
||||
# Act
|
||||
result = adapter.act(CommandInput(action="talk", target="Fargoth"))
|
||||
assert result.status == ActionStatus.SUCCESS
|
||||
|
||||
# Speak
|
||||
adapter.speak("Where is your ring, Fargoth?", target="Fargoth")
|
||||
assert len(adapter.speech_log) == 1
|
||||
|
||||
adapter.disconnect()
|
||||
assert adapter.is_connected is False
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user