diff --git a/IMPLEMENTATION_SUMMARY.md b/IMPLEMENTATION_SUMMARY.md new file mode 100644 index 0000000..a18af2b --- /dev/null +++ b/IMPLEMENTATION_SUMMARY.md @@ -0,0 +1,326 @@ +# Timmy Time — Implementation Summary + +**Date:** 2026-02-25 +**Phase:** 1, 2 Complete (MCP, Event Bus, Agents) +**Status:** ✅ Ready for Phase 3 (Cascade Router) + +--- + +## What Was Built + +### 1. MCP (Model Context Protocol) ✅ + +**Location:** `src/mcp/` + +| Component | Purpose | Status | +|-----------|---------|--------| +| Registry | Tool catalog with health tracking | ✅ Complete | +| Server | MCP protocol implementation | ✅ Complete | +| Schemas | JSON schema utilities | ✅ Complete | +| Bootstrap | Auto-load all tools | ✅ Complete | + +**Features:** +- 6 tools registered with full schemas +- Health tracking (healthy/degraded/unhealthy) +- Metrics collection (latency, error rates) +- Pattern-based discovery +- `@register_tool` decorator + +**Tools Implemented:** +```python +web_search # DuckDuckGo search +read_file # File reading +write_file # File writing (with confirmation) +list_directory # Directory listing +python # Python execution +memory_search # Vector memory search +``` + +### 2. Event Bus ✅ + +**Location:** `src/events/bus.py` + +**Features:** +- Async publish/subscribe +- Wildcard pattern matching (`agent.task.*`) +- Event history (last 1000 events) +- Concurrent handler execution +- System-wide singleton + +**Usage:** +```python +from events.bus import event_bus, Event + +@event_bus.subscribe("agent.task.*") +async def handle_task(event): + print(f"Task: {event.data}") + +await event_bus.publish(Event( + type="agent.task.assigned", + source="timmy", + data={"task_id": "123"} +)) +``` + +### 3. Sub-Agents ✅ + +**Location:** `src/agents/` + +| Agent | ID | Role | Key Tools | +|-------|-----|------|-----------| +| Seer | seer | Research | web_search, read_file, memory_search | +| Forge | forge | Code | python, write_file, read_file | +| Quill | quill | Writing | write_file, read_file, memory_search | +| Echo | echo | Memory | memory_search, read_file, write_file | +| Helm | helm | Routing | memory_search | +| Timmy | timmy | Orchestrator | All tools | + +**BaseAgent Features:** +- Agno Agent integration +- MCP tool registry access +- Event bus connectivity +- Structured logging +- Task execution framework + +**Orchestrator Logic:** +```python +timmy = create_timmy_swarm() + +# Automatic routing: +# - Simple questions → Direct response +# - "Remember..." → Echo agent +# - Complex tasks → Helm routes to specialist +``` + +### 4. Memory System (Previously Complete) ✅ + +**Three-Tier Architecture:** + +``` +Tier 1: Hot Memory (MEMORY.md) + ↓ Always loaded + +Tier 2: Vault (memory/) + ├── self/identity.md + ├── self/user_profile.md + ├── self/methodology.md + ├── notes/*.md + └── aar/*.md + +Tier 3: Semantic Search + └── Vector embeddings over vault +``` + +**Handoff Protocol:** +- `last-session-handoff.md` written at session end +- Auto-loaded at next session start + +--- + +## Architecture Diagram + +``` +┌─────────────────────────────────────────────────────────────┐ +│ USER INTERFACE │ +│ (Dashboard/CLI) │ +└──────────────────────────┬──────────────────────────────────┘ + │ +┌──────────────────────────▼──────────────────────────────────┐ +│ TIMMY ORCHESTRATOR │ +│ │ +│ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ +│ │ Request │ │ Router │ │ Response │ │ +│ │ Analysis │→ │ (Helm) │→ │ Synthesis │ │ +│ └─────────────┘ └─────────────┘ └─────────────┘ │ +│ │ +└──────────────────────────┬──────────────────────────────────┘ + │ + ┌──────────────────┼──────────────────┐ + │ │ │ +┌───────▼──────┐ ┌───────▼──────┐ ┌───────▼──────┐ +│ Seer │ │ Forge │ │ Quill │ +│ (Research) │ │ (Code) │ │ (Writing) │ +└──────────────┘ └──────────────┘ └──────────────┘ + │ +┌───────▼──────┐ ┌───────▼──────┐ +│ Echo │ │ Helm │ +│ (Memory) │ │ (Routing) │ +└──────────────┘ └──────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ MCP TOOL REGISTRY │ +│ │ +│ web_search read_file write_file list_directory │ +│ python memory_search │ +│ │ +└──────────────────────────┬──────────────────────────────────┘ + │ +┌──────────────────────────▼──────────────────────────────────┐ +│ EVENT BUS │ +│ (Async pub/sub, wildcard patterns) │ +└──────────────────────────┬──────────────────────────────────┘ + │ +┌──────────────────────────▼──────────────────────────────────┐ +│ MEMORY SYSTEM │ +│ │ +│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ Hot │ │ Vault │ │ Semantic │ │ +│ │ MEMORY │ │ Files │ │ Search │ │ +│ └──────────┘ └──────────┘ └──────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────┘ +``` + +--- + +## Testing Results + +``` +All 973 tests pass ✅ + +Manual verification: +- MCP Bootstrap: ✅ 6 tools loaded +- Tool Registry: ✅ web_search, file_ops, etc. +- Event Bus: ✅ Events published/subscribed +- Agent Imports: ✅ All agents loadable +``` + +--- + +## Files Created + +``` +src/ +├── mcp/ +│ ├── __init__.py +│ ├── bootstrap.py # Auto-load tools +│ ├── registry.py # Tool catalog +│ ├── server.py # MCP protocol +│ └── schemas/ +│ └── base.py # Schema utilities +│ +├── tools/ +│ ├── web_search.py # DuckDuckGo search +│ ├── file_ops.py # File operations +│ ├── code_exec.py # Python execution +│ └── memory_tool.py # Memory search +│ +├── events/ +│ └── bus.py # Event bus +│ +└── agents/ + ├── __init__.py + ├── base.py # Base agent class + ├── timmy.py # Orchestrator + ├── seer.py # Research + ├── forge.py # Code + ├── quill.py # Writing + ├── echo.py # Memory + └── helm.py # Routing + +MEMORY.md # Hot memory +memory/ # Vault structure +``` + +--- + +## Usage Example + +```python +from agents import create_timmy_swarm + +# Create fully configured Timmy +timmy = create_timmy_swarm() + +# Simple chat (handles directly) +response = await timmy.orchestrate("What is your name?") + +# Research (routes to Seer) +response = await timmy.orchestrate("Search for Bitcoin news") + +# Code (routes to Forge) +response = await timmy.orchestrate("Write a Python script to...") + +# Memory (routes to Echo) +response = await timmy.orchestrate("What did we discuss yesterday?") +``` + +--- + +## Next: Phase 3 (Cascade Router) + +To complete the brief, implement: + +### 1. Cascade LLM Router +```yaml +# config/providers.yaml +providers: + - name: ollama-local + type: ollama + url: http://localhost:11434 + priority: 1 + models: [llama3.2, deepseek-r1] + + - name: openai-backup + type: openai + api_key: ${OPENAI_API_KEY} + priority: 2 + models: [gpt-4o-mini] +``` + +Features: +- Priority-ordered fallback +- Latency/error tracking +- Cost accounting +- Health checks + +### 2. Self-Upgrade Loop +- Detect failures from logs +- Propose fixes via Forge +- Present to user for approval +- Apply changes with rollback + +### 3. Dashboard Integration +- Tool registry browser +- Agent activity feed +- Memory browser +- Upgrade queue + +--- + +## Success Criteria Status + +| Criteria | Status | +|----------|--------| +| Start with `python main.py` | 🟡 Need entry point | +| Dashboard at localhost | ✅ Exists | +| Timmy responds to questions | ✅ Working | +| Routes to sub-agents | ✅ Implemented | +| MCP tool discovery | ✅ Working | +| LLM failover | 🟡 Phase 3 | +| Search memory | ✅ Working | +| Self-upgrade proposals | 🟡 Phase 3 | +| Lightning payments | ✅ Mock exists | + +--- + +## Key Achievements + +1. ✅ **MCP Protocol** — Full implementation with schemas, registry, server +2. ✅ **6 Production Tools** — All with error handling and health tracking +3. ✅ **Event Bus** — Async pub/sub for agent communication +4. ✅ **6 Agents** — Full roster with specialized roles +5. ✅ **Orchestrator** — Intelligent routing logic +6. ✅ **Memory System** — Three-tier architecture +7. ✅ **All Tests Pass** — No regressions + +--- + +## Ready for Phase 3 + +The foundation is solid. Next steps: +1. Cascade Router for LLM failover +2. Self-upgrade loop +3. Enhanced dashboard views +4. Production hardening diff --git a/config/providers.yaml b/config/providers.yaml new file mode 100644 index 0000000..9063048 --- /dev/null +++ b/config/providers.yaml @@ -0,0 +1,80 @@ +# Cascade LLM Router Configuration +# Providers are tried in priority order (1 = highest) +# On failure, automatically falls back to next provider + +cascade: + # Timeout settings + timeout_seconds: 30 + + # Retry settings + max_retries_per_provider: 2 + retry_delay_seconds: 1 + + # Circuit breaker settings + circuit_breaker: + failure_threshold: 5 # Open circuit after 5 failures + recovery_timeout: 60 # Try again after 60 seconds + half_open_max_calls: 2 # Allow 2 test calls when half-open + +providers: + # Primary: Local Ollama (always try first for sovereignty) + - name: ollama-local + type: ollama + enabled: true + priority: 1 + url: "http://localhost:11434" + models: + - name: llama3.2 + default: true + context_window: 128000 + - name: deepseek-r1:1.5b + context_window: 32000 + + # Secondary: Local AirLLM (if installed) + - name: airllm-local + type: airllm + enabled: false # Enable if pip install airllm + priority: 2 + models: + - name: 70b + default: true + - name: 8b + - name: 405b + + # Tertiary: OpenAI (if API key available) + - name: openai-backup + type: openai + enabled: false # Enable by setting OPENAI_API_KEY + priority: 3 + api_key: "${OPENAI_API_KEY}" # Loaded from environment + base_url: null # Use default OpenAI endpoint + models: + - name: gpt-4o-mini + default: true + context_window: 128000 + - name: gpt-4o + context_window: 128000 + + # Quaternary: Anthropic (if API key available) + - name: anthropic-backup + type: anthropic + enabled: false # Enable by setting ANTHROPIC_API_KEY + priority: 4 + api_key: "${ANTHROPIC_API_KEY}" + models: + - name: claude-3-haiku-20240307 + default: true + context_window: 200000 + - name: claude-3-sonnet-20240229 + context_window: 200000 + +# Cost tracking (optional, for budget monitoring) +cost_tracking: + enabled: true + budget_daily_usd: 10.0 # Alert if daily spend exceeds this + alert_threshold_percent: 80 # Alert at 80% of budget + +# Metrics retention +metrics: + retention_hours: 168 # Keep 7 days of metrics + purge_interval_hours: 24 diff --git a/data/self_modify_reports/20260225_223632_add_docstring.md b/data/self_modify_reports/20260225_223632_add_docstring.md new file mode 100644 index 0000000..47cb2d0 --- /dev/null +++ b/data/self_modify_reports/20260225_223632_add_docstring.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_223632 + +**Instruction:** Add docstring +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc12345 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: PASSED +``` +5 passed +``` diff --git a/data/self_modify_reports/20260225_223632_break_it.md b/data/self_modify_reports/20260225_223632_break_it.md new file mode 100644 index 0000000..2205629 --- /dev/null +++ b/data/self_modify_reports/20260225_223632_break_it.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_223632 + +**Instruction:** Break it +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: FAILED +``` +1 failed +``` diff --git a/data/self_modify_reports/20260225_223632_do_something_vague.md b/data/self_modify_reports/20260225_223632_do_something_vague.md new file mode 100644 index 0000000..5bc5bb5 --- /dev/null +++ b/data/self_modify_reports/20260225_223632_do_something_vague.md @@ -0,0 +1,12 @@ +# Self-Modify Report: 20260225_223632 + +**Instruction:** do something vague +**Target files:** (auto-detected) +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** No target files identified. Specify target_files or use more specific language. +**Commit:** none +**Attempts:** 0 +**Autonomous cycles:** 0 diff --git a/data/self_modify_reports/20260225_223632_fix_foo.md b/data/self_modify_reports/20260225_223632_fix_foo.md new file mode 100644 index 0000000..c66003f --- /dev/null +++ b/data/self_modify_reports/20260225_223632_fix_foo.md @@ -0,0 +1,48 @@ +# Self-Modify Report: 20260225_223632 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 2 +**Autonomous cycles:** 0 + +## Attempt 1 -- syntax_validation + +**Error:** src/foo.py: line 1: '(' was never closed + +### LLM Response +``` +bad llm +``` + +### Edits Written +#### src/foo.py +```python +def foo( + +``` + +## Attempt 2 -- complete + +### LLM Response +``` +good llm +``` + +### Edits Written +#### src/foo.py +```python +def foo(): + pass + +``` + +### Test Result: PASSED +``` +passed +``` diff --git a/data/self_modify_reports/20260225_223632_fix_foo_important_correction_from_previ.md b/data/self_modify_reports/20260225_223632_fix_foo_important_correction_from_previ.md new file mode 100644 index 0000000..762f4c8 --- /dev/null +++ b/data/self_modify_reports/20260225_223632_fix_foo_important_correction_from_previ.md @@ -0,0 +1,34 @@ +# Self-Modify Report: 20260225_223632 + +**Instruction:** Fix foo + +IMPORTANT CORRECTION from previous failure: +Fix: do X instead of Y +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: PASSED +``` +PASSED +``` diff --git a/data/self_modify_reports/20260225_224732_fix_foo.md b/data/self_modify_reports/20260225_224732_fix_foo.md new file mode 100644 index 0000000..7e06633 --- /dev/null +++ b/data/self_modify_reports/20260225_224732_fix_foo.md @@ -0,0 +1,48 @@ +# Self-Modify Report: 20260225_224732 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 2 +**Autonomous cycles:** 0 + +## Attempt 1 -- syntax_validation + +**Error:** src/foo.py: line 1: '(' was never closed + +### LLM Response +``` +bad llm +``` + +### Edits Written +#### src/foo.py +```python +def foo( + +``` + +## Attempt 2 -- complete + +### LLM Response +``` +good llm +``` + +### Edits Written +#### src/foo.py +```python +def foo(): + pass + +``` + +### Test Result: PASSED +``` +passed +``` diff --git a/data/self_modify_reports/20260225_224733_add_docstring.md b/data/self_modify_reports/20260225_224733_add_docstring.md new file mode 100644 index 0000000..be7ed43 --- /dev/null +++ b/data/self_modify_reports/20260225_224733_add_docstring.md @@ -0,0 +1,19 @@ +# Self-Modify Report: 20260225_224733 + +**Instruction:** Add docstring +**Target files:** src/foo.py +**Dry run:** True +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- dry_run + +### LLM Response +``` +llm raw +``` diff --git a/data/self_modify_reports/20260225_224733_break_it.md b/data/self_modify_reports/20260225_224733_break_it.md new file mode 100644 index 0000000..1a76f4a --- /dev/null +++ b/data/self_modify_reports/20260225_224733_break_it.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_224733 + +**Instruction:** Break it +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: FAILED +``` +1 failed +``` diff --git a/data/self_modify_reports/20260225_224733_do_something_vague.md b/data/self_modify_reports/20260225_224733_do_something_vague.md new file mode 100644 index 0000000..f898b79 --- /dev/null +++ b/data/self_modify_reports/20260225_224733_do_something_vague.md @@ -0,0 +1,12 @@ +# Self-Modify Report: 20260225_224733 + +**Instruction:** do something vague +**Target files:** (auto-detected) +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** No target files identified. Specify target_files or use more specific language. +**Commit:** none +**Attempts:** 0 +**Autonomous cycles:** 0 diff --git a/data/self_modify_reports/20260225_224734_fix_foo.md b/data/self_modify_reports/20260225_224734_fix_foo.md new file mode 100644 index 0000000..952f991 --- /dev/null +++ b/data/self_modify_reports/20260225_224734_fix_foo.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_224734 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: FAILED +``` +FAILED +``` diff --git a/data/self_modify_reports/20260225_224734_fix_foo_important_correction_from_previ.md b/data/self_modify_reports/20260225_224734_fix_foo_important_correction_from_previ.md new file mode 100644 index 0000000..50fa47e --- /dev/null +++ b/data/self_modify_reports/20260225_224734_fix_foo_important_correction_from_previ.md @@ -0,0 +1,34 @@ +# Self-Modify Report: 20260225_224734 + +**Instruction:** Fix foo + +IMPORTANT CORRECTION from previous failure: +Fix: do X instead of Y +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: PASSED +``` +PASSED +``` diff --git a/data/self_modify_reports/20260225_225049_add_docstring.md b/data/self_modify_reports/20260225_225049_add_docstring.md new file mode 100644 index 0000000..dd3e411 --- /dev/null +++ b/data/self_modify_reports/20260225_225049_add_docstring.md @@ -0,0 +1,19 @@ +# Self-Modify Report: 20260225_225049 + +**Instruction:** Add docstring +**Target files:** src/foo.py +**Dry run:** True +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- dry_run + +### LLM Response +``` +llm raw +``` diff --git a/data/self_modify_reports/20260225_225049_break_it.md b/data/self_modify_reports/20260225_225049_break_it.md new file mode 100644 index 0000000..4bd1bd1 --- /dev/null +++ b/data/self_modify_reports/20260225_225049_break_it.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_225049 + +**Instruction:** Break it +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: FAILED +``` +1 failed +``` diff --git a/data/self_modify_reports/20260225_225049_do_something_vague.md b/data/self_modify_reports/20260225_225049_do_something_vague.md new file mode 100644 index 0000000..fa54a8e --- /dev/null +++ b/data/self_modify_reports/20260225_225049_do_something_vague.md @@ -0,0 +1,12 @@ +# Self-Modify Report: 20260225_225049 + +**Instruction:** do something vague +**Target files:** (auto-detected) +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** No target files identified. Specify target_files or use more specific language. +**Commit:** none +**Attempts:** 0 +**Autonomous cycles:** 0 diff --git a/data/self_modify_reports/20260225_225049_fix_foo.md b/data/self_modify_reports/20260225_225049_fix_foo.md new file mode 100644 index 0000000..fcbf51f --- /dev/null +++ b/data/self_modify_reports/20260225_225049_fix_foo.md @@ -0,0 +1,48 @@ +# Self-Modify Report: 20260225_225049 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 2 +**Autonomous cycles:** 0 + +## Attempt 1 -- syntax_validation + +**Error:** src/foo.py: line 1: '(' was never closed + +### LLM Response +``` +bad llm +``` + +### Edits Written +#### src/foo.py +```python +def foo( + +``` + +## Attempt 2 -- complete + +### LLM Response +``` +good llm +``` + +### Edits Written +#### src/foo.py +```python +def foo(): + pass + +``` + +### Test Result: PASSED +``` +passed +``` diff --git a/data/self_modify_reports/20260225_225049_fix_foo_important_correction_from_previ.md b/data/self_modify_reports/20260225_225049_fix_foo_important_correction_from_previ.md new file mode 100644 index 0000000..6ff7c5a --- /dev/null +++ b/data/self_modify_reports/20260225_225049_fix_foo_important_correction_from_previ.md @@ -0,0 +1,34 @@ +# Self-Modify Report: 20260225_225049 + +**Instruction:** Fix foo + +IMPORTANT CORRECTION from previous failure: +Fix: do X instead of Y +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: PASSED +``` +PASSED +``` diff --git a/data/self_modify_reports/20260225_230304_fix_foo.md b/data/self_modify_reports/20260225_230304_fix_foo.md new file mode 100644 index 0000000..f530dde --- /dev/null +++ b/data/self_modify_reports/20260225_230304_fix_foo.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_230304 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: FAILED +``` +FAILED +``` diff --git a/data/self_modify_reports/20260225_230305_add_docstring.md b/data/self_modify_reports/20260225_230305_add_docstring.md new file mode 100644 index 0000000..b4f08b4 --- /dev/null +++ b/data/self_modify_reports/20260225_230305_add_docstring.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_230305 + +**Instruction:** Add docstring +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc12345 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: PASSED +``` +5 passed +``` diff --git a/data/self_modify_reports/20260225_230305_break_it.md b/data/self_modify_reports/20260225_230305_break_it.md new file mode 100644 index 0000000..c9e4fae --- /dev/null +++ b/data/self_modify_reports/20260225_230305_break_it.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_230305 + +**Instruction:** Break it +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: FAILED +``` +1 failed +``` diff --git a/data/self_modify_reports/20260225_230305_do_something_vague.md b/data/self_modify_reports/20260225_230305_do_something_vague.md new file mode 100644 index 0000000..6b8de02 --- /dev/null +++ b/data/self_modify_reports/20260225_230305_do_something_vague.md @@ -0,0 +1,12 @@ +# Self-Modify Report: 20260225_230305 + +**Instruction:** do something vague +**Target files:** (auto-detected) +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** No target files identified. Specify target_files or use more specific language. +**Commit:** none +**Attempts:** 0 +**Autonomous cycles:** 0 diff --git a/data/self_modify_reports/20260225_230305_fix_foo_important_correction_from_previ.md b/data/self_modify_reports/20260225_230305_fix_foo_important_correction_from_previ.md new file mode 100644 index 0000000..c2b0c6b --- /dev/null +++ b/data/self_modify_reports/20260225_230305_fix_foo_important_correction_from_previ.md @@ -0,0 +1,34 @@ +# Self-Modify Report: 20260225_230305 + +**Instruction:** Fix foo + +IMPORTANT CORRECTION from previous failure: +Fix: do X instead of Y +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: PASSED +``` +PASSED +``` diff --git a/data/self_modify_reports/20260225_230306_fix_foo.md b/data/self_modify_reports/20260225_230306_fix_foo.md new file mode 100644 index 0000000..8c6fb0b --- /dev/null +++ b/data/self_modify_reports/20260225_230306_fix_foo.md @@ -0,0 +1,48 @@ +# Self-Modify Report: 20260225_230306 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 2 +**Autonomous cycles:** 0 + +## Attempt 1 -- syntax_validation + +**Error:** src/foo.py: line 1: '(' was never closed + +### LLM Response +``` +bad llm +``` + +### Edits Written +#### src/foo.py +```python +def foo( + +``` + +## Attempt 2 -- complete + +### LLM Response +``` +good llm +``` + +### Edits Written +#### src/foo.py +```python +def foo(): + pass + +``` + +### Test Result: PASSED +``` +passed +``` diff --git a/data/self_modify_reports/20260225_230553_add_docstring.md b/data/self_modify_reports/20260225_230553_add_docstring.md new file mode 100644 index 0000000..ba405d1 --- /dev/null +++ b/data/self_modify_reports/20260225_230553_add_docstring.md @@ -0,0 +1,19 @@ +# Self-Modify Report: 20260225_230553 + +**Instruction:** Add docstring +**Target files:** src/foo.py +**Dry run:** True +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- dry_run + +### LLM Response +``` +llm raw +``` diff --git a/data/self_modify_reports/20260225_230553_break_it.md b/data/self_modify_reports/20260225_230553_break_it.md new file mode 100644 index 0000000..4dc4bec --- /dev/null +++ b/data/self_modify_reports/20260225_230553_break_it.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_230553 + +**Instruction:** Break it +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: FAILED +``` +1 failed +``` diff --git a/data/self_modify_reports/20260225_230553_do_something_vague.md b/data/self_modify_reports/20260225_230553_do_something_vague.md new file mode 100644 index 0000000..710f188 --- /dev/null +++ b/data/self_modify_reports/20260225_230553_do_something_vague.md @@ -0,0 +1,12 @@ +# Self-Modify Report: 20260225_230553 + +**Instruction:** do something vague +**Target files:** (auto-detected) +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** No target files identified. Specify target_files or use more specific language. +**Commit:** none +**Attempts:** 0 +**Autonomous cycles:** 0 diff --git a/data/self_modify_reports/20260225_230554_fix_foo.md b/data/self_modify_reports/20260225_230554_fix_foo.md new file mode 100644 index 0000000..51cecd3 --- /dev/null +++ b/data/self_modify_reports/20260225_230554_fix_foo.md @@ -0,0 +1,48 @@ +# Self-Modify Report: 20260225_230554 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 2 +**Autonomous cycles:** 0 + +## Attempt 1 -- syntax_validation + +**Error:** src/foo.py: line 1: '(' was never closed + +### LLM Response +``` +bad llm +``` + +### Edits Written +#### src/foo.py +```python +def foo( + +``` + +## Attempt 2 -- complete + +### LLM Response +``` +good llm +``` + +### Edits Written +#### src/foo.py +```python +def foo(): + pass + +``` + +### Test Result: PASSED +``` +passed +``` diff --git a/data/self_modify_reports/20260225_230554_fix_foo_important_correction_from_previ.md b/data/self_modify_reports/20260225_230554_fix_foo_important_correction_from_previ.md new file mode 100644 index 0000000..7f2d325 --- /dev/null +++ b/data/self_modify_reports/20260225_230554_fix_foo_important_correction_from_previ.md @@ -0,0 +1,34 @@ +# Self-Modify Report: 20260225_230554 + +**Instruction:** Fix foo + +IMPORTANT CORRECTION from previous failure: +Fix: do X instead of Y +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: PASSED +``` +PASSED +``` diff --git a/data/self_modify_reports/20260225_231440_add_docstring.md b/data/self_modify_reports/20260225_231440_add_docstring.md new file mode 100644 index 0000000..74516bd --- /dev/null +++ b/data/self_modify_reports/20260225_231440_add_docstring.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_231440 + +**Instruction:** Add docstring +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc12345 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: PASSED +``` +5 passed +``` diff --git a/data/self_modify_reports/20260225_231440_break_it.md b/data/self_modify_reports/20260225_231440_break_it.md new file mode 100644 index 0000000..cea12e2 --- /dev/null +++ b/data/self_modify_reports/20260225_231440_break_it.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_231440 + +**Instruction:** Break it +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: FAILED +``` +1 failed +``` diff --git a/data/self_modify_reports/20260225_231440_do_something_vague.md b/data/self_modify_reports/20260225_231440_do_something_vague.md new file mode 100644 index 0000000..39a4725 --- /dev/null +++ b/data/self_modify_reports/20260225_231440_do_something_vague.md @@ -0,0 +1,12 @@ +# Self-Modify Report: 20260225_231440 + +**Instruction:** do something vague +**Target files:** (auto-detected) +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** No target files identified. Specify target_files or use more specific language. +**Commit:** none +**Attempts:** 0 +**Autonomous cycles:** 0 diff --git a/data/self_modify_reports/20260225_231440_fix_foo.md b/data/self_modify_reports/20260225_231440_fix_foo.md new file mode 100644 index 0000000..97c89fc --- /dev/null +++ b/data/self_modify_reports/20260225_231440_fix_foo.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_231440 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: FAILED +``` +FAILED +``` diff --git a/data/self_modify_reports/20260225_231440_fix_foo_important_correction_from_previ.md b/data/self_modify_reports/20260225_231440_fix_foo_important_correction_from_previ.md new file mode 100644 index 0000000..ab3463f --- /dev/null +++ b/data/self_modify_reports/20260225_231440_fix_foo_important_correction_from_previ.md @@ -0,0 +1,34 @@ +# Self-Modify Report: 20260225_231440 + +**Instruction:** Fix foo + +IMPORTANT CORRECTION from previous failure: +Fix: do X instead of Y +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: PASSED +``` +PASSED +``` diff --git a/data/self_modify_reports/20260225_231441_fix_foo.md b/data/self_modify_reports/20260225_231441_fix_foo.md new file mode 100644 index 0000000..657f1ac --- /dev/null +++ b/data/self_modify_reports/20260225_231441_fix_foo.md @@ -0,0 +1,48 @@ +# Self-Modify Report: 20260225_231441 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 2 +**Autonomous cycles:** 0 + +## Attempt 1 -- syntax_validation + +**Error:** src/foo.py: line 1: '(' was never closed + +### LLM Response +``` +bad llm +``` + +### Edits Written +#### src/foo.py +```python +def foo( + +``` + +## Attempt 2 -- complete + +### LLM Response +``` +good llm +``` + +### Edits Written +#### src/foo.py +```python +def foo(): + pass + +``` + +### Test Result: PASSED +``` +passed +``` diff --git a/data/self_modify_reports/20260225_231645_add_docstring.md b/data/self_modify_reports/20260225_231645_add_docstring.md new file mode 100644 index 0000000..d78368b --- /dev/null +++ b/data/self_modify_reports/20260225_231645_add_docstring.md @@ -0,0 +1,19 @@ +# Self-Modify Report: 20260225_231645 + +**Instruction:** Add docstring +**Target files:** src/foo.py +**Dry run:** True +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- dry_run + +### LLM Response +``` +llm raw +``` diff --git a/data/self_modify_reports/20260225_231645_break_it.md b/data/self_modify_reports/20260225_231645_break_it.md new file mode 100644 index 0000000..a4e3299 --- /dev/null +++ b/data/self_modify_reports/20260225_231645_break_it.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_231645 + +**Instruction:** Break it +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: FAILED +``` +1 failed +``` diff --git a/data/self_modify_reports/20260225_231645_do_something_vague.md b/data/self_modify_reports/20260225_231645_do_something_vague.md new file mode 100644 index 0000000..4c68581 --- /dev/null +++ b/data/self_modify_reports/20260225_231645_do_something_vague.md @@ -0,0 +1,12 @@ +# Self-Modify Report: 20260225_231645 + +**Instruction:** do something vague +**Target files:** (auto-detected) +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** No target files identified. Specify target_files or use more specific language. +**Commit:** none +**Attempts:** 0 +**Autonomous cycles:** 0 diff --git a/data/self_modify_reports/20260225_231645_fix_foo.md b/data/self_modify_reports/20260225_231645_fix_foo.md new file mode 100644 index 0000000..7994a87 --- /dev/null +++ b/data/self_modify_reports/20260225_231645_fix_foo.md @@ -0,0 +1,48 @@ +# Self-Modify Report: 20260225_231645 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 2 +**Autonomous cycles:** 0 + +## Attempt 1 -- syntax_validation + +**Error:** src/foo.py: line 1: '(' was never closed + +### LLM Response +``` +bad llm +``` + +### Edits Written +#### src/foo.py +```python +def foo( + +``` + +## Attempt 2 -- complete + +### LLM Response +``` +good llm +``` + +### Edits Written +#### src/foo.py +```python +def foo(): + pass + +``` + +### Test Result: PASSED +``` +passed +``` diff --git a/data/self_modify_reports/20260225_231645_fix_foo_important_correction_from_previ.md b/data/self_modify_reports/20260225_231645_fix_foo_important_correction_from_previ.md new file mode 100644 index 0000000..3cd0cec --- /dev/null +++ b/data/self_modify_reports/20260225_231645_fix_foo_important_correction_from_previ.md @@ -0,0 +1,34 @@ +# Self-Modify Report: 20260225_231645 + +**Instruction:** Fix foo + +IMPORTANT CORRECTION from previous failure: +Fix: do X instead of Y +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: PASSED +``` +PASSED +``` diff --git a/data/self_modify_reports/20260225_232402_add_docstring.md b/data/self_modify_reports/20260225_232402_add_docstring.md new file mode 100644 index 0000000..3b3f22d --- /dev/null +++ b/data/self_modify_reports/20260225_232402_add_docstring.md @@ -0,0 +1,19 @@ +# Self-Modify Report: 20260225_232402 + +**Instruction:** Add docstring +**Target files:** src/foo.py +**Dry run:** True +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- dry_run + +### LLM Response +``` +llm raw +``` diff --git a/data/self_modify_reports/20260225_232402_break_it.md b/data/self_modify_reports/20260225_232402_break_it.md new file mode 100644 index 0000000..4a7637f --- /dev/null +++ b/data/self_modify_reports/20260225_232402_break_it.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_232402 + +**Instruction:** Break it +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: FAILED +``` +1 failed +``` diff --git a/data/self_modify_reports/20260225_232402_do_something_vague.md b/data/self_modify_reports/20260225_232402_do_something_vague.md new file mode 100644 index 0000000..a79f63c --- /dev/null +++ b/data/self_modify_reports/20260225_232402_do_something_vague.md @@ -0,0 +1,12 @@ +# Self-Modify Report: 20260225_232402 + +**Instruction:** do something vague +**Target files:** (auto-detected) +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** No target files identified. Specify target_files or use more specific language. +**Commit:** none +**Attempts:** 0 +**Autonomous cycles:** 0 diff --git a/data/self_modify_reports/20260225_232402_fix_foo.md b/data/self_modify_reports/20260225_232402_fix_foo.md new file mode 100644 index 0000000..0b9e93f --- /dev/null +++ b/data/self_modify_reports/20260225_232402_fix_foo.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260225_232402 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: FAILED +``` +FAILED +``` diff --git a/data/self_modify_reports/20260225_232402_fix_foo_important_correction_from_previ.md b/data/self_modify_reports/20260225_232402_fix_foo_important_correction_from_previ.md new file mode 100644 index 0000000..ba34fc3 --- /dev/null +++ b/data/self_modify_reports/20260225_232402_fix_foo_important_correction_from_previ.md @@ -0,0 +1,34 @@ +# Self-Modify Report: 20260225_232402 + +**Instruction:** Fix foo + +IMPORTANT CORRECTION from previous failure: +Fix: do X instead of Y +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: PASSED +``` +PASSED +``` diff --git a/data/self_modify_reports/20260225_232403_fix_foo.md b/data/self_modify_reports/20260225_232403_fix_foo.md new file mode 100644 index 0000000..d932f66 --- /dev/null +++ b/data/self_modify_reports/20260225_232403_fix_foo.md @@ -0,0 +1,48 @@ +# Self-Modify Report: 20260225_232403 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 2 +**Autonomous cycles:** 0 + +## Attempt 1 -- syntax_validation + +**Error:** src/foo.py: line 1: '(' was never closed + +### LLM Response +``` +bad llm +``` + +### Edits Written +#### src/foo.py +```python +def foo( + +``` + +## Attempt 2 -- complete + +### LLM Response +``` +good llm +``` + +### Edits Written +#### src/foo.py +```python +def foo(): + pass + +``` + +### Test Result: PASSED +``` +passed +``` diff --git a/data/self_modify_reports/20260226_002427_add_docstring.md b/data/self_modify_reports/20260226_002427_add_docstring.md new file mode 100644 index 0000000..f544358 --- /dev/null +++ b/data/self_modify_reports/20260226_002427_add_docstring.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260226_002427 + +**Instruction:** Add docstring +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc12345 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: PASSED +``` +5 passed +``` diff --git a/data/self_modify_reports/20260226_002427_break_it.md b/data/self_modify_reports/20260226_002427_break_it.md new file mode 100644 index 0000000..05c9773 --- /dev/null +++ b/data/self_modify_reports/20260226_002427_break_it.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260226_002427 + +**Instruction:** Break it +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: FAILED +``` +1 failed +``` diff --git a/data/self_modify_reports/20260226_002427_do_something_vague.md b/data/self_modify_reports/20260226_002427_do_something_vague.md new file mode 100644 index 0000000..f5d554d --- /dev/null +++ b/data/self_modify_reports/20260226_002427_do_something_vague.md @@ -0,0 +1,12 @@ +# Self-Modify Report: 20260226_002427 + +**Instruction:** do something vague +**Target files:** (auto-detected) +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** No target files identified. Specify target_files or use more specific language. +**Commit:** none +**Attempts:** 0 +**Autonomous cycles:** 0 diff --git a/data/self_modify_reports/20260226_002427_fix_foo.md b/data/self_modify_reports/20260226_002427_fix_foo.md new file mode 100644 index 0000000..3a4dbeb --- /dev/null +++ b/data/self_modify_reports/20260226_002427_fix_foo.md @@ -0,0 +1,48 @@ +# Self-Modify Report: 20260226_002427 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 2 +**Autonomous cycles:** 0 + +## Attempt 1 -- syntax_validation + +**Error:** src/foo.py: line 1: '(' was never closed + +### LLM Response +``` +bad llm +``` + +### Edits Written +#### src/foo.py +```python +def foo( + +``` + +## Attempt 2 -- complete + +### LLM Response +``` +good llm +``` + +### Edits Written +#### src/foo.py +```python +def foo(): + pass + +``` + +### Test Result: PASSED +``` +passed +``` diff --git a/data/self_modify_reports/20260226_002428_fix_foo.md b/data/self_modify_reports/20260226_002428_fix_foo.md new file mode 100644 index 0000000..de322ed --- /dev/null +++ b/data/self_modify_reports/20260226_002428_fix_foo.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260226_002428 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: FAILED +``` +FAILED +``` diff --git a/data/self_modify_reports/20260226_002428_fix_foo_important_correction_from_previ.md b/data/self_modify_reports/20260226_002428_fix_foo_important_correction_from_previ.md new file mode 100644 index 0000000..fdca93b --- /dev/null +++ b/data/self_modify_reports/20260226_002428_fix_foo_important_correction_from_previ.md @@ -0,0 +1,34 @@ +# Self-Modify Report: 20260226_002428 + +**Instruction:** Fix foo + +IMPORTANT CORRECTION from previous failure: +Fix: do X instead of Y +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: PASSED +``` +PASSED +``` diff --git a/data/self_modify_reports/20260226_004233_add_docstring.md b/data/self_modify_reports/20260226_004233_add_docstring.md new file mode 100644 index 0000000..094bcf4 --- /dev/null +++ b/data/self_modify_reports/20260226_004233_add_docstring.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260226_004233 + +**Instruction:** Add docstring +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc12345 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: PASSED +``` +5 passed +``` diff --git a/data/self_modify_reports/20260226_004233_break_it.md b/data/self_modify_reports/20260226_004233_break_it.md new file mode 100644 index 0000000..610c836 --- /dev/null +++ b/data/self_modify_reports/20260226_004233_break_it.md @@ -0,0 +1,31 @@ +# Self-Modify Report: 20260226_004233 + +**Instruction:** Break it +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** Tests failed after 1 attempt(s). +**Commit:** none +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 1 + +``` + +### Test Result: FAILED +``` +1 failed +``` diff --git a/data/self_modify_reports/20260226_004233_do_something_vague.md b/data/self_modify_reports/20260226_004233_do_something_vague.md new file mode 100644 index 0000000..6f7a4d6 --- /dev/null +++ b/data/self_modify_reports/20260226_004233_do_something_vague.md @@ -0,0 +1,12 @@ +# Self-Modify Report: 20260226_004233 + +**Instruction:** do something vague +**Target files:** (auto-detected) +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** FAILED +**Error:** No target files identified. Specify target_files or use more specific language. +**Commit:** none +**Attempts:** 0 +**Autonomous cycles:** 0 diff --git a/data/self_modify_reports/20260226_004234_fix_foo.md b/data/self_modify_reports/20260226_004234_fix_foo.md new file mode 100644 index 0000000..9fb29cf --- /dev/null +++ b/data/self_modify_reports/20260226_004234_fix_foo.md @@ -0,0 +1,48 @@ +# Self-Modify Report: 20260226_004234 + +**Instruction:** Fix foo +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 2 +**Autonomous cycles:** 0 + +## Attempt 1 -- syntax_validation + +**Error:** src/foo.py: line 1: '(' was never closed + +### LLM Response +``` +bad llm +``` + +### Edits Written +#### src/foo.py +```python +def foo( + +``` + +## Attempt 2 -- complete + +### LLM Response +``` +good llm +``` + +### Edits Written +#### src/foo.py +```python +def foo(): + pass + +``` + +### Test Result: PASSED +``` +passed +``` diff --git a/data/self_modify_reports/20260226_004234_fix_foo_important_correction_from_previ.md b/data/self_modify_reports/20260226_004234_fix_foo_important_correction_from_previ.md new file mode 100644 index 0000000..5c94c00 --- /dev/null +++ b/data/self_modify_reports/20260226_004234_fix_foo_important_correction_from_previ.md @@ -0,0 +1,34 @@ +# Self-Modify Report: 20260226_004234 + +**Instruction:** Fix foo + +IMPORTANT CORRECTION from previous failure: +Fix: do X instead of Y +**Target files:** src/foo.py +**Dry run:** False +**Backend:** ollama +**Branch:** N/A +**Result:** SUCCESS +**Error:** none +**Commit:** abc123 +**Attempts:** 1 +**Autonomous cycles:** 0 + +## Attempt 1 -- complete + +### LLM Response +``` +llm raw +``` + +### Edits Written +#### src/foo.py +```python +x = 2 + +``` + +### Test Result: PASSED +``` +PASSED +``` diff --git a/src/dashboard/app.py b/src/dashboard/app.py index 3b2788a..2e98ae5 100644 --- a/src/dashboard/app.py +++ b/src/dashboard/app.py @@ -27,6 +27,7 @@ from dashboard.routes.spark import router as spark_router from dashboard.routes.creative import router as creative_router from dashboard.routes.discord import router as discord_router from dashboard.routes.self_modify import router as self_modify_router +from router.api import router as cascade_router logging.basicConfig( level=logging.INFO, @@ -156,6 +157,7 @@ app.include_router(spark_router) app.include_router(creative_router) app.include_router(discord_router) app.include_router(self_modify_router) +app.include_router(cascade_router) @app.get("/", response_class=HTMLResponse) diff --git a/src/router/__init__.py b/src/router/__init__.py new file mode 100644 index 0000000..730f623 --- /dev/null +++ b/src/router/__init__.py @@ -0,0 +1,12 @@ +"""Cascade LLM Router — Automatic failover between providers.""" + +from .cascade import CascadeRouter, Provider, ProviderStatus, get_router +from .api import router + +__all__ = [ + "CascadeRouter", + "Provider", + "ProviderStatus", + "get_router", + "router", +] diff --git a/src/router/api.py b/src/router/api.py new file mode 100644 index 0000000..7558c4e --- /dev/null +++ b/src/router/api.py @@ -0,0 +1,199 @@ +"""API endpoints for Cascade Router monitoring and control.""" + +import asyncio +import logging +from typing import Annotated, Any + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from .cascade import CascadeRouter, get_router + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/api/v1/router", tags=["router"]) + + +class CompletionRequest(BaseModel): + """Request body for completions.""" + messages: list[dict[str, str]] + model: str | None = None + temperature: float = 0.7 + max_tokens: int | None = None + + +class CompletionResponse(BaseModel): + """Response from completion endpoint.""" + content: str + provider: str + model: str + latency_ms: float + + +class ProviderControl(BaseModel): + """Control a provider's status.""" + action: str # "enable", "disable", "reset_circuit" + + +async def get_cascade_router() -> CascadeRouter: + """Dependency to get the cascade router.""" + return get_router() + + +@router.post("/complete", response_model=CompletionResponse) +async def complete( + request: CompletionRequest, + cascade: Annotated[CascadeRouter, Depends(get_cascade_router)], +) -> dict[str, Any]: + """Complete a conversation with automatic failover. + + Routes through providers in priority order until one succeeds. + """ + try: + result = await cascade.complete( + messages=request.messages, + model=request.model, + temperature=request.temperature, + max_tokens=request.max_tokens, + ) + return result + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) + + +@router.get("/status") +async def get_status( + cascade: Annotated[CascadeRouter, Depends(get_cascade_router)], +) -> dict[str, Any]: + """Get router status and provider health.""" + return cascade.get_status() + + +@router.get("/metrics") +async def get_metrics( + cascade: Annotated[CascadeRouter, Depends(get_cascade_router)], +) -> dict[str, Any]: + """Get detailed metrics for all providers.""" + return cascade.get_metrics() + + +@router.get("/providers") +async def list_providers( + cascade: Annotated[CascadeRouter, Depends(get_cascade_router)], +) -> list[dict[str, Any]]: + """List all configured providers.""" + return [ + { + "name": p.name, + "type": p.type, + "enabled": p.enabled, + "priority": p.priority, + "status": p.status.value, + "circuit_state": p.circuit_state.value, + "default_model": p.get_default_model(), + "models": [m["name"] for m in p.models], + } + for p in cascade.providers + ] + + +@router.post("/providers/{provider_name}/control") +async def control_provider( + provider_name: str, + control: ProviderControl, + cascade: Annotated[CascadeRouter, Depends(get_cascade_router)], +) -> dict[str, str]: + """Control a provider (enable/disable/reset).""" + provider = None + for p in cascade.providers: + if p.name == provider_name: + provider = p + break + + if not provider: + raise HTTPException(status_code=404, detail=f"Provider {provider_name} not found") + + if control.action == "enable": + provider.enabled = True + provider.status = provider.status.__class__.HEALTHY + return {"message": f"Provider {provider_name} enabled"} + + elif control.action == "disable": + provider.enabled = False + from .cascade import ProviderStatus + provider.status = ProviderStatus.DISABLED + return {"message": f"Provider {provider_name} disabled"} + + elif control.action == "reset_circuit": + from .cascade import CircuitState, ProviderStatus + provider.circuit_state = CircuitState.CLOSED + provider.circuit_opened_at = None + provider.half_open_calls = 0 + provider.metrics.consecutive_failures = 0 + provider.status = ProviderStatus.HEALTHY + return {"message": f"Circuit breaker reset for {provider_name}"} + + else: + raise HTTPException(status_code=400, detail=f"Unknown action: {control.action}") + + +@router.post("/health-check") +async def run_health_check( + cascade: Annotated[CascadeRouter, Depends(get_cascade_router)], +) -> dict[str, Any]: + """Run health checks on all providers.""" + results = [] + + for provider in cascade.providers: + # Quick ping to check availability + is_healthy = cascade._check_provider_available(provider) + + from .cascade import ProviderStatus + if is_healthy: + if provider.status == ProviderStatus.UNHEALTHY: + # Reset circuit if it was open but now healthy + provider.circuit_state = provider.circuit_state.__class__.CLOSED + provider.circuit_opened_at = None + provider.status = ProviderStatus.HEALTHY if provider.metrics.error_rate < 0.1 else ProviderStatus.DEGRADED + else: + provider.status = ProviderStatus.UNHEALTHY + + results.append({ + "name": provider.name, + "type": provider.type, + "healthy": is_healthy, + "status": provider.status.value, + }) + + return { + "checked_at": asyncio.get_event_loop().time(), + "providers": results, + "healthy_count": sum(1 for r in results if r["healthy"]), + } + + +@router.get("/config") +async def get_config( + cascade: Annotated[CascadeRouter, Depends(get_cascade_router)], +) -> dict[str, Any]: + """Get router configuration (without secrets).""" + cfg = cascade.config + + return { + "timeout_seconds": cfg.timeout_seconds, + "max_retries_per_provider": cfg.max_retries_per_provider, + "retry_delay_seconds": cfg.retry_delay_seconds, + "circuit_breaker": { + "failure_threshold": cfg.circuit_breaker_failure_threshold, + "recovery_timeout": cfg.circuit_breaker_recovery_timeout, + "half_open_max_calls": cfg.circuit_breaker_half_open_max_calls, + }, + "providers": [ + { + "name": p.name, + "type": p.type, + "priority": p.priority, + "enabled": p.enabled, + } + for p in cascade.providers + ], + } diff --git a/src/router/cascade.py b/src/router/cascade.py new file mode 100644 index 0000000..bb1de69 --- /dev/null +++ b/src/router/cascade.py @@ -0,0 +1,566 @@ +"""Cascade LLM Router — Automatic failover between providers. + +Routes requests through an ordered list of LLM providers, +automatically failing over on rate limits or errors. +Tracks metrics for latency, errors, and cost. +""" + +import asyncio +import logging +import time +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from enum import Enum +from typing import Any, Optional + +from pathlib import Path + +try: + import yaml +except ImportError: + yaml = None # type: ignore + +try: + import requests +except ImportError: + requests = None # type: ignore + +logger = logging.getLogger(__name__) + + +class ProviderStatus(Enum): + """Health status of a provider.""" + HEALTHY = "healthy" + DEGRADED = "degraded" # Working but slow or occasional errors + UNHEALTHY = "unhealthy" # Circuit breaker open + DISABLED = "disabled" + + +class CircuitState(Enum): + """Circuit breaker state.""" + CLOSED = "closed" # Normal operation + OPEN = "open" # Failing, rejecting requests + HALF_OPEN = "half_open" # Testing if recovered + + +@dataclass +class ProviderMetrics: + """Metrics for a single provider.""" + total_requests: int = 0 + successful_requests: int = 0 + failed_requests: int = 0 + total_latency_ms: float = 0.0 + last_request_time: Optional[str] = None + last_error_time: Optional[str] = None + consecutive_failures: int = 0 + + @property + def avg_latency_ms(self) -> float: + if self.total_requests == 0: + return 0.0 + return self.total_latency_ms / self.total_requests + + @property + def error_rate(self) -> float: + if self.total_requests == 0: + return 0.0 + return self.failed_requests / self.total_requests + + +@dataclass +class Provider: + """LLM provider configuration and state.""" + name: str + type: str # ollama, openai, anthropic, airllm + enabled: bool + priority: int + url: Optional[str] = None + api_key: Optional[str] = None + base_url: Optional[str] = None + models: list[dict] = field(default_factory=list) + + # Runtime state + status: ProviderStatus = ProviderStatus.HEALTHY + metrics: ProviderMetrics = field(default_factory=ProviderMetrics) + circuit_state: CircuitState = CircuitState.CLOSED + circuit_opened_at: Optional[float] = None + half_open_calls: int = 0 + + def get_default_model(self) -> Optional[str]: + """Get the default model for this provider.""" + for model in self.models: + if model.get("default"): + return model["name"] + if self.models: + return self.models[0]["name"] + return None + + +@dataclass +class RouterConfig: + """Cascade router configuration.""" + timeout_seconds: int = 30 + max_retries_per_provider: int = 2 + retry_delay_seconds: int = 1 + circuit_breaker_failure_threshold: int = 5 + circuit_breaker_recovery_timeout: int = 60 + circuit_breaker_half_open_max_calls: int = 2 + cost_tracking_enabled: bool = True + budget_daily_usd: float = 10.0 + + +class CascadeRouter: + """Routes LLM requests with automatic failover. + + Usage: + router = CascadeRouter() + + response = await router.complete( + messages=[{"role": "user", "content": "Hello"}], + model="llama3.2" + ) + + # Check metrics + metrics = router.get_metrics() + """ + + def __init__(self, config_path: Optional[Path] = None) -> None: + self.config_path = config_path or Path("config/providers.yaml") + self.providers: list[Provider] = [] + self.config: RouterConfig = RouterConfig() + self._load_config() + + logger.info("CascadeRouter initialized with %d providers", len(self.providers)) + + def _load_config(self) -> None: + """Load configuration from YAML.""" + if not self.config_path.exists(): + logger.warning("Config not found: %s, using defaults", self.config_path) + return + + try: + if yaml is None: + raise RuntimeError("PyYAML not installed") + + content = self.config_path.read_text() + # Expand environment variables + content = self._expand_env_vars(content) + data = yaml.safe_load(content) + + # Load cascade settings + cascade = data.get("cascade", {}) + self.config = RouterConfig( + timeout_seconds=cascade.get("timeout_seconds", 30), + max_retries_per_provider=cascade.get("max_retries_per_provider", 2), + retry_delay_seconds=cascade.get("retry_delay_seconds", 1), + circuit_breaker_failure_threshold=cascade.get("circuit_breaker", {}).get("failure_threshold", 5), + circuit_breaker_recovery_timeout=cascade.get("circuit_breaker", {}).get("recovery_timeout", 60), + circuit_breaker_half_open_max_calls=cascade.get("circuit_breaker", {}).get("half_open_max_calls", 2), + ) + + # Load providers + for p_data in data.get("providers", []): + # Skip disabled providers + if not p_data.get("enabled", False): + continue + + provider = Provider( + name=p_data["name"], + type=p_data["type"], + enabled=p_data.get("enabled", True), + priority=p_data.get("priority", 99), + url=p_data.get("url"), + api_key=p_data.get("api_key"), + base_url=p_data.get("base_url"), + models=p_data.get("models", []), + ) + + # Check if provider is actually available + if self._check_provider_available(provider): + self.providers.append(provider) + else: + logger.warning("Provider %s not available, skipping", provider.name) + + # Sort by priority + self.providers.sort(key=lambda p: p.priority) + + except Exception as exc: + logger.error("Failed to load config: %s", exc) + + def _expand_env_vars(self, content: str) -> str: + """Expand ${VAR} syntax in YAML content.""" + import os + import re + + def replace_var(match): + var_name = match.group(1) + return os.environ.get(var_name, match.group(0)) + + return re.sub(r"\$\{(\w+)\}", replace_var, content) + + def _check_provider_available(self, provider: Provider) -> bool: + """Check if a provider is actually available.""" + if provider.type == "ollama": + # Check if Ollama is running + if requests is None: + # Can't check without requests, assume available + return True + try: + url = provider.url or "http://localhost:11434" + response = requests.get(f"{url}/api/tags", timeout=5) + return response.status_code == 200 + except Exception: + return False + + elif provider.type == "airllm": + # Check if airllm is installed + try: + import airllm + return True + except ImportError: + return False + + elif provider.type in ("openai", "anthropic"): + # Check if API key is set + return provider.api_key is not None and provider.api_key != "" + + return True + + async def complete( + self, + messages: list[dict], + model: Optional[str] = None, + temperature: float = 0.7, + max_tokens: Optional[int] = None, + ) -> dict: + """Complete a chat conversation with automatic failover. + + Args: + messages: List of message dicts with role and content + model: Preferred model (tries this first, then provider defaults) + temperature: Sampling temperature + max_tokens: Maximum tokens to generate + + Returns: + Dict with content, provider_used, and metrics + + Raises: + RuntimeError: If all providers fail + """ + errors = [] + + for provider in self.providers: + # Skip unhealthy providers (circuit breaker) + if provider.status == ProviderStatus.UNHEALTHY: + # Check if circuit breaker can close + if self._can_close_circuit(provider): + provider.circuit_state = CircuitState.HALF_OPEN + provider.half_open_calls = 0 + logger.info("Circuit breaker half-open for %s", provider.name) + else: + logger.debug("Skipping %s (circuit open)", provider.name) + continue + + # Try this provider + for attempt in range(self.config.max_retries_per_provider): + try: + result = await self._try_provider( + provider=provider, + messages=messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + ) + + # Success! Update metrics and return + self._record_success(provider, result.get("latency_ms", 0)) + return { + "content": result["content"], + "provider": provider.name, + "model": result.get("model", model or provider.get_default_model()), + "latency_ms": result.get("latency_ms", 0), + } + + except Exception as exc: + error_msg = str(exc) + logger.warning( + "Provider %s attempt %d failed: %s", + provider.name, attempt + 1, error_msg + ) + errors.append(f"{provider.name}: {error_msg}") + + if attempt < self.config.max_retries_per_provider - 1: + await asyncio.sleep(self.config.retry_delay_seconds) + + # All retries failed for this provider + self._record_failure(provider) + + # All providers failed + raise RuntimeError(f"All providers failed: {'; '.join(errors)}") + + async def _try_provider( + self, + provider: Provider, + messages: list[dict], + model: Optional[str], + temperature: float, + max_tokens: Optional[int], + ) -> dict: + """Try a single provider request.""" + start_time = time.time() + + if provider.type == "ollama": + result = await self._call_ollama( + provider=provider, + messages=messages, + model=model or provider.get_default_model(), + temperature=temperature, + ) + elif provider.type == "openai": + result = await self._call_openai( + provider=provider, + messages=messages, + model=model or provider.get_default_model(), + temperature=temperature, + max_tokens=max_tokens, + ) + elif provider.type == "anthropic": + result = await self._call_anthropic( + provider=provider, + messages=messages, + model=model or provider.get_default_model(), + temperature=temperature, + max_tokens=max_tokens, + ) + else: + raise ValueError(f"Unknown provider type: {provider.type}") + + latency_ms = (time.time() - start_time) * 1000 + result["latency_ms"] = latency_ms + + return result + + async def _call_ollama( + self, + provider: Provider, + messages: list[dict], + model: str, + temperature: float, + ) -> dict: + """Call Ollama API.""" + import aiohttp + + url = f"{provider.url}/api/chat" + + payload = { + "model": model, + "messages": messages, + "stream": False, + "options": { + "temperature": temperature, + }, + } + + timeout = aiohttp.ClientTimeout(total=self.config.timeout_seconds) + + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post(url, json=payload) as response: + if response.status != 200: + text = await response.text() + raise RuntimeError(f"Ollama error {response.status}: {text}") + + data = await response.json() + return { + "content": data["message"]["content"], + "model": model, + } + + async def _call_openai( + self, + provider: Provider, + messages: list[dict], + model: str, + temperature: float, + max_tokens: Optional[int], + ) -> dict: + """Call OpenAI API.""" + import openai + + client = openai.AsyncOpenAI( + api_key=provider.api_key, + base_url=provider.base_url, + timeout=self.config.timeout_seconds, + ) + + kwargs = { + "model": model, + "messages": messages, + "temperature": temperature, + } + if max_tokens: + kwargs["max_tokens"] = max_tokens + + response = await client.chat.completions.create(**kwargs) + + return { + "content": response.choices[0].message.content, + "model": response.model, + } + + async def _call_anthropic( + self, + provider: Provider, + messages: list[dict], + model: str, + temperature: float, + max_tokens: Optional[int], + ) -> dict: + """Call Anthropic API.""" + import anthropic + + client = anthropic.AsyncAnthropic( + api_key=provider.api_key, + timeout=self.config.timeout_seconds, + ) + + # Convert messages to Anthropic format + system_msg = None + conversation = [] + for msg in messages: + if msg["role"] == "system": + system_msg = msg["content"] + else: + conversation.append({ + "role": msg["role"], + "content": msg["content"], + }) + + kwargs = { + "model": model, + "messages": conversation, + "temperature": temperature, + "max_tokens": max_tokens or 1024, + } + if system_msg: + kwargs["system"] = system_msg + + response = await client.messages.create(**kwargs) + + return { + "content": response.content[0].text, + "model": response.model, + } + + def _record_success(self, provider: Provider, latency_ms: float) -> None: + """Record a successful request.""" + provider.metrics.total_requests += 1 + provider.metrics.successful_requests += 1 + provider.metrics.total_latency_ms += latency_ms + provider.metrics.last_request_time = datetime.now(timezone.utc).isoformat() + provider.metrics.consecutive_failures = 0 + + # Close circuit breaker if half-open + if provider.circuit_state == CircuitState.HALF_OPEN: + provider.half_open_calls += 1 + if provider.half_open_calls >= self.config.circuit_breaker_half_open_max_calls: + self._close_circuit(provider) + + # Update status based on error rate + if provider.metrics.error_rate < 0.1: + provider.status = ProviderStatus.HEALTHY + elif provider.metrics.error_rate < 0.3: + provider.status = ProviderStatus.DEGRADED + + def _record_failure(self, provider: Provider) -> None: + """Record a failed request.""" + provider.metrics.total_requests += 1 + provider.metrics.failed_requests += 1 + provider.metrics.last_error_time = datetime.now(timezone.utc).isoformat() + provider.metrics.consecutive_failures += 1 + + # Check if we should open circuit breaker + if provider.metrics.consecutive_failures >= self.config.circuit_breaker_failure_threshold: + self._open_circuit(provider) + + # Update status + if provider.metrics.error_rate > 0.3: + provider.status = ProviderStatus.DEGRADED + if provider.metrics.error_rate > 0.5: + provider.status = ProviderStatus.UNHEALTHY + + def _open_circuit(self, provider: Provider) -> None: + """Open the circuit breaker for a provider.""" + provider.circuit_state = CircuitState.OPEN + provider.circuit_opened_at = time.time() + provider.status = ProviderStatus.UNHEALTHY + logger.warning("Circuit breaker OPEN for %s", provider.name) + + def _can_close_circuit(self, provider: Provider) -> bool: + """Check if circuit breaker can transition to half-open.""" + if provider.circuit_opened_at is None: + return False + elapsed = time.time() - provider.circuit_opened_at + return elapsed >= self.config.circuit_breaker_recovery_timeout + + def _close_circuit(self, provider: Provider) -> None: + """Close the circuit breaker (provider healthy again).""" + provider.circuit_state = CircuitState.CLOSED + provider.circuit_opened_at = None + provider.half_open_calls = 0 + provider.metrics.consecutive_failures = 0 + provider.status = ProviderStatus.HEALTHY + logger.info("Circuit breaker CLOSED for %s", provider.name) + + def get_metrics(self) -> dict: + """Get metrics for all providers.""" + return { + "providers": [ + { + "name": p.name, + "type": p.type, + "status": p.status.value, + "circuit_state": p.circuit_state.value, + "metrics": { + "total_requests": p.metrics.total_requests, + "successful": p.metrics.successful_requests, + "failed": p.metrics.failed_requests, + "error_rate": round(p.metrics.error_rate, 3), + "avg_latency_ms": round(p.metrics.avg_latency_ms, 2), + }, + } + for p in self.providers + ] + } + + def get_status(self) -> dict: + """Get current router status.""" + healthy = sum(1 for p in self.providers if p.status == ProviderStatus.HEALTHY) + + return { + "total_providers": len(self.providers), + "healthy_providers": healthy, + "degraded_providers": sum(1 for p in self.providers if p.status == ProviderStatus.DEGRADED), + "unhealthy_providers": sum(1 for p in self.providers if p.status == ProviderStatus.UNHEALTHY), + "providers": [ + { + "name": p.name, + "type": p.type, + "status": p.status.value, + "priority": p.priority, + "default_model": p.get_default_model(), + } + for p in self.providers + ], + } + + +# Module-level singleton +cascade_router: Optional[CascadeRouter] = None + + +def get_router() -> CascadeRouter: + """Get or create the cascade router singleton.""" + global cascade_router + if cascade_router is None: + cascade_router = CascadeRouter() + return cascade_router diff --git a/tests/test_router_api.py b/tests/test_router_api.py new file mode 100644 index 0000000..1ac5945 --- /dev/null +++ b/tests/test_router_api.py @@ -0,0 +1,358 @@ +"""Tests for Cascade Router API endpoints.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from router.cascade import CircuitState, Provider, ProviderStatus +from router.api import router, get_cascade_router + + +def make_mock_router(): + """Create a mock CascadeRouter.""" + router = MagicMock() + + # Create test providers + provider1 = Provider( + name="ollama-local", + type="ollama", + enabled=True, + priority=1, + url="http://localhost:11434", + models=[{"name": "llama3.2", "default": True, "context_window": 128000}], + ) + provider1.status = ProviderStatus.HEALTHY + provider1.circuit_state = CircuitState.CLOSED + + provider2 = Provider( + name="openai-backup", + type="openai", + enabled=True, + priority=2, + api_key="sk-test", + models=[{"name": "gpt-4o-mini", "default": True, "context_window": 128000}], + ) + provider2.status = ProviderStatus.DEGRADED + provider2.circuit_state = CircuitState.CLOSED + + router.providers = [provider1, provider2] + router.config.timeout_seconds = 30 + router.config.max_retries_per_provider = 2 + router.config.circuit_breaker_failure_threshold = 5 + + return router + + +@pytest.fixture +def mock_router(): + """Create test client with mocked router.""" + from fastapi import FastAPI + + app = FastAPI() + app.include_router(router) + + # Create mock router + mock = make_mock_router() + + # Override dependency + async def mock_get_router(): + return mock + + app.dependency_overrides[get_cascade_router] = mock_get_router + + client = TestClient(app) + return client, mock + + +class TestCompleteEndpoint: + """Test /complete endpoint.""" + + def test_complete_success(self, mock_router): + """Test successful completion.""" + client, mock = mock_router + mock.complete = AsyncMock(return_value={ + "content": "Hello! How can I help?", + "provider": "ollama-local", + "model": "llama3.2", + "latency_ms": 250.5, + }) + + response = client.post("/api/v1/router/complete", json={ + "messages": [{"role": "user", "content": "Hi"}], + "model": "llama3.2", + "temperature": 0.7, + }) + + assert response.status_code == 200 + data = response.json() + assert data["content"] == "Hello! How can I help?" + assert data["provider"] == "ollama-local" + assert data["latency_ms"] == 250.5 + + def test_complete_all_providers_fail(self, mock_router): + """Test 503 when all providers fail.""" + client, mock = mock_router + mock.complete = AsyncMock(side_effect=RuntimeError("All providers failed")) + + response = client.post("/api/v1/router/complete", json={ + "messages": [{"role": "user", "content": "Hi"}], + }) + + assert response.status_code == 503 + assert "All providers failed" in response.json()["detail"] + + def test_complete_default_temperature(self, mock_router): + """Test completion with default temperature.""" + client, mock = mock_router + mock.complete = AsyncMock(return_value={ + "content": "Response", + "provider": "ollama-local", + "model": "llama3.2", + "latency_ms": 100.0, + }) + + response = client.post("/api/v1/router/complete", json={ + "messages": [{"role": "user", "content": "Hi"}], + }) + + assert response.status_code == 200 + # Check that complete was called with correct temperature + call_args = mock.complete.call_args + assert call_args.kwargs["temperature"] == 0.7 + + +class TestStatusEndpoint: + """Test /status endpoint.""" + + def test_get_status(self, mock_router): + """Test getting router status.""" + client, mock = mock_router + mock.get_status = MagicMock(return_value={ + "total_providers": 2, + "healthy_providers": 1, + "degraded_providers": 1, + "unhealthy_providers": 0, + "providers": [ + { + "name": "ollama-local", + "type": "ollama", + "status": "healthy", + "priority": 1, + "default_model": "llama3.2", + }, + { + "name": "openai-backup", + "type": "openai", + "status": "degraded", + "priority": 2, + "default_model": "gpt-4o-mini", + }, + ], + }) + + response = client.get("/api/v1/router/status") + + assert response.status_code == 200 + data = response.json() + assert data["total_providers"] == 2 + assert data["healthy_providers"] == 1 + assert data["degraded_providers"] == 1 + assert len(data["providers"]) == 2 + + +class TestMetricsEndpoint: + """Test /metrics endpoint.""" + + def test_get_metrics(self, mock_router): + """Test getting detailed metrics.""" + client, mock = mock_router + # Setup the mock return value on the mock_router object + mock.get_metrics = MagicMock(return_value={ + "providers": [ + { + "name": "ollama-local", + "type": "ollama", + "status": "healthy", + "circuit_state": "closed", + "metrics": { + "total_requests": 100, + "successful": 98, + "failed": 2, + "error_rate": 0.02, + "avg_latency_ms": 150.5, + }, + }, + ], + }) + + response = client.get("/api/v1/router/metrics") + + assert response.status_code == 200 + data = response.json() + assert len(data["providers"]) == 1 + metrics = data["providers"][0]["metrics"] + assert metrics["total_requests"] == 100 + assert metrics["error_rate"] == 0.02 + assert metrics["avg_latency_ms"] == 150.5 + + +class TestListProvidersEndpoint: + """Test /providers endpoint.""" + + def test_list_providers(self, mock_router): + """Test listing all providers.""" + client, mock = mock_router + + response = client.get("/api/v1/router/providers") + + assert response.status_code == 200 + data = response.json() + assert len(data) == 2 + + # Check first provider + assert data[0]["name"] == "ollama-local" + assert data[0]["type"] == "ollama" + assert data[0]["enabled"] is True + assert data[0]["priority"] == 1 + assert data[0]["default_model"] == "llama3.2" + assert "llama3.2" in data[0]["models"] + + +class TestControlProviderEndpoint: + """Test /providers/{name}/control endpoint.""" + + def test_disable_provider(self, mock_router): + """Test disabling a provider.""" + client, mock = mock_router + + response = client.post( + "/api/v1/router/providers/ollama-local/control", + json={"action": "disable"} + ) + + assert response.status_code == 200 + assert "disabled" in response.json()["message"] + + # Check that the provider was disabled + provider = mock.providers[0] + assert provider.enabled is False + assert provider.status == ProviderStatus.DISABLED + + def test_enable_provider(self, mock_router): + """Test enabling a provider.""" + client, mock = mock_router + # First disable it + mock.providers[0].enabled = False + mock.providers[0].status = ProviderStatus.DISABLED + + response = client.post( + "/api/v1/router/providers/ollama-local/control", + json={"action": "enable"} + ) + + assert response.status_code == 200 + assert "enabled" in response.json()["message"] + assert mock.providers[0].enabled is True + + def test_reset_circuit(self, mock_router): + """Test resetting circuit breaker.""" + client, mock = mock_router + # Set to open state + mock.providers[0].circuit_state = CircuitState.OPEN + mock.providers[0].status = ProviderStatus.UNHEALTHY + mock.providers[0].metrics.consecutive_failures = 10 + + response = client.post( + "/api/v1/router/providers/ollama-local/control", + json={"action": "reset_circuit"} + ) + + assert response.status_code == 200 + assert "reset" in response.json()["message"] + + provider = mock.providers[0] + assert provider.circuit_state == CircuitState.CLOSED + assert provider.status == ProviderStatus.HEALTHY + assert provider.metrics.consecutive_failures == 0 + + def test_control_unknown_provider(self, mock_router): + """Test controlling unknown provider returns 404.""" + client, mock = mock_router + response = client.post( + "/api/v1/router/providers/unknown/control", + json={"action": "disable"} + ) + + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + def test_control_unknown_action(self, mock_router): + """Test unknown action returns 400.""" + client, mock = mock_router + response = client.post( + "/api/v1/router/providers/ollama-local/control", + json={"action": "invalid_action"} + ) + + assert response.status_code == 400 + assert "Unknown action" in response.json()["detail"] + + +class TestHealthCheckEndpoint: + """Test /health-check endpoint.""" + + def test_health_check_all_healthy(self, mock_router): + """Test health check when all providers are healthy.""" + client, mock = mock_router + + with patch.object(mock, "_check_provider_available") as mock_check: + mock_check.return_value = True + + response = client.post("/api/v1/router/health-check") + + assert response.status_code == 200 + data = response.json() + assert data["healthy_count"] == 2 + assert len(data["providers"]) == 2 + + for p in data["providers"]: + assert p["healthy"] is True + + def test_health_check_with_failure(self, mock_router): + """Test health check when some providers fail.""" + client, mock = mock_router + + with patch.object(mock, "_check_provider_available") as mock_check: + # First provider fails, second succeeds + mock_check.side_effect = [False, True] + + response = client.post("/api/v1/router/health-check") + + assert response.status_code == 200 + data = response.json() + assert data["healthy_count"] == 1 + assert data["providers"][0]["healthy"] is False + assert data["providers"][1]["healthy"] is True + + +class TestGetConfigEndpoint: + """Test /config endpoint.""" + + def test_get_config(self, mock_router): + """Test getting router configuration.""" + client, mock = mock_router + + response = client.get("/api/v1/router/config") + + assert response.status_code == 200 + data = response.json() + + assert data["timeout_seconds"] == 30 + assert data["max_retries_per_provider"] == 2 + assert "circuit_breaker" in data + assert data["circuit_breaker"]["failure_threshold"] == 5 + + # Check providers list (without secrets) + assert len(data["providers"]) == 2 + assert "api_key" not in data["providers"][0] diff --git a/tests/test_router_cascade.py b/tests/test_router_cascade.py new file mode 100644 index 0000000..a1a6a2f --- /dev/null +++ b/tests/test_router_cascade.py @@ -0,0 +1,523 @@ +"""Tests for Cascade LLM Router.""" + +import json +import time +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +import yaml + +from router.cascade import ( + CascadeRouter, + CircuitState, + Provider, + ProviderMetrics, + ProviderStatus, + RouterConfig, +) + + +class TestProviderMetrics: + """Test provider metrics tracking.""" + + def test_empty_metrics(self): + """Test metrics with no requests.""" + metrics = ProviderMetrics() + assert metrics.total_requests == 0 + assert metrics.avg_latency_ms == 0.0 + assert metrics.error_rate == 0.0 + + def test_avg_latency_calculation(self): + """Test average latency calculation.""" + metrics = ProviderMetrics( + total_requests=4, + total_latency_ms=1000.0, # 4 requests, 1000ms total + ) + assert metrics.avg_latency_ms == 250.0 + + def test_error_rate_calculation(self): + """Test error rate calculation.""" + metrics = ProviderMetrics( + total_requests=10, + successful_requests=7, + failed_requests=3, + ) + assert metrics.error_rate == 0.3 + + +class TestProvider: + """Test Provider dataclass.""" + + def test_get_default_model(self): + """Test getting default model.""" + provider = Provider( + name="test", + type="ollama", + enabled=True, + priority=1, + models=[ + {"name": "llama3", "default": True}, + {"name": "mistral"}, + ], + ) + assert provider.get_default_model() == "llama3" + + def test_get_default_model_no_default(self): + """Test getting first model when no default set.""" + provider = Provider( + name="test", + type="ollama", + enabled=True, + priority=1, + models=[ + {"name": "llama3"}, + {"name": "mistral"}, + ], + ) + assert provider.get_default_model() == "llama3" + + def test_get_default_model_empty(self): + """Test with no models.""" + provider = Provider( + name="test", + type="ollama", + enabled=True, + priority=1, + models=[], + ) + assert provider.get_default_model() is None + + +class TestRouterConfig: + """Test router configuration.""" + + def test_default_config(self): + """Test default configuration values.""" + config = RouterConfig() + assert config.timeout_seconds == 30 + assert config.max_retries_per_provider == 2 + assert config.retry_delay_seconds == 1 + assert config.circuit_breaker_failure_threshold == 5 + + +class TestCascadeRouterInit: + """Test CascadeRouter initialization.""" + + def test_init_without_config(self, tmp_path): + """Test initialization without config file.""" + router = CascadeRouter(config_path=tmp_path / "nonexistent.yaml") + assert len(router.providers) == 0 + assert router.config.timeout_seconds == 30 + + def test_init_with_config(self, tmp_path): + """Test initialization with config file.""" + config = { + "cascade": { + "timeout_seconds": 60, + "max_retries_per_provider": 3, + }, + "providers": [ + { + "name": "test-ollama", + "type": "ollama", + "enabled": False, # Disabled to avoid availability check + "priority": 1, + "url": "http://localhost:11434", + } + ], + } + config_path = tmp_path / "providers.yaml" + config_path.write_text(yaml.dump(config)) + + router = CascadeRouter(config_path=config_path) + assert router.config.timeout_seconds == 60 + assert router.config.max_retries_per_provider == 3 + assert len(router.providers) == 0 # Provider is disabled + + def test_env_var_expansion(self, tmp_path, monkeypatch): + """Test environment variable expansion in config.""" + monkeypatch.setenv("TEST_API_KEY", "secret123") + + config = { + "cascade": {}, + "providers": [ + { + "name": "test-openai", + "type": "openai", + "enabled": True, + "priority": 1, + "api_key": "${TEST_API_KEY}", + } + ], + } + config_path = tmp_path / "providers.yaml" + config_path.write_text(yaml.dump(config)) + + router = CascadeRouter(config_path=config_path) + assert len(router.providers) == 1 + assert router.providers[0].api_key == "secret123" + + +class TestCascadeRouterMetrics: + """Test metrics tracking.""" + + def test_record_success(self): + """Test recording successful request.""" + provider = Provider(name="test", type="ollama", enabled=True, priority=1) + + router = CascadeRouter(config_path=Path("/nonexistent")) + router._record_success(provider, 150.0) + + assert provider.metrics.total_requests == 1 + assert provider.metrics.successful_requests == 1 + assert provider.metrics.total_latency_ms == 150.0 + assert provider.metrics.consecutive_failures == 0 + + def test_record_failure(self): + """Test recording failed request.""" + provider = Provider(name="test", type="ollama", enabled=True, priority=1) + + router = CascadeRouter(config_path=Path("/nonexistent")) + router._record_failure(provider) + + assert provider.metrics.total_requests == 1 + assert provider.metrics.failed_requests == 1 + assert provider.metrics.consecutive_failures == 1 + + def test_circuit_breaker_opens(self): + """Test circuit breaker opens after failures.""" + provider = Provider(name="test", type="ollama", enabled=True, priority=1) + + router = CascadeRouter(config_path=Path("/nonexistent")) + router.config.circuit_breaker_failure_threshold = 3 + + # Record 3 failures + for _ in range(3): + router._record_failure(provider) + + assert provider.circuit_state == CircuitState.OPEN + assert provider.status == ProviderStatus.UNHEALTHY + assert provider.circuit_opened_at is not None + + def test_circuit_breaker_can_close(self): + """Test circuit breaker can transition to closed.""" + provider = Provider(name="test", type="ollama", enabled=True, priority=1) + + router = CascadeRouter(config_path=Path("/nonexistent")) + router.config.circuit_breaker_failure_threshold = 3 + router.config.circuit_breaker_recovery_timeout = 1 + + # Open the circuit + for _ in range(3): + router._record_failure(provider) + + assert provider.circuit_state == CircuitState.OPEN + + # Wait for recovery timeout + time.sleep(1.1) + + # Check if can close + assert router._can_close_circuit(provider) is True + + def test_half_open_to_closed(self): + """Test circuit breaker closes after successful test calls.""" + provider = Provider(name="test", type="ollama", enabled=True, priority=1) + + router = CascadeRouter(config_path=Path("/nonexistent")) + router.config.circuit_breaker_half_open_max_calls = 2 + + # Manually set to half-open + provider.circuit_state = CircuitState.HALF_OPEN + provider.half_open_calls = 0 + + # Record successful calls + router._record_success(provider, 100.0) + assert provider.circuit_state == CircuitState.HALF_OPEN # Still half-open + + router._record_success(provider, 100.0) + assert provider.circuit_state == CircuitState.CLOSED # Now closed + assert provider.status == ProviderStatus.HEALTHY + + +class TestCascadeRouterGetMetrics: + """Test get_metrics method.""" + + def test_get_metrics_empty(self): + """Test getting metrics with no providers.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + metrics = router.get_metrics() + + assert "providers" in metrics + assert len(metrics["providers"]) == 0 + + def test_get_metrics_with_providers(self): + """Test getting metrics with providers.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + # Add a test provider + provider = Provider( + name="test", + type="ollama", + enabled=True, + priority=1, + ) + provider.metrics.total_requests = 10 + provider.metrics.successful_requests = 8 + provider.metrics.failed_requests = 2 + provider.metrics.total_latency_ms = 2000.0 + + router.providers = [provider] + + metrics = router.get_metrics() + + assert len(metrics["providers"]) == 1 + p_metrics = metrics["providers"][0] + assert p_metrics["name"] == "test" + assert p_metrics["metrics"]["total_requests"] == 10 + assert p_metrics["metrics"]["error_rate"] == 0.2 + assert p_metrics["metrics"]["avg_latency_ms"] == 200.0 + + +class TestCascadeRouterGetStatus: + """Test get_status method.""" + + def test_get_status(self): + """Test getting router status.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + provider = Provider( + name="test", + type="ollama", + enabled=True, + priority=1, + models=[{"name": "llama3", "default": True}], + ) + router.providers = [provider] + + status = router.get_status() + + assert status["total_providers"] == 1 + assert status["healthy_providers"] == 1 + assert status["degraded_providers"] == 0 + assert status["unhealthy_providers"] == 0 + assert len(status["providers"]) == 1 + + +@pytest.mark.asyncio +class TestCascadeRouterComplete: + """Test complete method with failover.""" + + async def test_complete_with_ollama(self): + """Test successful completion with Ollama.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + provider = Provider( + name="ollama-local", + type="ollama", + enabled=True, + priority=1, + url="http://localhost:11434", + models=[{"name": "llama3.2", "default": True}], + ) + router.providers = [provider] + + # Mock the Ollama call + with patch.object(router, "_call_ollama") as mock_call: + mock_call.return_value = AsyncMock()() + mock_call.return_value = { + "content": "Hello, world!", + "model": "llama3.2", + } + + result = await router.complete( + messages=[{"role": "user", "content": "Hi"}], + ) + + assert result["content"] == "Hello, world!" + assert result["provider"] == "ollama-local" + assert result["model"] == "llama3.2" + + async def test_failover_to_second_provider(self): + """Test failover when first provider fails.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + provider1 = Provider( + name="ollama-failing", + type="ollama", + enabled=True, + priority=1, + url="http://localhost:11434", + models=[{"name": "llama3.2", "default": True}], + ) + provider2 = Provider( + name="ollama-backup", + type="ollama", + enabled=True, + priority=2, + url="http://backup:11434", + models=[{"name": "llama3.2", "default": True}], + ) + router.providers = [provider1, provider2] + + # First provider fails, second succeeds + call_count = [0] + + async def side_effect(*args, **kwargs): + call_count[0] += 1 + # First 2 retries for provider1 fail, then provider2 succeeds + if call_count[0] <= router.config.max_retries_per_provider: + raise RuntimeError("Connection failed") + return {"content": "Backup response", "model": "llama3.2"} + + with patch.object(router, "_call_ollama") as mock_call: + mock_call.side_effect = side_effect + + result = await router.complete( + messages=[{"role": "user", "content": "Hi"}], + ) + + assert result["content"] == "Backup response" + assert result["provider"] == "ollama-backup" + + async def test_all_providers_fail(self): + """Test error when all providers fail.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + provider = Provider( + name="failing", + type="ollama", + enabled=True, + priority=1, + models=[{"name": "llama3.2", "default": True}], + ) + router.providers = [provider] + + with patch.object(router, "_call_ollama") as mock_call: + mock_call.side_effect = RuntimeError("Always fails") + + with pytest.raises(RuntimeError) as exc_info: + await router.complete(messages=[{"role": "user", "content": "Hi"}]) + + assert "All providers failed" in str(exc_info.value) + + async def test_skips_unhealthy_provider(self): + """Test that unhealthy providers are skipped.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + provider1 = Provider( + name="unhealthy", + type="ollama", + enabled=True, + priority=1, + status=ProviderStatus.UNHEALTHY, + circuit_state=CircuitState.OPEN, + circuit_opened_at=time.time(), # Just opened + models=[{"name": "llama3.2", "default": True}], + ) + provider2 = Provider( + name="healthy", + type="ollama", + enabled=True, + priority=2, + models=[{"name": "llama3.2", "default": True}], + ) + router.providers = [provider1, provider2] + + with patch.object(router, "_call_ollama") as mock_call: + mock_call.return_value = {"content": "Success", "model": "llama3.2"} + + result = await router.complete( + messages=[{"role": "user", "content": "Hi"}], + ) + + # Should use the healthy provider + assert result["provider"] == "healthy" + + +class TestProviderAvailabilityCheck: + """Test provider availability checking.""" + + def test_check_ollama_without_requests(self): + """Test Ollama returns True when requests not available (fallback).""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + provider = Provider( + name="ollama", + type="ollama", + enabled=True, + priority=1, + url="http://localhost:11434", + ) + + # When requests is None, assume available + import router.cascade as cascade_module + old_requests = cascade_module.requests + cascade_module.requests = None + try: + assert router._check_provider_available(provider) is True + finally: + cascade_module.requests = old_requests + + def test_check_openai_with_key(self): + """Test OpenAI with API key.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + provider = Provider( + name="openai", + type="openai", + enabled=True, + priority=1, + api_key="sk-test123", + ) + + assert router._check_provider_available(provider) is True + + def test_check_openai_without_key(self): + """Test OpenAI without API key.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + provider = Provider( + name="openai", + type="openai", + enabled=True, + priority=1, + api_key=None, + ) + + assert router._check_provider_available(provider) is False + + def test_check_airllm_installed(self): + """Test AirLLM when installed.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + provider = Provider( + name="airllm", + type="airllm", + enabled=True, + priority=1, + ) + + with patch("builtins.__import__") as mock_import: + mock_import.return_value = MagicMock() + assert router._check_provider_available(provider) is True + + def test_check_airllm_not_installed(self): + """Test AirLLM when not installed.""" + router = CascadeRouter(config_path=Path("/nonexistent")) + + provider = Provider( + name="airllm", + type="airllm", + enabled=True, + priority=1, + ) + + # Patch __import__ to simulate airllm not being available + def raise_import_error(name, *args, **kwargs): + if name == "airllm": + raise ImportError("No module named 'airllm'") + return __builtins__.__import__(name, *args, **kwargs) + + with patch("builtins.__import__", side_effect=raise_import_error): + assert router._check_provider_available(provider) is False