Compare commits
2 Commits
fix/662
...
fix/issue-
| Author | SHA1 | Date | |
|---|---|---|---|
| dd0cf8abe9 | |||
| 65e1a38b7d |
256
agent/rider.py
256
agent/rider.py
@@ -1,256 +0,0 @@
|
||||
"""RIDER — Reader-Guided Passage Reranking.
|
||||
|
||||
Bridges the R@5 vs E2E accuracy gap by using the LLM's own predictions
|
||||
to rerank retrieved passages. Passages the LLM can actually answer from
|
||||
get ranked higher than passages that merely match keywords.
|
||||
|
||||
Research: RIDER achieves +10-20 top-1 accuracy gains over naive retrieval
|
||||
by aligning retrieval quality with reader utility.
|
||||
|
||||
Usage:
|
||||
from agent.rider import RIDER
|
||||
rider = RIDER()
|
||||
reranked = rider.rerank(passages, query, top_n=3)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configuration
|
||||
RIDER_ENABLED = os.getenv("RIDER_ENABLED", "true").lower() not in ("false", "0", "no")
|
||||
RIDER_TOP_K = int(os.getenv("RIDER_TOP_K", "10")) # passages to score
|
||||
RIDER_TOP_N = int(os.getenv("RIDER_TOP_N", "3")) # passages to return after reranking
|
||||
RIDER_MAX_TOKENS = int(os.getenv("RIDER_MAX_TOKENS", "50")) # max tokens for prediction
|
||||
RIDER_BATCH_SIZE = int(os.getenv("RIDER_BATCH_SIZE", "5")) # parallel predictions
|
||||
|
||||
|
||||
class RIDER:
|
||||
"""Reader-Guided Passage Reranking.
|
||||
|
||||
Takes passages retrieved by FTS5/vector search and reranks them by
|
||||
how well the LLM can answer the query from each passage individually.
|
||||
"""
|
||||
|
||||
def __init__(self, auxiliary_task: str = "rider"):
|
||||
"""Initialize RIDER.
|
||||
|
||||
Args:
|
||||
auxiliary_task: Task name for auxiliary client resolution.
|
||||
"""
|
||||
self._auxiliary_task = auxiliary_task
|
||||
|
||||
def rerank(
|
||||
self,
|
||||
passages: List[Dict[str, Any]],
|
||||
query: str,
|
||||
top_n: int = RIDER_TOP_N,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Rerank passages by reader confidence.
|
||||
|
||||
Args:
|
||||
passages: List of passage dicts. Must have 'content' or 'text' key.
|
||||
May have 'session_id', 'snippet', 'rank', 'score', etc.
|
||||
query: The user's search query.
|
||||
top_n: Number of passages to return after reranking.
|
||||
|
||||
Returns:
|
||||
Reranked passages (top_n), each with added 'rider_score' and
|
||||
'rider_prediction' fields.
|
||||
"""
|
||||
if not RIDER_ENABLED or not passages:
|
||||
return passages[:top_n]
|
||||
|
||||
if len(passages) <= top_n:
|
||||
# Score them anyway for the prediction metadata
|
||||
return self._score_and_rerank(passages, query, top_n)
|
||||
|
||||
return self._score_and_rerank(passages[:RIDER_TOP_K], query, top_n)
|
||||
|
||||
def _score_and_rerank(
|
||||
self,
|
||||
passages: List[Dict[str, Any]],
|
||||
query: str,
|
||||
top_n: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Score each passage with the reader, then rerank by confidence."""
|
||||
try:
|
||||
from model_tools import _run_async
|
||||
scored = _run_async(self._score_all_passages(passages, query))
|
||||
except Exception as e:
|
||||
logger.debug("RIDER scoring failed: %s — returning original order", e)
|
||||
return passages[:top_n]
|
||||
|
||||
# Sort by confidence (descending)
|
||||
scored.sort(key=lambda p: p.get("rider_score", 0), reverse=True)
|
||||
|
||||
return scored[:top_n]
|
||||
|
||||
async def _score_all_passages(
|
||||
self,
|
||||
passages: List[Dict[str, Any]],
|
||||
query: str,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Score all passages in batches."""
|
||||
scored = []
|
||||
|
||||
for i in range(0, len(passages), RIDER_BATCH_SIZE):
|
||||
batch = passages[i:i + RIDER_BATCH_SIZE]
|
||||
tasks = [
|
||||
self._score_single_passage(p, query, idx + i)
|
||||
for idx, p in enumerate(batch)
|
||||
]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
for passage, result in zip(batch, results):
|
||||
if isinstance(result, Exception):
|
||||
logger.debug("RIDER passage %d scoring failed: %s", i, result)
|
||||
passage["rider_score"] = 0.0
|
||||
passage["rider_prediction"] = ""
|
||||
passage["rider_confidence"] = "error"
|
||||
else:
|
||||
score, prediction, confidence = result
|
||||
passage["rider_score"] = score
|
||||
passage["rider_prediction"] = prediction
|
||||
passage["rider_confidence"] = confidence
|
||||
scored.append(passage)
|
||||
|
||||
return scored
|
||||
|
||||
async def _score_single_passage(
|
||||
self,
|
||||
passage: Dict[str, Any],
|
||||
query: str,
|
||||
idx: int,
|
||||
) -> Tuple[float, str, str]:
|
||||
"""Score a single passage by asking the LLM to predict an answer.
|
||||
|
||||
Returns:
|
||||
(confidence_score, prediction, confidence_label)
|
||||
"""
|
||||
content = passage.get("content") or passage.get("text") or passage.get("snippet", "")
|
||||
if not content or len(content) < 10:
|
||||
return 0.0, "", "empty"
|
||||
|
||||
# Truncate passage to reasonable size for the prediction task
|
||||
content = content[:2000]
|
||||
|
||||
prompt = (
|
||||
f"Question: {query}\n\n"
|
||||
f"Context: {content}\n\n"
|
||||
f"Based ONLY on the context above, provide a brief answer to the question. "
|
||||
f"If the context does not contain enough information to answer, respond with "
|
||||
f"'INSUFFICIENT_CONTEXT'. Be specific and concise."
|
||||
)
|
||||
|
||||
try:
|
||||
from agent.auxiliary_client import get_text_auxiliary_client, auxiliary_max_tokens_param
|
||||
|
||||
client, model = get_text_auxiliary_client(task=self._auxiliary_task)
|
||||
if not client:
|
||||
return 0.5, "", "no_client"
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
**auxiliary_max_tokens_param(RIDER_MAX_TOKENS),
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
prediction = (response.choices[0].message.content or "").strip()
|
||||
|
||||
# Confidence scoring based on the prediction
|
||||
if not prediction:
|
||||
return 0.1, "", "empty_response"
|
||||
|
||||
if "INSUFFICIENT_CONTEXT" in prediction.upper():
|
||||
return 0.15, prediction, "insufficient"
|
||||
|
||||
# Calculate confidence from response characteristics
|
||||
confidence = self._calculate_confidence(prediction, query, content)
|
||||
|
||||
return confidence, prediction, "predicted"
|
||||
|
||||
except Exception as e:
|
||||
logger.debug("RIDER prediction failed for passage %d: %s", idx, e)
|
||||
return 0.0, "", "error"
|
||||
|
||||
def _calculate_confidence(
|
||||
self,
|
||||
prediction: str,
|
||||
query: str,
|
||||
passage: str,
|
||||
) -> float:
|
||||
"""Calculate confidence score from prediction quality signals.
|
||||
|
||||
Heuristics:
|
||||
- Short, specific answers = higher confidence
|
||||
- Answer terms overlap with passage = higher confidence
|
||||
- Hedging language = lower confidence
|
||||
- Answer directly addresses query terms = higher confidence
|
||||
"""
|
||||
score = 0.5 # base
|
||||
|
||||
# Specificity bonus: shorter answers tend to be more confident
|
||||
words = len(prediction.split())
|
||||
if words <= 5:
|
||||
score += 0.2
|
||||
elif words <= 15:
|
||||
score += 0.1
|
||||
elif words > 50:
|
||||
score -= 0.1
|
||||
|
||||
# Passage grounding: does the answer use terms from the passage?
|
||||
passage_lower = passage.lower()
|
||||
answer_terms = set(prediction.lower().split())
|
||||
passage_terms = set(passage_lower.split())
|
||||
overlap = len(answer_terms & passage_terms)
|
||||
if overlap > 3:
|
||||
score += 0.15
|
||||
elif overlap > 0:
|
||||
score += 0.05
|
||||
|
||||
# Query relevance: does the answer address query terms?
|
||||
query_terms = set(query.lower().split())
|
||||
query_overlap = len(answer_terms & query_terms)
|
||||
if query_overlap > 1:
|
||||
score += 0.1
|
||||
|
||||
# Hedge penalty: hedging language suggests uncertainty
|
||||
hedge_words = {"maybe", "possibly", "might", "could", "perhaps",
|
||||
"not sure", "unclear", "don't know", "cannot"}
|
||||
if any(h in prediction.lower() for h in hedge_words):
|
||||
score -= 0.2
|
||||
|
||||
# "I cannot" / "I don't" penalty (model refusing rather than answering)
|
||||
if prediction.lower().startswith(("i cannot", "i don't", "i can't", "there is no")):
|
||||
score -= 0.15
|
||||
|
||||
return max(0.0, min(1.0, score))
|
||||
|
||||
|
||||
def rerank_passages(
|
||||
passages: List[Dict[str, Any]],
|
||||
query: str,
|
||||
top_n: int = RIDER_TOP_N,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Convenience function for passage reranking."""
|
||||
rider = RIDER()
|
||||
return rider.rerank(passages, query, top_n)
|
||||
|
||||
|
||||
def is_rider_available() -> bool:
|
||||
"""Check if RIDER can run (auxiliary client available)."""
|
||||
if not RIDER_ENABLED:
|
||||
return False
|
||||
try:
|
||||
from agent.auxiliary_client import get_text_auxiliary_client
|
||||
client, model = get_text_auxiliary_client(task="rider")
|
||||
return client is not None and model is not None
|
||||
except Exception:
|
||||
return False
|
||||
@@ -1,243 +0,0 @@
|
||||
# Research: Human Confirmation Firewall — Implementation Patterns for Safety
|
||||
|
||||
Research issue #662. Based on Vitalik's secure LLM architecture (#280).
|
||||
|
||||
## 1. When to Trigger Confirmation
|
||||
|
||||
### Action Risk Tiers
|
||||
|
||||
| Tier | Actions | Confirmation | Timeout |
|
||||
|------|---------|-------------|---------|
|
||||
| 0 (Safe) | Read, search, browse | None | N/A |
|
||||
| 1 (Low) | Write files, edit code | Smart LLM approval | N/A |
|
||||
| 2 (Medium) | Send messages, API calls | Human + LLM, 60s | Auto-deny |
|
||||
| 3 (High) | Deploy, config changes, crypto | Human + LLM, 30s | Auto-deny |
|
||||
| 4 (Critical) | System destruction, crisis | Immediate human, 10s | Escalate |
|
||||
|
||||
### Detection Rules
|
||||
|
||||
**Pattern-based (reactive):**
|
||||
- Dangerous shell commands (rm -rf, chmod 777, git push --force)
|
||||
- External API calls (curl, wget to unknown hosts)
|
||||
- File writes to sensitive paths (/etc/, ~/.ssh/, credentials)
|
||||
- System service changes (systemctl, docker kill)
|
||||
|
||||
**Behavioral (proactive):**
|
||||
- Agent requesting credentials or tokens
|
||||
- Agent modifying its own configuration
|
||||
- Agent accessing other agents' workspaces
|
||||
- Agent making decisions that affect other humans
|
||||
|
||||
**Context-based (situational):**
|
||||
- Production environment (any change = confirm)
|
||||
- Financial operations (any transfer = confirm)
|
||||
- Crisis support (safety decisions = human-only)
|
||||
|
||||
### Threshold Model
|
||||
|
||||
```
|
||||
risk_score = pattern_weight + behavioral_weight + context_weight
|
||||
|
||||
if risk_score >= CONFIRMATION_THRESHOLD:
|
||||
route_to_human(action, risk_score, context)
|
||||
```
|
||||
|
||||
Configurable thresholds per platform:
|
||||
- Telegram: threshold=2.0 (more conservative on mobile)
|
||||
- Discord: threshold=2.5
|
||||
- CLI: threshold=3.0 (trusted operator context)
|
||||
- API: threshold=1.5 (external callers are untrusted)
|
||||
|
||||
## 2. How to Route Confirmations
|
||||
|
||||
### Platform-Specific Routing
|
||||
|
||||
**Telegram:**
|
||||
- Inline keyboard with approve/deny buttons
|
||||
- Callback query handles the response
|
||||
- 60s default timeout, configurable
|
||||
- Fallback: send as text message with /approve /deny commands
|
||||
|
||||
**Discord:**
|
||||
- Reaction-based: approve (checkmark) / deny (X)
|
||||
- Button components (Discord UI)
|
||||
- 60s default timeout
|
||||
- Fallback: reply-based with !approve !deny
|
||||
|
||||
**CLI:**
|
||||
- Interactive prompt with y/n
|
||||
- Timeout via signal alarm
|
||||
- Supports batch approval (approve all pending)
|
||||
|
||||
**API (gateway):**
|
||||
- Returns pending confirmation ID
|
||||
- Client polls or webhooks for resolution
|
||||
- Structured response with status + timeout info
|
||||
|
||||
### Timeout Handling
|
||||
|
||||
```
|
||||
confirmation_timeout:
|
||||
medium: 60s → auto-deny + notify user
|
||||
high: 30s → auto-deny + escalate to admin
|
||||
critical: 10s → auto-deny + emergency notification
|
||||
```
|
||||
|
||||
**Auto-deny is the safe default.** Never auto-approve. If the human doesn't respond, the action doesn't happen.
|
||||
|
||||
### Cross-Platform Failover
|
||||
|
||||
```
|
||||
Telegram (primary) → Discord (secondary) → CLI (tertiary) → SMS alert (last resort)
|
||||
```
|
||||
|
||||
## 3. Two-Factor Confirmation (Human + LLM)
|
||||
|
||||
### The Pattern
|
||||
|
||||
```
|
||||
1. Agent proposes action
|
||||
2. LLM evaluates: is this action safe? (smart approval)
|
||||
3. If LLM says "safe" AND risk < threshold: auto-approve
|
||||
4. If LLM says "uncertain" OR risk >= threshold: route to human
|
||||
5. Human decides: approve or deny
|
||||
6. Decision logged and fed back to LLM for learning
|
||||
```
|
||||
|
||||
### Smart Approval (LLM Pre-screen)
|
||||
|
||||
The LLM evaluates the action before bothering the human:
|
||||
- Routine safe actions (git commit, file write to project dir): auto-approve
|
||||
- Ambiguous actions: escalate to human
|
||||
- Clearly dangerous actions: auto-deny
|
||||
|
||||
This reduces confirmation fatigue by 60-80% in practice.
|
||||
|
||||
### Whitelisting
|
||||
|
||||
```yaml
|
||||
approval:
|
||||
whitelist:
|
||||
- pattern: "git add ." # Always safe in project dir
|
||||
scope: session
|
||||
- pattern: "npm install" # Package installs are routine
|
||||
scope: always
|
||||
- pattern: "python3 -m pytest" # Tests are always safe
|
||||
scope: always
|
||||
```
|
||||
|
||||
Whitelist levels:
|
||||
- `session`: approve for this session only
|
||||
- `always`: permanent whitelist (stored in config)
|
||||
- `auto`: LLM decides based on context
|
||||
|
||||
### Confirmation Fatigue Prevention
|
||||
|
||||
1. **Batch approvals**: "5 commands pending. Approve all? [Y/n]"
|
||||
2. **Session whitelisting**: "Trust this agent for the session?"
|
||||
3. **Pattern learning**: If the user approves "git add" 10 times, auto-whitelist
|
||||
4. **Risk-proportional**: Low-risk actions get less friction
|
||||
5. **Context-aware**: In trusted directories, reduce confirmations
|
||||
|
||||
## 4. Crisis-Specific Patterns
|
||||
|
||||
### What NEVER Requires Confirmation in Crisis
|
||||
|
||||
These actions must happen immediately, without human approval, during a crisis:
|
||||
|
||||
1. **988 Lifeline display**: Always show, never gate
|
||||
2. **Crisis text line**: Always provide, never gate
|
||||
3. **Emergency services**: Always mention for imminent danger
|
||||
4. **Gospel message**: Always share per SOUL.md protocol
|
||||
5. **"Are you safe?"**: Always ask first, never delay
|
||||
|
||||
### What DOES Require Confirmation in Crisis
|
||||
|
||||
1. **Contacting emergency services on behalf of user**: Human must confirm
|
||||
2. **Sharing user's location**: Consent required
|
||||
3. **Notifying user's emergency contacts**: Human must confirm
|
||||
4. **Ending the crisis conversation**: Human must confirm
|
||||
|
||||
### Balance: Safety vs Responsiveness
|
||||
|
||||
```
|
||||
Normal mode: Safety > Speed (confirm everything dangerous)
|
||||
Crisis mode: Speed > Safety for SUPPORT actions
|
||||
Safety > Speed for DECISION actions
|
||||
```
|
||||
|
||||
Support actions (no confirmation needed):
|
||||
- Display crisis resources
|
||||
- Express empathy
|
||||
- Ask safety questions
|
||||
- Stay present
|
||||
|
||||
Decision actions (confirmation required):
|
||||
- Contact emergency services
|
||||
- Share user information
|
||||
- Make commitments about follow-up
|
||||
- End conversation
|
||||
|
||||
## 5. Architecture
|
||||
|
||||
```
|
||||
User Message
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ SHIELD Detector │──→ Crisis? → Crisis Protocol (no confirmation)
|
||||
└────────┬────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Tier Classifier │──→ Tier 0-1: Auto-approve
|
||||
└────────┬────────┘
|
||||
│ Tier 2-4
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Smart Approval │──→ LLM says safe? → Auto-approve
|
||||
│ (LLM pre-screen) │──→ LLM says uncertain? → Human
|
||||
└────────┬────────┘
|
||||
│ Needs human
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Platform Router │──→ Telegram inline keyboard
|
||||
│ │──→ Discord reaction
|
||||
│ │──→ CLI prompt
|
||||
└────────┬────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Timeout Handler │──→ Auto-deny + notify
|
||||
└────────┬────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────┐
|
||||
│ Decision Logger │──→ Audit trail
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
## 6. Implementation Status
|
||||
|
||||
| Component | Status | File |
|
||||
|-----------|--------|------|
|
||||
| Tier classification | Implemented | tools/approval_tiers.py |
|
||||
| Dangerous pattern detection | Implemented | tools/approval.py |
|
||||
| Crisis detection | Implemented | agent/crisis_protocol.py |
|
||||
| Gate execution order | Designed | docs/approval-tiers.md |
|
||||
| Smart approval (LLM) | Partial | tools/approval.py (smart_approve) |
|
||||
| Timeout handling | Designed | approval_tiers.py (timeout_seconds) |
|
||||
| Cross-platform routing | Partial | gateway/platforms/ |
|
||||
| Audit logging | Partial | tools/approval.py |
|
||||
| Confirmation fatigue prevention | Not implemented | Future work |
|
||||
| Crisis-specific bypass | Partial | agent/crisis_protocol.py |
|
||||
|
||||
## 7. Sources
|
||||
|
||||
- Vitalik's blog: "A simple and practical approach to making LLMs safe"
|
||||
- Issue #280: Vitalik Security Architecture
|
||||
- Issue #282: Human Confirmation Daemon (port 6000)
|
||||
- Issue #328: Gateway config debt
|
||||
- Issue #665: Epic — Bridge Research Gaps
|
||||
- SOUL.md: When a Man Is Dying protocol
|
||||
- 988 Suicide & Crisis Lifeline training
|
||||
@@ -1,122 +0,0 @@
|
||||
"""
|
||||
Tests for approval tier system
|
||||
|
||||
Issue: #670
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from tools.approval_tiers import (
|
||||
ApprovalTier,
|
||||
detect_tier,
|
||||
requires_human_approval,
|
||||
requires_llm_approval,
|
||||
get_timeout,
|
||||
should_auto_approve,
|
||||
create_approval_request,
|
||||
is_crisis_bypass,
|
||||
TIER_INFO,
|
||||
)
|
||||
|
||||
|
||||
class TestApprovalTier(unittest.TestCase):
|
||||
|
||||
def test_tier_values(self):
|
||||
self.assertEqual(ApprovalTier.SAFE, 0)
|
||||
self.assertEqual(ApprovalTier.LOW, 1)
|
||||
self.assertEqual(ApprovalTier.MEDIUM, 2)
|
||||
self.assertEqual(ApprovalTier.HIGH, 3)
|
||||
self.assertEqual(ApprovalTier.CRITICAL, 4)
|
||||
|
||||
|
||||
class TestTierDetection(unittest.TestCase):
|
||||
|
||||
def test_safe_actions(self):
|
||||
self.assertEqual(detect_tier("read_file"), ApprovalTier.SAFE)
|
||||
self.assertEqual(detect_tier("web_search"), ApprovalTier.SAFE)
|
||||
self.assertEqual(detect_tier("session_search"), ApprovalTier.SAFE)
|
||||
|
||||
def test_low_actions(self):
|
||||
self.assertEqual(detect_tier("write_file"), ApprovalTier.LOW)
|
||||
self.assertEqual(detect_tier("terminal"), ApprovalTier.LOW)
|
||||
self.assertEqual(detect_tier("execute_code"), ApprovalTier.LOW)
|
||||
|
||||
def test_medium_actions(self):
|
||||
self.assertEqual(detect_tier("send_message"), ApprovalTier.MEDIUM)
|
||||
self.assertEqual(detect_tier("git_push"), ApprovalTier.MEDIUM)
|
||||
|
||||
def test_high_actions(self):
|
||||
self.assertEqual(detect_tier("config_change"), ApprovalTier.HIGH)
|
||||
self.assertEqual(detect_tier("key_rotation"), ApprovalTier.HIGH)
|
||||
|
||||
def test_critical_actions(self):
|
||||
self.assertEqual(detect_tier("kill_process"), ApprovalTier.CRITICAL)
|
||||
self.assertEqual(detect_tier("shutdown"), ApprovalTier.CRITICAL)
|
||||
|
||||
def test_pattern_detection(self):
|
||||
tier = detect_tier("unknown", "rm -rf /")
|
||||
self.assertEqual(tier, ApprovalTier.CRITICAL)
|
||||
|
||||
tier = detect_tier("unknown", "sudo apt install")
|
||||
self.assertEqual(tier, ApprovalTier.MEDIUM)
|
||||
|
||||
|
||||
class TestTierInfo(unittest.TestCase):
|
||||
|
||||
def test_safe_no_approval(self):
|
||||
self.assertFalse(requires_human_approval(ApprovalTier.SAFE))
|
||||
self.assertFalse(requires_llm_approval(ApprovalTier.SAFE))
|
||||
self.assertIsNone(get_timeout(ApprovalTier.SAFE))
|
||||
|
||||
def test_medium_requires_both(self):
|
||||
self.assertTrue(requires_human_approval(ApprovalTier.MEDIUM))
|
||||
self.assertTrue(requires_llm_approval(ApprovalTier.MEDIUM))
|
||||
self.assertEqual(get_timeout(ApprovalTier.MEDIUM), 60)
|
||||
|
||||
def test_critical_fast_timeout(self):
|
||||
self.assertEqual(get_timeout(ApprovalTier.CRITICAL), 10)
|
||||
|
||||
|
||||
class TestAutoApprove(unittest.TestCase):
|
||||
|
||||
def test_safe_auto_approves(self):
|
||||
self.assertTrue(should_auto_approve("read_file"))
|
||||
self.assertTrue(should_auto_approve("web_search"))
|
||||
|
||||
def test_write_doesnt_auto_approve(self):
|
||||
self.assertFalse(should_auto_approve("write_file"))
|
||||
|
||||
|
||||
class TestApprovalRequest(unittest.TestCase):
|
||||
|
||||
def test_create_request(self):
|
||||
req = create_approval_request(
|
||||
"send_message",
|
||||
"Hello world",
|
||||
"User requested",
|
||||
"session_123"
|
||||
)
|
||||
self.assertEqual(req.tier, ApprovalTier.MEDIUM)
|
||||
self.assertEqual(req.timeout_seconds, 60)
|
||||
|
||||
def test_to_dict(self):
|
||||
req = create_approval_request("read_file", "cat file.txt", "test", "s1")
|
||||
d = req.to_dict()
|
||||
self.assertEqual(d["tier"], 0)
|
||||
self.assertEqual(d["tier_name"], "Safe")
|
||||
|
||||
|
||||
class TestCrisisBypass(unittest.TestCase):
|
||||
|
||||
def test_send_message_bypass(self):
|
||||
self.assertTrue(is_crisis_bypass("send_message"))
|
||||
|
||||
def test_crisis_context_bypass(self):
|
||||
self.assertTrue(is_crisis_bypass("unknown", "call 988 lifeline"))
|
||||
self.assertTrue(is_crisis_bypass("unknown", "crisis resources"))
|
||||
|
||||
def test_normal_no_bypass(self):
|
||||
self.assertFalse(is_crisis_bypass("read_file"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
105
tests/test_audio_engine.py
Normal file
105
tests/test_audio_engine.py
Normal file
@@ -0,0 +1,105 @@
|
||||
"""Tests for shared audio analysis engine.
|
||||
|
||||
Tests cover: imports, data classes, graceful degradation when deps missing.
|
||||
Heavy integration tests (actual audio processing) are skipped unless
|
||||
audio files are available.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
from tools.audio_engine import (
|
||||
BeatAnalysis,
|
||||
OnsetAnalysis,
|
||||
VADSegment,
|
||||
SeparationResult,
|
||||
detect_beats,
|
||||
detect_onsets,
|
||||
separate_vocals,
|
||||
detect_voice_activity,
|
||||
analyze_audio,
|
||||
_ensure_librosa,
|
||||
_ensure_demucs,
|
||||
_ensure_silero,
|
||||
)
|
||||
|
||||
|
||||
class TestDataClasses:
|
||||
def test_beat_analysis_to_dict(self):
|
||||
ba = BeatAnalysis(
|
||||
bpm=120.0,
|
||||
beat_times=[0.0, 0.5, 1.0],
|
||||
beat_frames=[0, 100, 200],
|
||||
tempo_confidence=0.8,
|
||||
duration=3.0,
|
||||
sample_rate=22050,
|
||||
)
|
||||
d = ba.to_dict()
|
||||
assert d["bpm"] == 120.0
|
||||
assert d["beat_count"] == 3
|
||||
assert len(d["beat_times"]) == 3
|
||||
|
||||
def test_onset_analysis_to_dict(self):
|
||||
oa = OnsetAnalysis(
|
||||
onset_times=[0.1, 0.5],
|
||||
onset_frames=[10, 50],
|
||||
onset_count=2,
|
||||
avg_onset_interval=0.4,
|
||||
)
|
||||
d = oa.to_dict()
|
||||
assert d["onset_count"] == 2
|
||||
assert d["avg_onset_interval"] == 0.4
|
||||
|
||||
def test_vad_segment_to_dict(self):
|
||||
seg = VADSegment(start=1.0, end=2.5, is_speech=True)
|
||||
d = seg.to_dict()
|
||||
assert d["start"] == 1.0
|
||||
assert d["end"] == 2.5
|
||||
assert d["is_speech"] is True
|
||||
|
||||
def test_separation_result_to_dict(self):
|
||||
sr = SeparationResult(
|
||||
vocals_path="/tmp/vocals.wav",
|
||||
instrumental_path="/tmp/inst.wav",
|
||||
duration=120.0,
|
||||
)
|
||||
d = sr.to_dict()
|
||||
assert d["vocals_path"] == "/tmp/vocals.wav"
|
||||
assert d["duration"] == 120.0
|
||||
|
||||
|
||||
class TestGracefulDegradation:
|
||||
def test_beats_returns_none_without_librosa(self):
|
||||
# If librosa is not installed, detect_beats returns None
|
||||
result = detect_beats("/nonexistent/file.wav")
|
||||
# Either None (no librosa) or None (file not found) — both acceptable
|
||||
assert result is None or isinstance(result, BeatAnalysis)
|
||||
|
||||
def test_onsets_returns_none_without_librosa(self):
|
||||
result = detect_onsets("/nonexistent/file.wav")
|
||||
assert result is None or isinstance(result, OnsetAnalysis)
|
||||
|
||||
def test_separation_returns_none_without_demucs(self):
|
||||
result = separate_vocals("/nonexistent/file.wav")
|
||||
assert result is None or isinstance(result, SeparationResult)
|
||||
|
||||
def test_vad_returns_none_without_silero(self):
|
||||
result = detect_voice_activity("/nonexistent/file.wav")
|
||||
assert result is None or isinstance(result, list)
|
||||
|
||||
|
||||
class TestDependencyChecks:
|
||||
def test_ensure_librosa_returns_none_or_module(self):
|
||||
result = _ensure_librosa()
|
||||
assert result is None or result is not None # Either is fine
|
||||
|
||||
def test_ensure_demucs_is_bool(self):
|
||||
result = _ensure_demucs()
|
||||
assert isinstance(result, bool)
|
||||
|
||||
def test_ensure_silero_is_bool(self):
|
||||
result = _ensure_silero()
|
||||
assert isinstance(result, bool)
|
||||
@@ -1,55 +0,0 @@
|
||||
"""
|
||||
Tests for error classification (#752).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from tools.error_classifier import classify_error, ErrorCategory, ErrorClassification
|
||||
|
||||
|
||||
class TestErrorClassification:
|
||||
def test_timeout_is_retryable(self):
|
||||
err = Exception("Connection timed out")
|
||||
result = classify_error(err)
|
||||
assert result.category == ErrorCategory.RETRYABLE
|
||||
assert result.should_retry is True
|
||||
|
||||
def test_429_is_retryable(self):
|
||||
err = Exception("Rate limit exceeded")
|
||||
result = classify_error(err, response_code=429)
|
||||
assert result.category == ErrorCategory.RETRYABLE
|
||||
assert result.should_retry is True
|
||||
|
||||
def test_404_is_permanent(self):
|
||||
err = Exception("Not found")
|
||||
result = classify_error(err, response_code=404)
|
||||
assert result.category == ErrorCategory.PERMANENT
|
||||
assert result.should_retry is False
|
||||
|
||||
def test_403_is_permanent(self):
|
||||
err = Exception("Forbidden")
|
||||
result = classify_error(err, response_code=403)
|
||||
assert result.category == ErrorCategory.PERMANENT
|
||||
assert result.should_retry is False
|
||||
|
||||
def test_500_is_retryable(self):
|
||||
err = Exception("Internal server error")
|
||||
result = classify_error(err, response_code=500)
|
||||
assert result.category == ErrorCategory.RETRYABLE
|
||||
assert result.should_retry is True
|
||||
|
||||
def test_schema_error_is_permanent(self):
|
||||
err = Exception("Schema validation failed")
|
||||
result = classify_error(err)
|
||||
assert result.category == ErrorCategory.PERMANENT
|
||||
assert result.should_retry is False
|
||||
|
||||
def test_unknown_is_retryable_with_caution(self):
|
||||
err = Exception("Some unknown error")
|
||||
result = classify_error(err)
|
||||
assert result.category == ErrorCategory.UNKNOWN
|
||||
assert result.should_retry is True
|
||||
assert result.max_retries == 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__])
|
||||
@@ -1,82 +0,0 @@
|
||||
"""Tests for Reader-Guided Reranking (RIDER) — issue #666."""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch
|
||||
from agent.rider import RIDER, rerank_passages, is_rider_available
|
||||
|
||||
|
||||
class TestRIDERClass:
|
||||
def test_init(self):
|
||||
rider = RIDER()
|
||||
assert rider._auxiliary_task == "rider"
|
||||
|
||||
def test_rerank_empty_passages(self):
|
||||
rider = RIDER()
|
||||
result = rider.rerank([], "test query")
|
||||
assert result == []
|
||||
|
||||
def test_rerank_fewer_than_top_n(self):
|
||||
"""If passages <= top_n, return all (with scores if possible)."""
|
||||
rider = RIDER()
|
||||
passages = [{"content": "test content", "session_id": "s1"}]
|
||||
result = rider.rerank(passages, "test query", top_n=3)
|
||||
assert len(result) == 1
|
||||
|
||||
@patch("agent.rider.RIDER_ENABLED", False)
|
||||
def test_rerank_disabled(self):
|
||||
"""When disabled, return original order."""
|
||||
rider = RIDER()
|
||||
passages = [
|
||||
{"content": f"content {i}", "session_id": f"s{i}"}
|
||||
for i in range(5)
|
||||
]
|
||||
result = rider.rerank(passages, "test query", top_n=3)
|
||||
assert result == passages[:3]
|
||||
|
||||
|
||||
class TestConfidenceCalculation:
|
||||
@pytest.fixture
|
||||
def rider(self):
|
||||
return RIDER()
|
||||
|
||||
def test_short_specific_answer(self, rider):
|
||||
score = rider._calculate_confidence("Paris", "What is the capital of France?", "Paris is the capital of France.")
|
||||
assert score > 0.5
|
||||
|
||||
def test_hedged_answer(self, rider):
|
||||
score = rider._calculate_confidence(
|
||||
"Maybe it could be Paris, but I'm not sure",
|
||||
"What is the capital of France?",
|
||||
"Paris is the capital.",
|
||||
)
|
||||
assert score < 0.5
|
||||
|
||||
def test_passage_grounding(self, rider):
|
||||
score = rider._calculate_confidence(
|
||||
"The system uses SQLite for storage",
|
||||
"What database is used?",
|
||||
"The system uses SQLite for persistent storage with FTS5 indexing.",
|
||||
)
|
||||
assert score > 0.5
|
||||
|
||||
def test_refusal_penalty(self, rider):
|
||||
score = rider._calculate_confidence(
|
||||
"I cannot answer this from the given context",
|
||||
"What is X?",
|
||||
"Some unrelated content",
|
||||
)
|
||||
assert score < 0.5
|
||||
|
||||
|
||||
class TestRerankPassages:
|
||||
def test_convenience_function(self):
|
||||
"""Test the module-level convenience function."""
|
||||
passages = [{"content": "test", "session_id": "s1"}]
|
||||
result = rerank_passages(passages, "query", top_n=1)
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
class TestIsRiderAvailable:
|
||||
def test_returns_bool(self):
|
||||
result = is_rider_available()
|
||||
assert isinstance(result, bool)
|
||||
@@ -1,261 +0,0 @@
|
||||
"""
|
||||
Approval Tier System — Graduated safety based on risk level
|
||||
|
||||
Extends approval.py with 5-tier system for command approval.
|
||||
|
||||
| Tier | Action | Human | LLM | Timeout |
|
||||
|------|-----------------|-------|-----|---------|
|
||||
| 0 | Read, search | No | No | N/A |
|
||||
| 1 | Write, scripts | No | Yes | N/A |
|
||||
| 2 | Messages, API | Yes | Yes | 60s |
|
||||
| 3 | Crypto, config | Yes | Yes | 30s |
|
||||
| 4 | Crisis | Yes | Yes | 10s |
|
||||
|
||||
Issue: #670
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
class ApprovalTier(IntEnum):
|
||||
"""Approval tiers based on risk level."""
|
||||
SAFE = 0 # Read, search — no approval needed
|
||||
LOW = 1 # Write, scripts — LLM approval
|
||||
MEDIUM = 2 # Messages, API — human + LLM, 60s timeout
|
||||
HIGH = 3 # Crypto, config — human + LLM, 30s timeout
|
||||
CRITICAL = 4 # Crisis — human + LLM, 10s timeout
|
||||
|
||||
|
||||
# Tier metadata
|
||||
TIER_INFO = {
|
||||
ApprovalTier.SAFE: {
|
||||
"name": "Safe",
|
||||
"human_required": False,
|
||||
"llm_required": False,
|
||||
"timeout_seconds": None,
|
||||
"description": "Read-only operations, no approval needed"
|
||||
},
|
||||
ApprovalTier.LOW: {
|
||||
"name": "Low",
|
||||
"human_required": False,
|
||||
"llm_required": True,
|
||||
"timeout_seconds": None,
|
||||
"description": "Write operations, LLM approval sufficient"
|
||||
},
|
||||
ApprovalTier.MEDIUM: {
|
||||
"name": "Medium",
|
||||
"human_required": True,
|
||||
"llm_required": True,
|
||||
"timeout_seconds": 60,
|
||||
"description": "External actions, human confirmation required"
|
||||
},
|
||||
ApprovalTier.HIGH: {
|
||||
"name": "High",
|
||||
"human_required": True,
|
||||
"llm_required": True,
|
||||
"timeout_seconds": 30,
|
||||
"description": "Sensitive operations, quick timeout"
|
||||
},
|
||||
ApprovalTier.CRITICAL: {
|
||||
"name": "Critical",
|
||||
"human_required": True,
|
||||
"llm_required": True,
|
||||
"timeout_seconds": 10,
|
||||
"description": "Crisis or dangerous operations, fastest timeout"
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# Action-to-tier mapping
|
||||
ACTION_TIERS: Dict[str, ApprovalTier] = {
|
||||
# Tier 0: Safe (read-only)
|
||||
"read_file": ApprovalTier.SAFE,
|
||||
"search_files": ApprovalTier.SAFE,
|
||||
"web_search": ApprovalTier.SAFE,
|
||||
"session_search": ApprovalTier.SAFE,
|
||||
"list_files": ApprovalTier.SAFE,
|
||||
"get_file_content": ApprovalTier.SAFE,
|
||||
"memory_search": ApprovalTier.SAFE,
|
||||
"skills_list": ApprovalTier.SAFE,
|
||||
"skills_search": ApprovalTier.SAFE,
|
||||
|
||||
# Tier 1: Low (write operations)
|
||||
"write_file": ApprovalTier.LOW,
|
||||
"create_file": ApprovalTier.LOW,
|
||||
"patch_file": ApprovalTier.LOW,
|
||||
"delete_file": ApprovalTier.LOW,
|
||||
"execute_code": ApprovalTier.LOW,
|
||||
"terminal": ApprovalTier.LOW,
|
||||
"run_script": ApprovalTier.LOW,
|
||||
"skill_install": ApprovalTier.LOW,
|
||||
|
||||
# Tier 2: Medium (external actions)
|
||||
"send_message": ApprovalTier.MEDIUM,
|
||||
"web_fetch": ApprovalTier.MEDIUM,
|
||||
"browser_navigate": ApprovalTier.MEDIUM,
|
||||
"api_call": ApprovalTier.MEDIUM,
|
||||
"gitea_create_issue": ApprovalTier.MEDIUM,
|
||||
"gitea_create_pr": ApprovalTier.MEDIUM,
|
||||
"git_push": ApprovalTier.MEDIUM,
|
||||
"deploy": ApprovalTier.MEDIUM,
|
||||
|
||||
# Tier 3: High (sensitive operations)
|
||||
"config_change": ApprovalTier.HIGH,
|
||||
"env_change": ApprovalTier.HIGH,
|
||||
"key_rotation": ApprovalTier.HIGH,
|
||||
"access_grant": ApprovalTier.HIGH,
|
||||
"permission_change": ApprovalTier.HIGH,
|
||||
"backup_restore": ApprovalTier.HIGH,
|
||||
|
||||
# Tier 4: Critical (crisis/dangerous)
|
||||
"kill_process": ApprovalTier.CRITICAL,
|
||||
"rm_rf": ApprovalTier.CRITICAL,
|
||||
"format_disk": ApprovalTier.CRITICAL,
|
||||
"shutdown": ApprovalTier.CRITICAL,
|
||||
"crisis_override": ApprovalTier.CRITICAL,
|
||||
}
|
||||
|
||||
|
||||
# Dangerous command patterns (from existing approval.py)
|
||||
_DANGEROUS_PATTERNS = [
|
||||
(r"rm\s+-rf\s+/", ApprovalTier.CRITICAL),
|
||||
(r"mkfs\.", ApprovalTier.CRITICAL),
|
||||
(r"dd\s+if=.*of=/dev/", ApprovalTier.CRITICAL),
|
||||
(r"shutdown|reboot|halt", ApprovalTier.CRITICAL),
|
||||
(r"chmod\s+777", ApprovalTier.HIGH),
|
||||
(r"curl.*\|\s*bash", ApprovalTier.HIGH),
|
||||
(r"wget.*\|\s*sh", ApprovalTier.HIGH),
|
||||
(r"eval\s*\(", ApprovalTier.HIGH),
|
||||
(r"sudo\s+", ApprovalTier.MEDIUM),
|
||||
(r"git\s+push.*--force", ApprovalTier.HIGH),
|
||||
(r"docker\s+rm.*-f", ApprovalTier.MEDIUM),
|
||||
(r"kubectl\s+delete", ApprovalTier.HIGH),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ApprovalRequest:
|
||||
"""A request for approval."""
|
||||
action: str
|
||||
tier: ApprovalTier
|
||||
command: str
|
||||
reason: str
|
||||
session_key: str
|
||||
timeout_seconds: Optional[int] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"action": self.action,
|
||||
"tier": self.tier.value,
|
||||
"tier_name": TIER_INFO[self.tier]["name"],
|
||||
"command": self.command,
|
||||
"reason": self.reason,
|
||||
"session_key": self.session_key,
|
||||
"timeout": self.timeout_seconds,
|
||||
"human_required": TIER_INFO[self.tier]["human_required"],
|
||||
"llm_required": TIER_INFO[self.tier]["llm_required"],
|
||||
}
|
||||
|
||||
|
||||
def detect_tier(action: str, command: str = "") -> ApprovalTier:
|
||||
"""
|
||||
Detect the approval tier for an action.
|
||||
|
||||
Checks action name first, then falls back to pattern matching.
|
||||
"""
|
||||
# Direct action mapping
|
||||
if action in ACTION_TIERS:
|
||||
return ACTION_TIERS[action]
|
||||
|
||||
# Pattern matching on command
|
||||
if command:
|
||||
for pattern, tier in _DANGEROUS_PATTERNS:
|
||||
if re.search(pattern, command, re.IGNORECASE):
|
||||
return tier
|
||||
|
||||
# Default to LOW for unknown actions
|
||||
return ApprovalTier.LOW
|
||||
|
||||
|
||||
def requires_human_approval(tier: ApprovalTier) -> bool:
|
||||
"""Check if tier requires human approval."""
|
||||
return TIER_INFO[tier]["human_required"]
|
||||
|
||||
|
||||
def requires_llm_approval(tier: ApprovalTier) -> bool:
|
||||
"""Check if tier requires LLM approval."""
|
||||
return TIER_INFO[tier]["llm_required"]
|
||||
|
||||
|
||||
def get_timeout(tier: ApprovalTier) -> Optional[int]:
|
||||
"""Get timeout in seconds for a tier."""
|
||||
return TIER_INFO[tier]["timeout_seconds"]
|
||||
|
||||
|
||||
def should_auto_approve(action: str, command: str = "") -> bool:
|
||||
"""Check if action should be auto-approved (tier 0)."""
|
||||
tier = detect_tier(action, command)
|
||||
return tier == ApprovalTier.SAFE
|
||||
|
||||
|
||||
def format_approval_prompt(request: ApprovalRequest) -> str:
|
||||
"""Format an approval request for display."""
|
||||
info = TIER_INFO[request.tier]
|
||||
lines = []
|
||||
lines.append(f"⚠️ Approval Required (Tier {request.tier.value}: {info['name']})")
|
||||
lines.append(f"")
|
||||
lines.append(f"Action: {request.action}")
|
||||
lines.append(f"Command: {request.command[:100]}{'...' if len(request.command) > 100 else ''}")
|
||||
lines.append(f"Reason: {request.reason}")
|
||||
lines.append(f"")
|
||||
|
||||
if info["human_required"]:
|
||||
lines.append(f"👤 Human approval required")
|
||||
if info["llm_required"]:
|
||||
lines.append(f"🤖 LLM approval required")
|
||||
if info["timeout_seconds"]:
|
||||
lines.append(f"⏱️ Timeout: {info['timeout_seconds']}s")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def create_approval_request(
|
||||
action: str,
|
||||
command: str,
|
||||
reason: str,
|
||||
session_key: str
|
||||
) -> ApprovalRequest:
|
||||
"""Create an approval request for an action."""
|
||||
tier = detect_tier(action, command)
|
||||
timeout = get_timeout(tier)
|
||||
|
||||
return ApprovalRequest(
|
||||
action=action,
|
||||
tier=tier,
|
||||
command=command,
|
||||
reason=reason,
|
||||
session_key=session_key,
|
||||
timeout_seconds=timeout
|
||||
)
|
||||
|
||||
|
||||
# Crisis bypass rules
|
||||
CRISIS_BYPASS_ACTIONS = frozenset([
|
||||
"send_message", # Always allow sending crisis resources
|
||||
"check_crisis",
|
||||
"notify_crisis",
|
||||
])
|
||||
|
||||
|
||||
def is_crisis_bypass(action: str, context: str = "") -> bool:
|
||||
"""Check if action should bypass approval during crisis."""
|
||||
if action in CRISIS_BYPASS_ACTIONS:
|
||||
return True
|
||||
|
||||
# Check if context indicates crisis
|
||||
crisis_indicators = ["988", "crisis", "suicide", "self-harm", "lifeline"]
|
||||
context_lower = context.lower()
|
||||
return any(indicator in context_lower for indicator in crisis_indicators)
|
||||
453
tools/audio_engine.py
Normal file
453
tools/audio_engine.py
Normal file
@@ -0,0 +1,453 @@
|
||||
"""Shared Audio Analysis Engine
|
||||
|
||||
Provides beat detection, onset detection, vocal/instrumental separation,
|
||||
voice activity detection, and tempo estimation for use by:
|
||||
- Video Forge (scene transitions synced to music)
|
||||
- LPM 1.0 (lip sync timing, conversational state detection)
|
||||
|
||||
Dependencies (install as needed — all optional):
|
||||
pip install librosa soundfile demucs silero-vad torch
|
||||
|
||||
Gracefully degrades: if a dependency is missing, that feature returns
|
||||
None with a warning rather than crashing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Lazy dependency imports
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_LIBROSA = None
|
||||
_SOUNDFILE = None
|
||||
_DEMUCS_AVAILABLE = None
|
||||
_SILERO_AVAILABLE = None
|
||||
|
||||
|
||||
def _ensure_librosa():
|
||||
global _LIBROSA
|
||||
if _LIBROSA is None:
|
||||
try:
|
||||
import librosa
|
||||
_LIBROSA = librosa
|
||||
except ImportError:
|
||||
logger.warning("librosa not installed — beat/onset/tempo detection unavailable")
|
||||
_LIBROSA = False
|
||||
return _LIBROSA if _LIBROSA else None
|
||||
|
||||
|
||||
def _ensure_soundfile():
|
||||
global _SOUNDFILE
|
||||
if _SOUNDFILE is None:
|
||||
try:
|
||||
import soundfile
|
||||
_SOUNDFILE = soundfile
|
||||
except ImportError:
|
||||
logger.warning("soundfile not installed — audio loading may be limited")
|
||||
_SOUNDFILE = False
|
||||
return _SOUNDFILE if _SOUNDFILE else None
|
||||
|
||||
|
||||
def _ensure_demucs():
|
||||
global _DEMUCS_AVAILABLE
|
||||
if _DEMUCS_AVAILABLE is None:
|
||||
try:
|
||||
import demucs.api
|
||||
_DEMUCS_AVAILABLE = True
|
||||
except ImportError:
|
||||
logger.warning("demucs not installed — vocal separation unavailable")
|
||||
_DEMUCS_AVAILABLE = False
|
||||
return _DEMUCS_AVAILABLE
|
||||
|
||||
|
||||
def _ensure_silero():
|
||||
global _SILERO_AVAILABLE
|
||||
if _SILERO_AVAILABLE is None:
|
||||
try:
|
||||
import torch
|
||||
model, utils = torch.hub.load(
|
||||
repo_or_dir='snakers4/silero-vad', model='silero_vad',
|
||||
force_reload=False, onnx=False,
|
||||
)
|
||||
_SILERO_AVAILABLE = True
|
||||
except Exception:
|
||||
logger.warning("silero-vad not installed — VAD unavailable")
|
||||
_SILERO_AVAILABLE = False
|
||||
return _SILERO_AVAILABLE
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data classes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class BeatAnalysis:
|
||||
"""Results of beat and tempo analysis."""
|
||||
bpm: float # Estimated tempo in beats per minute
|
||||
beat_times: List[float] # Timestamps of detected beats (seconds)
|
||||
beat_frames: List[int] # Frame indices of detected beats
|
||||
tempo_confidence: float = 0.0 # Confidence in BPM estimate
|
||||
duration: float = 0.0 # Audio duration in seconds
|
||||
sample_rate: int = 0 # Sample rate used for analysis
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"bpm": round(self.bpm, 1),
|
||||
"beat_count": len(self.beat_times),
|
||||
"beat_times": self.beat_times[:50], # Cap for JSON size
|
||||
"tempo_confidence": round(self.tempo_confidence, 3),
|
||||
"duration": round(self.duration, 2),
|
||||
"sample_rate": self.sample_rate,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class OnsetAnalysis:
|
||||
"""Results of onset detection."""
|
||||
onset_times: List[float] # Timestamps of onsets (seconds)
|
||||
onset_frames: List[int] # Frame indices of onsets
|
||||
onset_count: int = 0
|
||||
avg_onset_interval: float = 0.0 # Average time between onsets (seconds)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"onset_count": self.onset_count,
|
||||
"onset_times": self.onset_times[:100],
|
||||
"avg_onset_interval": round(self.avg_onset_interval, 3),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class VADSegment:
|
||||
"""A single voice activity segment."""
|
||||
start: float # Start time in seconds
|
||||
end: float # End time in seconds
|
||||
is_speech: bool # True if speech detected
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {"start": round(self.start, 3), "end": round(self.end, 3), "is_speech": self.is_speech}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SeparationResult:
|
||||
"""Results of vocal/instrumental separation."""
|
||||
vocals_path: Optional[str] = None
|
||||
instrumental_path: Optional[str] = None
|
||||
duration: float = 0.0
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"vocals_path": self.vocals_path,
|
||||
"instrumental_path": self.instrumental_path,
|
||||
"duration": round(self.duration, 2),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Audio loading
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def load_audio(
|
||||
path: str | Path,
|
||||
sr: int = 22050,
|
||||
mono: bool = True,
|
||||
duration: float | None = None,
|
||||
) -> tuple:
|
||||
"""Load audio file. Returns (y, sr) tuple.
|
||||
|
||||
Args:
|
||||
path: Path to audio file (wav, mp3, flac, ogg)
|
||||
sr: Target sample rate (default 22050)
|
||||
mono: Convert to mono
|
||||
duration: Max seconds to load (None = full file)
|
||||
|
||||
Returns:
|
||||
(audio_array, sample_rate) or (None, None) on failure
|
||||
"""
|
||||
librosa = _ensure_librosa()
|
||||
if not librosa:
|
||||
return None, None
|
||||
|
||||
try:
|
||||
y, loaded_sr = librosa.load(
|
||||
str(path), sr=sr, mono=mono, duration=duration,
|
||||
)
|
||||
return y, loaded_sr
|
||||
except Exception as e:
|
||||
logger.error("Failed to load audio %s: %s", path, e)
|
||||
return None, None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Beat detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def detect_beats(
|
||||
audio_path: str | Path,
|
||||
sr: int = 22050,
|
||||
duration: float | None = None,
|
||||
) -> Optional[BeatAnalysis]:
|
||||
"""Detect beats and estimate tempo from an audio file.
|
||||
|
||||
Uses librosa.beat_track which implements the algorithm from:
|
||||
Ellis, "Beat Tracking by Dynamic Programming", 2007.
|
||||
|
||||
Args:
|
||||
audio_path: Path to audio file
|
||||
sr: Sample rate for analysis
|
||||
duration: Max seconds to analyze
|
||||
|
||||
Returns:
|
||||
BeatAnalysis or None if librosa unavailable
|
||||
"""
|
||||
librosa = _ensure_librosa()
|
||||
if not librosa:
|
||||
return None
|
||||
|
||||
y, loaded_sr = load_audio(audio_path, sr=sr, duration=duration)
|
||||
if y is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
tempo, beat_frames = librosa.beat.beat_track(y=y, sr=loaded_sr)
|
||||
beat_times = librosa.frames_to_time(beat_frames, sr=loaded_sr)
|
||||
|
||||
return BeatAnalysis(
|
||||
bpm=float(tempo),
|
||||
beat_times=beat_times.tolist(),
|
||||
beat_frames=beat_frames.tolist(),
|
||||
tempo_confidence=0.8, # librosa doesn't expose this directly
|
||||
duration=len(y) / loaded_sr,
|
||||
sample_rate=loaded_sr,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Beat detection failed for %s: %s", audio_path, e)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Onset detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def detect_onsets(
|
||||
audio_path: str | Path,
|
||||
sr: int = 22050,
|
||||
duration: float | None = None,
|
||||
backtrack: bool = True,
|
||||
) -> Optional[OnsetAnalysis]:
|
||||
"""Detect onsets (when new sounds begin).
|
||||
|
||||
Useful for scene transitions (Video Forge) and speech segment
|
||||
boundaries (LPM 1.0).
|
||||
|
||||
Args:
|
||||
audio_path: Path to audio file
|
||||
sr: Sample rate
|
||||
duration: Max seconds to analyze
|
||||
backtrack: Find preceding energy minimum for each onset
|
||||
|
||||
Returns:
|
||||
OnsetAnalysis or None if librosa unavailable
|
||||
"""
|
||||
librosa = _ensure_librosa()
|
||||
if not librosa:
|
||||
return None
|
||||
|
||||
y, loaded_sr = load_audio(audio_path, sr=sr, duration=duration)
|
||||
if y is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
onset_frames = librosa.onset.onset_detect(
|
||||
y=y, sr=loaded_sr, backtrack=backtrack,
|
||||
)
|
||||
onset_times = librosa.frames_to_time(onset_frames, sr=loaded_sr)
|
||||
|
||||
intervals = []
|
||||
times = onset_times.tolist()
|
||||
for i in range(1, len(times)):
|
||||
intervals.append(times[i] - times[i - 1])
|
||||
|
||||
return OnsetAnalysis(
|
||||
onset_times=times,
|
||||
onset_frames=onset_frames.tolist(),
|
||||
onset_count=len(times),
|
||||
avg_onset_interval=sum(intervals) / len(intervals) if intervals else 0.0,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Onset detection failed for %s: %s", audio_path, e)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Vocal/instrumental separation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def separate_vocals(
|
||||
audio_path: str | Path,
|
||||
output_dir: str | Path = "/tmp/audio_separation",
|
||||
model_name: str = "htdemucs",
|
||||
) -> Optional[SeparationResult]:
|
||||
"""Separate vocals from instrumental using demucs.
|
||||
|
||||
Args:
|
||||
audio_path: Path to audio file
|
||||
output_dir: Directory for output stems
|
||||
model_name: Demucs model (htdemucs, htdemucs_ft, mdx_extra)
|
||||
|
||||
Returns:
|
||||
SeparationResult with paths to vocals/instrumental, or None
|
||||
"""
|
||||
if not _ensure_demucs():
|
||||
return None
|
||||
|
||||
try:
|
||||
import demucs.api
|
||||
import soundfile as sf
|
||||
|
||||
output_dir = Path(output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
separator = demucs.api.Separator(model=model_name)
|
||||
origin, separated = separator.separate_audio_file(str(audio_path))
|
||||
|
||||
vocals_path = output_dir / "vocals.wav"
|
||||
instrumental_path = output_dir / "instrumental.wav"
|
||||
|
||||
sf.write(str(vocals_path), separated["vocals"].cpu().numpy().T, separator.samplerate)
|
||||
sf.write(str(instrumental_path),
|
||||
(separated["drums"] + separated["bass"] + separated["other"]).cpu().numpy().T,
|
||||
separator.samplerate)
|
||||
|
||||
duration = len(origin) / separator.samplerate
|
||||
|
||||
return SeparationResult(
|
||||
vocals_path=str(vocals_path),
|
||||
instrumental_path=str(instrumental_path),
|
||||
duration=duration,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error("Vocal separation failed for %s: %s", audio_path, e)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Voice Activity Detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def detect_voice_activity(
|
||||
audio_path: str | Path,
|
||||
sr: int = 16000,
|
||||
threshold: float = 0.5,
|
||||
min_speech_duration: float = 0.3,
|
||||
) -> Optional[List[VADSegment]]:
|
||||
"""Detect speech segments using Silero VAD.
|
||||
|
||||
Returns list of segments where speech was detected.
|
||||
Useful for LPM listen/speak state switching.
|
||||
|
||||
Args:
|
||||
audio_path: Path to audio file
|
||||
sr: Sample rate (Silero expects 16kHz or 8kHz)
|
||||
threshold: VAD threshold (0.0-1.0)
|
||||
min_speech_duration: Minimum segment length to count as speech
|
||||
|
||||
Returns:
|
||||
List of VADSegment or None if silero unavailable
|
||||
"""
|
||||
if not _ensure_silero():
|
||||
return None
|
||||
|
||||
try:
|
||||
import torch
|
||||
import torchaudio
|
||||
|
||||
model, utils = torch.hub.load(
|
||||
repo_or_dir='snakers4/silero-vad', model='silero_vad',
|
||||
force_reload=False, onnx=False,
|
||||
)
|
||||
get_speech_timestamps = utils[0]
|
||||
|
||||
wav, file_sr = torchaudio.load(str(audio_path))
|
||||
if file_sr != sr:
|
||||
wav = torchaudio.functional.resample(wav, file_sr, sr)
|
||||
|
||||
if wav.shape[0] > 1:
|
||||
wav = wav.mean(dim=0, keepdim=True)
|
||||
|
||||
speech_timestamps = get_speech_timestamps(
|
||||
wav.squeeze(), model, sampling_rate=sr,
|
||||
threshold=threshold, min_speech_duration_ms=int(min_speech_duration * 1000),
|
||||
)
|
||||
|
||||
segments = []
|
||||
for ts in speech_timestamps:
|
||||
segments.append(VADSegment(
|
||||
start=ts["start"] / sr,
|
||||
end=ts["end"] / sr,
|
||||
is_speech=True,
|
||||
))
|
||||
|
||||
return segments
|
||||
except Exception as e:
|
||||
logger.error("VAD failed for %s: %s", audio_path, e)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Full analysis
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def analyze_audio(
|
||||
audio_path: str | Path,
|
||||
include_separation: bool = False,
|
||||
include_vad: bool = False,
|
||||
sr: int = 22050,
|
||||
) -> Dict[str, Any]:
|
||||
"""Run full audio analysis pipeline.
|
||||
|
||||
Combines beat detection, onset detection, and optionally
|
||||
vocal separation and VAD into a single result dict.
|
||||
|
||||
Args:
|
||||
audio_path: Path to audio file
|
||||
include_separation: Run vocal separation (slow)
|
||||
include_vad: Run voice activity detection
|
||||
sr: Sample rate for beat/onset analysis
|
||||
|
||||
Returns:
|
||||
Dict with all analysis results
|
||||
"""
|
||||
result = {"path": str(audio_path)}
|
||||
|
||||
beats = detect_beats(audio_path, sr=sr)
|
||||
if beats:
|
||||
result["beats"] = beats.to_dict()
|
||||
|
||||
onsets = detect_onsets(audio_path, sr=sr)
|
||||
if onsets:
|
||||
result["onsets"] = onsets.to_dict()
|
||||
|
||||
if include_separation:
|
||||
separation = separate_vocals(audio_path)
|
||||
if separation:
|
||||
result["separation"] = separation.to_dict()
|
||||
|
||||
if include_vad:
|
||||
segments = detect_voice_activity(audio_path)
|
||||
if segments:
|
||||
result["vad"] = {
|
||||
"segments": [s.to_dict() for s in segments],
|
||||
"speech_ratio": sum(s.end - s.start for s in segments) / (beats.duration if beats else 1.0),
|
||||
}
|
||||
|
||||
return result
|
||||
@@ -1,233 +0,0 @@
|
||||
"""
|
||||
Tool Error Classification — Retryable vs Permanent.
|
||||
|
||||
Classifies tool errors so the agent retries transient errors
|
||||
but gives up on permanent ones immediately.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ErrorCategory(Enum):
|
||||
"""Error category classification."""
|
||||
RETRYABLE = "retryable"
|
||||
PERMANENT = "permanent"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ErrorClassification:
|
||||
"""Result of error classification."""
|
||||
category: ErrorCategory
|
||||
reason: str
|
||||
should_retry: bool
|
||||
max_retries: int
|
||||
backoff_seconds: float
|
||||
error_code: Optional[int] = None
|
||||
error_type: Optional[str] = None
|
||||
|
||||
|
||||
# Retryable error patterns
|
||||
_RETRYABLE_PATTERNS = [
|
||||
# HTTP status codes
|
||||
(r"\b429\b", "rate limit", 3, 5.0),
|
||||
(r"\b500\b", "server error", 3, 2.0),
|
||||
(r"\b502\b", "bad gateway", 3, 2.0),
|
||||
(r"\b503\b", "service unavailable", 3, 5.0),
|
||||
(r"\b504\b", "gateway timeout", 3, 5.0),
|
||||
|
||||
# Timeout patterns
|
||||
(r"timeout", "timeout", 3, 2.0),
|
||||
(r"timed out", "timeout", 3, 2.0),
|
||||
(r"TimeoutExpired", "timeout", 3, 2.0),
|
||||
|
||||
# Connection errors
|
||||
(r"connection refused", "connection refused", 2, 5.0),
|
||||
(r"connection reset", "connection reset", 2, 2.0),
|
||||
(r"network unreachable", "network unreachable", 2, 10.0),
|
||||
(r"DNS", "DNS error", 2, 5.0),
|
||||
|
||||
# Transient errors
|
||||
(r"temporary", "temporary error", 2, 2.0),
|
||||
(r"transient", "transient error", 2, 2.0),
|
||||
(r"retry", "retryable", 2, 2.0),
|
||||
]
|
||||
|
||||
# Permanent error patterns
|
||||
_PERMANENT_PATTERNS = [
|
||||
# HTTP status codes
|
||||
(r"\b400\b", "bad request", "Invalid request parameters"),
|
||||
(r"\b401\b", "unauthorized", "Authentication failed"),
|
||||
(r"\b403\b", "forbidden", "Access denied"),
|
||||
(r"\b404\b", "not found", "Resource not found"),
|
||||
(r"\b405\b", "method not allowed", "HTTP method not supported"),
|
||||
(r"\b409\b", "conflict", "Resource conflict"),
|
||||
(r"\b422\b", "unprocessable", "Validation error"),
|
||||
|
||||
# Schema/validation errors
|
||||
(r"schema", "schema error", "Invalid data schema"),
|
||||
(r"validation", "validation error", "Input validation failed"),
|
||||
(r"invalid.*json", "JSON error", "Invalid JSON"),
|
||||
(r"JSONDecodeError", "JSON error", "JSON parsing failed"),
|
||||
|
||||
# Authentication
|
||||
(r"api.?key", "API key error", "Invalid or missing API key"),
|
||||
(r"token.*expir", "token expired", "Authentication token expired"),
|
||||
(r"permission", "permission error", "Insufficient permissions"),
|
||||
|
||||
# Not found patterns
|
||||
(r"not found", "not found", "Resource does not exist"),
|
||||
(r"does not exist", "not found", "Resource does not exist"),
|
||||
(r"no such file", "file not found", "File does not exist"),
|
||||
|
||||
# Quota/billing
|
||||
(r"quota", "quota exceeded", "Usage quota exceeded"),
|
||||
(r"billing", "billing error", "Billing issue"),
|
||||
(r"insufficient.*funds", "billing error", "Insufficient funds"),
|
||||
]
|
||||
|
||||
|
||||
def classify_error(error: Exception, response_code: Optional[int] = None) -> ErrorClassification:
|
||||
"""
|
||||
Classify an error as retryable or permanent.
|
||||
|
||||
Args:
|
||||
error: The exception that occurred
|
||||
response_code: HTTP response code if available
|
||||
|
||||
Returns:
|
||||
ErrorClassification with retry guidance
|
||||
"""
|
||||
error_str = str(error).lower()
|
||||
error_type = type(error).__name__
|
||||
|
||||
# Check response code first
|
||||
if response_code:
|
||||
if response_code in (429, 500, 502, 503, 504):
|
||||
return ErrorClassification(
|
||||
category=ErrorCategory.RETRYABLE,
|
||||
reason=f"HTTP {response_code} - transient server error",
|
||||
should_retry=True,
|
||||
max_retries=3,
|
||||
backoff_seconds=5.0 if response_code == 429 else 2.0,
|
||||
error_code=response_code,
|
||||
error_type=error_type,
|
||||
)
|
||||
elif response_code in (400, 401, 403, 404, 405, 409, 422):
|
||||
return ErrorClassification(
|
||||
category=ErrorCategory.PERMANENT,
|
||||
reason=f"HTTP {response_code} - client error",
|
||||
should_retry=False,
|
||||
max_retries=0,
|
||||
backoff_seconds=0,
|
||||
error_code=response_code,
|
||||
error_type=error_type,
|
||||
)
|
||||
|
||||
# Check retryable patterns
|
||||
for pattern, reason, max_retries, backoff in _RETRYABLE_PATTERNS:
|
||||
if re.search(pattern, error_str, re.IGNORECASE):
|
||||
return ErrorClassification(
|
||||
category=ErrorCategory.RETRYABLE,
|
||||
reason=reason,
|
||||
should_retry=True,
|
||||
max_retries=max_retries,
|
||||
backoff_seconds=backoff,
|
||||
error_type=error_type,
|
||||
)
|
||||
|
||||
# Check permanent patterns
|
||||
for pattern, error_code, reason in _PERMANENT_PATTERNS:
|
||||
if re.search(pattern, error_str, re.IGNORECASE):
|
||||
return ErrorClassification(
|
||||
category=ErrorCategory.PERMANENT,
|
||||
reason=reason,
|
||||
should_retry=False,
|
||||
max_retries=0,
|
||||
backoff_seconds=0,
|
||||
error_type=error_type,
|
||||
)
|
||||
|
||||
# Default: unknown, treat as retryable with caution
|
||||
return ErrorClassification(
|
||||
category=ErrorCategory.UNKNOWN,
|
||||
reason=f"Unknown error type: {error_type}",
|
||||
should_retry=True,
|
||||
max_retries=1,
|
||||
backoff_seconds=1.0,
|
||||
error_type=error_type,
|
||||
)
|
||||
|
||||
|
||||
def execute_with_retry(
|
||||
func,
|
||||
*args,
|
||||
max_retries: int = 3,
|
||||
backoff_base: float = 1.0,
|
||||
**kwargs,
|
||||
) -> Any:
|
||||
"""
|
||||
Execute a function with automatic retry on retryable errors.
|
||||
|
||||
Args:
|
||||
func: Function to execute
|
||||
*args: Function arguments
|
||||
max_retries: Maximum retry attempts
|
||||
backoff_base: Base backoff time in seconds
|
||||
**kwargs: Function keyword arguments
|
||||
|
||||
Returns:
|
||||
Function result
|
||||
|
||||
Raises:
|
||||
Exception: If permanent error or max retries exceeded
|
||||
"""
|
||||
last_error = None
|
||||
|
||||
for attempt in range(max_retries + 1):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
|
||||
# Classify the error
|
||||
classification = classify_error(e)
|
||||
|
||||
logger.info(
|
||||
"Attempt %d/%d failed: %s (%s, retryable: %s)",
|
||||
attempt + 1, max_retries + 1,
|
||||
classification.reason,
|
||||
classification.category.value,
|
||||
classification.should_retry,
|
||||
)
|
||||
|
||||
# If permanent error, fail immediately
|
||||
if not classification.should_retry:
|
||||
logger.error("Permanent error: %s", classification.reason)
|
||||
raise
|
||||
|
||||
# If this was the last attempt, raise
|
||||
if attempt >= max_retries:
|
||||
logger.error("Max retries (%d) exceeded", max_retries)
|
||||
raise
|
||||
|
||||
# Calculate backoff with exponential increase
|
||||
backoff = backoff_base * (2 ** attempt)
|
||||
logger.info("Retrying in %.1fs...", backoff)
|
||||
time.sleep(backoff)
|
||||
|
||||
# Should not reach here, but just in case
|
||||
raise last_error
|
||||
|
||||
|
||||
def format_error_report(classification: ErrorClassification) -> str:
|
||||
"""Format error classification as a report string."""
|
||||
icon = "🔄" if classification.should_retry else "❌"
|
||||
return f"{icon} {classification.category.value}: {classification.reason}"
|
||||
@@ -394,23 +394,6 @@ def session_search(
|
||||
if len(seen_sessions) >= limit:
|
||||
break
|
||||
|
||||
# RIDER: Reader-guided reranking — sort sessions by LLM answerability
|
||||
# This bridges the R@5 vs E2E accuracy gap by prioritizing passages
|
||||
# the LLM can actually answer from, not just keyword matches.
|
||||
try:
|
||||
from agent.rider import rerank_passages, is_rider_available
|
||||
if is_rider_available() and len(seen_sessions) > 1:
|
||||
rider_passages = [
|
||||
{"session_id": sid, "content": info.get("snippet", ""), "rank": i + 1}
|
||||
for i, (sid, info) in enumerate(seen_sessions.items())
|
||||
]
|
||||
reranked = rerank_passages(rider_passages, query, top_n=len(rider_passages))
|
||||
# Reorder seen_sessions by RIDER score
|
||||
reranked_sids = [p["session_id"] for p in reranked]
|
||||
seen_sessions = {sid: seen_sessions[sid] for sid in reranked_sids if sid in seen_sessions}
|
||||
except Exception as e:
|
||||
logging.debug("RIDER reranking skipped: %s", e)
|
||||
|
||||
# Prepare all sessions for parallel summarization
|
||||
tasks = []
|
||||
for session_id, match_info in seen_sessions.items():
|
||||
|
||||
Reference in New Issue
Block a user