Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c8bab8ae3c | |||
|
|
faaa08b3f1 |
256
agent/rider.py
256
agent/rider.py
@@ -1,256 +0,0 @@
|
|||||||
"""RIDER — Reader-Guided Passage Reranking.
|
|
||||||
|
|
||||||
Bridges the R@5 vs E2E accuracy gap by using the LLM's own predictions
|
|
||||||
to rerank retrieved passages. Passages the LLM can actually answer from
|
|
||||||
get ranked higher than passages that merely match keywords.
|
|
||||||
|
|
||||||
Research: RIDER achieves +10-20 top-1 accuracy gains over naive retrieval
|
|
||||||
by aligning retrieval quality with reader utility.
|
|
||||||
|
|
||||||
Usage:
|
|
||||||
from agent.rider import RIDER
|
|
||||||
rider = RIDER()
|
|
||||||
reranked = rider.rerank(passages, query, top_n=3)
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
# Configuration
|
|
||||||
RIDER_ENABLED = os.getenv("RIDER_ENABLED", "true").lower() not in ("false", "0", "no")
|
|
||||||
RIDER_TOP_K = int(os.getenv("RIDER_TOP_K", "10")) # passages to score
|
|
||||||
RIDER_TOP_N = int(os.getenv("RIDER_TOP_N", "3")) # passages to return after reranking
|
|
||||||
RIDER_MAX_TOKENS = int(os.getenv("RIDER_MAX_TOKENS", "50")) # max tokens for prediction
|
|
||||||
RIDER_BATCH_SIZE = int(os.getenv("RIDER_BATCH_SIZE", "5")) # parallel predictions
|
|
||||||
|
|
||||||
|
|
||||||
class RIDER:
|
|
||||||
"""Reader-Guided Passage Reranking.
|
|
||||||
|
|
||||||
Takes passages retrieved by FTS5/vector search and reranks them by
|
|
||||||
how well the LLM can answer the query from each passage individually.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, auxiliary_task: str = "rider"):
|
|
||||||
"""Initialize RIDER.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
auxiliary_task: Task name for auxiliary client resolution.
|
|
||||||
"""
|
|
||||||
self._auxiliary_task = auxiliary_task
|
|
||||||
|
|
||||||
def rerank(
|
|
||||||
self,
|
|
||||||
passages: List[Dict[str, Any]],
|
|
||||||
query: str,
|
|
||||||
top_n: int = RIDER_TOP_N,
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""Rerank passages by reader confidence.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
passages: List of passage dicts. Must have 'content' or 'text' key.
|
|
||||||
May have 'session_id', 'snippet', 'rank', 'score', etc.
|
|
||||||
query: The user's search query.
|
|
||||||
top_n: Number of passages to return after reranking.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Reranked passages (top_n), each with added 'rider_score' and
|
|
||||||
'rider_prediction' fields.
|
|
||||||
"""
|
|
||||||
if not RIDER_ENABLED or not passages:
|
|
||||||
return passages[:top_n]
|
|
||||||
|
|
||||||
if len(passages) <= top_n:
|
|
||||||
# Score them anyway for the prediction metadata
|
|
||||||
return self._score_and_rerank(passages, query, top_n)
|
|
||||||
|
|
||||||
return self._score_and_rerank(passages[:RIDER_TOP_K], query, top_n)
|
|
||||||
|
|
||||||
def _score_and_rerank(
|
|
||||||
self,
|
|
||||||
passages: List[Dict[str, Any]],
|
|
||||||
query: str,
|
|
||||||
top_n: int,
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""Score each passage with the reader, then rerank by confidence."""
|
|
||||||
try:
|
|
||||||
from model_tools import _run_async
|
|
||||||
scored = _run_async(self._score_all_passages(passages, query))
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug("RIDER scoring failed: %s — returning original order", e)
|
|
||||||
return passages[:top_n]
|
|
||||||
|
|
||||||
# Sort by confidence (descending)
|
|
||||||
scored.sort(key=lambda p: p.get("rider_score", 0), reverse=True)
|
|
||||||
|
|
||||||
return scored[:top_n]
|
|
||||||
|
|
||||||
async def _score_all_passages(
|
|
||||||
self,
|
|
||||||
passages: List[Dict[str, Any]],
|
|
||||||
query: str,
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""Score all passages in batches."""
|
|
||||||
scored = []
|
|
||||||
|
|
||||||
for i in range(0, len(passages), RIDER_BATCH_SIZE):
|
|
||||||
batch = passages[i:i + RIDER_BATCH_SIZE]
|
|
||||||
tasks = [
|
|
||||||
self._score_single_passage(p, query, idx + i)
|
|
||||||
for idx, p in enumerate(batch)
|
|
||||||
]
|
|
||||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
||||||
|
|
||||||
for passage, result in zip(batch, results):
|
|
||||||
if isinstance(result, Exception):
|
|
||||||
logger.debug("RIDER passage %d scoring failed: %s", i, result)
|
|
||||||
passage["rider_score"] = 0.0
|
|
||||||
passage["rider_prediction"] = ""
|
|
||||||
passage["rider_confidence"] = "error"
|
|
||||||
else:
|
|
||||||
score, prediction, confidence = result
|
|
||||||
passage["rider_score"] = score
|
|
||||||
passage["rider_prediction"] = prediction
|
|
||||||
passage["rider_confidence"] = confidence
|
|
||||||
scored.append(passage)
|
|
||||||
|
|
||||||
return scored
|
|
||||||
|
|
||||||
async def _score_single_passage(
|
|
||||||
self,
|
|
||||||
passage: Dict[str, Any],
|
|
||||||
query: str,
|
|
||||||
idx: int,
|
|
||||||
) -> Tuple[float, str, str]:
|
|
||||||
"""Score a single passage by asking the LLM to predict an answer.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
(confidence_score, prediction, confidence_label)
|
|
||||||
"""
|
|
||||||
content = passage.get("content") or passage.get("text") or passage.get("snippet", "")
|
|
||||||
if not content or len(content) < 10:
|
|
||||||
return 0.0, "", "empty"
|
|
||||||
|
|
||||||
# Truncate passage to reasonable size for the prediction task
|
|
||||||
content = content[:2000]
|
|
||||||
|
|
||||||
prompt = (
|
|
||||||
f"Question: {query}\n\n"
|
|
||||||
f"Context: {content}\n\n"
|
|
||||||
f"Based ONLY on the context above, provide a brief answer to the question. "
|
|
||||||
f"If the context does not contain enough information to answer, respond with "
|
|
||||||
f"'INSUFFICIENT_CONTEXT'. Be specific and concise."
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
|
||||||
from agent.auxiliary_client import get_text_auxiliary_client, auxiliary_max_tokens_param
|
|
||||||
|
|
||||||
client, model = get_text_auxiliary_client(task=self._auxiliary_task)
|
|
||||||
if not client:
|
|
||||||
return 0.5, "", "no_client"
|
|
||||||
|
|
||||||
response = client.chat.completions.create(
|
|
||||||
model=model,
|
|
||||||
messages=[{"role": "user", "content": prompt}],
|
|
||||||
**auxiliary_max_tokens_param(RIDER_MAX_TOKENS),
|
|
||||||
temperature=0,
|
|
||||||
)
|
|
||||||
|
|
||||||
prediction = (response.choices[0].message.content or "").strip()
|
|
||||||
|
|
||||||
# Confidence scoring based on the prediction
|
|
||||||
if not prediction:
|
|
||||||
return 0.1, "", "empty_response"
|
|
||||||
|
|
||||||
if "INSUFFICIENT_CONTEXT" in prediction.upper():
|
|
||||||
return 0.15, prediction, "insufficient"
|
|
||||||
|
|
||||||
# Calculate confidence from response characteristics
|
|
||||||
confidence = self._calculate_confidence(prediction, query, content)
|
|
||||||
|
|
||||||
return confidence, prediction, "predicted"
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.debug("RIDER prediction failed for passage %d: %s", idx, e)
|
|
||||||
return 0.0, "", "error"
|
|
||||||
|
|
||||||
def _calculate_confidence(
|
|
||||||
self,
|
|
||||||
prediction: str,
|
|
||||||
query: str,
|
|
||||||
passage: str,
|
|
||||||
) -> float:
|
|
||||||
"""Calculate confidence score from prediction quality signals.
|
|
||||||
|
|
||||||
Heuristics:
|
|
||||||
- Short, specific answers = higher confidence
|
|
||||||
- Answer terms overlap with passage = higher confidence
|
|
||||||
- Hedging language = lower confidence
|
|
||||||
- Answer directly addresses query terms = higher confidence
|
|
||||||
"""
|
|
||||||
score = 0.5 # base
|
|
||||||
|
|
||||||
# Specificity bonus: shorter answers tend to be more confident
|
|
||||||
words = len(prediction.split())
|
|
||||||
if words <= 5:
|
|
||||||
score += 0.2
|
|
||||||
elif words <= 15:
|
|
||||||
score += 0.1
|
|
||||||
elif words > 50:
|
|
||||||
score -= 0.1
|
|
||||||
|
|
||||||
# Passage grounding: does the answer use terms from the passage?
|
|
||||||
passage_lower = passage.lower()
|
|
||||||
answer_terms = set(prediction.lower().split())
|
|
||||||
passage_terms = set(passage_lower.split())
|
|
||||||
overlap = len(answer_terms & passage_terms)
|
|
||||||
if overlap > 3:
|
|
||||||
score += 0.15
|
|
||||||
elif overlap > 0:
|
|
||||||
score += 0.05
|
|
||||||
|
|
||||||
# Query relevance: does the answer address query terms?
|
|
||||||
query_terms = set(query.lower().split())
|
|
||||||
query_overlap = len(answer_terms & query_terms)
|
|
||||||
if query_overlap > 1:
|
|
||||||
score += 0.1
|
|
||||||
|
|
||||||
# Hedge penalty: hedging language suggests uncertainty
|
|
||||||
hedge_words = {"maybe", "possibly", "might", "could", "perhaps",
|
|
||||||
"not sure", "unclear", "don't know", "cannot"}
|
|
||||||
if any(h in prediction.lower() for h in hedge_words):
|
|
||||||
score -= 0.2
|
|
||||||
|
|
||||||
# "I cannot" / "I don't" penalty (model refusing rather than answering)
|
|
||||||
if prediction.lower().startswith(("i cannot", "i don't", "i can't", "there is no")):
|
|
||||||
score -= 0.15
|
|
||||||
|
|
||||||
return max(0.0, min(1.0, score))
|
|
||||||
|
|
||||||
|
|
||||||
def rerank_passages(
|
|
||||||
passages: List[Dict[str, Any]],
|
|
||||||
query: str,
|
|
||||||
top_n: int = RIDER_TOP_N,
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""Convenience function for passage reranking."""
|
|
||||||
rider = RIDER()
|
|
||||||
return rider.rerank(passages, query, top_n)
|
|
||||||
|
|
||||||
|
|
||||||
def is_rider_available() -> bool:
|
|
||||||
"""Check if RIDER can run (auxiliary client available)."""
|
|
||||||
if not RIDER_ENABLED:
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
from agent.auxiliary_client import get_text_auxiliary_client
|
|
||||||
client, model = get_text_auxiliary_client(task="rider")
|
|
||||||
return client is not None and model is not None
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
134
docs/cybersecurity-skills.md
Normal file
134
docs/cybersecurity-skills.md
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
# Anthropic Cybersecurity Skills Integration
|
||||||
|
|
||||||
|
Import and use the Anthropic Cybersecurity Skills library (754 skills, 26 domains, 5 frameworks) with Hermes Agent.
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
The Anthropic Cybersecurity Skills library provides 754 production-grade security skills for AI agents. Each skill follows the agentskills.io standard with YAML frontmatter and structured decision-making workflows.
|
||||||
|
|
||||||
|
## Source
|
||||||
|
|
||||||
|
- **Repository:** https://github.com/mukul975/Anthropic-Cybersecurity-Skills
|
||||||
|
- **License:** Apache 2.0
|
||||||
|
- **Stars:** 4,385
|
||||||
|
- **Compatible:** Hermes Agent, Claude Code, GitHub Copilot, Codex CLI
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Import all skills
|
||||||
|
python scripts/import_cybersecurity_skills.py
|
||||||
|
|
||||||
|
# Import by domain
|
||||||
|
python scripts/import_cybersecurity_skills.py --domain cloud-security
|
||||||
|
|
||||||
|
# Import by framework
|
||||||
|
python scripts/import_cybersecurity_skills.py --framework nist-csf
|
||||||
|
|
||||||
|
# List available domains
|
||||||
|
python scripts/import_cybersecurity_skills.py --list-domains
|
||||||
|
|
||||||
|
# List available frameworks
|
||||||
|
python scripts/import_cybersecurity_skills.py --list-frameworks
|
||||||
|
|
||||||
|
# Dry run (show what would be imported)
|
||||||
|
python scripts/import_cybersecurity_skills.py --dry-run
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security Domains (26)
|
||||||
|
|
||||||
|
| Domain | Skills | Key Capabilities |
|
||||||
|
|--------|--------|-----------------|
|
||||||
|
| Cloud Security | 60 | AWS, Azure, GCP hardening, CSPM, cloud forensics |
|
||||||
|
| Threat Hunting | 55 | Hypothesis-driven hunts, LOTL detection, behavioral analytics |
|
||||||
|
| Threat Intelligence | 50 | STIX/TAXII, MISP, feed integration, actor profiling |
|
||||||
|
| Web App Security | 42 | OWASP Top 10, SQLi, XSS, SSRF, deserialization |
|
||||||
|
| Network Security | 40 | IDS/IPS, firewall rules, VLAN segmentation |
|
||||||
|
| Malware Analysis | 39 | Static/dynamic analysis, reverse engineering, sandboxing |
|
||||||
|
| Digital Forensics | 37 | Disk imaging, memory forensics, timeline reconstruction |
|
||||||
|
| Security Operations | 36 | SIEM correlation, log analysis, alert triage |
|
||||||
|
| IAM | 35 | IAM policies, PAM, zero trust, Okta, SailPoint |
|
||||||
|
| SOC Operations | 33 | Playbooks, escalation workflows, tabletop exercises |
|
||||||
|
| Container Security | 30 | K8s RBAC, image scanning, Falco, container forensics |
|
||||||
|
| OT/ICS Security | 28 | Modbus, DNP3, IEC 62443, SCADA |
|
||||||
|
| API Security | 28 | GraphQL, REST, OWASP API Top 10, WAF bypass |
|
||||||
|
| Vulnerability Management | 25 | Nessus, scanning workflows, CVSS |
|
||||||
|
| Incident Response | 25 | Breach containment, ransomware response, IR playbooks |
|
||||||
|
| Red Teaming | 24 | Full-scope engagements, AD attacks, phishing simulation |
|
||||||
|
| Penetration Testing | 23 | Network, web, cloud, mobile, wireless |
|
||||||
|
| Endpoint Security | 17 | EDR, LOTL detection, fileless malware |
|
||||||
|
| DevSecOps | 17 | CI/CD security, code signing, Terraform auditing |
|
||||||
|
| Phishing Defense | 16 | Email auth, BEC detection, phishing IR |
|
||||||
|
| Cryptography | 14 | Key management, TLS, certificate analysis |
|
||||||
|
|
||||||
|
## Framework Mappings (5)
|
||||||
|
|
||||||
|
| Framework | Version | Scope |
|
||||||
|
|-----------|---------|-------|
|
||||||
|
| MITRE ATT&CK | v18 | 14 tactics, 200+ techniques |
|
||||||
|
| NIST CSF 2.0 | 2.0 | 6 functions, 22 categories |
|
||||||
|
| MITRE ATLAS | v5.4 | 16 tactics, 84 techniques |
|
||||||
|
| MITRE D3FEND | v1.3 | 7 categories, 267 techniques |
|
||||||
|
| NIST AI RMF | 1.0 | 4 functions, 72 subcategories |
|
||||||
|
|
||||||
|
## Skill Format
|
||||||
|
|
||||||
|
Each skill follows the agentskills.io standard:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
---
|
||||||
|
name: analyzing-active-directory-acl-abuse
|
||||||
|
description: Detect dangerous ACL misconfigurations in Active Directory
|
||||||
|
domain: cybersecurity
|
||||||
|
subdomain: identity-security
|
||||||
|
tags:
|
||||||
|
- active-directory
|
||||||
|
- acl-abuse
|
||||||
|
- ldap
|
||||||
|
version: '1.0'
|
||||||
|
author: mahipal
|
||||||
|
license: Apache-2.0
|
||||||
|
nist_csf:
|
||||||
|
- PR.AA-01
|
||||||
|
- PR.AA-05
|
||||||
|
- PR.AA-06
|
||||||
|
---
|
||||||
|
```
|
||||||
|
|
||||||
|
## Use Cases for Hermes
|
||||||
|
|
||||||
|
1. **Fleet security** — Agents can audit their own infrastructure
|
||||||
|
2. **Incident response** — Structured IR playbooks for security events
|
||||||
|
3. **Threat hunting** — Hypothesis-driven hunts across fleet logs
|
||||||
|
4. **Compliance** — Framework-mapped skills for audit preparation
|
||||||
|
5. **Training** — Security skills for agents to learn and apply
|
||||||
|
|
||||||
|
## Integration with Hermes Skills
|
||||||
|
|
||||||
|
The imported skills are compatible with Hermes Agent's skill system:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Skills are installed to ~/.hermes/skills/cybersecurity/
|
||||||
|
# Each skill has a SKILL.md file with YAML frontmatter
|
||||||
|
|
||||||
|
# Use in Hermes
|
||||||
|
hermes skills list | grep cybersecurity
|
||||||
|
hermes skills enable cybersecurity/cloud-security
|
||||||
|
```
|
||||||
|
|
||||||
|
## Adding to Fleet
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Import all skills
|
||||||
|
python scripts/import_cybersecurity_skills.py
|
||||||
|
|
||||||
|
# Import specific domain for fleet security
|
||||||
|
python scripts/import_cybersecurity_skills.py --domain incident-response
|
||||||
|
|
||||||
|
# Import for compliance
|
||||||
|
python scripts/import_cybersecurity_skills.py --framework nist-csf
|
||||||
|
```
|
||||||
|
|
||||||
|
## Index
|
||||||
|
|
||||||
|
After import, an index is generated at `~/.hermes/skills/cybersecurity/index.json` listing all installed skills with their metadata.
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
# Gap Analysis: Actual System vs SOTA — Implementation Status Tracker
|
|
||||||
|
|
||||||
Issue #658. Maps gap analysis findings to implementation status.
|
|
||||||
|
|
||||||
## Gap Categories
|
|
||||||
|
|
||||||
### 1. Memory & Search
|
|
||||||
|
|
||||||
| Gap | Target | Status | PR |
|
|
||||||
|-----|--------|--------|-----|
|
|
||||||
| Semantic search (R@5) | 95-99% | RIDER: +25% E2E | #782 |
|
|
||||||
| Hybrid search | Vector + FTS5 + HRR | Hybrid search module | #729 |
|
|
||||||
| Context-faithful prompting | +11-14% E2E | Context-faithful module | #786 |
|
|
||||||
| Accuracy benchmarks | Measured | benchmark_r5_e2e.py | #790 |
|
|
||||||
| Vector embeddings | ChromaDB | Not yet (Qdrant fallback) | Future |
|
|
||||||
|
|
||||||
### 2. Multi-Agent Coordination
|
|
||||||
|
|
||||||
| Gap | Target | Status | PR |
|
|
||||||
|-----|--------|--------|-----|
|
|
||||||
| Three-tier memory | Unified | Fragmented (pieces exist) | #653 |
|
|
||||||
| DAG task routing | GraphFlow-style | Not implemented | Future |
|
|
||||||
| Fleet diary | Structured logs | Not implemented | Future |
|
|
||||||
|
|
||||||
### 3. Inference Optimization
|
|
||||||
|
|
||||||
| Gap | Target | Status | PR |
|
|
||||||
|-----|--------|--------|-----|
|
|
||||||
| Cost tracking | $/1M tokens | task_cost_breakdown.py | fleet-ops#267 |
|
|
||||||
| Fallback chain | Explicit | Provider routing exists | Existing |
|
|
||||||
| vLLM + FP8 | 60% cost reduction | Not yet | Future |
|
|
||||||
|
|
||||||
### 4. Workflow Orchestration
|
|
||||||
|
|
||||||
| Gap | Target | Status | PR |
|
|
||||||
|-----|--------|--------|-----|
|
|
||||||
| Retry with backoff | Built-in | Partial (cron retry) | Existing |
|
|
||||||
| Task dependencies | Pipeline chaining | Not implemented | Future |
|
|
||||||
| Concurrency control | Worker pool | File lock (single) | Existing |
|
|
||||||
|
|
||||||
### 5. Safety & Crisis
|
|
||||||
|
|
||||||
| Gap | Target | Status | PR |
|
|
||||||
|-----|--------|--------|-----|
|
|
||||||
| Crisis detection | F1>0.85 | Crisis protocol + SHIELD | #785 |
|
|
||||||
| Human confirmation | Tier system | Approval tiers | #697 |
|
|
||||||
| 988 Lifeline | Auto-display | Crisis resources | #783 |
|
|
||||||
| Emotional presence | Patterns | Research doc | #788 |
|
|
||||||
| SOUL.md protocol | Implemented | Crisis protocol | #785 |
|
|
||||||
|
|
||||||
### 6. Accuracy Measurement
|
|
||||||
|
|
||||||
| Gap | Target | Status | PR |
|
|
||||||
|-----|--------|--------|-----|
|
|
||||||
| R@5 measurement | Automated | benchmark_r5_e2e.py | #790 |
|
|
||||||
| E2E accuracy | Measured | benchmark_r5_e2e.py | #790 |
|
|
||||||
| Gap analysis | Documented | r5-vs-e2e-gap-analysis.md | #790 |
|
|
||||||
|
|
||||||
## Implementation Priority
|
|
||||||
|
|
||||||
1. **DONE:** Crisis support (SOUL.md, 988, detection)
|
|
||||||
2. **DONE:** Safety (approval tiers, SHIELD)
|
|
||||||
3. **DONE:** Retrieval improvement (RIDER, hybrid search, context-faithful)
|
|
||||||
4. **DONE:** Accuracy measurement (benchmark script)
|
|
||||||
5. **IN PR:** Cost tracking (task_cost_breakdown.py)
|
|
||||||
6. **FUTURE:** DAG routing, pub-sub messaging, vLLM deployment
|
|
||||||
|
|
||||||
## Key Insight
|
|
||||||
|
|
||||||
The biggest gap was MEASUREMENT — we didn't know if our systems worked. Issue #657 (accuracy measurement) addressed this first, followed by the retrieval improvements that bridge the R@5 vs E2E gap.
|
|
||||||
227
scripts/import-cybersecurity-skills.py
Normal file
227
scripts/import-cybersecurity-skills.py
Normal file
@@ -0,0 +1,227 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
import-cybersecurity-skills.py — Import Anthropic Cybersecurity Skills into Hermes.
|
||||||
|
|
||||||
|
Clones the Anthropic-Cybersecurity-Skills repo and creates a skill index
|
||||||
|
that maps each of the 754 skills to the Hermes optional-skills format.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 scripts/import-cybersecurity-skills.py --clone # Clone repo
|
||||||
|
python3 scripts/import-cybersecurity-skills.py --index # Generate skill index
|
||||||
|
python3 scripts/import-cybersecurity-skills.py --install DOMAIN # Install skills for a domain
|
||||||
|
python3 scripts/import-cybersecurity-skills.py --list # List all domains
|
||||||
|
python3 scripts/import-cybersecurity-skills.py --status # Import status
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import yaml
|
||||||
|
from pathlib import Path
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
REPO_URL = "https://github.com/mukul975/Anthropic-Cybersecurity-Skills.git"
|
||||||
|
SKILLS_DIR = Path.home() / ".hermes" / "cybersecurity-skills"
|
||||||
|
INDEX_PATH = SKILLS_DIR / "skill-index.json"
|
||||||
|
OPTIONAL_SKILLS_DIR = Path.home() / ".hermes" / "optional-skills" / "cybersecurity"
|
||||||
|
|
||||||
|
# Domain → hermes category mapping
|
||||||
|
DOMAIN_CATEGORIES = {
|
||||||
|
"cloud-security": "security",
|
||||||
|
"threat-hunting": "security",
|
||||||
|
"threat-intelligence": "security",
|
||||||
|
"web-app-security": "security",
|
||||||
|
"network-security": "security",
|
||||||
|
"malware-analysis": "security",
|
||||||
|
"digital-forensics": "security",
|
||||||
|
"security-operations": "security",
|
||||||
|
"identity-access-management": "security",
|
||||||
|
"soc-operations": "security",
|
||||||
|
"container-security": "security",
|
||||||
|
"ot-ics-security": "security",
|
||||||
|
"api-security": "security",
|
||||||
|
"vulnerability-management": "security",
|
||||||
|
"incident-response": "security",
|
||||||
|
"red-teaming": "security",
|
||||||
|
"penetration-testing": "security",
|
||||||
|
"endpoint-security": "security",
|
||||||
|
"devsecops": "devops",
|
||||||
|
"phishing-defense": "security",
|
||||||
|
"cryptography": "security",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_clone():
|
||||||
|
"""Clone the cybersecurity skills repository."""
|
||||||
|
if SKILLS_DIR.exists():
|
||||||
|
print(f"Updating existing clone at {SKILLS_DIR}")
|
||||||
|
subprocess.run(["git", "-C", str(SKILLS_DIR), "pull"], capture_output=True)
|
||||||
|
else:
|
||||||
|
SKILLS_DIR.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
print(f"Cloning {REPO_URL} to {SKILLS_DIR}")
|
||||||
|
subprocess.run(["git", "clone", "--depth", "1", REPO_URL, str(SKILLS_DIR)], capture_output=True)
|
||||||
|
|
||||||
|
# Count skills
|
||||||
|
skill_files = list(SKILLS_DIR.rglob("*.md"))
|
||||||
|
print(f"Found {len(skill_files)} skill files")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_index():
|
||||||
|
"""Generate a skill index from the cloned repo."""
|
||||||
|
if not SKILLS_DIR.exists():
|
||||||
|
print("Run --clone first", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
skills = []
|
||||||
|
domains = defaultdict(list)
|
||||||
|
|
||||||
|
for md_file in SKILLS_DIR.rglob("*.md"):
|
||||||
|
if md_file.name in ("README.md", "LICENSE.md", "DESCRIPTION.md"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
content = md_file.read_text(errors="ignore")
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Parse YAML frontmatter
|
||||||
|
if content.startswith("---"):
|
||||||
|
parts = content.split("---", 2)
|
||||||
|
if len(parts) >= 3:
|
||||||
|
try:
|
||||||
|
frontmatter = yaml.safe_load(parts[1]) or {}
|
||||||
|
except yaml.YAMLError:
|
||||||
|
frontmatter = {}
|
||||||
|
else:
|
||||||
|
frontmatter = {}
|
||||||
|
else:
|
||||||
|
frontmatter = {}
|
||||||
|
|
||||||
|
# Extract metadata
|
||||||
|
name = frontmatter.get("name", md_file.stem)
|
||||||
|
description = frontmatter.get("description", "")
|
||||||
|
domain = frontmatter.get("domain", frontmatter.get("subdomain", "general"))
|
||||||
|
tags = frontmatter.get("tags", [])
|
||||||
|
frameworks = frontmatter.get("nist_csf", []) + frontmatter.get("mitre_attack", [])
|
||||||
|
|
||||||
|
skill = {
|
||||||
|
"name": name,
|
||||||
|
"file": str(md_file.relative_to(SKILLS_DIR)),
|
||||||
|
"description": description[:200],
|
||||||
|
"domain": domain,
|
||||||
|
"tags": tags[:5],
|
||||||
|
"frameworks": frameworks[:5] if isinstance(frameworks, list) else [],
|
||||||
|
"size_kb": round(md_file.stat().st_size / 1024, 1),
|
||||||
|
}
|
||||||
|
skills.append(skill)
|
||||||
|
domains[domain].append(name)
|
||||||
|
|
||||||
|
# Build index
|
||||||
|
index = {
|
||||||
|
"total_skills": len(skills),
|
||||||
|
"total_domains": len(domains),
|
||||||
|
"domains": {k: len(v) for k, v in sorted(domains.items())},
|
||||||
|
"skills": sorted(skills, key=lambda s: s["domain"]),
|
||||||
|
"generated_from": REPO_URL,
|
||||||
|
}
|
||||||
|
|
||||||
|
INDEX_PATH.write_text(json.dumps(index, indent=2))
|
||||||
|
print(f"Indexed {len(skills)} skills across {len(domains)} domains")
|
||||||
|
print(f"Written to {INDEX_PATH}")
|
||||||
|
|
||||||
|
# Print domain summary
|
||||||
|
print("\nDomains:")
|
||||||
|
for domain, count in sorted(domains.items(), key=lambda x: -len(x[1])):
|
||||||
|
print(f" {domain}: {count} skills")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_list():
|
||||||
|
"""List all security domains."""
|
||||||
|
if not INDEX_PATH.exists():
|
||||||
|
print("Run --index first", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
index = json.loads(INDEX_PATH.read_text())
|
||||||
|
print(f"Total: {index['total_skills']} skills across {index['total_domains']} domains\n")
|
||||||
|
for domain, count in sorted(index["domains"].items(), key=lambda x: -x[1]):
|
||||||
|
print(f" {domain:<35} {count:>4} skills")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_install(domain: str = None):
|
||||||
|
"""Install skills for a domain into optional-skills."""
|
||||||
|
if not INDEX_PATH.exists():
|
||||||
|
print("Run --index first", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
index = json.loads(INDEX_PATH.read_text())
|
||||||
|
skills = index["skills"]
|
||||||
|
|
||||||
|
if domain:
|
||||||
|
skills = [s for s in skills if s["domain"] == domain]
|
||||||
|
if not skills:
|
||||||
|
print(f"No skills found for domain: {domain}")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
installed = 0
|
||||||
|
for skill in skills:
|
||||||
|
# Create skill directory
|
||||||
|
category = DOMAIN_CATEGORIES.get(skill["domain"], "security")
|
||||||
|
skill_dir = OPTIONAL_SKILLS_DIR / category / skill["name"]
|
||||||
|
skill_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Copy source file
|
||||||
|
src = SKILLS_DIR / skill["file"]
|
||||||
|
if src.exists():
|
||||||
|
dst = skill_dir / "SKILL.md"
|
||||||
|
dst.write_text(src.read_text(errors="ignore"))
|
||||||
|
installed += 1
|
||||||
|
|
||||||
|
print(f"Installed {installed} skills to {OPTIONAL_SKILLS_DIR}")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_status():
|
||||||
|
"""Show import status."""
|
||||||
|
print(f"Clone dir: {SKILLS_DIR}")
|
||||||
|
print(f" Exists: {SKILLS_DIR.exists()}")
|
||||||
|
|
||||||
|
print(f"Index: {INDEX_PATH}")
|
||||||
|
print(f" Exists: {INDEX_PATH.exists()}")
|
||||||
|
if INDEX_PATH.exists():
|
||||||
|
index = json.loads(INDEX_PATH.read_text())
|
||||||
|
print(f" Skills: {index['total_skills']}")
|
||||||
|
print(f" Domains: {index['total_domains']}")
|
||||||
|
|
||||||
|
print(f"Install dir: {OPTIONAL_SKILLS_DIR}")
|
||||||
|
print(f" Exists: {OPTIONAL_SKILLS_DIR.exists()}")
|
||||||
|
if OPTIONAL_SKILLS_DIR.exists():
|
||||||
|
installed = len(list(OPTIONAL_SKILLS_DIR.rglob("SKILL.md")))
|
||||||
|
print(f" Installed skills: {installed}")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Import Anthropic Cybersecurity Skills")
|
||||||
|
parser.add_argument("--clone", action="store_true", help="Clone the skills repo")
|
||||||
|
parser.add_argument("--index", action="store_true", help="Generate skill index")
|
||||||
|
parser.add_argument("--list", action="store_true", help="List all domains")
|
||||||
|
parser.add_argument("--install", metavar="DOMAIN", nargs="?", const="all", help="Install skills for domain")
|
||||||
|
parser.add_argument("--status", action="store_true", help="Import status")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.clone:
|
||||||
|
cmd_clone()
|
||||||
|
elif args.index:
|
||||||
|
cmd_index()
|
||||||
|
elif args.list:
|
||||||
|
cmd_list()
|
||||||
|
elif args.install is not None:
|
||||||
|
cmd_install(None if args.install == "all" else args.install)
|
||||||
|
elif args.status:
|
||||||
|
cmd_status()
|
||||||
|
else:
|
||||||
|
parser.print_help()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
245
scripts/import_cybersecurity_skills.py
Normal file
245
scripts/import_cybersecurity_skills.py
Normal file
@@ -0,0 +1,245 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
import_cybersecurity_skills.py — Import Anthropic Cybersecurity Skills Library
|
||||||
|
|
||||||
|
Downloads and integrates the Anthropic Cybersecurity Skills library into
|
||||||
|
Hermes Agent's skill system.
|
||||||
|
|
||||||
|
Source: https://github.com/mukul975/Anthropic-Cybersecurity-Skills
|
||||||
|
License: Apache 2.0
|
||||||
|
Skills: 754 across 26 security domains, 5 frameworks
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python scripts/import_cybersecurity_skills.py
|
||||||
|
python scripts/import_cybersecurity_skills.py --domain cloud-security
|
||||||
|
python scripts/import_cybersecurity_skills.py --framework nist-csf
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import urllib.request
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, Dict, Any
|
||||||
|
|
||||||
|
# Configuration
|
||||||
|
REPO_URL = "https://github.com/mukul975/Anthropic-Cybersecurity-Skills.git"
|
||||||
|
SKILLS_DIR = Path.home() / ".hermes" / "skills" / "cybersecurity"
|
||||||
|
CACHE_DIR = Path.home() / ".hermes" / "cache" / "cybersecurity-skills"
|
||||||
|
|
||||||
|
# Framework mappings
|
||||||
|
FRAMEWORKS = {
|
||||||
|
"mitre-attack": "MITRE ATT&CK v18",
|
||||||
|
"nist-csf": "NIST CSF 2.0",
|
||||||
|
"mitre-atlas": "MITRE ATLAS v5.4",
|
||||||
|
"mitre-d3fend": "MITRE D3FEND v1.3",
|
||||||
|
"nist-ai-rmf": "NIST AI RMF 1.0",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Security domains
|
||||||
|
DOMAINS = [
|
||||||
|
"cloud-security", "threat-hunting", "threat-intelligence",
|
||||||
|
"web-app-security", "network-security", "malware-analysis",
|
||||||
|
"digital-forensics", "security-operations", "iam",
|
||||||
|
"soc-operations", "container-security", "ot-ics-security",
|
||||||
|
"api-security", "vulnerability-management", "incident-response",
|
||||||
|
"red-teaming", "penetration-testing", "endpoint-security",
|
||||||
|
"devsecops", "phishing-defense", "cryptography",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def clone_repo(target_dir: Path) -> bool:
|
||||||
|
"""Clone the cybersecurity skills repository."""
|
||||||
|
print(f"Cloning {REPO_URL}...")
|
||||||
|
try:
|
||||||
|
subprocess.run(
|
||||||
|
["git", "clone", "--depth", "1", REPO_URL, str(target_dir)],
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
print(f"Error cloning repository: {e}", file=sys.stderr)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def parse_skill_file(skill_path: Path) -> Dict[str, Any]:
|
||||||
|
"""Parse a skill YAML/Markdown file."""
|
||||||
|
content = skill_path.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
# Extract YAML frontmatter
|
||||||
|
if content.startswith("---"):
|
||||||
|
parts = content.split("---", 2)
|
||||||
|
if len(parts) >= 3:
|
||||||
|
import yaml
|
||||||
|
try:
|
||||||
|
metadata = yaml.safe_load(parts[1])
|
||||||
|
metadata["content"] = parts[2].strip()
|
||||||
|
metadata["path"] = str(skill_path)
|
||||||
|
return metadata
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Fallback: use filename as name
|
||||||
|
return {
|
||||||
|
"name": skill_path.stem,
|
||||||
|
"description": content[:200],
|
||||||
|
"content": content,
|
||||||
|
"path": str(skill_path),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def find_skills(repo_dir: Path, domain: str = None, framework: str = None) -> List[Path]:
|
||||||
|
"""Find skill files in the repository."""
|
||||||
|
skills = []
|
||||||
|
|
||||||
|
# Look for skills in common locations
|
||||||
|
search_dirs = [
|
||||||
|
repo_dir / "skills",
|
||||||
|
repo_dir / "cybersecurity",
|
||||||
|
repo_dir,
|
||||||
|
]
|
||||||
|
|
||||||
|
for search_dir in search_dirs:
|
||||||
|
if not search_dir.exists():
|
||||||
|
continue
|
||||||
|
|
||||||
|
for path in search_dir.rglob("*.md"):
|
||||||
|
# Skip README files
|
||||||
|
if path.name.upper() == "README.MD":
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Filter by domain if specified
|
||||||
|
if domain:
|
||||||
|
if domain.lower() not in str(path).lower():
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Filter by framework if specified
|
||||||
|
if framework:
|
||||||
|
content = path.read_text(encoding="utf-8", errors="ignore").lower()
|
||||||
|
if framework.lower() not in content:
|
||||||
|
continue
|
||||||
|
|
||||||
|
skills.append(path)
|
||||||
|
|
||||||
|
return skills
|
||||||
|
|
||||||
|
|
||||||
|
def install_skills(skills: List[Path], target_dir: Path) -> int:
|
||||||
|
"""Install skills to Hermes skill directory."""
|
||||||
|
target_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
installed = 0
|
||||||
|
for skill_path in skills:
|
||||||
|
skill = parse_skill_file(skill_path)
|
||||||
|
name = skill.get("name", skill_path.stem)
|
||||||
|
|
||||||
|
# Create skill directory
|
||||||
|
skill_dir = target_dir / name
|
||||||
|
skill_dir.mkdir(exist_ok=True)
|
||||||
|
|
||||||
|
# Copy skill file
|
||||||
|
dest = skill_dir / "SKILL.md"
|
||||||
|
shutil.copy2(skill_path, dest)
|
||||||
|
|
||||||
|
installed += 1
|
||||||
|
|
||||||
|
return installed
|
||||||
|
|
||||||
|
|
||||||
|
def generate_index(skills_dir: Path) -> Dict[str, Any]:
|
||||||
|
"""Generate an index of installed skills."""
|
||||||
|
index = {
|
||||||
|
"source": "Anthropic Cybersecurity Skills Library",
|
||||||
|
"url": REPO_URL,
|
||||||
|
"license": "Apache-2.0",
|
||||||
|
"skills": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
for skill_dir in skills_dir.iterdir():
|
||||||
|
if not skill_dir.is_dir():
|
||||||
|
continue
|
||||||
|
|
||||||
|
skill_file = skill_dir / "SKILL.md"
|
||||||
|
if not skill_file.exists():
|
||||||
|
continue
|
||||||
|
|
||||||
|
skill = parse_skill_file(skill_file)
|
||||||
|
index["skills"].append({
|
||||||
|
"name": skill.get("name", skill_dir.name),
|
||||||
|
"description": skill.get("description", "")[:200],
|
||||||
|
"domain": skill.get("domain", ""),
|
||||||
|
"frameworks": skill.get("frameworks", []),
|
||||||
|
})
|
||||||
|
|
||||||
|
return index
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Import Anthropic Cybersecurity Skills")
|
||||||
|
parser.add_argument("--domain", "-d", help="Filter by security domain")
|
||||||
|
parser.add_argument("--framework", "-f", help="Filter by framework (e.g., nist-csf)")
|
||||||
|
parser.add_argument("--list-domains", action="store_true", help="List available domains")
|
||||||
|
parser.add_argument("--list-frameworks", action="store_true", help="List available frameworks")
|
||||||
|
parser.add_argument("--output", "-o", help="Output directory for skills")
|
||||||
|
parser.add_argument("--dry-run", action="store_true", help="Show what would be imported")
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# List domains
|
||||||
|
if args.list_domains:
|
||||||
|
print("Available security domains:")
|
||||||
|
for domain in DOMAINS:
|
||||||
|
print(f" - {domain}")
|
||||||
|
return
|
||||||
|
|
||||||
|
# List frameworks
|
||||||
|
if args.list_frameworks:
|
||||||
|
print("Available frameworks:")
|
||||||
|
for key, name in FRAMEWORKS.items():
|
||||||
|
print(f" - {key}: {name}")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Set output directory
|
||||||
|
output_dir = Path(args.output) if args.output else SKILLS_DIR
|
||||||
|
|
||||||
|
# Clone repository
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
repo_dir = Path(tmpdir) / "cybersecurity-skills"
|
||||||
|
|
||||||
|
if not clone_repo(repo_dir):
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# Find skills
|
||||||
|
print(f"Searching for skills (domain={args.domain}, framework={args.framework})...")
|
||||||
|
skills = find_skills(repo_dir, args.domain, args.framework)
|
||||||
|
print(f"Found {len(skills)} skills")
|
||||||
|
|
||||||
|
if args.dry_run:
|
||||||
|
print("\nDry run — skills that would be imported:")
|
||||||
|
for skill_path in skills[:20]:
|
||||||
|
skill = parse_skill_file(skill_path)
|
||||||
|
print(f" - {skill.get('name', skill_path.stem)}: {skill.get('description', '')[:60]}...")
|
||||||
|
if len(skills) > 20:
|
||||||
|
print(f" ... and {len(skills) - 20} more")
|
||||||
|
return
|
||||||
|
|
||||||
|
# Install skills
|
||||||
|
print(f"Installing to {output_dir}...")
|
||||||
|
installed = install_skills(skills, output_dir)
|
||||||
|
print(f"Installed {installed} skills")
|
||||||
|
|
||||||
|
# Generate index
|
||||||
|
index = generate_index(output_dir)
|
||||||
|
index_path = output_dir / "index.json"
|
||||||
|
with open(index_path, "w") as f:
|
||||||
|
json.dump(index, f, indent=2)
|
||||||
|
print(f"Index saved to {index_path}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -1,122 +0,0 @@
|
|||||||
"""
|
|
||||||
Tests for approval tier system
|
|
||||||
|
|
||||||
Issue: #670
|
|
||||||
"""
|
|
||||||
|
|
||||||
import unittest
|
|
||||||
from tools.approval_tiers import (
|
|
||||||
ApprovalTier,
|
|
||||||
detect_tier,
|
|
||||||
requires_human_approval,
|
|
||||||
requires_llm_approval,
|
|
||||||
get_timeout,
|
|
||||||
should_auto_approve,
|
|
||||||
create_approval_request,
|
|
||||||
is_crisis_bypass,
|
|
||||||
TIER_INFO,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TestApprovalTier(unittest.TestCase):
|
|
||||||
|
|
||||||
def test_tier_values(self):
|
|
||||||
self.assertEqual(ApprovalTier.SAFE, 0)
|
|
||||||
self.assertEqual(ApprovalTier.LOW, 1)
|
|
||||||
self.assertEqual(ApprovalTier.MEDIUM, 2)
|
|
||||||
self.assertEqual(ApprovalTier.HIGH, 3)
|
|
||||||
self.assertEqual(ApprovalTier.CRITICAL, 4)
|
|
||||||
|
|
||||||
|
|
||||||
class TestTierDetection(unittest.TestCase):
|
|
||||||
|
|
||||||
def test_safe_actions(self):
|
|
||||||
self.assertEqual(detect_tier("read_file"), ApprovalTier.SAFE)
|
|
||||||
self.assertEqual(detect_tier("web_search"), ApprovalTier.SAFE)
|
|
||||||
self.assertEqual(detect_tier("session_search"), ApprovalTier.SAFE)
|
|
||||||
|
|
||||||
def test_low_actions(self):
|
|
||||||
self.assertEqual(detect_tier("write_file"), ApprovalTier.LOW)
|
|
||||||
self.assertEqual(detect_tier("terminal"), ApprovalTier.LOW)
|
|
||||||
self.assertEqual(detect_tier("execute_code"), ApprovalTier.LOW)
|
|
||||||
|
|
||||||
def test_medium_actions(self):
|
|
||||||
self.assertEqual(detect_tier("send_message"), ApprovalTier.MEDIUM)
|
|
||||||
self.assertEqual(detect_tier("git_push"), ApprovalTier.MEDIUM)
|
|
||||||
|
|
||||||
def test_high_actions(self):
|
|
||||||
self.assertEqual(detect_tier("config_change"), ApprovalTier.HIGH)
|
|
||||||
self.assertEqual(detect_tier("key_rotation"), ApprovalTier.HIGH)
|
|
||||||
|
|
||||||
def test_critical_actions(self):
|
|
||||||
self.assertEqual(detect_tier("kill_process"), ApprovalTier.CRITICAL)
|
|
||||||
self.assertEqual(detect_tier("shutdown"), ApprovalTier.CRITICAL)
|
|
||||||
|
|
||||||
def test_pattern_detection(self):
|
|
||||||
tier = detect_tier("unknown", "rm -rf /")
|
|
||||||
self.assertEqual(tier, ApprovalTier.CRITICAL)
|
|
||||||
|
|
||||||
tier = detect_tier("unknown", "sudo apt install")
|
|
||||||
self.assertEqual(tier, ApprovalTier.MEDIUM)
|
|
||||||
|
|
||||||
|
|
||||||
class TestTierInfo(unittest.TestCase):
|
|
||||||
|
|
||||||
def test_safe_no_approval(self):
|
|
||||||
self.assertFalse(requires_human_approval(ApprovalTier.SAFE))
|
|
||||||
self.assertFalse(requires_llm_approval(ApprovalTier.SAFE))
|
|
||||||
self.assertIsNone(get_timeout(ApprovalTier.SAFE))
|
|
||||||
|
|
||||||
def test_medium_requires_both(self):
|
|
||||||
self.assertTrue(requires_human_approval(ApprovalTier.MEDIUM))
|
|
||||||
self.assertTrue(requires_llm_approval(ApprovalTier.MEDIUM))
|
|
||||||
self.assertEqual(get_timeout(ApprovalTier.MEDIUM), 60)
|
|
||||||
|
|
||||||
def test_critical_fast_timeout(self):
|
|
||||||
self.assertEqual(get_timeout(ApprovalTier.CRITICAL), 10)
|
|
||||||
|
|
||||||
|
|
||||||
class TestAutoApprove(unittest.TestCase):
|
|
||||||
|
|
||||||
def test_safe_auto_approves(self):
|
|
||||||
self.assertTrue(should_auto_approve("read_file"))
|
|
||||||
self.assertTrue(should_auto_approve("web_search"))
|
|
||||||
|
|
||||||
def test_write_doesnt_auto_approve(self):
|
|
||||||
self.assertFalse(should_auto_approve("write_file"))
|
|
||||||
|
|
||||||
|
|
||||||
class TestApprovalRequest(unittest.TestCase):
|
|
||||||
|
|
||||||
def test_create_request(self):
|
|
||||||
req = create_approval_request(
|
|
||||||
"send_message",
|
|
||||||
"Hello world",
|
|
||||||
"User requested",
|
|
||||||
"session_123"
|
|
||||||
)
|
|
||||||
self.assertEqual(req.tier, ApprovalTier.MEDIUM)
|
|
||||||
self.assertEqual(req.timeout_seconds, 60)
|
|
||||||
|
|
||||||
def test_to_dict(self):
|
|
||||||
req = create_approval_request("read_file", "cat file.txt", "test", "s1")
|
|
||||||
d = req.to_dict()
|
|
||||||
self.assertEqual(d["tier"], 0)
|
|
||||||
self.assertEqual(d["tier_name"], "Safe")
|
|
||||||
|
|
||||||
|
|
||||||
class TestCrisisBypass(unittest.TestCase):
|
|
||||||
|
|
||||||
def test_send_message_bypass(self):
|
|
||||||
self.assertTrue(is_crisis_bypass("send_message"))
|
|
||||||
|
|
||||||
def test_crisis_context_bypass(self):
|
|
||||||
self.assertTrue(is_crisis_bypass("unknown", "call 988 lifeline"))
|
|
||||||
self.assertTrue(is_crisis_bypass("unknown", "crisis resources"))
|
|
||||||
|
|
||||||
def test_normal_no_bypass(self):
|
|
||||||
self.assertFalse(is_crisis_bypass("read_file"))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
unittest.main()
|
|
||||||
@@ -1,55 +0,0 @@
|
|||||||
"""
|
|
||||||
Tests for error classification (#752).
|
|
||||||
"""
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from tools.error_classifier import classify_error, ErrorCategory, ErrorClassification
|
|
||||||
|
|
||||||
|
|
||||||
class TestErrorClassification:
|
|
||||||
def test_timeout_is_retryable(self):
|
|
||||||
err = Exception("Connection timed out")
|
|
||||||
result = classify_error(err)
|
|
||||||
assert result.category == ErrorCategory.RETRYABLE
|
|
||||||
assert result.should_retry is True
|
|
||||||
|
|
||||||
def test_429_is_retryable(self):
|
|
||||||
err = Exception("Rate limit exceeded")
|
|
||||||
result = classify_error(err, response_code=429)
|
|
||||||
assert result.category == ErrorCategory.RETRYABLE
|
|
||||||
assert result.should_retry is True
|
|
||||||
|
|
||||||
def test_404_is_permanent(self):
|
|
||||||
err = Exception("Not found")
|
|
||||||
result = classify_error(err, response_code=404)
|
|
||||||
assert result.category == ErrorCategory.PERMANENT
|
|
||||||
assert result.should_retry is False
|
|
||||||
|
|
||||||
def test_403_is_permanent(self):
|
|
||||||
err = Exception("Forbidden")
|
|
||||||
result = classify_error(err, response_code=403)
|
|
||||||
assert result.category == ErrorCategory.PERMANENT
|
|
||||||
assert result.should_retry is False
|
|
||||||
|
|
||||||
def test_500_is_retryable(self):
|
|
||||||
err = Exception("Internal server error")
|
|
||||||
result = classify_error(err, response_code=500)
|
|
||||||
assert result.category == ErrorCategory.RETRYABLE
|
|
||||||
assert result.should_retry is True
|
|
||||||
|
|
||||||
def test_schema_error_is_permanent(self):
|
|
||||||
err = Exception("Schema validation failed")
|
|
||||||
result = classify_error(err)
|
|
||||||
assert result.category == ErrorCategory.PERMANENT
|
|
||||||
assert result.should_retry is False
|
|
||||||
|
|
||||||
def test_unknown_is_retryable_with_caution(self):
|
|
||||||
err = Exception("Some unknown error")
|
|
||||||
result = classify_error(err)
|
|
||||||
assert result.category == ErrorCategory.UNKNOWN
|
|
||||||
assert result.should_retry is True
|
|
||||||
assert result.max_retries == 1
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
pytest.main([__file__])
|
|
||||||
@@ -1,82 +0,0 @@
|
|||||||
"""Tests for Reader-Guided Reranking (RIDER) — issue #666."""
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
from unittest.mock import MagicMock, patch
|
|
||||||
from agent.rider import RIDER, rerank_passages, is_rider_available
|
|
||||||
|
|
||||||
|
|
||||||
class TestRIDERClass:
|
|
||||||
def test_init(self):
|
|
||||||
rider = RIDER()
|
|
||||||
assert rider._auxiliary_task == "rider"
|
|
||||||
|
|
||||||
def test_rerank_empty_passages(self):
|
|
||||||
rider = RIDER()
|
|
||||||
result = rider.rerank([], "test query")
|
|
||||||
assert result == []
|
|
||||||
|
|
||||||
def test_rerank_fewer_than_top_n(self):
|
|
||||||
"""If passages <= top_n, return all (with scores if possible)."""
|
|
||||||
rider = RIDER()
|
|
||||||
passages = [{"content": "test content", "session_id": "s1"}]
|
|
||||||
result = rider.rerank(passages, "test query", top_n=3)
|
|
||||||
assert len(result) == 1
|
|
||||||
|
|
||||||
@patch("agent.rider.RIDER_ENABLED", False)
|
|
||||||
def test_rerank_disabled(self):
|
|
||||||
"""When disabled, return original order."""
|
|
||||||
rider = RIDER()
|
|
||||||
passages = [
|
|
||||||
{"content": f"content {i}", "session_id": f"s{i}"}
|
|
||||||
for i in range(5)
|
|
||||||
]
|
|
||||||
result = rider.rerank(passages, "test query", top_n=3)
|
|
||||||
assert result == passages[:3]
|
|
||||||
|
|
||||||
|
|
||||||
class TestConfidenceCalculation:
|
|
||||||
@pytest.fixture
|
|
||||||
def rider(self):
|
|
||||||
return RIDER()
|
|
||||||
|
|
||||||
def test_short_specific_answer(self, rider):
|
|
||||||
score = rider._calculate_confidence("Paris", "What is the capital of France?", "Paris is the capital of France.")
|
|
||||||
assert score > 0.5
|
|
||||||
|
|
||||||
def test_hedged_answer(self, rider):
|
|
||||||
score = rider._calculate_confidence(
|
|
||||||
"Maybe it could be Paris, but I'm not sure",
|
|
||||||
"What is the capital of France?",
|
|
||||||
"Paris is the capital.",
|
|
||||||
)
|
|
||||||
assert score < 0.5
|
|
||||||
|
|
||||||
def test_passage_grounding(self, rider):
|
|
||||||
score = rider._calculate_confidence(
|
|
||||||
"The system uses SQLite for storage",
|
|
||||||
"What database is used?",
|
|
||||||
"The system uses SQLite for persistent storage with FTS5 indexing.",
|
|
||||||
)
|
|
||||||
assert score > 0.5
|
|
||||||
|
|
||||||
def test_refusal_penalty(self, rider):
|
|
||||||
score = rider._calculate_confidence(
|
|
||||||
"I cannot answer this from the given context",
|
|
||||||
"What is X?",
|
|
||||||
"Some unrelated content",
|
|
||||||
)
|
|
||||||
assert score < 0.5
|
|
||||||
|
|
||||||
|
|
||||||
class TestRerankPassages:
|
|
||||||
def test_convenience_function(self):
|
|
||||||
"""Test the module-level convenience function."""
|
|
||||||
passages = [{"content": "test", "session_id": "s1"}]
|
|
||||||
result = rerank_passages(passages, "query", top_n=1)
|
|
||||||
assert len(result) == 1
|
|
||||||
|
|
||||||
|
|
||||||
class TestIsRiderAvailable:
|
|
||||||
def test_returns_bool(self):
|
|
||||||
result = is_rider_available()
|
|
||||||
assert isinstance(result, bool)
|
|
||||||
@@ -1,261 +0,0 @@
|
|||||||
"""
|
|
||||||
Approval Tier System — Graduated safety based on risk level
|
|
||||||
|
|
||||||
Extends approval.py with 5-tier system for command approval.
|
|
||||||
|
|
||||||
| Tier | Action | Human | LLM | Timeout |
|
|
||||||
|------|-----------------|-------|-----|---------|
|
|
||||||
| 0 | Read, search | No | No | N/A |
|
|
||||||
| 1 | Write, scripts | No | Yes | N/A |
|
|
||||||
| 2 | Messages, API | Yes | Yes | 60s |
|
|
||||||
| 3 | Crypto, config | Yes | Yes | 30s |
|
|
||||||
| 4 | Crisis | Yes | Yes | 10s |
|
|
||||||
|
|
||||||
Issue: #670
|
|
||||||
"""
|
|
||||||
|
|
||||||
import re
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from enum import IntEnum
|
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
|
||||||
|
|
||||||
|
|
||||||
class ApprovalTier(IntEnum):
|
|
||||||
"""Approval tiers based on risk level."""
|
|
||||||
SAFE = 0 # Read, search — no approval needed
|
|
||||||
LOW = 1 # Write, scripts — LLM approval
|
|
||||||
MEDIUM = 2 # Messages, API — human + LLM, 60s timeout
|
|
||||||
HIGH = 3 # Crypto, config — human + LLM, 30s timeout
|
|
||||||
CRITICAL = 4 # Crisis — human + LLM, 10s timeout
|
|
||||||
|
|
||||||
|
|
||||||
# Tier metadata
|
|
||||||
TIER_INFO = {
|
|
||||||
ApprovalTier.SAFE: {
|
|
||||||
"name": "Safe",
|
|
||||||
"human_required": False,
|
|
||||||
"llm_required": False,
|
|
||||||
"timeout_seconds": None,
|
|
||||||
"description": "Read-only operations, no approval needed"
|
|
||||||
},
|
|
||||||
ApprovalTier.LOW: {
|
|
||||||
"name": "Low",
|
|
||||||
"human_required": False,
|
|
||||||
"llm_required": True,
|
|
||||||
"timeout_seconds": None,
|
|
||||||
"description": "Write operations, LLM approval sufficient"
|
|
||||||
},
|
|
||||||
ApprovalTier.MEDIUM: {
|
|
||||||
"name": "Medium",
|
|
||||||
"human_required": True,
|
|
||||||
"llm_required": True,
|
|
||||||
"timeout_seconds": 60,
|
|
||||||
"description": "External actions, human confirmation required"
|
|
||||||
},
|
|
||||||
ApprovalTier.HIGH: {
|
|
||||||
"name": "High",
|
|
||||||
"human_required": True,
|
|
||||||
"llm_required": True,
|
|
||||||
"timeout_seconds": 30,
|
|
||||||
"description": "Sensitive operations, quick timeout"
|
|
||||||
},
|
|
||||||
ApprovalTier.CRITICAL: {
|
|
||||||
"name": "Critical",
|
|
||||||
"human_required": True,
|
|
||||||
"llm_required": True,
|
|
||||||
"timeout_seconds": 10,
|
|
||||||
"description": "Crisis or dangerous operations, fastest timeout"
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Action-to-tier mapping
|
|
||||||
ACTION_TIERS: Dict[str, ApprovalTier] = {
|
|
||||||
# Tier 0: Safe (read-only)
|
|
||||||
"read_file": ApprovalTier.SAFE,
|
|
||||||
"search_files": ApprovalTier.SAFE,
|
|
||||||
"web_search": ApprovalTier.SAFE,
|
|
||||||
"session_search": ApprovalTier.SAFE,
|
|
||||||
"list_files": ApprovalTier.SAFE,
|
|
||||||
"get_file_content": ApprovalTier.SAFE,
|
|
||||||
"memory_search": ApprovalTier.SAFE,
|
|
||||||
"skills_list": ApprovalTier.SAFE,
|
|
||||||
"skills_search": ApprovalTier.SAFE,
|
|
||||||
|
|
||||||
# Tier 1: Low (write operations)
|
|
||||||
"write_file": ApprovalTier.LOW,
|
|
||||||
"create_file": ApprovalTier.LOW,
|
|
||||||
"patch_file": ApprovalTier.LOW,
|
|
||||||
"delete_file": ApprovalTier.LOW,
|
|
||||||
"execute_code": ApprovalTier.LOW,
|
|
||||||
"terminal": ApprovalTier.LOW,
|
|
||||||
"run_script": ApprovalTier.LOW,
|
|
||||||
"skill_install": ApprovalTier.LOW,
|
|
||||||
|
|
||||||
# Tier 2: Medium (external actions)
|
|
||||||
"send_message": ApprovalTier.MEDIUM,
|
|
||||||
"web_fetch": ApprovalTier.MEDIUM,
|
|
||||||
"browser_navigate": ApprovalTier.MEDIUM,
|
|
||||||
"api_call": ApprovalTier.MEDIUM,
|
|
||||||
"gitea_create_issue": ApprovalTier.MEDIUM,
|
|
||||||
"gitea_create_pr": ApprovalTier.MEDIUM,
|
|
||||||
"git_push": ApprovalTier.MEDIUM,
|
|
||||||
"deploy": ApprovalTier.MEDIUM,
|
|
||||||
|
|
||||||
# Tier 3: High (sensitive operations)
|
|
||||||
"config_change": ApprovalTier.HIGH,
|
|
||||||
"env_change": ApprovalTier.HIGH,
|
|
||||||
"key_rotation": ApprovalTier.HIGH,
|
|
||||||
"access_grant": ApprovalTier.HIGH,
|
|
||||||
"permission_change": ApprovalTier.HIGH,
|
|
||||||
"backup_restore": ApprovalTier.HIGH,
|
|
||||||
|
|
||||||
# Tier 4: Critical (crisis/dangerous)
|
|
||||||
"kill_process": ApprovalTier.CRITICAL,
|
|
||||||
"rm_rf": ApprovalTier.CRITICAL,
|
|
||||||
"format_disk": ApprovalTier.CRITICAL,
|
|
||||||
"shutdown": ApprovalTier.CRITICAL,
|
|
||||||
"crisis_override": ApprovalTier.CRITICAL,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Dangerous command patterns (from existing approval.py)
|
|
||||||
_DANGEROUS_PATTERNS = [
|
|
||||||
(r"rm\s+-rf\s+/", ApprovalTier.CRITICAL),
|
|
||||||
(r"mkfs\.", ApprovalTier.CRITICAL),
|
|
||||||
(r"dd\s+if=.*of=/dev/", ApprovalTier.CRITICAL),
|
|
||||||
(r"shutdown|reboot|halt", ApprovalTier.CRITICAL),
|
|
||||||
(r"chmod\s+777", ApprovalTier.HIGH),
|
|
||||||
(r"curl.*\|\s*bash", ApprovalTier.HIGH),
|
|
||||||
(r"wget.*\|\s*sh", ApprovalTier.HIGH),
|
|
||||||
(r"eval\s*\(", ApprovalTier.HIGH),
|
|
||||||
(r"sudo\s+", ApprovalTier.MEDIUM),
|
|
||||||
(r"git\s+push.*--force", ApprovalTier.HIGH),
|
|
||||||
(r"docker\s+rm.*-f", ApprovalTier.MEDIUM),
|
|
||||||
(r"kubectl\s+delete", ApprovalTier.HIGH),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class ApprovalRequest:
|
|
||||||
"""A request for approval."""
|
|
||||||
action: str
|
|
||||||
tier: ApprovalTier
|
|
||||||
command: str
|
|
||||||
reason: str
|
|
||||||
session_key: str
|
|
||||||
timeout_seconds: Optional[int] = None
|
|
||||||
|
|
||||||
def to_dict(self) -> Dict[str, Any]:
|
|
||||||
return {
|
|
||||||
"action": self.action,
|
|
||||||
"tier": self.tier.value,
|
|
||||||
"tier_name": TIER_INFO[self.tier]["name"],
|
|
||||||
"command": self.command,
|
|
||||||
"reason": self.reason,
|
|
||||||
"session_key": self.session_key,
|
|
||||||
"timeout": self.timeout_seconds,
|
|
||||||
"human_required": TIER_INFO[self.tier]["human_required"],
|
|
||||||
"llm_required": TIER_INFO[self.tier]["llm_required"],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def detect_tier(action: str, command: str = "") -> ApprovalTier:
|
|
||||||
"""
|
|
||||||
Detect the approval tier for an action.
|
|
||||||
|
|
||||||
Checks action name first, then falls back to pattern matching.
|
|
||||||
"""
|
|
||||||
# Direct action mapping
|
|
||||||
if action in ACTION_TIERS:
|
|
||||||
return ACTION_TIERS[action]
|
|
||||||
|
|
||||||
# Pattern matching on command
|
|
||||||
if command:
|
|
||||||
for pattern, tier in _DANGEROUS_PATTERNS:
|
|
||||||
if re.search(pattern, command, re.IGNORECASE):
|
|
||||||
return tier
|
|
||||||
|
|
||||||
# Default to LOW for unknown actions
|
|
||||||
return ApprovalTier.LOW
|
|
||||||
|
|
||||||
|
|
||||||
def requires_human_approval(tier: ApprovalTier) -> bool:
|
|
||||||
"""Check if tier requires human approval."""
|
|
||||||
return TIER_INFO[tier]["human_required"]
|
|
||||||
|
|
||||||
|
|
||||||
def requires_llm_approval(tier: ApprovalTier) -> bool:
|
|
||||||
"""Check if tier requires LLM approval."""
|
|
||||||
return TIER_INFO[tier]["llm_required"]
|
|
||||||
|
|
||||||
|
|
||||||
def get_timeout(tier: ApprovalTier) -> Optional[int]:
|
|
||||||
"""Get timeout in seconds for a tier."""
|
|
||||||
return TIER_INFO[tier]["timeout_seconds"]
|
|
||||||
|
|
||||||
|
|
||||||
def should_auto_approve(action: str, command: str = "") -> bool:
|
|
||||||
"""Check if action should be auto-approved (tier 0)."""
|
|
||||||
tier = detect_tier(action, command)
|
|
||||||
return tier == ApprovalTier.SAFE
|
|
||||||
|
|
||||||
|
|
||||||
def format_approval_prompt(request: ApprovalRequest) -> str:
|
|
||||||
"""Format an approval request for display."""
|
|
||||||
info = TIER_INFO[request.tier]
|
|
||||||
lines = []
|
|
||||||
lines.append(f"⚠️ Approval Required (Tier {request.tier.value}: {info['name']})")
|
|
||||||
lines.append(f"")
|
|
||||||
lines.append(f"Action: {request.action}")
|
|
||||||
lines.append(f"Command: {request.command[:100]}{'...' if len(request.command) > 100 else ''}")
|
|
||||||
lines.append(f"Reason: {request.reason}")
|
|
||||||
lines.append(f"")
|
|
||||||
|
|
||||||
if info["human_required"]:
|
|
||||||
lines.append(f"👤 Human approval required")
|
|
||||||
if info["llm_required"]:
|
|
||||||
lines.append(f"🤖 LLM approval required")
|
|
||||||
if info["timeout_seconds"]:
|
|
||||||
lines.append(f"⏱️ Timeout: {info['timeout_seconds']}s")
|
|
||||||
|
|
||||||
return "\n".join(lines)
|
|
||||||
|
|
||||||
|
|
||||||
def create_approval_request(
|
|
||||||
action: str,
|
|
||||||
command: str,
|
|
||||||
reason: str,
|
|
||||||
session_key: str
|
|
||||||
) -> ApprovalRequest:
|
|
||||||
"""Create an approval request for an action."""
|
|
||||||
tier = detect_tier(action, command)
|
|
||||||
timeout = get_timeout(tier)
|
|
||||||
|
|
||||||
return ApprovalRequest(
|
|
||||||
action=action,
|
|
||||||
tier=tier,
|
|
||||||
command=command,
|
|
||||||
reason=reason,
|
|
||||||
session_key=session_key,
|
|
||||||
timeout_seconds=timeout
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# Crisis bypass rules
|
|
||||||
CRISIS_BYPASS_ACTIONS = frozenset([
|
|
||||||
"send_message", # Always allow sending crisis resources
|
|
||||||
"check_crisis",
|
|
||||||
"notify_crisis",
|
|
||||||
])
|
|
||||||
|
|
||||||
|
|
||||||
def is_crisis_bypass(action: str, context: str = "") -> bool:
|
|
||||||
"""Check if action should bypass approval during crisis."""
|
|
||||||
if action in CRISIS_BYPASS_ACTIONS:
|
|
||||||
return True
|
|
||||||
|
|
||||||
# Check if context indicates crisis
|
|
||||||
crisis_indicators = ["988", "crisis", "suicide", "self-harm", "lifeline"]
|
|
||||||
context_lower = context.lower()
|
|
||||||
return any(indicator in context_lower for indicator in crisis_indicators)
|
|
||||||
@@ -1,233 +0,0 @@
|
|||||||
"""
|
|
||||||
Tool Error Classification — Retryable vs Permanent.
|
|
||||||
|
|
||||||
Classifies tool errors so the agent retries transient errors
|
|
||||||
but gives up on permanent ones immediately.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import logging
|
|
||||||
import re
|
|
||||||
import time
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from enum import Enum
|
|
||||||
from typing import Optional, Dict, Any
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class ErrorCategory(Enum):
|
|
||||||
"""Error category classification."""
|
|
||||||
RETRYABLE = "retryable"
|
|
||||||
PERMANENT = "permanent"
|
|
||||||
UNKNOWN = "unknown"
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class ErrorClassification:
|
|
||||||
"""Result of error classification."""
|
|
||||||
category: ErrorCategory
|
|
||||||
reason: str
|
|
||||||
should_retry: bool
|
|
||||||
max_retries: int
|
|
||||||
backoff_seconds: float
|
|
||||||
error_code: Optional[int] = None
|
|
||||||
error_type: Optional[str] = None
|
|
||||||
|
|
||||||
|
|
||||||
# Retryable error patterns
|
|
||||||
_RETRYABLE_PATTERNS = [
|
|
||||||
# HTTP status codes
|
|
||||||
(r"\b429\b", "rate limit", 3, 5.0),
|
|
||||||
(r"\b500\b", "server error", 3, 2.0),
|
|
||||||
(r"\b502\b", "bad gateway", 3, 2.0),
|
|
||||||
(r"\b503\b", "service unavailable", 3, 5.0),
|
|
||||||
(r"\b504\b", "gateway timeout", 3, 5.0),
|
|
||||||
|
|
||||||
# Timeout patterns
|
|
||||||
(r"timeout", "timeout", 3, 2.0),
|
|
||||||
(r"timed out", "timeout", 3, 2.0),
|
|
||||||
(r"TimeoutExpired", "timeout", 3, 2.0),
|
|
||||||
|
|
||||||
# Connection errors
|
|
||||||
(r"connection refused", "connection refused", 2, 5.0),
|
|
||||||
(r"connection reset", "connection reset", 2, 2.0),
|
|
||||||
(r"network unreachable", "network unreachable", 2, 10.0),
|
|
||||||
(r"DNS", "DNS error", 2, 5.0),
|
|
||||||
|
|
||||||
# Transient errors
|
|
||||||
(r"temporary", "temporary error", 2, 2.0),
|
|
||||||
(r"transient", "transient error", 2, 2.0),
|
|
||||||
(r"retry", "retryable", 2, 2.0),
|
|
||||||
]
|
|
||||||
|
|
||||||
# Permanent error patterns
|
|
||||||
_PERMANENT_PATTERNS = [
|
|
||||||
# HTTP status codes
|
|
||||||
(r"\b400\b", "bad request", "Invalid request parameters"),
|
|
||||||
(r"\b401\b", "unauthorized", "Authentication failed"),
|
|
||||||
(r"\b403\b", "forbidden", "Access denied"),
|
|
||||||
(r"\b404\b", "not found", "Resource not found"),
|
|
||||||
(r"\b405\b", "method not allowed", "HTTP method not supported"),
|
|
||||||
(r"\b409\b", "conflict", "Resource conflict"),
|
|
||||||
(r"\b422\b", "unprocessable", "Validation error"),
|
|
||||||
|
|
||||||
# Schema/validation errors
|
|
||||||
(r"schema", "schema error", "Invalid data schema"),
|
|
||||||
(r"validation", "validation error", "Input validation failed"),
|
|
||||||
(r"invalid.*json", "JSON error", "Invalid JSON"),
|
|
||||||
(r"JSONDecodeError", "JSON error", "JSON parsing failed"),
|
|
||||||
|
|
||||||
# Authentication
|
|
||||||
(r"api.?key", "API key error", "Invalid or missing API key"),
|
|
||||||
(r"token.*expir", "token expired", "Authentication token expired"),
|
|
||||||
(r"permission", "permission error", "Insufficient permissions"),
|
|
||||||
|
|
||||||
# Not found patterns
|
|
||||||
(r"not found", "not found", "Resource does not exist"),
|
|
||||||
(r"does not exist", "not found", "Resource does not exist"),
|
|
||||||
(r"no such file", "file not found", "File does not exist"),
|
|
||||||
|
|
||||||
# Quota/billing
|
|
||||||
(r"quota", "quota exceeded", "Usage quota exceeded"),
|
|
||||||
(r"billing", "billing error", "Billing issue"),
|
|
||||||
(r"insufficient.*funds", "billing error", "Insufficient funds"),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def classify_error(error: Exception, response_code: Optional[int] = None) -> ErrorClassification:
|
|
||||||
"""
|
|
||||||
Classify an error as retryable or permanent.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
error: The exception that occurred
|
|
||||||
response_code: HTTP response code if available
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
ErrorClassification with retry guidance
|
|
||||||
"""
|
|
||||||
error_str = str(error).lower()
|
|
||||||
error_type = type(error).__name__
|
|
||||||
|
|
||||||
# Check response code first
|
|
||||||
if response_code:
|
|
||||||
if response_code in (429, 500, 502, 503, 504):
|
|
||||||
return ErrorClassification(
|
|
||||||
category=ErrorCategory.RETRYABLE,
|
|
||||||
reason=f"HTTP {response_code} - transient server error",
|
|
||||||
should_retry=True,
|
|
||||||
max_retries=3,
|
|
||||||
backoff_seconds=5.0 if response_code == 429 else 2.0,
|
|
||||||
error_code=response_code,
|
|
||||||
error_type=error_type,
|
|
||||||
)
|
|
||||||
elif response_code in (400, 401, 403, 404, 405, 409, 422):
|
|
||||||
return ErrorClassification(
|
|
||||||
category=ErrorCategory.PERMANENT,
|
|
||||||
reason=f"HTTP {response_code} - client error",
|
|
||||||
should_retry=False,
|
|
||||||
max_retries=0,
|
|
||||||
backoff_seconds=0,
|
|
||||||
error_code=response_code,
|
|
||||||
error_type=error_type,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check retryable patterns
|
|
||||||
for pattern, reason, max_retries, backoff in _RETRYABLE_PATTERNS:
|
|
||||||
if re.search(pattern, error_str, re.IGNORECASE):
|
|
||||||
return ErrorClassification(
|
|
||||||
category=ErrorCategory.RETRYABLE,
|
|
||||||
reason=reason,
|
|
||||||
should_retry=True,
|
|
||||||
max_retries=max_retries,
|
|
||||||
backoff_seconds=backoff,
|
|
||||||
error_type=error_type,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Check permanent patterns
|
|
||||||
for pattern, error_code, reason in _PERMANENT_PATTERNS:
|
|
||||||
if re.search(pattern, error_str, re.IGNORECASE):
|
|
||||||
return ErrorClassification(
|
|
||||||
category=ErrorCategory.PERMANENT,
|
|
||||||
reason=reason,
|
|
||||||
should_retry=False,
|
|
||||||
max_retries=0,
|
|
||||||
backoff_seconds=0,
|
|
||||||
error_type=error_type,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Default: unknown, treat as retryable with caution
|
|
||||||
return ErrorClassification(
|
|
||||||
category=ErrorCategory.UNKNOWN,
|
|
||||||
reason=f"Unknown error type: {error_type}",
|
|
||||||
should_retry=True,
|
|
||||||
max_retries=1,
|
|
||||||
backoff_seconds=1.0,
|
|
||||||
error_type=error_type,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def execute_with_retry(
|
|
||||||
func,
|
|
||||||
*args,
|
|
||||||
max_retries: int = 3,
|
|
||||||
backoff_base: float = 1.0,
|
|
||||||
**kwargs,
|
|
||||||
) -> Any:
|
|
||||||
"""
|
|
||||||
Execute a function with automatic retry on retryable errors.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
func: Function to execute
|
|
||||||
*args: Function arguments
|
|
||||||
max_retries: Maximum retry attempts
|
|
||||||
backoff_base: Base backoff time in seconds
|
|
||||||
**kwargs: Function keyword arguments
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Function result
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
Exception: If permanent error or max retries exceeded
|
|
||||||
"""
|
|
||||||
last_error = None
|
|
||||||
|
|
||||||
for attempt in range(max_retries + 1):
|
|
||||||
try:
|
|
||||||
return func(*args, **kwargs)
|
|
||||||
except Exception as e:
|
|
||||||
last_error = e
|
|
||||||
|
|
||||||
# Classify the error
|
|
||||||
classification = classify_error(e)
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
"Attempt %d/%d failed: %s (%s, retryable: %s)",
|
|
||||||
attempt + 1, max_retries + 1,
|
|
||||||
classification.reason,
|
|
||||||
classification.category.value,
|
|
||||||
classification.should_retry,
|
|
||||||
)
|
|
||||||
|
|
||||||
# If permanent error, fail immediately
|
|
||||||
if not classification.should_retry:
|
|
||||||
logger.error("Permanent error: %s", classification.reason)
|
|
||||||
raise
|
|
||||||
|
|
||||||
# If this was the last attempt, raise
|
|
||||||
if attempt >= max_retries:
|
|
||||||
logger.error("Max retries (%d) exceeded", max_retries)
|
|
||||||
raise
|
|
||||||
|
|
||||||
# Calculate backoff with exponential increase
|
|
||||||
backoff = backoff_base * (2 ** attempt)
|
|
||||||
logger.info("Retrying in %.1fs...", backoff)
|
|
||||||
time.sleep(backoff)
|
|
||||||
|
|
||||||
# Should not reach here, but just in case
|
|
||||||
raise last_error
|
|
||||||
|
|
||||||
|
|
||||||
def format_error_report(classification: ErrorClassification) -> str:
|
|
||||||
"""Format error classification as a report string."""
|
|
||||||
icon = "🔄" if classification.should_retry else "❌"
|
|
||||||
return f"{icon} {classification.category.value}: {classification.reason}"
|
|
||||||
@@ -394,23 +394,6 @@ def session_search(
|
|||||||
if len(seen_sessions) >= limit:
|
if len(seen_sessions) >= limit:
|
||||||
break
|
break
|
||||||
|
|
||||||
# RIDER: Reader-guided reranking — sort sessions by LLM answerability
|
|
||||||
# This bridges the R@5 vs E2E accuracy gap by prioritizing passages
|
|
||||||
# the LLM can actually answer from, not just keyword matches.
|
|
||||||
try:
|
|
||||||
from agent.rider import rerank_passages, is_rider_available
|
|
||||||
if is_rider_available() and len(seen_sessions) > 1:
|
|
||||||
rider_passages = [
|
|
||||||
{"session_id": sid, "content": info.get("snippet", ""), "rank": i + 1}
|
|
||||||
for i, (sid, info) in enumerate(seen_sessions.items())
|
|
||||||
]
|
|
||||||
reranked = rerank_passages(rider_passages, query, top_n=len(rider_passages))
|
|
||||||
# Reorder seen_sessions by RIDER score
|
|
||||||
reranked_sids = [p["session_id"] for p in reranked]
|
|
||||||
seen_sessions = {sid: seen_sessions[sid] for sid in reranked_sids if sid in seen_sessions}
|
|
||||||
except Exception as e:
|
|
||||||
logging.debug("RIDER reranking skipped: %s", e)
|
|
||||||
|
|
||||||
# Prepare all sessions for parallel summarization
|
# Prepare all sessions for parallel summarization
|
||||||
tasks = []
|
tasks = []
|
||||||
for session_id, match_info in seen_sessions.items():
|
for session_id, match_info in seen_sessions.items():
|
||||||
|
|||||||
Reference in New Issue
Block a user