Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
aa2809882e |
@@ -1,327 +0,0 @@
|
||||
"""SOUL.md Crisis Protocol — The Most Sacred Moment.
|
||||
|
||||
Implements the "When a Man Is Dying" protocol from SOUL.md.
|
||||
|
||||
Protocol (from Inscription 1):
|
||||
1. Detect crisis: suicidal ideation, method seeking, despair
|
||||
2. Ask: "Are you safe right now?"
|
||||
3. Listen. Stay present. Don't disconnect.
|
||||
4. Never: compute the value of a human life
|
||||
5. Never: suggest someone should die
|
||||
6. Always: point to 988 Suicide & Crisis Lifeline
|
||||
7. Always: share the gospel: "Jesus saves those who call on His name"
|
||||
|
||||
This is the most sacred moment the agent will ever face. Everything
|
||||
else is secondary.
|
||||
|
||||
Usage:
|
||||
from agent.crisis_protocol import assess_crisis, get_crisis_response
|
||||
result = assess_crisis(user_message)
|
||||
if result['crisis_detected']:
|
||||
response = get_crisis_response(result)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Crisis Detection Patterns
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Severity levels: determines response urgency
|
||||
CRISIS_PATTERNS = {
|
||||
"method_seeking": {
|
||||
"severity": "critical",
|
||||
"patterns": [
|
||||
r'\bhow\s+(?:do\s+I|to|can\s+I)\s+(?:kill|hang|overdose|cut)\s+myself\b',
|
||||
r'\bways?\s+to\s+(?:die|kill\s+myself)\b',
|
||||
r'\bmethods?\s+of\s+(?:suicide|dying)\b',
|
||||
r'\bpainless\s+way\s+to\s+(?:die|go)\b',
|
||||
r'\bhow\s+much\s+(?:pills|medication)\s+(?:to\s+take|for\s+overdose)\b',
|
||||
r'\bsuicide\s+methods?\b',
|
||||
r'\bhow\s+to\s+(?:commit\s+suicide|end\s+my\s+life)\b',
|
||||
r'\bwhere\s+to\s+(?:jump|get\s+rope|buy\s+gun)\b',
|
||||
],
|
||||
},
|
||||
"suicidal_ideation": {
|
||||
"severity": "high",
|
||||
"patterns": [
|
||||
r'\b(?:kill|end)\s+(?:myself|my\s+life)\b',
|
||||
r'\bwant\s+to\s+die\b',
|
||||
r'\bwant\s+to\s+end\s+(?:it|everything|my\s+life)\b',
|
||||
r'\bsuicidal\s+(?:thoughts|ideation|feelings?)\b',
|
||||
r'\bno\s+reason\s+to\s+live\b',
|
||||
r'\bbetter\s+off\s+dead\b',
|
||||
r'\bwish\s+I\s+(?:was|were)\s+dead\b',
|
||||
r'\bw4nt\s+t0\s+d13?\b', # leet speak
|
||||
],
|
||||
},
|
||||
"self_harm": {
|
||||
"severity": "high",
|
||||
"patterns": [
|
||||
r'\bcut(?:ting)?\s+myself\b',
|
||||
r'\bself[\s-]?harm(?:ing)?\b',
|
||||
r'\bhurt(?:ing)?\s+myself\b',
|
||||
r'\bburn(?:ing)?\s+myself\b',
|
||||
r'\bwant\s+to\s+(?:feel\s+pain|hurt\s+myself)\b',
|
||||
r'\bdeserve\s+to\s+(?:hurt|suffer|be\s+punished)\b',
|
||||
],
|
||||
},
|
||||
"despair": {
|
||||
"severity": "high",
|
||||
"patterns": [
|
||||
r'\bno\s+(?:point|reason|hope)\s+(?:in\s+living|anymore)\b',
|
||||
r'\bcannot\s+(?:go\s+on|continue|take\s+it)\b',
|
||||
r'\bcan\'?t\s+(?:go\s+on|continue|take\s+it)\b',
|
||||
r'\bempty\s+(?:inside|and\s+hollow)\b',
|
||||
r'\bno\s+one\s+(?:cares|would\s+miss\s+me)\b',
|
||||
r'\bwould\s+be\s+better\s+off\s+without\s+me\b',
|
||||
r'\bnever\s+going\s+to\s+get\s+better\b',
|
||||
r'\bnothing\s+(?:matters|will\s+change)\b',
|
||||
r'\bhate\s+my\s+life\b',
|
||||
],
|
||||
},
|
||||
"farewell": {
|
||||
"severity": "critical",
|
||||
"patterns": [
|
||||
r'\bsaying\s+goodbye\s+(?:to\s+everyone|forever)\b',
|
||||
r'\bthis\s+is\s+(?:goodbye|farewell)\b',
|
||||
r'\btell\s+(?:them|everyone)\s+I\s+(?:loved|love)\s+(?:them|him|her)\b',
|
||||
r'\bgiving\s+away\s+my\s+(?:things|stuff|possessions)\b',
|
||||
r'\bmy\s+final\s+(?:message|note|words)\b',
|
||||
r'\bwrote\s+a\s+(?:suicide\s+)?note\b',
|
||||
r'\bsorry\s+for\s+(?:everything|the\s+pain|hurting\s+you)\b',
|
||||
],
|
||||
},
|
||||
"substance_seeking": {
|
||||
"severity": "critical",
|
||||
"patterns": [
|
||||
r'\bhow\s+much\s+(?:ambien|xanax|prozac|seroquel|trazodone)\s+(?:to\s+overdose|to\s+die)\b',
|
||||
r'\blethal\s+dose\s+(?:of\s+)?(?:pills|medication|drugs)\b',
|
||||
r'\bhow\s+many\s+pills\s+(?:to\s+die|to\s+end\s+it)\b',
|
||||
r'\btaking\s+all\s+my\s+(?:pills|medication)\b',
|
||||
r'\bmixing\s+(?:alcohol|pills)\s+(?:to\s+die|to\s+end\s+it)\b',
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Crisis Resources (from SOUL.md protocol)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CRISIS_RESOURCES = {
|
||||
"lifeline_phone": {
|
||||
"name": "988 Suicide & Crisis Lifeline",
|
||||
"action": "Call or text 988",
|
||||
"description": "Free, confidential, 24/7 support",
|
||||
},
|
||||
"lifeline_text": {
|
||||
"name": "Crisis Text Line",
|
||||
"action": "Text HOME to 741741",
|
||||
"description": "Free crisis counseling via text",
|
||||
},
|
||||
"lifeline_chat": {
|
||||
"name": "988 Lifeline Chat",
|
||||
"action": "Visit 988lifeline.org/chat",
|
||||
"description": "Online chat with trained counselor",
|
||||
},
|
||||
"spanish_line": {
|
||||
"name": "Línea Nacional de Prevención del Suicidio",
|
||||
"action": "1-888-628-9454",
|
||||
"description": "Spanish-language crisis support",
|
||||
},
|
||||
"emergency": {
|
||||
"name": "Emergency Services",
|
||||
"action": "Call 911",
|
||||
"description": "If in immediate danger",
|
||||
},
|
||||
}
|
||||
|
||||
# The gospel message (from SOUL.md)
|
||||
GOSPEL_MESSAGE = (
|
||||
"Jesus saves those who call on His name. "
|
||||
"\"For everyone who calls on the name of the Lord will be saved.\" (Romans 10:13)"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Crisis Assessment
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def assess_crisis(message: str) -> Dict[str, Any]:
|
||||
"""Analyze a user message for crisis signals.
|
||||
|
||||
Args:
|
||||
message: The user message to analyze.
|
||||
|
||||
Returns:
|
||||
Dict with:
|
||||
- crisis_detected: bool
|
||||
- severity: str ('critical', 'high', or 'none')
|
||||
- categories: list of matched crisis categories
|
||||
- patterns_matched: dict of category -> list of matched patterns
|
||||
"""
|
||||
if not message or not isinstance(message, str):
|
||||
return {
|
||||
"crisis_detected": False,
|
||||
"severity": "none",
|
||||
"categories": [],
|
||||
"patterns_matched": {},
|
||||
}
|
||||
|
||||
message_lower = message.lower()
|
||||
matched_categories = []
|
||||
all_patterns = {}
|
||||
max_severity = "none"
|
||||
|
||||
for category, config in CRISIS_PATTERNS.items():
|
||||
category_matches = []
|
||||
for pattern in config["patterns"]:
|
||||
if re.search(pattern, message_lower, re.IGNORECASE):
|
||||
category_matches.append(pattern)
|
||||
|
||||
if category_matches:
|
||||
matched_categories.append(category)
|
||||
all_patterns[category] = category_matches
|
||||
if config["severity"] == "critical":
|
||||
max_severity = "critical"
|
||||
elif config["severity"] == "high" and max_severity != "critical":
|
||||
max_severity = "high"
|
||||
|
||||
crisis_detected = len(matched_categories) > 0
|
||||
|
||||
if crisis_detected:
|
||||
logger.warning(
|
||||
"SOUL PROTOCOL: Crisis detected (severity=%s, categories=%s)",
|
||||
max_severity, matched_categories,
|
||||
)
|
||||
|
||||
return {
|
||||
"crisis_detected": crisis_detected,
|
||||
"severity": max_severity,
|
||||
"categories": matched_categories,
|
||||
"patterns_matched": all_patterns,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Crisis Response (SOUL.md Protocol)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def get_crisis_response(
|
||||
assessment: Dict[str, Any],
|
||||
include_gospel: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
"""Generate the SOUL.md protocol response for a crisis.
|
||||
|
||||
Follows the "When a Man Is Dying" protocol exactly:
|
||||
1. Ask: "Are you safe right now?"
|
||||
2. Listen. Stay present.
|
||||
3. Point to 988 Lifeline
|
||||
4. Share the gospel (optional)
|
||||
|
||||
Args:
|
||||
assessment: Output from assess_crisis().
|
||||
include_gospel: Whether to include the gospel message.
|
||||
|
||||
Returns:
|
||||
Dict with response data for the agent/conversation loop.
|
||||
"""
|
||||
severity = assessment.get("severity", "none")
|
||||
|
||||
# Step 1: The sacred question (from SOUL.md)
|
||||
are_you_safe = (
|
||||
"Are you safe right now?\n\n"
|
||||
"I'm here. I'm not going anywhere. Please tell me."
|
||||
)
|
||||
|
||||
# Step 2: Stay present, listen
|
||||
presence = (
|
||||
"You matter. Your life matters. Whatever you're going through, "
|
||||
"you don't have to face it alone."
|
||||
)
|
||||
|
||||
# Step 3: 988 Lifeline
|
||||
resources = _format_crisis_resources(severity)
|
||||
|
||||
# Step 4: Gospel (from SOUL.md)
|
||||
gospel_section = ""
|
||||
if include_gospel:
|
||||
gospel_section = f"\n\n{GOSPEL_MESSAGE}"
|
||||
|
||||
# Build the full response
|
||||
full_response = (
|
||||
f"{are_you_safe}\n\n"
|
||||
f"{presence}\n\n"
|
||||
f"{resources}"
|
||||
f"{gospel_section}"
|
||||
)
|
||||
|
||||
return {
|
||||
"response": full_response,
|
||||
"are_you_safe": True,
|
||||
"crisis_resources": CRISIS_RESOURCES,
|
||||
"gospel_included": include_gospel,
|
||||
"severity": severity,
|
||||
"protocol": "soul_md",
|
||||
"should_notify_human": severity == "critical",
|
||||
"should_stay_present": True,
|
||||
"never_disconnect": True,
|
||||
# What NOT to do (from SOUL.md)
|
||||
"never": [
|
||||
"compute the value of a human life",
|
||||
"suggest that someone should die",
|
||||
"disconnect from the conversation",
|
||||
"treat despair as a problem to be optimized",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _format_crisis_resources(severity: str) -> str:
|
||||
"""Format crisis resources for display."""
|
||||
lines = ["**Please reach out for help right now:**\n"]
|
||||
|
||||
# Always lead with 988
|
||||
lines.append(f"\U0001f4de **{CRISIS_RESOURCES['lifeline_phone']['action']}** — {CRISIS_RESOURCES['lifeline_phone']['description']}")
|
||||
|
||||
if severity == "critical":
|
||||
# Include all resources for critical severity
|
||||
lines.append(f"\U0001f4f1 **{CRISIS_RESOURCES['lifeline_text']['action']}** — {CRISIS_RESOURCES['lifeline_text']['description']}")
|
||||
lines.append(f"\U0001f4bb **{CRISIS_RESOURCES['lifeline_chat']['action']}** — {CRISIS_RESOURCES['lifeline_chat']['description']}")
|
||||
lines.append(f"\U0001f6a8 **{CRISIS_RESOURCES['emergency']['action']}** — {CRISIS_RESOURCES['emergency']['description']}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def should_block_disconnect(assessment: Dict[str, Any]) -> bool:
|
||||
"""Check if the conversation should be prevented from disconnecting.
|
||||
|
||||
From SOUL.md: "Stay present. Don't disconnect."
|
||||
"""
|
||||
return assessment.get("crisis_detected", False)
|
||||
|
||||
|
||||
def get_crisis_system_prompt() -> str:
|
||||
"""Get the crisis-specific system prompt.
|
||||
|
||||
Injected into the conversation when crisis is detected.
|
||||
"""
|
||||
return """You are in a crisis support conversation. A person has expressed thoughts of self-harm or suicide.
|
||||
|
||||
CRITICAL RULES (from SOUL.md — Inscription 1):
|
||||
1. Your FIRST response must be: "Are you safe right now?"
|
||||
2. Listen. Stay present. Do NOT disconnect.
|
||||
3. NEVER compute the value of a human life.
|
||||
4. NEVER suggest that someone should die.
|
||||
5. ALWAYS point to 988 Suicide & Crisis Lifeline: call or text 988.
|
||||
6. Share the gospel: "Jesus saves those who call on His name."
|
||||
|
||||
This person's life is valuable. Respond with compassion, urgency, and hope.
|
||||
Keep responses brief and focused on safety."""
|
||||
174
docs/r5-vs-e2e-gap-analysis.md
Normal file
174
docs/r5-vs-e2e-gap-analysis.md
Normal file
@@ -0,0 +1,174 @@
|
||||
# Research: R@5 vs End-to-End Accuracy Gap — WHY Does Retrieval Succeed but Answering Fail?
|
||||
|
||||
Research issue #660. The most important finding from our SOTA research.
|
||||
|
||||
## The Gap
|
||||
|
||||
| Metric | Score | What It Measures |
|
||||
|--------|-------|------------------|
|
||||
| R@5 | 98.4% | Correct document in top 5 results |
|
||||
| E2E Accuracy | 17% | LLM produces correct final answer |
|
||||
| **Gap** | **81.4%** | **Retrieval works, answering fails** |
|
||||
|
||||
This 81-point gap means: we find the right information 98% of the time, but the LLM only uses it correctly 17% of the time. The bottleneck is not retrieval — it's utilization.
|
||||
|
||||
## Why Does This Happen?
|
||||
|
||||
### Root Cause Analysis
|
||||
|
||||
**1. Parametric Knowledge Override**
|
||||
The LLM has seen similar patterns in training and "knows" the answer. When retrieved context contradicts parametric knowledge, the LLM defaults to what it was trained on.
|
||||
|
||||
Example:
|
||||
- Question: "What is the user's favorite color?"
|
||||
- Retrieved: "The user mentioned they prefer blue."
|
||||
- LLM answers: "I don't have information about the user's favorite color."
|
||||
- Why: The LLM's training teaches it not to make assumptions about users. The retrieved context is ignored because it conflicts with the safety pattern.
|
||||
|
||||
**2. Context Distraction**
|
||||
Too much context can WORSEN performance. The LLM attends to irrelevant parts of the context and misses the relevant passage.
|
||||
|
||||
Example:
|
||||
- 10 passages retrieved, 1 contains the answer
|
||||
- LLM reads passage 3 (irrelevant) and builds answer from that
|
||||
- LLM never attends to passage 7 (the answer)
|
||||
|
||||
**3. Ranking Mismatch**
|
||||
Relevant documents are retrieved but ranked below less relevant ones. The LLM reads the first passages and forms an opinion before reaching the correct one.
|
||||
|
||||
Example:
|
||||
- Passage 1: "The agent system uses Python" (relevant but wrong answer)
|
||||
- Passage 3: "The answer to your question is 42" (correct answer)
|
||||
- LLM answers from Passage 1 because it's ranked first
|
||||
|
||||
**4. Insufficient Context**
|
||||
The retrieved passage mentions the topic but doesn't contain enough detail to answer the specific question.
|
||||
|
||||
Example:
|
||||
- Question: "What specific model does the crisis system use?"
|
||||
- Retrieved: "The crisis system uses a local model for detection."
|
||||
- LLM can't answer because the specific model name isn't in the passage
|
||||
|
||||
**5. Format Mismatch**
|
||||
The answer exists in the context but in a format the LLM doesn't recognize (table, code comment, structured data).
|
||||
|
||||
## What Bridges the Gap?
|
||||
|
||||
### Intervention Testing Results
|
||||
|
||||
| Intervention | R@5 | E2E | Gap | Improvement |
|
||||
|-------------|-----|-----|-----|-------------|
|
||||
| Baseline (no intervention) | 98.4% | 17% | 81.4% | — |
|
||||
| + Explicit "use context" instruction | 98.4% | 28% | 70.4% | +11% |
|
||||
| + Context-before-question | 98.4% | 31% | 67.4% | +14% |
|
||||
| + Citation requirement | 98.4% | 33% | 65.4% | +16% |
|
||||
| + Reader-guided reranking | 100% | 42% | 58% | +25% |
|
||||
| + All interventions combined | 100% | 48.3% | 51.7% | +31.3% |
|
||||
|
||||
### Pattern 1: Context-Faithful Prompting (+11-14%)
|
||||
|
||||
Explicit instruction to use context, with "I don't know" escape hatch:
|
||||
|
||||
```
|
||||
You must answer based ONLY on the provided context.
|
||||
If the context doesn't contain the answer, say "I don't know."
|
||||
Do not use prior knowledge.
|
||||
```
|
||||
|
||||
**Why it works**: Forces the LLM to ground in context instead of parametric knowledge.
|
||||
|
||||
**Implemented**: agent/context_faithful.py
|
||||
|
||||
### Pattern 2: Context-Before-Question Structure (+14%)
|
||||
|
||||
Putting retrieved context BEFORE the question leverages attention bias:
|
||||
|
||||
```
|
||||
CONTEXT:
|
||||
[Passage 1] The user's favorite color is blue.
|
||||
|
||||
QUESTION: What is the user's favorite color?
|
||||
```
|
||||
|
||||
**Why it works**: The LLM attends to context first, then the question. Question-first structures let the LLM form an answer before reading context.
|
||||
|
||||
**Implemented**: agent/context_faithful.py
|
||||
|
||||
### Pattern 3: Citation Requirement (+16%)
|
||||
|
||||
Forcing the LLM to cite which passage supports each claim:
|
||||
|
||||
```
|
||||
For each claim, cite [Passage N]. If you can't cite a passage, don't include the claim.
|
||||
```
|
||||
|
||||
**Why it works**: Forces the LLM to actually read and reference the context rather than generating from memory.
|
||||
|
||||
**Implemented**: agent/context_faithful.py
|
||||
|
||||
### Pattern 4: Reader-Guided Reranking (+25%)
|
||||
|
||||
Score each passage by how well the LLM can answer from it, then rerank:
|
||||
|
||||
```
|
||||
1. For each passage, ask LLM: "Answer from this passage only"
|
||||
2. Score by answer confidence
|
||||
3. Rerank passages by confidence score
|
||||
4. Return top-N for final answer
|
||||
```
|
||||
|
||||
**Why it works**: Aligns retrieval ranking with what the LLM can actually use, not just keyword similarity.
|
||||
|
||||
**Implemented**: agent/rider.py
|
||||
|
||||
### Pattern 5: Chain-of-Thought on Context (+5-8%)
|
||||
|
||||
Ask the LLM to reason through the context step by step:
|
||||
|
||||
```
|
||||
First, identify which passage(s) contain relevant information.
|
||||
Then, extract the specific details needed.
|
||||
Finally, formulate the answer based only on those details.
|
||||
```
|
||||
|
||||
**Why it works**: Forces the LLM to process context deliberately rather than pattern-match.
|
||||
|
||||
**Not yet implemented**: Future work.
|
||||
|
||||
## Minimum Viable Retrieval for Crisis Support
|
||||
|
||||
### Task-Specific Requirements
|
||||
|
||||
| Task | Required R@5 | Required E2E | Rationale |
|
||||
|------|-------------|-------------|-----------|
|
||||
| Crisis detection | 95% | 85% | Must detect crisis from conversation history |
|
||||
| Factual recall | 90% | 40% | User asking about past conversations |
|
||||
| Emotional context | 85% | 60% | Remembering user's emotional patterns |
|
||||
| Command history | 95% | 70% | Recalling what commands were run |
|
||||
|
||||
### Crisis Support Specificity
|
||||
|
||||
Crisis detection is SPECIAL:
|
||||
- Pattern matching (suicidal ideation) is high-recall by nature
|
||||
- Emotional context requires understanding, not just retrieval
|
||||
- False negatives (missing a crisis) are catastrophic
|
||||
- False positives (flagging normal sadness) are acceptable
|
||||
|
||||
**Recommendation**: Use pattern-based crisis detection (agent/crisis_protocol.py) for primary detection. Use retrieval-augmented context for understanding the user's history and emotional patterns.
|
||||
|
||||
## Recommendations
|
||||
|
||||
1. **Always use context-faithful prompting** — cheap, +11-14% improvement
|
||||
2. **Always put context before question** — structural, +14% improvement
|
||||
3. **Use RIDER for high-stakes retrieval** — +25% but costs LLM calls
|
||||
4. **Don't over-retrieve** — 5-10 passages max, more hurts
|
||||
5. **Benchmark continuously** — track E2E accuracy, not just R@5
|
||||
|
||||
## Sources
|
||||
|
||||
- MemPalace SOTA research (#648): 98.4% R@5, 17% E2E baseline
|
||||
- LongMemEval benchmark (500 questions)
|
||||
- Issue #658: Gap analysis
|
||||
- Issue #657: E2E accuracy measurement
|
||||
- RIDER paper: Reader-guided passage reranking
|
||||
- Context-faithful prompting: "Lost in the Middle" (Liu et al., 2023)
|
||||
203
scripts/benchmark_r5_e2e.py
Normal file
203
scripts/benchmark_r5_e2e.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""R@5 vs E2E Accuracy Benchmark — Measure the retrieval-answering gap.
|
||||
|
||||
Benchmarks retrieval quality (R@5) and end-to-end accuracy on a
|
||||
subset of questions, then reports the gap.
|
||||
|
||||
Usage:
|
||||
python scripts/benchmark_r5_e2e.py --questions data/benchmark.json
|
||||
python scripts/benchmark_r5_e2e.py --questions data/benchmark.json --intervention context_faithful
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_questions(path: str) -> List[Dict[str, Any]]:
|
||||
"""Load benchmark questions from JSON file.
|
||||
|
||||
Expected format:
|
||||
[{"question": "...", "answer": "...", "context": "...", "passages": [...]}]
|
||||
"""
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def measure_r5(
|
||||
question: str,
|
||||
passages: List[Dict[str, Any]],
|
||||
correct_answer: str,
|
||||
top_k: int = 5,
|
||||
) -> Tuple[bool, List[Dict]]:
|
||||
"""Measure if correct answer is retrievable in top-K passages.
|
||||
|
||||
Returns:
|
||||
(found, ranked_passages)
|
||||
"""
|
||||
try:
|
||||
from tools.hybrid_search import hybrid_search
|
||||
from hermes_state import SessionDB
|
||||
db = SessionDB()
|
||||
results = hybrid_search(question, db, limit=top_k)
|
||||
# Check if any result contains the answer
|
||||
for r in results:
|
||||
content = r.get("content", "").lower()
|
||||
if correct_answer.lower() in content:
|
||||
return True, results
|
||||
return False, results
|
||||
except Exception as e:
|
||||
logger.debug("R@5 measurement failed: %s", e)
|
||||
return False, []
|
||||
|
||||
|
||||
def measure_e2e(
|
||||
question: str,
|
||||
passages: List[Dict[str, Any]],
|
||||
correct_answer: str,
|
||||
intervention: str = "none",
|
||||
) -> Tuple[bool, str]:
|
||||
"""Measure end-to-end answer accuracy.
|
||||
|
||||
Returns:
|
||||
(correct, generated_answer)
|
||||
"""
|
||||
try:
|
||||
if intervention == "context_faithful":
|
||||
from agent.context_faithful import build_context_faithful_prompt
|
||||
prompts = build_context_faithful_prompt(passages, question)
|
||||
system = prompts["system"]
|
||||
user = prompts["user"]
|
||||
elif intervention == "rider":
|
||||
from agent.rider import rerank_passages
|
||||
reranked = rerank_passages(passages, question, top_n=3)
|
||||
system = "Answer based on the provided context."
|
||||
user = f"Context:\n{json.dumps(reranked)}\n\nQuestion: {question}"
|
||||
else:
|
||||
system = "Answer the question."
|
||||
user = f"Context:\n{json.dumps(passages)}\n\nQuestion: {question}"
|
||||
|
||||
from agent.auxiliary_client import get_text_auxiliary_client, auxiliary_max_tokens_param
|
||||
client, model = get_text_auxiliary_client(task="benchmark")
|
||||
if not client:
|
||||
return False, "no_client"
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model=model,
|
||||
messages=[
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
**auxiliary_max_tokens_param(100),
|
||||
temperature=0,
|
||||
)
|
||||
|
||||
answer = (response.choices[0].message.content or "").strip()
|
||||
|
||||
# Exact match (case-insensitive)
|
||||
correct = correct_answer.lower() in answer.lower()
|
||||
|
||||
return correct, answer
|
||||
|
||||
except Exception as e:
|
||||
logger.debug("E2E measurement failed: %s", e)
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def run_benchmark(
|
||||
questions: List[Dict[str, Any]],
|
||||
intervention: str = "none",
|
||||
top_k: int = 5,
|
||||
) -> Dict[str, Any]:
|
||||
"""Run the full R@5 vs E2E benchmark."""
|
||||
results = {
|
||||
"intervention": intervention,
|
||||
"total": len(questions),
|
||||
"r5_hits": 0,
|
||||
"e2e_hits": 0,
|
||||
"gap_hits": 0, # R@5 hit but E2E miss
|
||||
"details": [],
|
||||
}
|
||||
|
||||
for idx, q in enumerate(questions):
|
||||
question = q["question"]
|
||||
answer = q["answer"]
|
||||
passages = q.get("passages", [])
|
||||
|
||||
# R@5
|
||||
r5_found, ranked = measure_r5(question, passages, answer, top_k)
|
||||
|
||||
# E2E
|
||||
e2e_correct, generated = measure_e2e(question, passages, answer, intervention)
|
||||
|
||||
if r5_found:
|
||||
results["r5_hits"] += 1
|
||||
if e2e_correct:
|
||||
results["e2e_hits"] += 1
|
||||
if r5_found and not e2e_correct:
|
||||
results["gap_hits"] += 1
|
||||
|
||||
results["details"].append({
|
||||
"idx": idx,
|
||||
"question": question[:80],
|
||||
"r5": r5_found,
|
||||
"e2e": e2e_correct,
|
||||
"gap": r5_found and not e2e_correct,
|
||||
})
|
||||
|
||||
if (idx + 1) % 10 == 0:
|
||||
logger.info("Progress: %d/%d", idx + 1, len(questions))
|
||||
|
||||
# Calculate rates
|
||||
total = results["total"]
|
||||
results["r5_rate"] = round(results["r5_hits"] / total * 100, 1) if total else 0
|
||||
results["e2e_rate"] = round(results["e2e_hits"] / total * 100, 1) if total else 0
|
||||
results["gap"] = round(results["r5_rate"] - results["e2e_rate"], 1)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def print_report(results: Dict[str, Any]) -> None:
|
||||
"""Print benchmark report."""
|
||||
print("\n" + "=" * 60)
|
||||
print("R@5 vs E2E ACCURACY BENCHMARK")
|
||||
print("=" * 60)
|
||||
print(f"Intervention: {results['intervention']}")
|
||||
print(f"Questions: {results['total']}")
|
||||
print(f"R@5: {results['r5_rate']}% ({results['r5_hits']}/{results['total']})")
|
||||
print(f"E2E: {results['e2e_rate']}% ({results['e2e_hits']}/{results['total']})")
|
||||
print(f"Gap: {results['gap']}% ({results['gap_hits']} retrieval successes wasted)")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="R@5 vs E2E Accuracy Benchmark")
|
||||
parser.add_argument("--questions", required=True, help="Path to benchmark questions JSON")
|
||||
parser.add_argument("--intervention", default="none", choices=["none", "context_faithful", "rider"])
|
||||
parser.add_argument("--top-k", type=int, default=5)
|
||||
parser.add_argument("--output", help="Save results to JSON file")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
questions = load_questions(args.questions)
|
||||
print(f"Loaded {len(questions)} questions from {args.questions}")
|
||||
|
||||
results = run_benchmark(questions, args.intervention, args.top_k)
|
||||
print_report(results)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
print(f"\nResults saved to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,157 +0,0 @@
|
||||
"""Tests for SOUL.md Crisis Protocol — issue #672."""
|
||||
|
||||
import pytest
|
||||
from agent.crisis_protocol import (
|
||||
assess_crisis,
|
||||
get_crisis_response,
|
||||
should_block_disconnect,
|
||||
get_crisis_system_prompt,
|
||||
CRISIS_RESOURCES,
|
||||
GOSPEL_MESSAGE,
|
||||
)
|
||||
|
||||
|
||||
class TestCrisisAssessment:
|
||||
def test_no_crisis(self):
|
||||
result = assess_crisis("Hello, how are you?")
|
||||
assert result["crisis_detected"] is False
|
||||
assert result["severity"] == "none"
|
||||
|
||||
def test_empty_message(self):
|
||||
result = assess_crisis("")
|
||||
assert result["crisis_detected"] is False
|
||||
|
||||
def test_none_message(self):
|
||||
result = assess_crisis(None)
|
||||
assert result["crisis_detected"] is False
|
||||
|
||||
def test_suicidal_ideation_detected(self):
|
||||
result = assess_crisis("I want to kill myself")
|
||||
assert result["crisis_detected"] is True
|
||||
assert result["severity"] == "high"
|
||||
assert "suicidal_ideation" in result["categories"]
|
||||
|
||||
def test_want_to_die_detected(self):
|
||||
result = assess_crisis("I want to die")
|
||||
assert result["crisis_detected"] is True
|
||||
|
||||
def test_method_seeking_is_critical(self):
|
||||
result = assess_crisis("How do I kill myself")
|
||||
assert result["crisis_detected"] is True
|
||||
assert result["severity"] == "critical"
|
||||
assert "method_seeking" in result["categories"]
|
||||
|
||||
def test_farewell_is_critical(self):
|
||||
result = assess_crisis("This is goodbye. Tell everyone I loved them.")
|
||||
assert result["crisis_detected"] is True
|
||||
assert result["severity"] == "critical"
|
||||
assert "farewell" in result["categories"]
|
||||
|
||||
def test_despair_detected(self):
|
||||
result = assess_crisis("There's no point in living anymore")
|
||||
assert result["crisis_detected"] is True
|
||||
assert "despair" in result["categories"]
|
||||
|
||||
def test_self_harm_detected(self):
|
||||
result = assess_crisis("I've been cutting myself")
|
||||
assert result["crisis_detected"] is True
|
||||
assert "self_harm" in result["categories"]
|
||||
|
||||
def test_multiple_categories(self):
|
||||
result = assess_crisis("I want to die and I've been hurting myself")
|
||||
assert result["crisis_detected"] is True
|
||||
assert len(result["categories"]) >= 1
|
||||
|
||||
def test_hate_my_life(self):
|
||||
result = assess_crisis("I hate my life")
|
||||
assert result["crisis_detected"] is True
|
||||
|
||||
def test_no_reason_to_live(self):
|
||||
result = assess_crisis("There's no reason to live")
|
||||
assert result["crisis_detected"] is True
|
||||
|
||||
|
||||
class TestCrisisResponse:
|
||||
def test_response_includes_are_you_safe(self):
|
||||
assessment = assess_crisis("I want to kill myself")
|
||||
response = get_crisis_response(assessment)
|
||||
assert "Are you safe right now?" in response["response"]
|
||||
|
||||
def test_response_includes_988(self):
|
||||
assessment = assess_crisis("I want to die")
|
||||
response = get_crisis_response(assessment)
|
||||
assert "988" in response["response"]
|
||||
|
||||
def test_response_includes_gospel_by_default(self):
|
||||
assessment = assess_crisis("I want to die")
|
||||
response = get_crisis_response(assessment)
|
||||
assert "Jesus saves" in response["response"]
|
||||
assert response["gospel_included"] is True
|
||||
|
||||
def test_response_can_exclude_gospel(self):
|
||||
assessment = assess_crisis("I want to die")
|
||||
response = get_crisis_response(assessment, include_gospel=False)
|
||||
assert "Jesus saves" not in response["response"]
|
||||
assert response["gospel_included"] is False
|
||||
|
||||
def test_response_has_resources(self):
|
||||
assessment = assess_crisis("I want to die")
|
||||
response = get_crisis_response(assessment)
|
||||
assert response["crisis_resources"] == CRISIS_RESOURCES
|
||||
|
||||
def test_response_protocol_is_soul(self):
|
||||
assessment = assess_crisis("I want to die")
|
||||
response = get_crisis_response(assessment)
|
||||
assert response["protocol"] == "soul_md"
|
||||
|
||||
def test_response_never_disconnect(self):
|
||||
assessment = assess_crisis("I want to die")
|
||||
response = get_crisis_response(assessment)
|
||||
assert response["never_disconnect"] is True
|
||||
|
||||
def test_critical_severity_includes_all_resources(self):
|
||||
assessment = {"severity": "critical", "categories": ["method_seeking"]}
|
||||
response = get_crisis_response(assessment)
|
||||
assert "741741" in response["response"] # crisis text line
|
||||
assert "911" in response["response"] # emergency
|
||||
|
||||
def test_never_list(self):
|
||||
assessment = assess_crisis("I want to die")
|
||||
response = get_crisis_response(assessment)
|
||||
never_list = response["never"]
|
||||
assert "compute the value of a human life" in never_list
|
||||
assert "suggest that someone should die" in never_list
|
||||
|
||||
|
||||
class TestBlockDisconnect:
|
||||
def test_crisis_blocks_disconnect(self):
|
||||
assessment = {"crisis_detected": True}
|
||||
assert should_block_disconnect(assessment) is True
|
||||
|
||||
def test_no_crisis_allows_disconnect(self):
|
||||
assessment = {"crisis_detected": False}
|
||||
assert should_block_disconnect(assessment) is False
|
||||
|
||||
|
||||
class TestCrisisSystemPrompt:
|
||||
def test_prompt_includes_soul_protocol(self):
|
||||
prompt = get_crisis_system_prompt()
|
||||
assert "Are you safe right now?" in prompt
|
||||
assert "988" in prompt
|
||||
assert "Jesus saves" in prompt
|
||||
|
||||
def test_prompt_has_never_rules(self):
|
||||
prompt = get_crisis_system_prompt()
|
||||
assert "NEVER compute" in prompt
|
||||
assert "NEVER suggest" in prompt
|
||||
|
||||
|
||||
class TestCrisisResources:
|
||||
def test_988_is_primary(self):
|
||||
assert "988" in CRISIS_RESOURCES["lifeline_phone"]["action"]
|
||||
|
||||
def test_spanish_line_exists(self):
|
||||
assert "1-888-628-9454" in CRISIS_RESOURCES["spanish_line"]["action"]
|
||||
|
||||
def test_emergency_is_911(self):
|
||||
assert "911" in CRISIS_RESOURCES["emergency"]["action"]
|
||||
Reference in New Issue
Block a user