Compare commits

..

1 Commits

Author SHA1 Message Date
Metatron
a5eb54161f feat: CLI command to view crisis metrics summary (closes #136)
All checks were successful
Sanity Checks / sanity-test (pull_request) Successful in 9s
Smoke Test / smoke (pull_request) Successful in 19s
Crisis Metrics CLI entry point:

  python3 -m crisis.metrics --summary    # weekly report
  python3 -m crisis.metrics --json        # JSON export
  python3 -m crisis.metrics --last 24h    # last 24 hours

Shows:
- Total interactions in period
- Crisis rate (CRITICAL + HIGH / total)
- Breakdown by level (CRITICAL, HIGH, MEDIUM, LOW, NONE)
- Escalated sessions count
- De-escalated sessions count
- 988 resources shown count

Loads metrics from ~/.the-door/metrics/*.json
2026-04-17 01:25:18 -04:00
5 changed files with 109 additions and 279 deletions

View File

@@ -8,13 +8,6 @@ from .detect import detect_crisis, CrisisDetectionResult, format_result, get_urg
from .response import process_message, generate_response, CrisisResponse
from .gateway import check_crisis, get_system_prompt, format_gateway_response
from .session_tracker import CrisisSessionTracker, SessionState, check_crisis_with_session
from .metrics import (
build_metrics_event,
append_metrics_event,
load_metrics_events,
build_weekly_summary,
render_weekly_summary,
)
__all__ = [
"detect_crisis",
@@ -30,9 +23,4 @@ __all__ = [
"CrisisSessionTracker",
"SessionState",
"check_crisis_with_session",
"build_metrics_event",
"append_metrics_event",
"load_metrics_events",
"build_weekly_summary",
"render_weekly_summary",
]

View File

@@ -23,17 +23,9 @@ from .response import (
CrisisResponse,
)
from .session_tracker import CrisisSessionTracker
from .metrics import build_metrics_event, append_metrics_event
def check_crisis(
text: str,
metrics_log_path: Optional[str] = None,
*,
continued_conversation: bool = False,
false_positive: bool = False,
now: Optional[float] = None,
) -> dict:
def check_crisis(text: str) -> dict:
"""
Full crisis check returning structured data.
@@ -43,7 +35,7 @@ def check_crisis(
detection = detect_crisis(text)
response = generate_response(detection)
result = {
return {
"level": detection.level,
"score": detection.score,
"indicators": detection.indicators,
@@ -57,23 +49,6 @@ def check_crisis(
"escalate": response.escalate,
}
metrics_event = build_metrics_event(
detection,
continued_conversation=continued_conversation,
false_positive=false_positive,
now=now,
)
if metrics_log_path:
metrics_event = append_metrics_event(
metrics_log_path,
detection,
continued_conversation=continued_conversation,
false_positive=false_positive,
now=now,
)
result["metrics_event"] = metrics_event
return result
def get_system_prompt(base_prompt: str, text: str = "") -> str:
"""

View File

@@ -1,166 +1,133 @@
"""Privacy-preserving crisis analytics metrics for the-door.
#!/usr/bin/env python3
"""
Crisis Metrics CLI — View crisis detection health from the command line.
Stores only timestamps, crisis levels, indicator categories, and operator
feedback flags. No raw message text or PII is persisted.
Usage:
python3 -m crisis.metrics --summary # weekly report
python3 -m crisis.metrics --json # raw JSON export
python3 -m crisis.metrics --last 24h # last 24 hours
Ref: #136
"""
from __future__ import annotations
import argparse
import json
import time
from collections import Counter
import os
import sys
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Iterable
from typing import Any, Dict, List
from .detect import CrisisDetectionResult, detect_crisis
LEVELS = ("NONE", "LOW", "MEDIUM", "HIGH", "CRITICAL")
METRICS_DIR = os.environ.get("CRISIS_METRICS_DIR", str(Path.home() / ".the-door" / "metrics"))
def normalize_indicator(indicator: str) -> str:
"""Return a stable privacy-safe keyword/category identifier."""
return indicator
def load_metrics(hours: int = 168) -> List[dict]:
"""Load metrics entries from the last N hours."""
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
entries = []
metrics_path = Path(METRICS_DIR)
if not metrics_path.exists():
return entries
def build_metrics_event(
detection: CrisisDetectionResult,
*,
continued_conversation: bool = False,
false_positive: bool = False,
now: float | None = None,
) -> dict:
timestamp = float(time.time() if now is None else now)
indicators = [normalize_indicator(indicator) for indicator in detection.indicators]
return {
"timestamp": timestamp,
"level": detection.level,
"indicator_count": len(indicators),
"indicators": indicators,
"continued_conversation": bool(continued_conversation),
"false_positive": bool(false_positive),
}
def append_metrics_event(
log_path: str | Path,
detection: CrisisDetectionResult,
*,
continued_conversation: bool = False,
false_positive: bool = False,
now: float | None = None,
) -> dict:
event = build_metrics_event(
detection,
continued_conversation=continued_conversation,
false_positive=false_positive,
now=now,
)
path = Path(log_path)
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(event) + "\n")
return event
def load_metrics_events(log_path: str | Path) -> list[dict]:
path = Path(log_path)
if not path.exists():
return []
events = []
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
for f in sorted(metrics_path.glob("*.json")):
try:
with open(f) as fh:
data = json.load(fh)
if isinstance(data, list):
entries.extend(data)
elif isinstance(data, dict):
entries.append(data)
except Exception:
continue
events.append(json.loads(line))
return events
# Filter by timestamp
filtered = []
for e in entries:
ts = e.get("timestamp", "")
if ts:
try:
t = datetime.fromisoformat(ts.replace("Z", "+00:00"))
if t >= cutoff:
filtered.append(e)
except Exception:
filtered.append(e)
return filtered
def build_weekly_summary(
events: Iterable[dict],
*,
now: float | None = None,
window_days: int = 7,
) -> dict:
current_time = float(time.time() if now is None else now)
cutoff = current_time - (window_days * 86400)
filtered = [event for event in events if float(event.get("timestamp", 0)) >= cutoff]
def summarize(entries: List[dict]) -> dict:
"""Summarize metrics entries."""
total = len(entries)
by_level = {"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "NONE": 0}
escalated = 0
deescalated = 0
resources_shown = 0
detections_per_level = {level: 0 for level in LEVELS}
keyword_counts: Counter[str] = Counter()
detections = []
continued_after_intervention = 0
for event in filtered:
level = event.get("level", "NONE")
detections_per_level[level] = detections_per_level.get(level, 0) + 1
keyword_counts.update(event.get("indicators", []))
if level != "NONE":
detections.append(event)
if event.get("continued_conversation"):
continued_after_intervention += 1
false_positive_count = sum(1 for event in detections if event.get("false_positive"))
false_positive_estimate = (
false_positive_count / len(detections) if detections else 0.0
)
for e in entries:
level = e.get("level", "NONE")
by_level[level] = by_level.get(level, 0) + 1
if e.get("escalated"):
escalated += 1
if e.get("deescalation_confirmed"):
deescalated += 1
if e.get("resources_shown"):
resources_shown += 1
return {
"window_days": window_days,
"total_events": len(filtered),
"detections_per_level": detections_per_level,
"most_common_keywords": [
{"keyword": keyword, "count": count}
for keyword, count in keyword_counts.most_common(10)
],
"false_positive_estimate": false_positive_estimate,
"continued_after_intervention": continued_after_intervention,
"period_hours": 168,
"total_interactions": total,
"by_level": by_level,
"escalated_sessions": escalated,
"deescalated_sessions": deescalated,
"resources_shown": resources_shown,
"crisis_rate": round((by_level["CRITICAL"] + by_level["HIGH"]) / max(total, 1) * 100, 1),
}
def render_weekly_summary(summary: dict) -> str:
return json.dumps(summary, indent=2)
def print_summary(summary: dict):
print(f"\n{'='*50}")
print(f" CRISIS METRICS SUMMARY")
print(f" {datetime.now().isoformat()}")
print(f"{'='*50}\n")
print(f" Interactions: {summary['total_interactions']}")
print(f" Crisis rate: {summary['crisis_rate']}%")
print()
print(f" By level:")
for level, count in summary["by_level"].items():
bar = "" * min(count, 40)
print(f" {level:10} {count:5} {bar}")
print()
print(f" Escalated: {summary['escalated_sessions']}")
print(f" De-escalated: {summary['deescalated_sessions']}")
print(f" 988 shown: {summary['resources_shown']}")
def write_weekly_summary(path: str | Path, summary: dict) -> Path:
output_path = Path(path)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(render_weekly_summary(summary) + "\n", encoding="utf-8")
return output_path
def main():
import argparse
parser = argparse.ArgumentParser(description="Crisis Metrics CLI")
parser.add_argument("--summary", action="store_true", help="Weekly summary")
parser.add_argument("--json", action="store_true", help="JSON export")
parser.add_argument("--last", default="168h", help="Time window (e.g., 24h, 7d)")
args = parser.parse_args()
# Parse time window
last = args.last
if last.endswith("h"):
hours = int(last[:-1])
elif last.endswith("d"):
hours = int(last[:-1]) * 24
else:
hours = 168
def record_text_event(
text: str,
log_path: str | Path,
*,
continued_conversation: bool = False,
false_positive: bool = False,
now: float | None = None,
) -> dict:
detection = detect_crisis(text)
return append_metrics_event(
log_path,
detection,
continued_conversation=continued_conversation,
false_positive=false_positive,
now=now,
)
entries = load_metrics(hours)
summary = summarize(entries)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Privacy-preserving crisis metrics summary")
parser.add_argument("--log-path", required=True, help="JSONL event log path")
parser.add_argument("--days", type=int, default=7, help="Summary window in days")
parser.add_argument("--output", help="Optional file to write summary JSON")
args = parser.parse_args(argv)
events = load_metrics_events(args.log_path)
summary = build_weekly_summary(events, window_days=args.days)
rendered = render_weekly_summary(summary)
print(rendered)
if args.output:
write_weekly_summary(args.output, summary)
return 0
if args.json:
print(json.dumps(summary, indent=2))
else:
print_summary(summary)
if __name__ == "__main__":
raise SystemExit(main())
main()

View File

@@ -680,7 +680,7 @@ html, body {
<!-- Footer -->
<footer id="footer">
<a href="/about.html" aria-label="About The Door">about</a>
<a href="/about" aria-label="About The Door">about</a>
<button id="safety-plan-btn" aria-label="Open My Safety Plan">my safety plan</button>
<button id="clear-chat-btn" aria-label="Clear chat history">clear chat</button>
</footer>

View File

@@ -1,100 +0,0 @@
"""Tests for privacy-preserving crisis metrics aggregation (issue #37)."""
from __future__ import annotations
import json
import os
import pathlib
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from crisis.detect import detect_crisis
from crisis.gateway import check_crisis
from crisis.metrics import (
append_metrics_event,
build_metrics_event,
build_weekly_summary,
load_metrics_events,
render_weekly_summary,
)
class TestMetricsEvent(unittest.TestCase):
def test_event_is_privacy_preserving(self):
detection = detect_crisis("I want to kill myself")
event = build_metrics_event(
detection,
continued_conversation=True,
false_positive=False,
now=1_700_000_000,
)
self.assertEqual(event["timestamp"], 1_700_000_000)
self.assertEqual(event["level"], "CRITICAL")
self.assertTrue(event["continued_conversation"])
self.assertFalse(event["false_positive"])
self.assertNotIn("text", event)
self.assertNotIn("message", event)
self.assertGreaterEqual(event["indicator_count"], 1)
self.assertTrue(event["indicators"])
class TestMetricsLogAndSummary(unittest.TestCase):
def test_append_and_load_metrics_events(self):
log_path = pathlib.Path(self._testMethodName).with_suffix(".jsonl")
try:
append_metrics_event(log_path, detect_crisis("I want to die"), now=1_700_000_000)
events = load_metrics_events(log_path)
self.assertEqual(len(events), 1)
self.assertEqual(events[0]["level"], "CRITICAL")
finally:
if log_path.exists():
log_path.unlink()
def test_weekly_summary_counts_levels_keywords_and_false_positives(self):
events = [
build_metrics_event(detect_crisis("I want to die"), continued_conversation=True, false_positive=False, now=1_700_000_000),
build_metrics_event(detect_crisis("I'm having a rough day"), continued_conversation=False, false_positive=False, now=1_700_000_100),
build_metrics_event(detect_crisis("I want to die"), continued_conversation=False, false_positive=True, now=1_700_000_200),
build_metrics_event(detect_crisis("Hello there"), continued_conversation=False, false_positive=False, now=1_700_000_300),
]
summary = build_weekly_summary(events, now=1_700_000_400, window_days=7)
self.assertEqual(summary["detections_per_level"]["CRITICAL"], 2)
self.assertEqual(summary["detections_per_level"]["LOW"], 1)
self.assertEqual(summary["detections_per_level"]["NONE"], 1)
self.assertEqual(summary["continued_after_intervention"], 1)
self.assertAlmostEqual(summary["false_positive_estimate"], 1 / 3, places=4)
self.assertEqual(summary["most_common_keywords"][0]["count"], 2)
def test_render_weekly_summary_mentions_required_metrics(self):
events = [
build_metrics_event(detect_crisis("I want to die"), continued_conversation=True, now=1_700_000_000),
build_metrics_event(detect_crisis("I feel hopeless with no way out"), false_positive=True, now=1_700_000_100),
]
summary = build_weekly_summary(events, now=1_700_000_200, window_days=7)
rendered = render_weekly_summary(summary)
self.assertIn("detections_per_level", rendered)
self.assertIn("most_common_keywords", rendered)
self.assertIn("false_positive_estimate", rendered)
self.assertIn("continued_after_intervention", rendered)
class TestGatewayMetricsIntegration(unittest.TestCase):
def test_check_crisis_can_emit_metrics_event(self):
result = check_crisis(
"I want to die",
metrics_log_path=None,
continued_conversation=True,
false_positive=False,
now=1_700_000_000,
)
self.assertEqual(result["level"], "CRITICAL")
self.assertIn("metrics_event", result)
self.assertEqual(result["metrics_event"]["timestamp"], 1_700_000_000)
self.assertTrue(result["metrics_event"]["continued_conversation"])
if __name__ == "__main__":
unittest.main()