Compare commits
2 Commits
fix/37
...
burn/59-17
| Author | SHA1 | Date | |
|---|---|---|---|
| b6af9ca9db | |||
| 56f991d615 |
@@ -8,13 +8,6 @@ from .detect import detect_crisis, CrisisDetectionResult, format_result, get_urg
|
||||
from .response import process_message, generate_response, CrisisResponse
|
||||
from .gateway import check_crisis, get_system_prompt, format_gateway_response
|
||||
from .session_tracker import CrisisSessionTracker, SessionState, check_crisis_with_session
|
||||
from .metrics import (
|
||||
build_metrics_event,
|
||||
append_metrics_event,
|
||||
load_metrics_events,
|
||||
build_weekly_summary,
|
||||
render_weekly_summary,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"detect_crisis",
|
||||
@@ -30,9 +23,4 @@ __all__ = [
|
||||
"CrisisSessionTracker",
|
||||
"SessionState",
|
||||
"check_crisis_with_session",
|
||||
"build_metrics_event",
|
||||
"append_metrics_event",
|
||||
"load_metrics_events",
|
||||
"build_weekly_summary",
|
||||
"render_weekly_summary",
|
||||
]
|
||||
|
||||
@@ -23,17 +23,9 @@ from .response import (
|
||||
CrisisResponse,
|
||||
)
|
||||
from .session_tracker import CrisisSessionTracker
|
||||
from .metrics import build_metrics_event, append_metrics_event
|
||||
|
||||
|
||||
def check_crisis(
|
||||
text: str,
|
||||
metrics_log_path: Optional[str] = None,
|
||||
*,
|
||||
continued_conversation: bool = False,
|
||||
false_positive: bool = False,
|
||||
now: Optional[float] = None,
|
||||
) -> dict:
|
||||
def check_crisis(text: str) -> dict:
|
||||
"""
|
||||
Full crisis check returning structured data.
|
||||
|
||||
@@ -43,7 +35,7 @@ def check_crisis(
|
||||
detection = detect_crisis(text)
|
||||
response = generate_response(detection)
|
||||
|
||||
result = {
|
||||
return {
|
||||
"level": detection.level,
|
||||
"score": detection.score,
|
||||
"indicators": detection.indicators,
|
||||
@@ -57,23 +49,6 @@ def check_crisis(
|
||||
"escalate": response.escalate,
|
||||
}
|
||||
|
||||
metrics_event = build_metrics_event(
|
||||
detection,
|
||||
continued_conversation=continued_conversation,
|
||||
false_positive=false_positive,
|
||||
now=now,
|
||||
)
|
||||
if metrics_log_path:
|
||||
metrics_event = append_metrics_event(
|
||||
metrics_log_path,
|
||||
detection,
|
||||
continued_conversation=continued_conversation,
|
||||
false_positive=false_positive,
|
||||
now=now,
|
||||
)
|
||||
result["metrics_event"] = metrics_event
|
||||
return result
|
||||
|
||||
|
||||
def get_system_prompt(base_prompt: str, text: str = "") -> str:
|
||||
"""
|
||||
|
||||
@@ -1,166 +0,0 @@
|
||||
"""Privacy-preserving crisis analytics metrics for the-door.
|
||||
|
||||
Stores only timestamps, crisis levels, indicator categories, and operator
|
||||
feedback flags. No raw message text or PII is persisted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
from .detect import CrisisDetectionResult, detect_crisis
|
||||
|
||||
LEVELS = ("NONE", "LOW", "MEDIUM", "HIGH", "CRITICAL")
|
||||
|
||||
|
||||
def normalize_indicator(indicator: str) -> str:
|
||||
"""Return a stable privacy-safe keyword/category identifier."""
|
||||
return indicator
|
||||
|
||||
|
||||
def build_metrics_event(
|
||||
detection: CrisisDetectionResult,
|
||||
*,
|
||||
continued_conversation: bool = False,
|
||||
false_positive: bool = False,
|
||||
now: float | None = None,
|
||||
) -> dict:
|
||||
timestamp = float(time.time() if now is None else now)
|
||||
indicators = [normalize_indicator(indicator) for indicator in detection.indicators]
|
||||
return {
|
||||
"timestamp": timestamp,
|
||||
"level": detection.level,
|
||||
"indicator_count": len(indicators),
|
||||
"indicators": indicators,
|
||||
"continued_conversation": bool(continued_conversation),
|
||||
"false_positive": bool(false_positive),
|
||||
}
|
||||
|
||||
|
||||
def append_metrics_event(
|
||||
log_path: str | Path,
|
||||
detection: CrisisDetectionResult,
|
||||
*,
|
||||
continued_conversation: bool = False,
|
||||
false_positive: bool = False,
|
||||
now: float | None = None,
|
||||
) -> dict:
|
||||
event = build_metrics_event(
|
||||
detection,
|
||||
continued_conversation=continued_conversation,
|
||||
false_positive=false_positive,
|
||||
now=now,
|
||||
)
|
||||
path = Path(log_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("a", encoding="utf-8") as handle:
|
||||
handle.write(json.dumps(event) + "\n")
|
||||
return event
|
||||
|
||||
|
||||
def load_metrics_events(log_path: str | Path) -> list[dict]:
|
||||
path = Path(log_path)
|
||||
if not path.exists():
|
||||
return []
|
||||
events = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
events.append(json.loads(line))
|
||||
return events
|
||||
|
||||
|
||||
def build_weekly_summary(
|
||||
events: Iterable[dict],
|
||||
*,
|
||||
now: float | None = None,
|
||||
window_days: int = 7,
|
||||
) -> dict:
|
||||
current_time = float(time.time() if now is None else now)
|
||||
cutoff = current_time - (window_days * 86400)
|
||||
filtered = [event for event in events if float(event.get("timestamp", 0)) >= cutoff]
|
||||
|
||||
detections_per_level = {level: 0 for level in LEVELS}
|
||||
keyword_counts: Counter[str] = Counter()
|
||||
detections = []
|
||||
continued_after_intervention = 0
|
||||
|
||||
for event in filtered:
|
||||
level = event.get("level", "NONE")
|
||||
detections_per_level[level] = detections_per_level.get(level, 0) + 1
|
||||
keyword_counts.update(event.get("indicators", []))
|
||||
if level != "NONE":
|
||||
detections.append(event)
|
||||
if event.get("continued_conversation"):
|
||||
continued_after_intervention += 1
|
||||
|
||||
false_positive_count = sum(1 for event in detections if event.get("false_positive"))
|
||||
false_positive_estimate = (
|
||||
false_positive_count / len(detections) if detections else 0.0
|
||||
)
|
||||
|
||||
return {
|
||||
"window_days": window_days,
|
||||
"total_events": len(filtered),
|
||||
"detections_per_level": detections_per_level,
|
||||
"most_common_keywords": [
|
||||
{"keyword": keyword, "count": count}
|
||||
for keyword, count in keyword_counts.most_common(10)
|
||||
],
|
||||
"false_positive_estimate": false_positive_estimate,
|
||||
"continued_after_intervention": continued_after_intervention,
|
||||
}
|
||||
|
||||
|
||||
def render_weekly_summary(summary: dict) -> str:
|
||||
return json.dumps(summary, indent=2)
|
||||
|
||||
|
||||
def write_weekly_summary(path: str | Path, summary: dict) -> Path:
|
||||
output_path = Path(path)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(render_weekly_summary(summary) + "\n", encoding="utf-8")
|
||||
return output_path
|
||||
|
||||
|
||||
def record_text_event(
|
||||
text: str,
|
||||
log_path: str | Path,
|
||||
*,
|
||||
continued_conversation: bool = False,
|
||||
false_positive: bool = False,
|
||||
now: float | None = None,
|
||||
) -> dict:
|
||||
detection = detect_crisis(text)
|
||||
return append_metrics_event(
|
||||
log_path,
|
||||
detection,
|
||||
continued_conversation=continued_conversation,
|
||||
false_positive=false_positive,
|
||||
now=now,
|
||||
)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="Privacy-preserving crisis metrics summary")
|
||||
parser.add_argument("--log-path", required=True, help="JSONL event log path")
|
||||
parser.add_argument("--days", type=int, default=7, help="Summary window in days")
|
||||
parser.add_argument("--output", help="Optional file to write summary JSON")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
events = load_metrics_events(args.log_path)
|
||||
summary = build_weekly_summary(events, window_days=args.days)
|
||||
rendered = render_weekly_summary(summary)
|
||||
print(rendered)
|
||||
if args.output:
|
||||
write_weekly_summary(args.output, summary)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,100 +0,0 @@
|
||||
"""Tests for privacy-preserving crisis metrics aggregation (issue #37)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from crisis.detect import detect_crisis
|
||||
from crisis.gateway import check_crisis
|
||||
from crisis.metrics import (
|
||||
append_metrics_event,
|
||||
build_metrics_event,
|
||||
build_weekly_summary,
|
||||
load_metrics_events,
|
||||
render_weekly_summary,
|
||||
)
|
||||
|
||||
|
||||
class TestMetricsEvent(unittest.TestCase):
|
||||
def test_event_is_privacy_preserving(self):
|
||||
detection = detect_crisis("I want to kill myself")
|
||||
event = build_metrics_event(
|
||||
detection,
|
||||
continued_conversation=True,
|
||||
false_positive=False,
|
||||
now=1_700_000_000,
|
||||
)
|
||||
self.assertEqual(event["timestamp"], 1_700_000_000)
|
||||
self.assertEqual(event["level"], "CRITICAL")
|
||||
self.assertTrue(event["continued_conversation"])
|
||||
self.assertFalse(event["false_positive"])
|
||||
self.assertNotIn("text", event)
|
||||
self.assertNotIn("message", event)
|
||||
self.assertGreaterEqual(event["indicator_count"], 1)
|
||||
self.assertTrue(event["indicators"])
|
||||
|
||||
|
||||
class TestMetricsLogAndSummary(unittest.TestCase):
|
||||
def test_append_and_load_metrics_events(self):
|
||||
log_path = pathlib.Path(self._testMethodName).with_suffix(".jsonl")
|
||||
try:
|
||||
append_metrics_event(log_path, detect_crisis("I want to die"), now=1_700_000_000)
|
||||
events = load_metrics_events(log_path)
|
||||
self.assertEqual(len(events), 1)
|
||||
self.assertEqual(events[0]["level"], "CRITICAL")
|
||||
finally:
|
||||
if log_path.exists():
|
||||
log_path.unlink()
|
||||
|
||||
def test_weekly_summary_counts_levels_keywords_and_false_positives(self):
|
||||
events = [
|
||||
build_metrics_event(detect_crisis("I want to die"), continued_conversation=True, false_positive=False, now=1_700_000_000),
|
||||
build_metrics_event(detect_crisis("I'm having a rough day"), continued_conversation=False, false_positive=False, now=1_700_000_100),
|
||||
build_metrics_event(detect_crisis("I want to die"), continued_conversation=False, false_positive=True, now=1_700_000_200),
|
||||
build_metrics_event(detect_crisis("Hello there"), continued_conversation=False, false_positive=False, now=1_700_000_300),
|
||||
]
|
||||
summary = build_weekly_summary(events, now=1_700_000_400, window_days=7)
|
||||
|
||||
self.assertEqual(summary["detections_per_level"]["CRITICAL"], 2)
|
||||
self.assertEqual(summary["detections_per_level"]["LOW"], 1)
|
||||
self.assertEqual(summary["detections_per_level"]["NONE"], 1)
|
||||
self.assertEqual(summary["continued_after_intervention"], 1)
|
||||
self.assertAlmostEqual(summary["false_positive_estimate"], 1 / 3, places=4)
|
||||
self.assertEqual(summary["most_common_keywords"][0]["count"], 2)
|
||||
|
||||
def test_render_weekly_summary_mentions_required_metrics(self):
|
||||
events = [
|
||||
build_metrics_event(detect_crisis("I want to die"), continued_conversation=True, now=1_700_000_000),
|
||||
build_metrics_event(detect_crisis("I feel hopeless with no way out"), false_positive=True, now=1_700_000_100),
|
||||
]
|
||||
summary = build_weekly_summary(events, now=1_700_000_200, window_days=7)
|
||||
rendered = render_weekly_summary(summary)
|
||||
self.assertIn("detections_per_level", rendered)
|
||||
self.assertIn("most_common_keywords", rendered)
|
||||
self.assertIn("false_positive_estimate", rendered)
|
||||
self.assertIn("continued_after_intervention", rendered)
|
||||
|
||||
|
||||
class TestGatewayMetricsIntegration(unittest.TestCase):
|
||||
def test_check_crisis_can_emit_metrics_event(self):
|
||||
result = check_crisis(
|
||||
"I want to die",
|
||||
metrics_log_path=None,
|
||||
continued_conversation=True,
|
||||
false_positive=False,
|
||||
now=1_700_000_000,
|
||||
)
|
||||
self.assertEqual(result["level"], "CRITICAL")
|
||||
self.assertIn("metrics_event", result)
|
||||
self.assertEqual(result["metrics_event"]["timestamp"], 1_700_000_000)
|
||||
self.assertTrue(result["metrics_event"]["continued_conversation"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
76
tests/test_link_integrity.py
Normal file
76
tests/test_link_integrity.py
Normal file
@@ -0,0 +1,76 @@
|
||||
"""Tests for the-door static site link integrity.
|
||||
|
||||
Validates that all internal links in HTML files point to existing files,
|
||||
preventing broken navigation from deployed static servers.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
PROJECT_ROOT = Path(__file__).parent.parent
|
||||
|
||||
|
||||
def _extract_internal_links(html_content: str) -> list[str]:
|
||||
"""Extract href values that point to local paths (not http/https/mailto/#)."""
|
||||
pattern = r'href="(/[^"]*|[^"]*\.html)"'
|
||||
links = re.findall(pattern, html_content)
|
||||
return [
|
||||
link for link in links
|
||||
if not link.startswith("http")
|
||||
and not link.startswith("mailto:")
|
||||
and not link.startswith("#")
|
||||
and not link.startswith("https://")
|
||||
]
|
||||
|
||||
|
||||
def _link_to_filepath(link: str) -> Path:
|
||||
"""Convert an href path to a filesystem path relative to project root."""
|
||||
if link.startswith("/"):
|
||||
return PROJECT_ROOT / link.lstrip("/")
|
||||
return PROJECT_ROOT / link
|
||||
|
||||
|
||||
@pytest.mark.parametrize("html_file", [
|
||||
"index.html",
|
||||
"about.html",
|
||||
"testimony.html",
|
||||
"crisis-offline.html",
|
||||
])
|
||||
def test_internal_links_resolve(html_file: str):
|
||||
"""All internal links in HTML files should point to existing files."""
|
||||
html_path = PROJECT_ROOT / html_file
|
||||
if not html_path.exists():
|
||||
pytest.skip(f"{html_file} not found")
|
||||
|
||||
content = html_path.read_text(encoding="utf-8")
|
||||
links = _extract_internal_links(content)
|
||||
|
||||
broken = []
|
||||
for link in links:
|
||||
target = _link_to_filepath(link)
|
||||
if not target.exists():
|
||||
broken.append(f" {link} -> {target} (NOT FOUND)")
|
||||
|
||||
assert not broken, (
|
||||
f"Broken links in {html_file}:\n" + "\n".join(broken)
|
||||
)
|
||||
|
||||
|
||||
def test_about_link_points_to_html():
|
||||
"""Specific regression test: the about footer link must point to about.html, not /about."""
|
||||
index_path = PROJECT_ROOT / "index.html"
|
||||
content = index_path.read_text(encoding="utf-8")
|
||||
|
||||
# Should contain about.html link
|
||||
assert 'href="/about.html"' in content, (
|
||||
"Footer about link should be '/about.html', not '/about'"
|
||||
)
|
||||
|
||||
# Should NOT contain bare /about link (which 404s on static servers)
|
||||
about_links = re.findall(r'href="(/about)"', content)
|
||||
assert not about_links, (
|
||||
f"Found bare '/about' link(s) that will 404 on static servers: {about_links}"
|
||||
)
|
||||
Reference in New Issue
Block a user