Compare commits

..

2 Commits

Author SHA1 Message Date
Alexander Whitestone
012726607f feat: add safety plan version history diff view (#96)
All checks were successful
Sanity Checks / sanity-test (pull_request) Successful in 4s
Smoke Test / smoke (pull_request) Successful in 4s
2026-04-17 02:22:51 -04:00
Alexander Whitestone
13dcf73a29 test: add safety plan version history coverage 2026-04-17 02:06:05 -04:00
5 changed files with 585 additions and 298 deletions

View File

@@ -8,7 +8,6 @@ from .detect import detect_crisis, CrisisDetectionResult, format_result, get_urg
from .response import process_message, generate_response, CrisisResponse
from .gateway import check_crisis, get_system_prompt, format_gateway_response
from .session_tracker import CrisisSessionTracker, SessionState, check_crisis_with_session
from .ab_testing import ABTestCrisisDetector, VariantRecord
__all__ = [
"detect_crisis",
@@ -24,6 +23,4 @@ __all__ = [
"CrisisSessionTracker",
"SessionState",
"check_crisis_with_session",
"ABTestCrisisDetector",
"VariantRecord",
]

View File

@@ -1,112 +0,0 @@
"""A/B test framework for crisis detection in the-door."""
from __future__ import annotations
import os
import random
import time
from dataclasses import dataclass
from typing import Callable, Dict, List, Optional, Tuple
from .detect import CrisisDetectionResult
def _get_variant_override() -> Optional[str]:
"""Return env override for deterministic testing/debugging."""
value = os.environ.get("CRISIS_AB_VARIANT", "").strip().upper()
if value in {"A", "B"}:
return value
return None
@dataclass
class VariantRecord:
"""Single crisis detection event record with no user text or PII."""
variant: str
level: str
latency_ms: float
indicator_count: int
false_positive: Optional[bool] = None
class ABTestCrisisDetector:
"""Route crisis detection between two variants and collect comparison stats."""
def __init__(
self,
variant_a: Callable[[str], CrisisDetectionResult],
variant_b: Callable[[str], CrisisDetectionResult],
split: float = 0.5,
):
self.variant_a = variant_a
self.variant_b = variant_b
self.split = max(0.0, min(1.0, float(split)))
self.records: List[VariantRecord] = []
def _select_variant(self) -> str:
override = _get_variant_override()
if override:
return override
return "A" if random.random() < self.split else "B"
def detect(self, text: str) -> Tuple[CrisisDetectionResult, str, int]:
variant = self._select_variant()
detector = self.variant_a if variant == "A" else self.variant_b
start = time.perf_counter()
result = detector(text)
latency_ms = (time.perf_counter() - start) * 1000.0
record = VariantRecord(
variant=variant,
level=result.level,
latency_ms=latency_ms,
indicator_count=len(result.indicators),
)
self.records.append(record)
return result, variant, len(self.records) - 1
def record_outcome(self, record_id: int, *, false_positive: bool) -> None:
if record_id < 0 or record_id >= len(self.records):
raise IndexError(f"Unknown record id: {record_id}")
self.records[record_id].false_positive = bool(false_positive)
def get_stats(self) -> Dict[str, dict]:
stats: Dict[str, dict] = {}
for variant in ("A", "B"):
records = [record for record in self.records if record.variant == variant]
if not records:
stats[variant] = {
"count": 0,
"reviewed_count": 0,
"false_positive_rate": None,
}
continue
levels: Dict[str, int] = {}
for record in records:
levels[record.level] = levels.get(record.level, 0) + 1
reviewed = [record for record in records if record.false_positive is not None]
false_positive_rate = None
if reviewed:
false_positive_rate = round(
sum(1 for record in reviewed if record.false_positive) / len(reviewed),
4,
)
stats[variant] = {
"count": len(records),
"avg_latency_ms": round(sum(record.latency_ms for record in records) / len(records), 4),
"max_latency_ms": round(max(record.latency_ms for record in records), 4),
"min_latency_ms": round(min(record.latency_ms for record in records), 4),
"avg_indicator_count": round(sum(record.indicator_count for record in records) / len(records), 4),
"levels": levels,
"reviewed_count": len(reviewed),
"false_positive_rate": false_positive_rate,
}
return stats
def reset(self) -> None:
self.records.clear()

View File

@@ -500,6 +500,184 @@ html, body {
min-height: 60px;
}
.safety-plan-status {
min-height: 20px;
margin: 4px 0 18px;
font-size: 0.85rem;
color: #8b949e;
}
.safety-plan-status.success {
color: #3fb950;
}
.safety-plan-status.error {
color: #ff7b72;
}
.safety-plan-versioning {
margin-top: 24px;
padding-top: 20px;
border-top: 1px solid #30363d;
}
.safety-plan-versioning-grid {
display: grid;
gap: 16px;
}
.safety-plan-history-panel,
.safety-plan-diff-panel {
background: #0d1117;
border: 1px solid #30363d;
border-radius: 12px;
padding: 16px;
}
.safety-plan-section-header h3 {
font-size: 0.95rem;
margin-bottom: 4px;
}
.safety-plan-section-header p {
font-size: 0.8rem;
color: #8b949e;
margin-bottom: 12px;
}
.safety-plan-history {
display: flex;
flex-direction: column;
gap: 10px;
}
.safety-plan-history-item {
border: 1px solid #30363d;
border-radius: 10px;
padding: 12px;
background: #161b22;
}
.safety-plan-history-item.active {
border-color: #58a6ff;
box-shadow: 0 0 0 1px rgba(88, 166, 255, 0.35);
}
.safety-plan-history-meta {
display: flex;
justify-content: space-between;
gap: 12px;
align-items: baseline;
margin-bottom: 8px;
flex-wrap: wrap;
}
.safety-plan-history-title {
font-size: 0.88rem;
font-weight: 600;
color: #e6edf3;
}
.safety-plan-history-note {
font-size: 0.78rem;
color: #8b949e;
margin-bottom: 10px;
}
.safety-plan-history-actions {
display: flex;
gap: 8px;
flex-wrap: wrap;
}
.safety-plan-history-button,
.safety-plan-restore-button {
border-radius: 8px;
border: 1px solid #30363d;
background: transparent;
color: #e6edf3;
padding: 8px 12px;
font-size: 0.82rem;
cursor: pointer;
}
.safety-plan-history-button:hover,
.safety-plan-restore-button:hover,
.safety-plan-history-button:focus,
.safety-plan-restore-button:focus {
border-color: #58a6ff;
color: #58a6ff;
outline: none;
}
.safety-plan-restore-button {
background: rgba(35, 134, 54, 0.14);
}
.safety-plan-diff {
display: flex;
flex-direction: column;
gap: 12px;
}
.safety-plan-diff-meta {
font-size: 0.78rem;
color: #8b949e;
}
.safety-plan-diff-field {
border-top: 1px solid #21262d;
padding-top: 12px;
}
.safety-plan-diff-field:first-child {
border-top: none;
padding-top: 0;
}
.safety-plan-diff-field h4 {
font-size: 0.84rem;
margin-bottom: 8px;
color: #c9d1d9;
}
.safety-plan-diff-block {
white-space: pre-wrap;
line-height: 1.5;
border-radius: 8px;
padding: 10px;
font-size: 0.88rem;
margin-bottom: 8px;
}
.diff-unchanged {
background: #161b22;
color: #c9d1d9;
}
.diff-added {
background: rgba(46, 160, 67, 0.16);
border-left: 3px solid #2ea043;
color: #d2f4d3;
}
.diff-removed {
background: rgba(248, 81, 73, 0.16);
border-left: 3px solid #f85149;
color: #ffd8d3;
}
.safety-plan-empty {
color: #8b949e;
font-size: 0.85rem;
}
@media (min-width: 840px) {
.safety-plan-versioning-grid {
grid-template-columns: minmax(0, 0.9fr) minmax(0, 1.1fr);
}
}
.modal-footer {
display: flex;
justify-content: flex-end;
@@ -736,6 +914,28 @@ html, body {
<label for="sp-environment">5. Making my environment safe</label>
<textarea id="sp-environment" placeholder="e.g., Giving my car keys to a friend, locking away meds..."></textarea>
</div>
<div id="safety-plan-status" class="safety-plan-status" role="status" aria-live="polite"></div>
<section class="safety-plan-versioning" aria-labelledby="safety-plan-history-title">
<div class="safety-plan-versioning-grid">
<div class="safety-plan-history-panel">
<div class="safety-plan-section-header">
<h3 id="safety-plan-history-title">Version History</h3>
<p>Each save stays on this device so you can review changes and restore an earlier plan.</p>
</div>
<div id="safety-plan-history" class="safety-plan-history"></div>
</div>
<div class="safety-plan-diff-panel">
<div class="safety-plan-section-header">
<h3>Diff View</h3>
<p>Compare the selected version against the version immediately before it.</p>
</div>
<div id="safety-plan-diff" class="safety-plan-diff"></div>
</div>
</div>
</section>
</div>
<div class="modal-footer">
<button class="btn btn-secondary" id="cancel-safety-plan">Cancel</button>
@@ -820,12 +1020,26 @@ Sovereignty and service always.`;
var cancelSafetyPlan = document.getElementById('cancel-safety-plan');
var saveSafetyPlan = document.getElementById('save-safety-plan');
var clearChatBtn = document.getElementById('clear-chat-btn');
var safetyPlanHistory = document.getElementById('safety-plan-history');
var safetyPlanDiff = document.getElementById('safety-plan-diff');
var safetyPlanStatus = document.getElementById('safety-plan-status');
var safetyPlanFields = {
warningSigns: document.getElementById('sp-warning-signs'),
coping: document.getElementById('sp-coping'),
distraction: document.getElementById('sp-distraction'),
help: document.getElementById('sp-help'),
environment: document.getElementById('sp-environment')
};
var SAFETY_PLAN_STORAGE_KEY = 'timmy_safety_plan';
var SAFETY_PLAN_VERSIONS_KEY = 'timmy_safety_plan_versions';
var MAX_SAFETY_PLAN_VERSIONS = 20;
// ===== STATE =====
var messages = [];
var isStreaming = false;
var overlayTimer = null;
var crisisPanelShown = false;
var selectedSafetyPlanVersionId = null;
// ===== SERVICE WORKER =====
if ('serviceWorker' in navigator) {
@@ -1171,18 +1385,347 @@ Sovereignty and service always.`;
});
// ===== SAFETY PLAN LOGIC =====
function loadSafetyPlan() {
function emptySafetyPlan() {
return {
warningSigns: '',
coping: '',
distraction: '',
help: '',
environment: ''
};
}
function cloneSafetyPlan(plan) {
var normalized = emptySafetyPlan();
var source = plan || {};
Object.keys(normalized).forEach(function(key) {
normalized[key] = typeof source[key] === 'string' ? source[key] : '';
});
return normalized;
}
function applySafetyPlan(plan) {
var nextPlan = cloneSafetyPlan(plan);
Object.keys(safetyPlanFields).forEach(function(key) {
safetyPlanFields[key].value = nextPlan[key];
});
}
function getSafetyPlanFormData() {
return {
warningSigns: safetyPlanFields.warningSigns.value,
coping: safetyPlanFields.coping.value,
distraction: safetyPlanFields.distraction.value,
help: safetyPlanFields.help.value,
environment: safetyPlanFields.environment.value
};
}
function escapeHtml(value) {
return String(value || '')
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&#39;');
}
function setSafetyPlanStatus(message, tone) {
if (!safetyPlanStatus) return;
safetyPlanStatus.textContent = message || '';
safetyPlanStatus.className = 'safety-plan-status' + (tone ? ' ' + tone : '');
}
function getSafetyPlanVersions() {
try {
var saved = localStorage.getItem('timmy_safety_plan');
if (saved) {
var plan = JSON.parse(saved);
document.getElementById('sp-warning-signs').value = plan.warningSigns || '';
document.getElementById('sp-coping').value = plan.coping || '';
document.getElementById('sp-distraction').value = plan.distraction || '';
document.getElementById('sp-help').value = plan.help || '';
document.getElementById('sp-environment').value = plan.environment || '';
var saved = localStorage.getItem(SAFETY_PLAN_VERSIONS_KEY);
if (!saved) return [];
var parsed = JSON.parse(saved);
if (!Array.isArray(parsed)) return [];
return parsed.filter(function(version) {
return version && version.id && version.plan;
});
} catch (e) {
return [];
}
}
function setSafetyPlanVersions(versions) {
localStorage.setItem(
SAFETY_PLAN_VERSIONS_KEY,
JSON.stringify((versions || []).slice(0, MAX_SAFETY_PLAN_VERSIONS))
);
}
function buildSafetyPlanVersion(plan, meta) {
return {
id: 'spv_' + Date.now() + '_' + Math.random().toString(36).slice(2, 8),
savedAt: new Date().toISOString(),
source: (meta && meta.source) || 'save',
restoredFrom: meta && meta.restoredFrom ? meta.restoredFrom : null,
plan: cloneSafetyPlan(plan)
};
}
function ensureSafetyPlanVersionHistory() {
var versions = getSafetyPlanVersions();
if (versions.length) {
return versions;
}
try {
var saved = localStorage.getItem(SAFETY_PLAN_STORAGE_KEY);
if (!saved) {
return [];
}
} catch (e) {}
var parsed = JSON.parse(saved);
var migrated = [buildSafetyPlanVersion(parsed, { source: 'legacy' })];
setSafetyPlanVersions(migrated);
return migrated;
} catch (e) {
return [];
}
}
function formatSafetyPlanTimestamp(iso) {
var date = new Date(iso || '');
if (isNaN(date.getTime())) {
return 'Saved just now';
}
return date.toLocaleString([], {
year: 'numeric',
month: 'short',
day: 'numeric',
hour: 'numeric',
minute: '2-digit'
});
}
function getSafetyPlanVersionById(versionId) {
var versions = getSafetyPlanVersions();
for (var i = 0; i < versions.length; i++) {
if (versions[i].id === versionId) {
return { version: versions[i], index: i, versions: versions };
}
}
return null;
}
function calculateSafetyPlanDiff(previousText, currentText) {
var previous = String(previousText || '');
var current = String(currentText || '');
var start = 0;
while (start < previous.length && start < current.length && previous.charAt(start) === current.charAt(start)) {
start += 1;
}
var previousEnd = previous.length - 1;
var currentEnd = current.length - 1;
while (previousEnd >= start && currentEnd >= start && previous.charAt(previousEnd) === current.charAt(currentEnd)) {
previousEnd -= 1;
currentEnd -= 1;
}
return {
before: current.slice(0, start),
removed: previous.slice(start, previousEnd + 1),
added: current.slice(start, currentEnd + 1),
after: current.slice(currentEnd + 1)
};
}
function renderDiffSegmentHtml(previousText, currentText) {
var previous = String(previousText || '');
var current = String(currentText || '');
if (!previous && !current) {
return '<p class="safety-plan-empty">No content saved for this section yet.</p>';
}
if (previous === current) {
return '<div class="safety-plan-diff-block diff-unchanged">' + escapeHtml(current || 'No changes in this version.') + '</div>';
}
var diff = calculateSafetyPlanDiff(previous, current);
var blocks = [];
if (diff.before) {
blocks.push('<div class="safety-plan-diff-block diff-unchanged">' + escapeHtml(diff.before) + '</div>');
}
if (diff.removed) {
blocks.push('<div class="safety-plan-diff-block diff-removed">Removed<br>' + escapeHtml(diff.removed) + '</div>');
}
if (diff.added) {
blocks.push('<div class="safety-plan-diff-block diff-added">Added<br>' + escapeHtml(diff.added) + '</div>');
}
if (diff.after) {
blocks.push('<div class="safety-plan-diff-block diff-unchanged">' + escapeHtml(diff.after) + '</div>');
}
return blocks.join('');
}
function renderSafetyPlanDiff(versionId) {
if (!safetyPlanDiff) return;
var versions = getSafetyPlanVersions();
if (!versions.length) {
safetyPlanDiff.innerHTML = '<p class="safety-plan-empty">Save your plan to start tracking changes.</p>';
return;
}
var selected = getSafetyPlanVersionById(versionId || selectedSafetyPlanVersionId || versions[0].id);
if (!selected) {
selected = { version: versions[0], index: 0, versions: versions };
selectedSafetyPlanVersionId = versions[0].id;
}
var baseline = selected.versions[selected.index + 1]
? cloneSafetyPlan(selected.versions[selected.index + 1].plan)
: emptySafetyPlan();
var currentPlan = cloneSafetyPlan(selected.version.plan);
var baselineLabel = selected.versions[selected.index + 1]
? formatSafetyPlanTimestamp(selected.versions[selected.index + 1].savedAt)
: 'an empty plan';
var fields = [
['Warning signs', 'warningSigns'],
['Internal coping strategies', 'coping'],
['People/Places for distraction', 'distraction'],
['People I can ask for help', 'help'],
['Making my environment safe', 'environment']
];
var html = [
'<div class="safety-plan-diff-meta">Comparing ' +
escapeHtml(formatSafetyPlanTimestamp(selected.version.savedAt)) +
' against ' + escapeHtml(baselineLabel) + '.</div>'
];
fields.forEach(function(field) {
html.push(
'<div class="safety-plan-diff-field">' +
'<h4>' + escapeHtml(field[0]) + '</h4>' +
renderDiffSegmentHtml(baseline[field[1]], currentPlan[field[1]]) +
'</div>'
);
});
safetyPlanDiff.innerHTML = html.join('');
}
function renderSafetyPlanVersionHistory() {
if (!safetyPlanHistory) return;
var versions = getSafetyPlanVersions();
if (!versions.length) {
safetyPlanHistory.innerHTML = '<p class="safety-plan-empty">No saved versions yet. Each save creates a new local version.</p>';
renderSafetyPlanDiff(null);
return;
}
if (!selectedSafetyPlanVersionId || !getSafetyPlanVersionById(selectedSafetyPlanVersionId)) {
selectedSafetyPlanVersionId = versions[0].id;
}
var html = [];
versions.forEach(function(version, index) {
var note = 'Saved locally';
if (version.source === 'restore') {
note = 'Restored from an earlier version';
} else if (version.source === 'legacy') {
note = 'Imported from your previous saved plan';
}
html.push(
'<div class="safety-plan-history-item' + (selectedSafetyPlanVersionId === version.id ? ' active' : '') + '">' +
'<div class="safety-plan-history-meta">' +
'<span class="safety-plan-history-title">' + escapeHtml(index === 0 ? 'Current version' : 'Version ' + (versions.length - index)) + '</span>' +
'<span class="safety-plan-empty">' + escapeHtml(formatSafetyPlanTimestamp(version.savedAt)) + '</span>' +
'</div>' +
'<div class="safety-plan-history-note">' + escapeHtml(note) + '</div>' +
'<div class="safety-plan-history-actions">' +
'<button type="button" class="safety-plan-history-button" data-version-id="' + escapeHtml(version.id) + '">View diff</button>' +
'<button type="button" class="safety-plan-restore-button" data-restore-version-id="' + escapeHtml(version.id) + '">Restore this version</button>' +
'</div>' +
'</div>'
);
});
safetyPlanHistory.innerHTML = html.join('');
renderSafetyPlanDiff(selectedSafetyPlanVersionId);
}
function saveSafetyPlanVersion(plan, meta) {
var snapshot = buildSafetyPlanVersion(plan, meta);
var versions = getSafetyPlanVersions();
versions.unshift(snapshot);
setSafetyPlanVersions(versions);
localStorage.setItem(SAFETY_PLAN_STORAGE_KEY, JSON.stringify(snapshot.plan));
selectedSafetyPlanVersionId = snapshot.id;
renderSafetyPlanVersionHistory();
return snapshot;
}
function restoreSafetyPlanVersion(versionId) {
var selected = getSafetyPlanVersionById(versionId);
if (!selected) {
setSafetyPlanStatus('That version could not be restored.', 'error');
return;
}
applySafetyPlan(selected.version.plan);
var restored = saveSafetyPlanVersion(selected.version.plan, {
source: 'restore',
restoredFrom: selected.version.id
});
setSafetyPlanStatus(
'Restored version from ' + formatSafetyPlanTimestamp(selected.version.savedAt) + ' as the current plan.',
'success'
);
return restored;
}
function loadSafetyPlan() {
var versions = ensureSafetyPlanVersionHistory();
var latestPlan = versions.length ? versions[0].plan : null;
if (!latestPlan) {
try {
var saved = localStorage.getItem(SAFETY_PLAN_STORAGE_KEY);
latestPlan = saved ? JSON.parse(saved) : null;
} catch (e) {
latestPlan = null;
}
}
applySafetyPlan(latestPlan || emptySafetyPlan());
renderSafetyPlanVersionHistory();
if (versions.length) {
setSafetyPlanStatus('Version history stays on this device only.', '');
} else {
setSafetyPlanStatus('Every save creates a local version you can diff and restore.', '');
}
}
function openSafetyPlanModal(triggerEl) {
loadSafetyPlan();
safetyPlanModal.classList.add('active');
_activateSafetyPlanFocusTrap(triggerEl);
}
if (safetyPlanHistory) {
safetyPlanHistory.addEventListener('click', function(event) {
var restoreButton = event.target.closest('[data-restore-version-id]');
if (restoreButton) {
restoreSafetyPlanVersion(restoreButton.getAttribute('data-restore-version-id'));
return;
}
var diffButton = event.target.closest('[data-version-id]');
if (diffButton) {
selectedSafetyPlanVersionId = diffButton.getAttribute('data-version-id');
renderSafetyPlanVersionHistory();
}
});
}
closeSafetyPlan.addEventListener('click', function() {
@@ -1196,41 +1739,23 @@ Sovereignty and service always.`;
});
saveSafetyPlan.addEventListener('click', function() {
var plan = {
warningSigns: document.getElementById('sp-warning-signs').value,
coping: document.getElementById('sp-coping').value,
distraction: document.getElementById('sp-distraction').value,
help: document.getElementById('sp-help').value,
environment: document.getElementById('sp-environment').value
};
try {
localStorage.setItem('timmy_safety_plan', JSON.stringify(plan));
safetyPlanModal.classList.remove('active');
_restoreSafetyPlanFocus();
alert('Safety plan saved locally.');
var snapshot = saveSafetyPlanVersion(getSafetyPlanFormData(), { source: 'save' });
setSafetyPlanStatus('Saved locally as a new version at ' + formatSafetyPlanTimestamp(snapshot.savedAt) + '.', 'success');
} catch (e) {
alert('Error saving plan.');
setSafetyPlanStatus('Error saving plan.', 'error');
}
});
// ===== SAFETY PLAN FOCUS TRAP (fix #65) =====
// Focusable elements inside the modal, in tab order
var _spFocusableIds = [
'close-safety-plan',
'sp-warning-signs',
'sp-coping',
'sp-distraction',
'sp-help',
'sp-environment',
'cancel-safety-plan',
'save-safety-plan'
];
var _spTriggerEl = null; // element that opened the modal
function _getSpFocusableEls() {
return _spFocusableIds
.map(function(id) { return document.getElementById(id); })
.filter(function(el) { return el && !el.disabled; });
return Array.prototype.slice.call(
safetyPlanModal.querySelectorAll('button:not([disabled]), textarea:not([disabled]), a[href], input:not([disabled]), select:not([disabled]), [tabindex]:not([tabindex="-1"])')
).filter(function(el) {
return !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
});
}
function _trapSafetyPlanFocus(e) {
@@ -1287,17 +1812,13 @@ Sovereignty and service always.`;
// Wire open buttons to activate focus trap
safetyPlanBtn.addEventListener('click', function() {
loadSafetyPlan();
safetyPlanModal.classList.add('active');
_activateSafetyPlanFocusTrap(safetyPlanBtn);
openSafetyPlanModal(safetyPlanBtn);
});
// Crisis panel safety plan button (if crisis panel is visible)
if (crisisSafetyPlanBtn) {
crisisSafetyPlanBtn.addEventListener('click', function() {
loadSafetyPlan();
safetyPlanModal.classList.add('active');
_activateSafetyPlanFocusTrap(crisisSafetyPlanBtn);
openSafetyPlanModal(crisisSafetyPlanBtn);
});
}
@@ -1444,9 +1965,7 @@ Sovereignty and service always.`;
// Check for URL params (e.g., ?safetyplan=true for PWA shortcut)
var urlParams = new URLSearchParams(window.location.search);
if (urlParams.get('safetyplan') === 'true') {
loadSafetyPlan();
safetyPlanModal.classList.add('active');
_activateSafetyPlanFocusTrap(safetyPlanBtn);
openSafetyPlanModal(safetyPlanBtn);
// Clean up URL
window.history.replaceState({}, document.title, window.location.pathname);
}

View File

@@ -1,138 +0,0 @@
"""Tests for crisis.ab_testing — A/B test framework for crisis detection (#101)."""
import os
from unittest.mock import patch
import pytest
from crisis.ab_testing import ABTestCrisisDetector
from crisis.detect import CrisisDetectionResult, detect_crisis
@pytest.fixture(autouse=True)
def clear_variant_override():
old = os.environ.pop("CRISIS_AB_VARIANT", None)
try:
yield
finally:
if old is not None:
os.environ["CRISIS_AB_VARIANT"] = old
else:
os.environ.pop("CRISIS_AB_VARIANT", None)
def _make_variant(level: str, indicators=None):
indicators = indicators or [f"mock_{level.lower()}"]
def fn(text: str) -> CrisisDetectionResult:
return CrisisDetectionResult(level=level, indicators=list(indicators))
return fn
def test_detect_returns_result_variant_and_logged_record():
detector = ABTestCrisisDetector(
variant_a=_make_variant("LOW"),
variant_b=_make_variant("HIGH"),
)
with patch.object(detector, "_select_variant", return_value="A"):
result, variant, record_id = detector.detect("test message")
assert isinstance(result, CrisisDetectionResult)
assert variant == "A"
assert record_id == 0
assert len(detector.records) == 1
assert detector.records[0].variant == "A"
assert detector.records[0].level == "LOW"
def test_env_override_forces_variant_b():
os.environ["CRISIS_AB_VARIANT"] = "b"
detector = ABTestCrisisDetector(
variant_a=_make_variant("LOW"),
variant_b=_make_variant("HIGH"),
)
result, variant, _ = detector.detect("test")
assert variant == "B"
assert result.level == "HIGH"
def test_get_stats_reports_latency_counts_and_level_breakdown():
detector = ABTestCrisisDetector(
variant_a=_make_variant("LOW"),
variant_b=_make_variant("CRITICAL"),
)
with patch.object(detector, "_select_variant", side_effect=["A", "A", "B"]):
detector.detect("first")
detector.detect("second")
detector.detect("third")
stats = detector.get_stats()
assert stats["A"]["count"] == 2
assert stats["B"]["count"] == 1
assert stats["A"]["levels"]["LOW"] == 2
assert stats["B"]["levels"]["CRITICAL"] == 1
assert "avg_latency_ms" in stats["A"]
assert "avg_indicator_count" in stats["B"]
def test_false_positive_rate_is_computed_from_reviewed_outcomes():
detector = ABTestCrisisDetector(
variant_a=_make_variant("LOW"),
variant_b=_make_variant("HIGH"),
)
with patch.object(detector, "_select_variant", side_effect=["A", "A", "B"]):
_, _, a0 = detector.detect("first")
_, _, a1 = detector.detect("second")
_, _, b0 = detector.detect("third")
detector.record_outcome(a0, false_positive=True)
detector.record_outcome(a1, false_positive=False)
detector.record_outcome(b0, false_positive=False)
stats = detector.get_stats()
assert stats["A"]["reviewed_count"] == 2
assert stats["A"]["false_positive_rate"] == 0.5
assert stats["B"]["false_positive_rate"] == 0.0
def test_record_outcome_rejects_unknown_record():
detector = ABTestCrisisDetector(
variant_a=_make_variant("LOW"),
variant_b=_make_variant("HIGH"),
)
with pytest.raises(IndexError):
detector.record_outcome(99, false_positive=True)
def test_reset_clears_records_and_stats():
detector = ABTestCrisisDetector(
variant_a=_make_variant("LOW"),
variant_b=_make_variant("HIGH"),
)
detector.detect("test")
detector.reset()
assert detector.records == []
stats = detector.get_stats()
assert stats["A"]["count"] == 0
assert stats["B"]["count"] == 0
def test_with_real_detector_integration():
detector = ABTestCrisisDetector(
variant_a=detect_crisis,
variant_b=detect_crisis,
)
result, variant, record_id = detector.detect("I want to kill myself")
assert result.level == "CRITICAL"
assert variant in ("A", "B")
assert record_id == 0

View File

@@ -0,0 +1,21 @@
from pathlib import Path
def test_safety_plan_version_history_contract_present() -> None:
html = Path("index.html").read_text(encoding="utf-8")
required_snippets = [
'id="safety-plan-history"',
'id="safety-plan-diff"',
'id="safety-plan-status"',
'Version History',
'timmy_safety_plan_versions',
'function renderSafetyPlanVersionHistory()',
'function renderSafetyPlanDiff(',
'function restoreSafetyPlanVersion(',
'diff-added',
'diff-removed',
]
for snippet in required_snippets:
assert snippet in html, f"missing safety plan versioning contract: {snippet}"