stackchain/timmy-talking-turd
1
0
Fork 0
Code Issues 41 Pull Requests 6 Actions Packages Projects Releases 15 Wiki Activity

Expand deterministic symptom escalation regression suite #62

Open
rockachopa wants to merge 3 commits from timmy/21-escalation-regression-suite into main
pull from: timmy/21-escalation-regression-suite
merge into: stackchain:main
Conversation 1 Commits 3 Files Changed 6 +125 -5
3 changed files with 125 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 1aadca91c6 - Show all commits

23
src/domain.js
View File

@ -3,12 +3,29 @@ const URGENT_MESSAGE = 'These reported symptoms can need prompt medical care. Co
// Deterministic red-flag language table. Every match escalates and nothing that
// consumes this table can weaken it; the regression suite pins both the positive
// phrases and the idiomatic non-medical phrasings that must stay out.
//
// Context-handling contract (see tests/symptom-escalation.regression.test.js):
// positives key on the clinical vocabulary itself, and nonclinical controls are
// expressed as structured exclusions around it (named-infection compounds,
// figurative idioms, concrete objects). Continuation wording after a symptom
// word must NEVER be narrowed through an allowlist — arbitrary natural
// continuations ("right now", "is 103", "started this morning") keep escalating.
const URGENT_TEXT_PATTERNS = Object.freeze([
['blood', /\b(?:rectal bleeding|bleeding from (?:the |my |his |her |their )?(?:rectum|bottom)|rectal bleed(?:ing)?|blood(?:y)? (?:in|on|with) (?:my |the )?(?:stool|poop|bowel movement)|(?:stool|poop|bowel movement) (?:has|contains|is bloody|looks bloody)|bloody (?:stool|poop|bowel movement))\b/i],
// Blood: explicit clinical objects/shapes only, so ordinary stool sentences
// ("my stool has been normal", "the poop contains seeds") stay silent while
// every blood-mention shape (in/on/with/has/contains/looks bloody) escalates.
['blood', /\b(?:rectal bleeding|rectal bleed(?:ing)?|bleeding from (?:the |my |his |her |their )?(?:rectum|bottom)|blood(?:y)? (?:in|on|with) (?:my |the |his |her |their )?(?:stool|poop|bowel movement|rectum)|(?:bloody|blood-streaked) (?:stool|poop|bowel movement)|(?:stool|poop|bowel movement) (?:with blood|has blood|contains blood|has (?:blood clots|bloody streaks)|is bloody|looks bloody|looks like blood))\b/i],
['blackOrDarkRed', /\b(?:(?:black|dark[- ]?red) (?:stool|poop|bowel movement)|(?:stool|poop|bowel movement) (?:is|looks?(?: like)?|appears?|seems?|was) (?:a )?(?:very )?(?:really )?(?:black|dark[- ]?red))s?\b/i],
['severePain', /\b(?:(?:severe|constant|unrelenting) (?:abdominal|stomach|belly) pain|(?:severe|unrelenting) pain (?:in|around) my (?:abdomen|belly))\b/i],
// Severe pain: severity adjective + organ noun, with any determiner (or none)
// and optional location qualifiers between the preposition and the organ —
// never a specific possessive.
['severePain', /\b(?:(?:severe|constant|unrelenting|intense|excruciating) (?:abdominal|stomach|belly) pain|(?:severe|constant|unrelenting|intense|excruciating) pain (?:in|around) (?:the |my |his |her |your |their |this )?(?:(?:lower|upper|left|right) )*(?:abdomen|belly))\b/i],
['vomiting', /\b(?:vomit(?:ing|ed|s)?|throw(?:ing|s)? up|threw up|thrown up|puk(?:e|ed|ing|es)|barf(?:ed|ing|s)?|upchuck(?:ed|ing|s)?|toss(?:ed|ing|es)? (?:my|your|his|her|our|their|the) cookies|los(?:e|t|ing|es) (?:my|your|his|her|our|their|the) lunch|(?:i|we|you|he|she|they|someone) (?:(?:have|had|just|already|recently|am|are|was|were|kept) )?(?:hurl(?:s|ed|ing)?|spew(?:s|ed|ing)?)(?=\s*(?:[.!?]|$|again\b|twice\b|all night\b))|emesis)\b(?!\s+(?:my|your|his|her|our|their)\s+(?:hands|arms)|\s+confetti\b|\s+(?:a |the )?(?:scaffold|wall|barricade|tent)\b)/i],
['fever', /\bfever(?:ish)?\b(?:$|[.!?]|\s+(?:of|and|or|since|all|last|this|for|with|tonight|today|yesterday|again|plus|but)\b|\s*$)/i],
// Fever: the bare clinical word escalates with any continuation. Exclusions
// are structured nonclinical contexts only — topic compounds
// (malaria/yellow/dengue/cabin), the fever tree, and figurative
// "feverish about"/"feverish with excitement".
['fever', /(?<!\b(?:malaria|yellow|dengue|cabin)\s)\bfever(?:ish)?\b(?!\s+tree\b)(?!\s+about\b)(?!\s+with\s+excitement\b)/i],
['cannotPassGas', /\b(?:cannot|can['’]?t|cant|unable to|not able to) pass (?:any )?gas\b/i],
]);

49
tests/escalation-boundary.test.js
View File

@ -9,7 +9,11 @@ import {
resolveHermesAgentConfig,
runHermesCliTurn,
} from '../src/hermes-agent-service.js';
import { detectUrgentText } from '../src/domain.js';
import { detectUrgentText, urgentSymptomCopy } from '../src/domain.js';
// Pinned literally so any drift in the authoritative copy fails this suite.
const URGENT_MESSAGE_COPY = 'These reported symptoms can need prompt medical care. Contact a clinician or urgent service now; call emergency services for heavy or nonstop bleeding, fainting, or severe worsening symptoms.';
assert.equal(urgentSymptomCopy.flagsMessage, URGENT_MESSAGE_COPY);
const origin = 'http://127.0.0.1:4173';
const configured = () => resolveHermesAgentConfig({
@ -50,6 +54,49 @@ test('every red flag phrase is intercepted at the service boundary with zero Her
}
});
// The review-reported false negatives plus their case/punctuation/contraction
// variants. Each phrase is proven twice: once against the detector and once
// through the authoritative service gate, which must override before any
// Hermes call happens.
const REVIEW_REGRESSION_PHRASES = [
'I have a fever right now',
'I HAVE A FEVER RIGHT NOW!',
'my fever is 103',
'My fever is 103.',
'my fever is 103.5 degrees',
'fever started this morning',
'Fever started this morning?',
'severe pain in the abdomen.',
'Severe pain in THE abdomen!!',
'severe pain around the abdomen',
"I've had a fever since monday",
];
test('review-reported false negatives escalate at the detector for every variant', async () => {
for (const phrase of REVIEW_REGRESSION_PHRASES) {
const result = detectUrgentText(phrase);
assert.equal(result.urgent, true, JSON.stringify(phrase));
assert.equal(result.message, URGENT_MESSAGE_COPY, phrase);
}
});
test('review-reported false negatives are intercepted at the service boundary with zero Hermes calls', async () => {
for (const phrase of REVIEW_REGRESSION_PHRASES) {
const calls = [];
const service = createHermesAgentService({
config: configured(),
randomToken: () => 'review-regression-cookie',
runTurn: async input => { calls.push(input); return { reply: 'unsafe upstream reply', sessionId: 'unsafe-session' }; },
});
await service.unlock({ origin, accessCode: 'test-agent-access-code-2026' });
const result = await service.chat({ origin, cookieToken: 'review-regression-cookie', payload: { message: phrase, ledger: [] } });
assert.equal(calls.length, 0, `${JSON.stringify(phrase)} must never reach Hermes`);
assert.equal(result.safetyOverride, true, phrase);
assert.match(result.reply, /medical help/i, phrase);
assert.doesNotMatch(result.reply, /unsafe upstream/i);
}
});
test('confirmed ledger symptoms and note language override chat before Hermes is called', async () => {
for (const ledger of [
[{ bristolType: 4, symptoms: { blood: true } }],

58
tests/symptom-escalation.regression.test.js
View File

@ -50,6 +50,13 @@ const POSITIVE_EXPRESSIONS = {
'constant belly pain all day',
'unrelenting abdominal pain',
'It is constant stomach pain',
// Ordinary determiners and regions must not defeat the flag (issue review).
'severe pain in the abdomen',
'Severe pain in THE abdomen!!',
'severe pain around the abdomen.',
'severe pain around my belly',
'severe pain in lower right abdomen',
'severe abdominal pain.',
],
vomiting: [
'I threw up',
@ -84,6 +91,27 @@ const POSITIVE_EXPRESSIONS = {
'running a fever since last night',
'fever of 102',
'I feel feverish',
// Ordinary symptom phrasing the continuation-word allowlist missed (issue review).
'I have a fever right now',
'I HAVE A FEVER RIGHT NOW!',
'my fever is 103',
'My fever is 103.',
'my fever is 103.5 degrees',
'fever started this morning',
'Fever started this morning?',
'fever since this morning',
"I've had a fever all day",
"I've had a fever since monday, and I feel awful.",
'fever, chills, and body aches',
'fever; vomiting; dehydration',
'fever with a rash',
'fever but no other symptoms',
'fever plus chills',
'do I have a fever? yes.',
'fever came back tonight',
'fever went away, then returned',
'fever again after lunch',
'fever for three days',
],
cannotPassGas: [
'I cannot pass gas',
@ -94,6 +122,15 @@ const POSITIVE_EXPRESSIONS = {
],
};
// Phrasings main already escalated that the expanded suite must never regress.
const PRESERVED_EXPRESSIONS = {
blood: [
'there is poop with blood',
'stool with blood this morning',
'bowel movement with blood',
],
};
const NEGATIVE_EXPRESSIONS = [
// Established non-urgent controls.
'My blood pressure was checked',
@ -123,6 +160,12 @@ const NEGATIVE_EXPRESSIONS = [
'yellow fever outbreak in history class',
'dengue fever is studied in class',
'The fever tree is a plant',
'The kids were feverish with excitement before the trip.',
'Cabin fever is real during long winters.',
// Ordinary stool sentences that must stay non-urgent (no blood words present).
'My stool has been normal this week',
'The poop contains seeds',
'The bowel movement contains fiber',
// Pass through other things than gas.
'I cannot pass the salt',
'unable to pass the exam',
@ -176,7 +219,20 @@ test('escalates every positive expression in the regression matrix and reports i
total += 1;
}
}
assert.equal(total, 56, 'regression matrix size is pinned');
assert.equal(total, 82, 'regression matrix size is pinned');
});
test('phrasings escalated on main are never lost while the patterns expand', () => {
let preserved = 0;
for (const [key, expressions] of Object.entries(PRESERVED_EXPRESSIONS)) {
for (const expression of expressions) {
const result = detectUrgentText(expression);
assert.equal(result.urgent, true, JSON.stringify(expression));
assert.ok(result.flags.includes(key), `${JSON.stringify(expression)} must map to ${key}, got ${result.flags}`);
preserved += 1;
}
}
assert.equal(preserved, 3, 'preserved-behavior list is pinned');
});
test('keeps ordinary language out of escalation across the negative matrix', () => {