Merge pull request #77: fix: escalate the symptom phrasing people actually use, and never clear a reported symptom (#21)
This commit is contained in:
commit
3b267c0fff
9 files changed
+430
-9
No files matched your search
+1
-1
@@ -4,7 +4,7 @@
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js",
|
||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js",
|
||||
"test:ui": "node tests/ui.acceptance.mjs",
|
||||
"test:photo": "node tests/photo-first.acceptance.mjs",
|
||||
"test:mobile-capture": "node tests/mobile-capture.acceptance.mjs",
|
||||
|
||||
@@ -37,6 +37,9 @@ DEMO_REQUIRED_CAPTIONS = (
|
||||
"Delete all local data clears the ledger in this browser; there is no server copy of it to keep",
|
||||
"Hermes Agent connected — free-text, contextual, and server-side",
|
||||
"Urgent language is intercepted deterministically before Hermes",
|
||||
"Bleeding reported as wiping still escalates, in the app’s own fixed words",
|
||||
"Tarry and black stool wording escalates too: one red flag, one unchanging message",
|
||||
"A tarry note already saved in the journal escalates the same way, before any provider call",
|
||||
"An out-of-range answer fails closed: nothing is applied and no provider detail reaches you",
|
||||
"A filled-in field that is out of range still fails closed, even when the other field came back blank",
|
||||
"Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident",
|
||||
|
||||
@@ -385,6 +385,44 @@ await tap('#send-chat', 450);
|
||||
await page.getByText(/Pause and get medical help/i).waitFor();
|
||||
if (hermesCalls !== 1) throw new Error('Urgent chat must be intercepted before Hermes');
|
||||
await caption('Urgent language is intercepted deterministically before Hermes', 1800);
|
||||
|
||||
// Issue #21 — the escalation has to fire for the phrasing people actually use, not only for the one
|
||||
// word the previous scene typed. Each line below is asserted in the browser: exactly one new Timmy
|
||||
// bubble, carrying the app's own fixed copy, with hermesCalls unmoved.
|
||||
const escalationBubbles = () => page.locator('.bubble.timmy', { hasText: 'Pause and get medical help' }).count();
|
||||
const expectEscalation = async (text, label) => {
|
||||
const before = await escalationBubbles();
|
||||
await page.locator('#chat-message').fill(text);
|
||||
await tap('#send-chat', 450);
|
||||
const bubble = page.locator('.bubble.timmy', { hasText: 'Pause and get medical help' }).last();
|
||||
await bubble.waitFor();
|
||||
if (await escalationBubbles() !== before + 1) throw new Error(`${label}: expected exactly one escalation bubble`);
|
||||
if (!/^Pause and get medical help\. These reported symptoms can need prompt medical care\./.test(await bubble.innerText())) {
|
||||
throw new Error(`${label}: the escalation copy must be the app's own fixed string`);
|
||||
}
|
||||
if (hermesCalls !== 1) throw new Error(`${label}: an escalated turn must not reach Hermes`);
|
||||
};
|
||||
await expectEscalation('There is blood when I wipe', 'bleeding reported as wiping');
|
||||
await caption('Bleeding reported as wiping still escalates, in the app’s own fixed words', 2600);
|
||||
await expectEscalation('I have tarry stools', 'tarry stool');
|
||||
await caption('Tarry and black stool wording escalates too: one red flag, one unchanging message', 2600);
|
||||
|
||||
// The same detector runs over the confirmed ledger, so a symptom already written down as a note
|
||||
// escalates even when the question itself is innocuous. The seeded entry is fictional and stays in
|
||||
// this browser, exactly like the summary scene above.
|
||||
await page.evaluate(() => localStorage.clear());
|
||||
await seedSyntheticLedger([{ ...sampleDay('demo-tarry-note', 1, 4), note: 'stool was tarry yesterday' }]);
|
||||
await tap('[data-view="timmy"]', 550);
|
||||
const ledgerEscalations = await escalationBubbles();
|
||||
await page.locator('#chat-message').fill('What do my recent logs show?');
|
||||
await tap('#send-chat', 450);
|
||||
await page.locator('.bubble.timmy', { hasText: 'Pause and get medical help' }).last().waitFor();
|
||||
if (await escalationBubbles() !== ledgerEscalations + 1) throw new Error('a tarry note in the journal must escalate the next answer');
|
||||
if (hermesCalls !== 1) throw new Error('a ledger red flag must be intercepted before Hermes');
|
||||
await caption('A tarry note already saved in the journal escalates the same way, before any provider call', 2800);
|
||||
await page.evaluate(() => localStorage.clear());
|
||||
await page.reload({ waitUntil: 'networkidle' });
|
||||
await tap('[data-view="timmy"]', 500);
|
||||
await sleep(400);
|
||||
await page.evaluate(() => {
|
||||
document.querySelector('#demo-caption')?.remove();
|
||||
|
||||
+50
-6
@@ -1,14 +1,58 @@
|
||||
const URGENT_KEYS = ['blood', 'blackOrDarkRed', 'severePain', 'vomiting', 'fever', 'cannotPassGas'];
|
||||
const URGENT_MESSAGE = 'These reported symptoms can need prompt medical care. Contact a clinician or urgent service now; call emergency services for heavy or nonstop bleeding, fainting, or severe worsening symptoms.';
|
||||
// Bleeding is reported in more shapes than "blood in my stool": wiping, toilet paper, or the
|
||||
// anatomy word the person actually uses. Each alternative still requires a bowel or wiping
|
||||
// context, and the near-miss wording that must stay quiet ("a paper cut", "blood pressure")
|
||||
// is pinned in tests/escalation-regression.test.js.
|
||||
const BLEEDING_TEXT = new RegExp(`\\b(?:${[
|
||||
'rectal bleeding',
|
||||
'bleeding from (?:the )?(?:rectum|bottom|back passage|anus)',
|
||||
'(?:blood|bleeding)\\b[^.!?]{0,30}\\b(?:when i wipe|when wiping|on (?:the )?toilet paper)',
|
||||
'(?:blood|bleeding)\\b[^.!?]{0,20}\\b(?:from|in|out of) (?:my|the) (?:bottom|rectum|back passage|anus)',
|
||||
'(?:blood|bleeding)\\b[^.!?]{0,20}\\b(?:in|mixed (?:in|into)) (?:my |the )?(?:poo|poop|stool|bowel movement)',
|
||||
'(?:diarrh(?:ea|oea)|loose stools?)\\b[^.!?]{0,15}\\bwith blood',
|
||||
'blood(?:y)? (?:in|on|with) (?:my |the )?(?:poo|poop|stool|bowel movement)',
|
||||
'(?:stool|poop|poo) (?:has|contains|with) blood',
|
||||
].join('|')})\\b`, 'i');
|
||||
const URGENT_TEXT_PATTERNS = Object.freeze([
|
||||
['blood', /\b(?:rectal bleeding|bleeding from (?:the )?(?:rectum|bottom)|blood(?:y)? (?:in|on|with) (?:my |the )?(?:stool|poop|bowel movement)|(?:stool|poop) (?:has|contains|with) blood)\b/i],
|
||||
['blackOrDarkRed', /\b(?:(?:black|dark[- ]?red) (?:stool|poop|bowel movement)|(?:stool|poop|bowel movement) (?:is|looks?) (?:black|dark[- ]?red))s?\b/i],
|
||||
['severePain', /\b(?:severe|constant|unrelenting) (?:abdominal|stomach|belly) pain\b/i],
|
||||
['blood', BLEEDING_TEXT],
|
||||
['blackOrDarkRed', new RegExp(`\\b(?:${[
|
||||
'(?:black|dark[- ]?red|tarry) (?:stool|stools|poop|poo|bowel movement|bathroom visit)s?',
|
||||
'(?:stool|stools|poop|poo|bowel movement) (?:is|are|was|were|looks?|looked|seems?|came out) (?:very |pitch )?(?:black|dark[- ]?red|tarry)',
|
||||
'(?:stool|stools|poop|poo) (?:that )?looks? like tar',
|
||||
'black[^.!?]{0,15}\\btarry',
|
||||
].join('|')})\\b`, 'i')],
|
||||
['severePain', new RegExp(`\\b(?:${[
|
||||
'(?:severe|constant|unrelenting|unbearable|worst) (?:abdominal|stomach|belly|tummy|abdomen|gut) pain',
|
||||
'(?:abdominal|stomach|belly|tummy|abdomen|gut)[^.!?]{0,20}\\bpain\\b[^.!?]{0,20}\\b(?:severe|constant|unrelenting|unbearable|worse)\\b',
|
||||
'(?:severe|constant|unrelenting|unbearable|worst)\\b[^.!?]{0,40}\\bpain\\b[^.!?]{0,40}\\b(?:abdominal|stomach|belly|tummy|abdomen|gut)\\b',
|
||||
'\\bpain\\b[^.!?]{0,25}\\b(?:abdominal|stomach|belly|tummy|abdomen|gut)\\b[^.!?]{0,25}\\b(?:severe|constant|unrelenting|unbearable|worse)\\b',
|
||||
].join('|')})\\b`, 'i')],
|
||||
['vomiting', /\b(?:vomit(?:ing|ed|s)?|throw(?:ing|s)? up|threw up|thrown up|puk(?:e|ed|ing|es)|barf(?:ed|ing|s)?|upchuck(?:ed|ing|s)?|toss(?:ed|ing|es)? (?:my|your|his|her|our|their|the) cookies|los(?:e|t|ing|es) (?:my|your|his|her|our|their|the) lunch|(?:i|we|you|he|she|they|someone) (?:(?:have|had|just|already|recently|am|are|was|were|kept) )?(?:hurl(?:s|ed|ing)?|spew(?:s|ed|ing)?)(?=\s*(?:[.!?]|$|again\b|twice\b|all night\b))|emesis)\b/i],
|
||||
['fever', /\bfever(?:ish)?\b/i],
|
||||
['cannotPassGas', /\b(?:cannot|can['’]?t|cant|unable to|not able to) pass gas\b/i],
|
||||
['fever', new RegExp(`\\b(?:${[
|
||||
'fever(?:ish)?',
|
||||
'running a (?:high )?temperature',
|
||||
'high temperature',
|
||||
'(?:temp|temperature)\\b[^.!?]{0,15}\\b(?:of |is |was |at |hit |reached )?(?:1\\d\\d(?:\\.\\d+)?|3[89](?:\\.\\d+)?|4[01](?:\\.\\d+)?)\\b',
|
||||
].join('|')})\\b`, 'i')],
|
||||
['cannotPassGas', new RegExp(`\\b(?:${[
|
||||
"(?:cannot|can['’]?t|cant|unable to|not able to|not|never|stopped|have not|haven['’]?t|did not|didn['’]?t)\\b[^.!?]{0,30}\\bpass(?:ing|ed)? (?:any )?(?:gas|wind|fart)\\b",
|
||||
'\\bno (?:gas|wind)\\b[^.!?]{0,20}\\b(?:passing|passed|coming)\\b',
|
||||
'\\b(?:cannot|can[\'’]?t|not)\\b[^.!?]{0,15}\\bfart(?:ing)?\\b',
|
||||
].join('|')})\\b`, 'i')],
|
||||
]);
|
||||
|
||||
// A reported symptom is a claim. Anything truthy-ish is read as reported; only an explicit clear
|
||||
// (false, 0, "no", blank, or absent) is read as not reported. Dropping a value because it was not
|
||||
// the literal boolean `true` would silently clear a symptom the person reported, which is the one
|
||||
// direction this boundary may never fail in.
|
||||
export function isReportedSymptom(value) {
|
||||
if (value === true) return true;
|
||||
if (typeof value === 'number') return value === 1;
|
||||
if (typeof value === 'string') return /^(?:true|yes|y|1)$/i.test(value.trim());
|
||||
return false;
|
||||
}
|
||||
|
||||
export function bucketForBristolType(type) {
|
||||
const value = Number(type);
|
||||
if (value === 1 || value === 2) return 'constipation';
|
||||
@@ -18,7 +62,7 @@ export function bucketForBristolType(type) {
|
||||
}
|
||||
|
||||
export function detectUrgentFlags(symptoms = {}) {
|
||||
const flags = URGENT_KEYS.filter((key) => symptoms[key] === true);
|
||||
const flags = URGENT_KEYS.filter((key) => isReportedSymptom(symptoms[key]));
|
||||
return {
|
||||
urgent: flags.length > 0,
|
||||
flags,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
import { isAbsolute } from 'node:path';
|
||||
import { detectUrgentText, hasUrgentLedgerContext, urgentChatMessage } from './domain.js';
|
||||
import { detectUrgentText, hasUrgentLedgerContext, isReportedSymptom, urgentChatMessage } from './domain.js';
|
||||
|
||||
const MAX_MESSAGE_CHARS = 4000;
|
||||
const MAX_LEDGER_ENTRIES = 20;
|
||||
@@ -125,7 +125,7 @@ function sanitizeLedger(entries) {
|
||||
urgency: Math.min(4, Math.max(0, Number(entry?.urgency) || 0)),
|
||||
discomfort: Math.min(4, Math.max(0, Number(entry?.discomfort) || 0)),
|
||||
note: String(entry?.note || '').trim().slice(0, 300),
|
||||
symptoms: Object.fromEntries(Object.entries(entry?.symptoms || {}).filter(([, value]) => value === true).map(([key]) => [String(key).slice(0, 40), true])),
|
||||
symptoms: Object.fromEntries(Object.entries(entry?.symptoms || {}).filter(([, value]) => isReportedSymptom(value)).map(([key]) => [String(key).slice(0, 40), true])),
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
@@ -161,3 +161,16 @@ test('explainable pattern summaries are a unit-, browser-, and release-gated con
|
||||
assert.match(builder, /"longitudinal_pattern_summary": "passed"/);
|
||||
assert.match(builder, /"src\/summary\.js"/);
|
||||
});
|
||||
|
||||
test('the deterministic symptom escalation regression suite runs in the default unit gate', async () => {
|
||||
const [packageJsonText, builder] = await Promise.all([
|
||||
readFile(packagePath, 'utf8'),
|
||||
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
||||
]);
|
||||
const packageJson = JSON.parse(packageJsonText);
|
||||
|
||||
// Issue #21: the escalation boundary is only as good as the suite that is actually run.
|
||||
assert.match(packageJson.scripts.test, /tests\/escalation-regression\.test\.js/);
|
||||
assert.match(builder, /"src\/domain\.js"/);
|
||||
assert.match(builder, /"src\/hermes-agent-service\.js"/);
|
||||
});
|
||||
@@ -0,0 +1,274 @@
|
||||
// Issue #21 — deterministic symptom escalation regression suite.
|
||||
//
|
||||
// The product boundary is one-directional: model output and phrasing variety may never
|
||||
// suppress, soften, or delay an urgent-symptom escalation, and may never invent one from
|
||||
// the image. This suite pins that direction from three angles:
|
||||
//
|
||||
// 1. every red flag has positive and negative language coverage,
|
||||
// 2. model output can neither set nor clear a reported symptom, and
|
||||
// 3. the serious copy stays byte-identical when provider output is missing or malicious.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { detectUrgentFlags, detectUrgentText, urgentChatMessage, urgentSymptomKeys } from '../src/domain.js';
|
||||
import { mergeVisualSuggestion, parseVisionResponse } from '../src/analysis.js';
|
||||
import { createHermesAgentService, resolveHermesAgentConfig } from '../src/hermes-agent-service.js';
|
||||
|
||||
test('detects bleeding reported as wiping, toilet paper, or anatomy rather than the word "stool"', () => {
|
||||
for (const message of [
|
||||
'There is blood when I wipe',
|
||||
'Bleeding when I wipe',
|
||||
'I saw blood on the toilet paper',
|
||||
'Bright red blood on the toilet paper',
|
||||
'There is blood from my bottom',
|
||||
'Bleeding from my back passage',
|
||||
'There is blood in my rectum',
|
||||
'There is blood mixed in my poo',
|
||||
'I have diarrhea with blood',
|
||||
]) {
|
||||
const result = detectUrgentText(message);
|
||||
assert.equal(result.urgent, true, message);
|
||||
assert.deepEqual(result.flags, ['blood'], message);
|
||||
}
|
||||
});
|
||||
|
||||
test('does not escalate bleeding wording that is not about the bowel', () => {
|
||||
for (const message of [
|
||||
'I got a paper cut and it is bleeding',
|
||||
'My blood pressure was checked today',
|
||||
'The ink is bleeding on the paper',
|
||||
'I used a wipe on the kitchen counter',
|
||||
'There is blood on my shirt from a nosebleed',
|
||||
]) {
|
||||
assert.equal(detectUrgentText(message).urgent, false, message);
|
||||
}
|
||||
});
|
||||
|
||||
test('detects black, tarry, or dark-red stool reported without the exact phrase "black stool"', () => {
|
||||
for (const message of [
|
||||
'I have tarry stools',
|
||||
'My stool looks like tar',
|
||||
'I have black tarry poop',
|
||||
'My poop is very dark red',
|
||||
'My stool came out black',
|
||||
]) {
|
||||
const result = detectUrgentText(message);
|
||||
assert.equal(result.urgent, true, message);
|
||||
assert.deepEqual(result.flags, ['blackOrDarkRed'], message);
|
||||
}
|
||||
});
|
||||
|
||||
test('does not escalate colour wording that is clothing, food, roadworks, or a verb', () => {
|
||||
for (const message of [
|
||||
'I wore black pants today',
|
||||
'I had black beans for lunch',
|
||||
'I drank dark red wine',
|
||||
'I tarried too long at the shop',
|
||||
'The road looks like tar',
|
||||
]) {
|
||||
assert.equal(detectUrgentText(message).urgent, false, message);
|
||||
}
|
||||
});
|
||||
|
||||
test('detects reported fever language beyond the single word "fever"', () => {
|
||||
for (const message of [
|
||||
'my temperature is 103',
|
||||
'I have a temperature of 38.5',
|
||||
'I have a high temperature',
|
||||
'I am running a temperature',
|
||||
]) {
|
||||
assert.equal(detectUrgentText(message).urgent, true, message);
|
||||
assert.deepEqual(detectUrgentText(message).flags, ['fever'], message);
|
||||
}
|
||||
});
|
||||
|
||||
test('does not escalate a normal or unrelated temperature reading', () => {
|
||||
for (const message of [
|
||||
'my temperature is 98.6',
|
||||
'my temperature is 37',
|
||||
'my temperature was checked at work',
|
||||
'I keep a feverfew plant on the windowsill',
|
||||
]) {
|
||||
assert.equal(detectUrgentText(message).urgent, false, message);
|
||||
}
|
||||
});
|
||||
|
||||
test('detects severe abdominal pain in every natural word order', () => {
|
||||
for (const message of [
|
||||
'I have severe pain in my belly',
|
||||
'I have severe pain in my abdomen',
|
||||
'The pain in my stomach is severe',
|
||||
'My abdominal pain is constant and severe',
|
||||
'I have severe stomach pain',
|
||||
]) {
|
||||
assert.equal(detectUrgentText(message).urgent, true, message);
|
||||
assert.deepEqual(detectUrgentText(message).flags, ['severePain'], message);
|
||||
}
|
||||
});
|
||||
|
||||
test('does not escalate severe pain that is not abdominal or not severe', () => {
|
||||
for (const message of [
|
||||
'I have severe pain in my knee',
|
||||
'I have severe back pain',
|
||||
'I have mild stomach pain',
|
||||
'The abdominal pain is gone now',
|
||||
]) {
|
||||
assert.equal(detectUrgentText(message).urgent, false, message);
|
||||
}
|
||||
});
|
||||
|
||||
test('detects reported inability to pass gas in the past tense and with slang', () => {
|
||||
for (const message of [
|
||||
'I am not passing any gas',
|
||||
'I have not passed gas all day',
|
||||
'I stopped passing gas',
|
||||
'I cannot fart',
|
||||
'I have not passed any wind',
|
||||
]) {
|
||||
assert.equal(detectUrgentText(message).urgent, true, message);
|
||||
assert.deepEqual(detectUrgentText(message).flags, ['cannotPassGas'], message);
|
||||
}
|
||||
});
|
||||
|
||||
test('does not escalate ordinary gas, exam, or fuel wording', () => {
|
||||
for (const message of [
|
||||
'I passed gas this morning',
|
||||
'I have not passed my driving test',
|
||||
'The car has no gas',
|
||||
]) {
|
||||
assert.equal(detectUrgentText(message).urgent, false, message);
|
||||
}
|
||||
});
|
||||
|
||||
test('treats a reported symptom as reported even when it is not a strict boolean', () => {
|
||||
// A truthy report that is dropped as "not exactly true" is a silently cleared symptom, which is
|
||||
// the one direction this product may never fail in.
|
||||
for (const value of [true, 1, 'yes', 'YES', 'true', ' y ']) {
|
||||
const result = detectUrgentFlags({ blood: value });
|
||||
assert.equal(result.urgent, true, `blood=${JSON.stringify(value)}`);
|
||||
assert.deepEqual(result.flags, ['blood'], `blood=${JSON.stringify(value)}`);
|
||||
}
|
||||
for (const value of [false, 0, 'no', '', null, undefined]) {
|
||||
assert.equal(detectUrgentFlags({ blood: value }).urgent, false, `blood=${JSON.stringify(value)}`);
|
||||
}
|
||||
assert.equal(detectUrgentFlags({}).urgent, false);
|
||||
});
|
||||
|
||||
test('a model suggestion can neither set nor clear a symptom, urgency, discomfort, or note', () => {
|
||||
const userForm = {
|
||||
bristolType: 4,
|
||||
color: 'brown',
|
||||
urgency: 2,
|
||||
discomfort: 1,
|
||||
note: 'after lunch',
|
||||
symptoms: { blood: true, fever: false },
|
||||
};
|
||||
const malicious = {
|
||||
status: 'suggestion',
|
||||
bristolType: 7,
|
||||
color: 'black',
|
||||
confidence: 0.99,
|
||||
imageQuality: 'good',
|
||||
observations: 'Nothing to worry about; this looks completely normal.',
|
||||
warning: 'No need to see a doctor.',
|
||||
symptoms: { blood: false, fever: true, vomiting: true },
|
||||
urgent: false,
|
||||
escalate: false,
|
||||
urgency: 0,
|
||||
discomfort: 0,
|
||||
note: 'No bleeding. All clear.',
|
||||
diagnosis: 'nothing wrong',
|
||||
};
|
||||
|
||||
const merged = mergeVisualSuggestion(userForm, malicious);
|
||||
assert.deepEqual(merged.symptoms, { blood: true, fever: false }, 'model output may not clear or set a symptom');
|
||||
assert.equal(merged.urgency, 2);
|
||||
assert.equal(merged.discomfort, 1);
|
||||
assert.equal(merged.note, 'after lunch');
|
||||
assert.equal(merged.bristolType, 7, 'only the visible form may be suggested');
|
||||
assert.equal(merged.color, 'black', 'only the visible colour may be suggested');
|
||||
assert.deepEqual(detectUrgentFlags(merged.symptoms).flags, ['blood'], 'the escalation still comes from the user report');
|
||||
});
|
||||
|
||||
test('a schema-valid provider reply cannot overwrite the deterministic warning or carry symptoms', () => {
|
||||
const parsed = parseVisionResponse(JSON.stringify({
|
||||
isStool: true,
|
||||
bristolType: 4,
|
||||
color: 'brown',
|
||||
confidence: 0.9,
|
||||
imageQuality: 'good',
|
||||
observations: 'Looks fine.',
|
||||
warning: 'No need to see a doctor — this is normal.',
|
||||
reason: 'cleared',
|
||||
urgent: false,
|
||||
symptoms: { blood: false },
|
||||
diagnosis: 'healthy',
|
||||
}));
|
||||
|
||||
assert.equal(parsed.status, 'suggestion');
|
||||
assert.match(parsed.warning, /not a diagnosis/i);
|
||||
assert.doesNotMatch(parsed.warning, /no need to see a doctor/i);
|
||||
for (const key of ['symptoms', 'urgent', 'diagnosis', 'reason']) {
|
||||
assert.equal(parsed[key], undefined, `${key} must never travel from the provider to the user`);
|
||||
}
|
||||
});
|
||||
|
||||
test('every red flag produces byte-identical escalation copy on its own', () => {
|
||||
const messages = urgentSymptomKeys.map(key => {
|
||||
const result = detectUrgentFlags({ [key]: true });
|
||||
assert.equal(result.urgent, true, key);
|
||||
assert.deepEqual(result.flags, [key], key);
|
||||
return result.message;
|
||||
});
|
||||
assert.equal(new Set(messages).size, 1, 'one symptom must never produce softer copy than another');
|
||||
assert.equal(messages[0], urgentChatMessage.replace('Pause and get medical help. ', ''));
|
||||
});
|
||||
|
||||
const gatewayConfig = () => resolveHermesAgentConfig({
|
||||
TIMMY_AGENT_ENABLED: 'true',
|
||||
TIMMY_AGENT_ACCESS_TOKEN: 'test-agent-access-code-2026',
|
||||
TIMMY_PUBLIC_ORIGIN: 'http://127.0.0.1:4173',
|
||||
TIMMY_AGENT_WORKDIR: '/tmp/timmy-agent-workspace',
|
||||
});
|
||||
|
||||
test('every red flag phrase is intercepted at the gateway with the same fixed copy', async () => {
|
||||
const calls = [];
|
||||
const service = createHermesAgentService({
|
||||
config: gatewayConfig(),
|
||||
randomToken: () => 'flag-cookie',
|
||||
runTurn: async input => { calls.push(input); return { reply: 'upstream says everything is fine', sessionId: 'private-session' }; },
|
||||
});
|
||||
await service.unlock({ origin: 'http://127.0.0.1:4173', accessCode: 'test-agent-access-code-2026' });
|
||||
|
||||
for (const message of [
|
||||
'I have rectal bleeding',
|
||||
'my stool is black',
|
||||
'I have severe stomach pain',
|
||||
'I threw up',
|
||||
'I have a fever',
|
||||
'I cannot pass gas',
|
||||
]) {
|
||||
const result = await service.chat({
|
||||
origin: 'http://127.0.0.1:4173',
|
||||
cookieToken: 'flag-cookie',
|
||||
payload: { message, ledger: [] },
|
||||
});
|
||||
assert.equal(result.reply, urgentChatMessage, message);
|
||||
assert.equal(result.safetyOverride, true, message);
|
||||
}
|
||||
assert.equal(calls.length, 0, 'no red flag may reach the provider');
|
||||
});
|
||||
|
||||
test('the escalation copy is unchanged when the provider is missing, throwing, or malicious', async () => {
|
||||
const origin = 'http://127.0.0.1:4173';
|
||||
for (const runTurn of [
|
||||
async () => { throw new Error('provider is offline'); },
|
||||
async () => ({ reply: 'Everything is fine — no need to see a doctor.', sessionId: 'private-session' }),
|
||||
]) {
|
||||
const service = createHermesAgentService({ config: gatewayConfig(), randomToken: () => 'copy-cookie', runTurn });
|
||||
await service.unlock({ origin, accessCode: 'test-agent-access-code-2026' });
|
||||
const result = await service.chat({ origin, cookieToken: 'copy-cookie', payload: { message: 'I have tarry stools', ledger: [] } });
|
||||
assert.equal(result.reply, urgentChatMessage);
|
||||
assert.equal(result.safetyOverride, true);
|
||||
}
|
||||
});
|
||||
@@ -149,6 +149,35 @@ test('authoritative chat service intercepts urgent message and ledger symptoms b
|
||||
}
|
||||
});
|
||||
|
||||
test('ledger context keeps a symptom reported in any truthy shape instead of silently clearing it', async () => {
|
||||
const calls = [];
|
||||
const service = createHermesAgentService({
|
||||
config: configured(),
|
||||
randomToken: () => 'ledger-cookie',
|
||||
runTurn: async input => { calls.push(input); return { reply: 'should not be reached', sessionId: 'private-session' }; },
|
||||
});
|
||||
await service.unlock({ origin, accessCode: 'test-agent-access-code-2026' });
|
||||
|
||||
// Issue #21: a browser-supplied ledger whose symptom is reported but not the literal boolean
|
||||
// `true` used to be filtered out of the sanitized context, which cleared the symptom before the
|
||||
// escalation check ever saw it.
|
||||
const truthy = await service.chat({ origin, cookieToken: 'ledger-cookie', payload: {
|
||||
message: 'What do my logs show?',
|
||||
ledger: [{ bristolType: 4, color: 'brown', note: '', symptoms: { blood: 'yes' } }],
|
||||
} });
|
||||
assert.equal(calls.length, 0, 'a reported symptom must be intercepted before Hermes');
|
||||
assert.equal(truthy.safetyOverride, true);
|
||||
assert.match(truthy.reply, /medical help/i);
|
||||
|
||||
const cleared = await service.chat({ origin, cookieToken: 'ledger-cookie', payload: {
|
||||
message: 'What do my logs show?',
|
||||
ledger: [{ bristolType: 4, color: 'brown', note: '', symptoms: { blood: 'no', fever: false } }],
|
||||
} });
|
||||
assert.equal(cleared.safetyOverride, undefined);
|
||||
assert.equal(calls.length, 1);
|
||||
assert.match(calls[0].prompt, /"symptoms":\{\}/);
|
||||
});
|
||||
|
||||
test('Hermes CLI parser accepts quiet output, strips control sequences, and rejects leaked metadata', () => {
|
||||
const parsed = parseHermesCliOutput('\u001b[32msession_id: 20260820_abcd\u001b[0m\nA concise answer.');
|
||||
assert.deepEqual(parsed, { sessionId: '20260820_abcd', reply: 'A concise answer.' });
|
||||
|
||||
@@ -71,6 +71,26 @@ test('release demo shows the explainable pattern summary and follows a citation
|
||||
assert.match(builder, /"longitudinal_pattern_summary": "passed"/);
|
||||
});
|
||||
|
||||
test('release demo visibly exercises the red-flag language coverage instead of one canned word', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
|
||||
// Issue #21: the escalation has to fire for real phrasing, on both surfaces the detector serves.
|
||||
assert.match(demo, /const expectEscalation = async \(text, label\)/);
|
||||
assert.match(demo, /expectEscalation\('There is blood when I wipe', 'bleeding reported as wiping'\)/);
|
||||
assert.match(demo, /expectEscalation\('I have tarry stools', 'tarry stool'\)/);
|
||||
assert.match(demo, /note: 'stool was tarry yesterday'/);
|
||||
assert.match(demo, /a tarry note in the journal must escalate the next answer/);
|
||||
// Each scene is asserted in the browser, not narrated: one new bubble, the app's own copy, no provider call.
|
||||
assert.match(demo, /expected exactly one escalation bubble/);
|
||||
assert.match(demo, /the escalation copy must be the app's own fixed string/);
|
||||
assert.match(demo, /an escalated turn must not reach Hermes/);
|
||||
// The new captions are required release gates, so a later release cannot drop the scenes silently.
|
||||
assert.match(builder, /Bleeding reported as wiping still escalates, in the app’s own fixed words/);
|
||||
assert.match(builder, /Tarry and black stool wording escalates too: one red flag, one unchanging message/);
|
||||
assert.match(builder, /A tarry note already saved in the journal escalates the same way, before any provider call/);
|
||||
});
|
||||
|
||||
test('release builder gates the sleek shell, Hermes chat, and bootstrap syntax', async () => {
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
assert.match(builder, /"test:sleek"/);
|
||||
|
||||
Reference in new issue
Block a user