timmy
b4be7049ea
Quality gates / quality (pull_request) Successful in 2m38s
Deterministic escalation only fired on a narrow set of literal phrases, so real reports like "there is blood when I wipe" or "I have tarry stools" reached the chat provider without the fixed medical guidance. Expanding the suite found 18 of 18 checked clinical phrasings missing. - domain: cover bleeding reported by wiping/toilet paper/anatomy/mixed-in phrasing, black-tarry-dark-red stool, reported temperature and high temperature, abdominal-scoped severe pain in every natural word order, and past-tense/slang inability to pass gas. Near-miss wording that must stay quiet (paper cut, blood pressure, black beans, dark red wine, a normal temperature, knee or back pain, passed gas, no gas in the car) is pinned. - domain: a symptom reported in any truthy shape (1, "yes", "true") now counts as reported. Reading only the literal boolean `true` silently cleared a symptom, which is the one direction this boundary may never fail in. - hermes-agent-service: the gateway ledger sanitizer used the same strict check, so a browser-supplied ledger entry could have its symptom cleared before the escalation check. - tests/escalation-regression.test.js: per-flag positive and negative language coverage, model output can neither set nor clear a symptom/urgency/discomfort/note, and the escalation copy is byte-identical for every flag and under a missing, throwing, or malicious provider. Registered in the default unit gate. - scripts/record_release_demo.mjs: the recording now types the new bleeding and tarry phrasing, and seeds a fictional ledger whose note says "stool was tarry yesterday", then asserts in the browser that exactly one new Timmy bubble carries the app's own copy and that the turn never reaches Hermes. Three new captions are required release gates. Boundary unchanged: no new inference, no diagnosis, no image claim. Escalation only ever gets broader on user-reported symptoms.
246 lines
15 KiB
JavaScript
246 lines
15 KiB
JavaScript
import test from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import { readFile } from 'node:fs/promises';
|
||
|
||
const demoPath = new URL('../scripts/record_release_demo.mjs', import.meta.url);
|
||
const builderPath = new URL('../scripts/build_release.py', import.meta.url);
|
||
|
||
test('release demo visibly explains the CI-protected browser path without overstating safety', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /Automated checks replay this synthetic path before review/);
|
||
assert.match(demo, /One clear photo action\. Manual logging stays one tap away\./);
|
||
assert.match(demo, /Camera closed cleanly — gallery and manual logging are still available/);
|
||
assert.match(demo, /#camera-photo.*dispatchEvent\('cancel'\)/s);
|
||
assert.match(demo, /#gallery-photo.*synthetic-type4\.jpg/s);
|
||
assert.match(demo, /tests\/fixtures\/synthetic-type4\.jpg/);
|
||
assert.match(demo, /The pinned bootstrap verifies both model files before starting on private loopback/);
|
||
assert.match(demo, /AI may suggest visible form, broad color, and image quality — never symptoms or diagnosis/);
|
||
assert.match(demo, /Hermes Agent connected/);
|
||
assert.match(demo, /Photos stay out of chat/);
|
||
assert.match(demo, /I barfed/);
|
||
assert.match(demo, /Urgent language is intercepted deterministically before Hermes/);
|
||
assert.match(demo, /SLEEK\. SIMPLE\.<br>HERMES-POWERED\./);
|
||
});
|
||
|
||
test('release demo exercises the vision-offline manual path and the retained draft', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /Vision worker offline — the AI and consent steps disappear instead of failing after upload/);
|
||
assert.match(demo, /No consent prompt and no upload when analysis cannot run/);
|
||
assert.match(demo, /One tap still reaches manual Bristol entry/);
|
||
assert.match(demo, /Closing the log keeps the draft in this session instead of throwing it away/);
|
||
assert.match(demo, /Start over clears the kept draft — nothing lingers silently/);
|
||
assert.match(demo, /#manual-from-scan/);
|
||
assert.match(demo, /#discard-draft/);
|
||
assert.match(demo, /\.draft-resume/);
|
||
// The offline pass must be a real state change driven by the app's own readiness contract.
|
||
assert.match(demo, /providerReady: false|visionOffline/);
|
||
});
|
||
|
||
test('release demo shows the provisional, confirmed, and corrected suggestion states on camera', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /\[data-provenance="pending"\]/);
|
||
assert.match(demo, /The suggestion stays provisional — nothing is applied or saved until you confirm it/);
|
||
assert.match(demo, /Confirming is explicit, and every suggested value is still yours to overrule/);
|
||
assert.match(demo, /Change a suggested value and Timmy clears the AI-confirmed state/);
|
||
assert.match(demo, /\[data-provenance="corrected"\]/);
|
||
assert.match(demo, /Only visible form and color were ever suggested — urgency, discomfort, and symptoms stay yours/);
|
||
assert.match(demo, /Type 2/, 'the retained draft in the demo must show the value the user corrected to');
|
||
});
|
||
|
||
test('release demo shows the explainable pattern summary and follows a citation to its own records', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #34: the recording must exercise the summary, its citation sheet, and the sparse case.
|
||
assert.match(demo, /\[data-pattern-summary\]/);
|
||
assert.match(demo, /\[data-statement-link="typical"\]/);
|
||
assert.match(demo, /\[data-summary-record\]/);
|
||
assert.match(demo, /\[data-summary-insufficient\]/);
|
||
assert.match(demo, /The summary says which confirmed logs it read, and a citation opens exactly those records/);
|
||
assert.match(demo, /Every pattern line names the confirmed logs behind it, and following one shows exactly those records/);
|
||
assert.match(demo, /Fictional sample logs are seeded into this browser's own storage — no real health data appears here/);
|
||
assert.match(demo, /Two logs is not a pattern: the summary refuses to describe a trend it cannot support/);
|
||
// The cited records are asserted in the browser, not narrated.
|
||
assert.match(demo, /must list exactly the records it cites/);
|
||
// Both captions are required release gates, so a later release cannot drop the scene silently.
|
||
assert.match(builder, /Every pattern line names the confirmed logs behind it, and following one shows exactly those records/);
|
||
assert.match(builder, /Two logs is not a pattern: the summary refuses to describe a trend it cannot support/);
|
||
assert.match(builder, /"longitudinal_pattern_summary": "passed"/);
|
||
});
|
||
|
||
test('release demo visibly exercises the red-flag language coverage instead of one canned word', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #21: the escalation has to fire for real phrasing, on both surfaces the detector serves.
|
||
assert.match(demo, /const expectEscalation = async \(text, label\)/);
|
||
assert.match(demo, /expectEscalation\('There is blood when I wipe', 'bleeding reported as wiping'\)/);
|
||
assert.match(demo, /expectEscalation\('I have tarry stools', 'tarry stool'\)/);
|
||
assert.match(demo, /note: 'stool was tarry yesterday'/);
|
||
assert.match(demo, /a tarry note in the journal must escalate the next answer/);
|
||
// Each scene is asserted in the browser, not narrated: one new bubble, the app's own copy, no provider call.
|
||
assert.match(demo, /expected exactly one escalation bubble/);
|
||
assert.match(demo, /the escalation copy must be the app's own fixed string/);
|
||
assert.match(demo, /an escalated turn must not reach Hermes/);
|
||
// The new captions are required release gates, so a later release cannot drop the scenes silently.
|
||
assert.match(builder, /Bleeding reported as wiping still escalates, in the app’s own fixed words/);
|
||
assert.match(builder, /Tarry and black stool wording escalates too: one red flag, one unchanging message/);
|
||
assert.match(builder, /A tarry note already saved in the journal escalates the same way, before any provider call/);
|
||
});
|
||
|
||
test('release builder gates the sleek shell, Hermes chat, and bootstrap syntax', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
assert.match(builder, /"test:sleek"/);
|
||
assert.match(builder, /"test:mobile-capture"/);
|
||
assert.match(builder, /"test:manual-fallback"/);
|
||
assert.match(builder, /Manual fallback: vision-offline logging stays one action away with an in-session draft/);
|
||
assert.match(builder, /"manual_fallback_draft_recovery": "passed"/);
|
||
assert.match(builder, /"src\/draft\.js"/);
|
||
assert.match(builder, /"src\/vision-readiness\.js"/);
|
||
assert.match(builder, /offline vision status/);
|
||
assert.match(builder, /"sleek_hermes_chat_acceptance": "passed"/);
|
||
assert.match(builder, /Sleek three-destination shell/);
|
||
assert.match(builder, /"bash", "-n", "scripts\/bootstrap_selfhost_smolvlm\.sh"/);
|
||
});
|
||
|
||
test('release builder gates the provisional suggestion states the same way CI does', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
assert.match(builder, /"test:provisional"/);
|
||
assert.match(builder, /"provisional_suggestion_state": "passed"/);
|
||
assert.match(builder, /"src\/suggestion\.js"/);
|
||
assert.match(builder, /AI suggestions stay provisional/);
|
||
});
|
||
|
||
test('release demo visibly exercises the user-reported hard-stool follow-up and its red-flag handoff', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /\[data-followup-block\]/);
|
||
assert.match(demo, /Type 2 is hard or lumpy, so Timmy asks about soreness, hydration, blood, and black-or-tarry — never guessed from the photo/);
|
||
assert.match(demo, /\[data-followup="bloodVisible"\]\[data-answer="no"\]/);
|
||
assert.match(demo, /Every answer is yours to give, and a skipped question stays unanswered/);
|
||
assert.match(demo, /#followup-guidance/);
|
||
assert.match(demo, /Mild report: steady water, no straining, gentle care — general self-care, not a diagnosis/);
|
||
assert.match(demo, /And it never claims the photo ruled out bleeding/);
|
||
assert.match(demo, /\[data-followup="tarryBlack"\]\[data-answer="yes"\]/);
|
||
assert.match(demo, /Say it was truly black and tarry and the same deterministic red-flag path fires instead/);
|
||
assert.match(demo, /Dark brown is not silently escalated, and the flag clears the moment the answer changes/);
|
||
});
|
||
|
||
test('release builder gates the hard-stool follow-up the same way CI does', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
assert.match(builder, /"test:followup"/);
|
||
assert.match(builder, /"hard_stool_followup": "passed"/);
|
||
assert.match(builder, /"src\/followup\.js"/);
|
||
assert.match(builder, /Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions/);
|
||
});
|
||
|
||
test('release demo states the declared release delta on camera and proves it with a receipt', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
// The delta comes from the builder, so a documentation-only release cannot imply a new feature.
|
||
assert.match(demo, /TIMMY_RELEASE_DEMO_DELTA/);
|
||
assert.match(demo, /This release: \$\{releaseDelta\}/);
|
||
assert.match(demo, /no new feature is claimed/);
|
||
// A caption alone is not evidence; the demo must emit a machine-readable receipt of what rendered.
|
||
assert.match(demo, /TIMMY_DEMO_CAPTIONS_OUT/);
|
||
assert.match(demo, /captionsShown\.push\(text\)/);
|
||
assert.match(demo, /writeFile\(captionsOut/);
|
||
});
|
||
|
||
test('release demo shows the byte-bearing controls the threat model governs, without inventing UI', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /\[data-view="privacy"\]/);
|
||
assert.match(demo, /Every byte-bearing control the model names/);
|
||
assert.match(demo, /indicate\('#export'\)/);
|
||
assert.match(demo, /indicate\('#delete-all'\)/);
|
||
assert.match(demo, /Delete all local data clears the ledger in this browser; there is no server copy of it to keep/);
|
||
// Real controls only: the selectors must exist in the app shell, not be demo-only scaffolding.
|
||
const app = await readFile(new URL('../app.js', import.meta.url), 'utf8');
|
||
assert.match(app, /id="export"/);
|
||
assert.match(app, /id="delete-all"/);
|
||
});
|
||
|
||
test('release builder refuses to publish without the on-camera delta receipt', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
assert.match(builder, /TIMMY_RELEASE_DELTA/);
|
||
assert.match(builder, /TIMMY_DEMO_CAPTIONS_OUT/);
|
||
assert.match(builder, /Release demo produced no on-camera caption receipt/);
|
||
assert.match(builder, /Release demo did not state the declared release delta on camera/);
|
||
assert.match(builder, /Release demo is missing a required on-camera caption/);
|
||
assert.match(builder, /"release_delta_visible_on_camera": "passed"/);
|
||
assert.match(builder, /"release_delta": release_delta/);
|
||
});
|
||
|
||
test('release demo captions expire and the contact sheet spans the whole recording', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// A caption that outlives its stated duration hangs the previous view's claim over the next screen.
|
||
assert.match(demo, /creates via removal after duration|A caption must not outlive its stated duration/);
|
||
assert.match(demo, /setTimeout\(\(\) => node\.remove\(\), 220\)/);
|
||
// The sheet must cover the entire demo, not just its opening.
|
||
assert.match(builder, /12 \/ demo_duration/);
|
||
assert.match(builder, /Release demo contact sheet was not produced/);
|
||
});
|
||
|
||
test('release demo visibly exercises the corpus refusal paths instead of only the happy path', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// The refusal responses must come from the recorded corpus and the real analysis code path,
|
||
// never from copy pasted into the recording script.
|
||
assert.match(demo, /loadVisionCorpus/);
|
||
assert.match(demo, /visionFailureResponse/);
|
||
assert.match(demo, /corpusAnalyzeResult/);
|
||
assert.match(demo, /tests\/fixtures\/vision-corpus\/non-stool-scene\.png/);
|
||
assert.match(demo, /tests\/fixtures\/vision-corpus\/out-of-range-fields\.png/);
|
||
assert.match(demo, /getByText\(\/No confident match\/i\)/);
|
||
assert.match(demo, /couldn’t analyze that safely/i);
|
||
|
||
assert.match(demo, /A non-stool photo gets a refusal, never a guess/);
|
||
assert.match(demo, /Unreadable visual fields abstain as well/);
|
||
assert.match(demo, /An out-of-range answer fails closed/);
|
||
assert.match(demo, /no real photos in the corpus|no real photos in the recording/);
|
||
// The refusal copy the user sees is gated on camera, so a release cannot drop it silently.
|
||
assert.match(builder, /An out-of-range answer fails closed/);
|
||
});
|
||
|
||
test('release demo shows a filled-in out-of-range field failing closed even beside a blank one', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #73's escape class, replayed on camera from the committed corpus rather than described.
|
||
assert.match(demo, /corpusCase\('blank-plus-out-of-range'\)/);
|
||
assert.match(demo, /tests\/fixtures\/vision-corpus\/blank-plus-out-of-range\.png/);
|
||
assert.match(demo, /A filled-in field that is out of range still fails closed, even when the other field came back blank/);
|
||
assert.match(demo, /no longer told the photo was unreadable when the model returned an impossible value/);
|
||
// The extra fail-closed scene is a real 503, so the demo's error budget must count it.
|
||
assert.match(demo, /expectedFailClosed\.length !== 2/);
|
||
// The caption is a required gate, so a later release cannot silently drop the scene.
|
||
assert.match(builder, /A filled-in field that is out of range still fails closed/);
|
||
});
|
||
|
||
test('release demo shows readiness waiting for the loaded pinned model before AI is offered', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #15: reachable is not loaded, and the recording must prove the app withholds AI.
|
||
assert.match(demo, /modelLoaded = false/);
|
||
assert.match(demo, /getByText\(\/model is not loaded yet\/i\)/);
|
||
assert.match(demo, /model-status\.loading/);
|
||
assert.match(demo, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/);
|
||
assert.match(demo, /No AI control may exist before the model is loaded/);
|
||
assert.match(demo, /Once the pinned model is loaded, the same screen offers the AI analysis again/);
|
||
// The scene is asserted in the browser, not narrated: the demo throws if the state is wrong.
|
||
assert.match(demo, /must never be reported ready/);
|
||
// The caption is a required release gate, so the scene cannot be dropped silently.
|
||
assert.match(builder, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/);
|
||
});
|