timmy
ef22ea8970
Quality gates / quality (pull_request) Successful in 3m44s
A journal save now records no training contribution. A contribution is a second consent: the privacy page states consent version training-consent-v1 and the receipt (contribution id, included fields, never-included fields) before the confirm control is enabled, the receipt is queued on this device only and copies no photo bytes, note text, or symptom answers, and withdrawal keeps a tombstone naming every derived artifact by contribution id. Delete-all clears the receipts with the ledger. Closes #24
346 lines
21 KiB
JavaScript
346 lines
21 KiB
JavaScript
import test from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import { readFile } from 'node:fs/promises';
|
||
|
||
const demoPath = new URL('../scripts/record_release_demo.mjs', import.meta.url);
|
||
const builderPath = new URL('../scripts/build_release.py', import.meta.url);
|
||
|
||
test('release demo visibly explains the CI-protected browser path without overstating safety', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /Automated checks replay this synthetic path before review/);
|
||
assert.match(demo, /One clear photo action\. Manual logging stays one tap away\./);
|
||
assert.match(demo, /Camera closed cleanly — gallery and manual logging are still available/);
|
||
assert.match(demo, /#camera-photo.*dispatchEvent\('cancel'\)/s);
|
||
assert.match(demo, /#gallery-photo.*synthetic-type4\.jpg/s);
|
||
assert.match(demo, /tests\/fixtures\/synthetic-type4\.jpg/);
|
||
assert.match(demo, /The pinned bootstrap verifies both model files before starting on private loopback/);
|
||
assert.match(demo, /AI may suggest visible form, broad color, and image quality — never symptoms or diagnosis/);
|
||
assert.match(demo, /Hermes Agent connected/);
|
||
assert.match(demo, /Photos stay out of chat/);
|
||
assert.match(demo, /I barfed/);
|
||
assert.match(demo, /Urgent language is intercepted deterministically before Hermes/);
|
||
assert.match(demo, /SLEEK\. SIMPLE\.<br>HERMES-POWERED\./);
|
||
});
|
||
|
||
test('release demo exercises the vision-offline manual path and the retained draft', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /Vision worker offline — the AI and consent steps disappear instead of failing after upload/);
|
||
assert.match(demo, /No consent prompt and no upload when analysis cannot run/);
|
||
assert.match(demo, /One tap still reaches manual Bristol entry/);
|
||
assert.match(demo, /Closing the log keeps the draft in this session instead of throwing it away/);
|
||
assert.match(demo, /Start over clears the kept draft — nothing lingers silently/);
|
||
assert.match(demo, /#manual-from-scan/);
|
||
assert.match(demo, /#discard-draft/);
|
||
assert.match(demo, /\.draft-resume/);
|
||
// The offline pass must be a real state change driven by the app's own readiness contract.
|
||
assert.match(demo, /providerReady: false|visionOffline/);
|
||
});
|
||
|
||
test('release demo shows the provisional, confirmed, and corrected suggestion states on camera', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /\[data-provenance="pending"\]/);
|
||
assert.match(demo, /The suggestion stays provisional — nothing is applied or saved until you confirm it/);
|
||
assert.match(demo, /Confirming is explicit, and every suggested value is still yours to overrule/);
|
||
assert.match(demo, /Change a suggested value and Timmy clears the AI-confirmed state/);
|
||
assert.match(demo, /\[data-provenance="corrected"\]/);
|
||
assert.match(demo, /Only visible form and color were ever suggested — urgency, discomfort, and symptoms stay yours/);
|
||
assert.match(demo, /Type 2/, 'the retained draft in the demo must show the value the user corrected to');
|
||
});
|
||
|
||
test('release demo shows the explainable pattern summary and follows a citation to its own records', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #34: the recording must exercise the summary, its citation sheet, and the sparse case.
|
||
assert.match(demo, /\[data-pattern-summary\]/);
|
||
assert.match(demo, /\[data-statement-link="typical"\]/);
|
||
assert.match(demo, /\[data-summary-record\]/);
|
||
assert.match(demo, /\[data-summary-insufficient\]/);
|
||
assert.match(demo, /The summary says which confirmed logs it read, and a citation opens exactly those records/);
|
||
assert.match(demo, /Every pattern line names the confirmed logs behind it, and following one shows exactly those records/);
|
||
assert.match(demo, /Fictional sample logs are seeded into this browser's own storage — no real health data appears here/);
|
||
assert.match(demo, /Two logs is not a pattern: the summary refuses to describe a trend it cannot support/);
|
||
// The cited records are asserted in the browser, not narrated.
|
||
assert.match(demo, /must list exactly the records it cites/);
|
||
// Both captions are required release gates, so a later release cannot drop the scene silently.
|
||
assert.match(builder, /Every pattern line names the confirmed logs behind it, and following one shows exactly those records/);
|
||
assert.match(builder, /Two logs is not a pattern: the summary refuses to describe a trend it cannot support/);
|
||
assert.match(builder, /"longitudinal_pattern_summary": "passed"/);
|
||
});
|
||
|
||
test('release demo shows the chat answering from cited records and refusing what it cannot cite', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #37: the recording must exercise the grounded local answer, its citation sheet, the
|
||
// refusal, and the Hermes reply whose refs resolve to the same records.
|
||
assert.match(demo, /\[data-chat-citation\]/);
|
||
assert.match(demo, /\[data-chat-record\]/);
|
||
assert.match(demo, /\[data-chat-record-sheet\]/);
|
||
assert.match(demo, /A grounded chat answer must cite every record it counted/);
|
||
assert.match(demo, /A chat citation must open exactly the one record it names/);
|
||
assert.match(demo, /A refusal must not borrow journal citations/);
|
||
assert.match(demo, /The answer states the count and cites the four confirmed logs it counted/);
|
||
assert.match(demo, /Following a citation opens exactly the confirmed log behind that answer/);
|
||
assert.match(demo, /Ask something the journal cannot answer and Timmy refuses instead of reassuring/);
|
||
// The local surface is a real state toggle, not narrated: the agent route is flipped off on camera.
|
||
assert.match(demo, /agentConnected = false/);
|
||
assert.match(demo, /entry:L1/);
|
||
assert.match(builder, /cites the four confirmed logs it counted/);
|
||
assert.match(builder, /"grounded_chat_citations": "passed"/);
|
||
});
|
||
|
||
test('release demo visibly exercises the red-flag language coverage instead of one canned word', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #21: the escalation has to fire for real phrasing, on both surfaces the detector serves.
|
||
assert.match(demo, /const expectEscalation = async \(text, label\)/);
|
||
assert.match(demo, /expectEscalation\('There is blood when I wipe', 'bleeding reported as wiping'\)/);
|
||
assert.match(demo, /expectEscalation\('I have tarry stools', 'tarry stool'\)/);
|
||
assert.match(demo, /note: 'stool was tarry yesterday'/);
|
||
assert.match(demo, /a tarry note in the journal must escalate the next answer/);
|
||
// Each scene is asserted in the browser, not narrated: one new bubble, the app's own copy, no provider call.
|
||
assert.match(demo, /expected exactly one escalation bubble/);
|
||
assert.match(demo, /the escalation copy must be the app's own fixed string/);
|
||
assert.match(demo, /an escalated turn must not reach Hermes/);
|
||
// The new captions are required release gates, so a later release cannot drop the scenes silently.
|
||
assert.match(builder, /Bleeding reported as wiping still escalates, in the app’s own fixed words/);
|
||
assert.match(builder, /Tarry and black stool wording escalates too: one red flag, one unchanging message/);
|
||
assert.match(builder, /A tarry note already saved in the journal escalates the same way, before any provider call/);
|
||
});
|
||
|
||
test('release builder gates the sleek shell, Hermes chat, and bootstrap syntax', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
assert.match(builder, /"test:sleek"/);
|
||
assert.match(builder, /"test:mobile-capture"/);
|
||
assert.match(builder, /"test:manual-fallback"/);
|
||
assert.match(builder, /Manual fallback: vision-offline logging stays one action away with an in-session draft/);
|
||
assert.match(builder, /"manual_fallback_draft_recovery": "passed"/);
|
||
assert.match(builder, /"src\/draft\.js"/);
|
||
assert.match(builder, /"src\/vision-readiness\.js"/);
|
||
assert.match(builder, /offline vision status/);
|
||
assert.match(builder, /"sleek_hermes_chat_acceptance": "passed"/);
|
||
assert.match(builder, /Sleek three-destination shell/);
|
||
assert.match(builder, /"bash", "-n", "scripts\/bootstrap_selfhost_smolvlm\.sh"/);
|
||
});
|
||
|
||
test('release builder gates the provisional suggestion states the same way CI does', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
assert.match(builder, /"test:provisional"/);
|
||
assert.match(builder, /"provisional_suggestion_state": "passed"/);
|
||
assert.match(builder, /"src\/suggestion\.js"/);
|
||
assert.match(builder, /AI suggestions stay provisional/);
|
||
});
|
||
|
||
test('release demo visibly exercises the user-reported hard-stool follow-up and its red-flag handoff', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /\[data-followup-block\]/);
|
||
assert.match(demo, /Type 2 is hard or lumpy, so Timmy asks about soreness, hydration, blood, and black-or-tarry — never guessed from the photo/);
|
||
assert.match(demo, /\[data-followup="bloodVisible"\]\[data-answer="no"\]/);
|
||
assert.match(demo, /Every answer is yours to give, and a skipped question stays unanswered/);
|
||
assert.match(demo, /#followup-guidance/);
|
||
assert.match(demo, /Mild report: steady water, no straining, gentle care — general self-care, not a diagnosis/);
|
||
assert.match(demo, /And it never claims the photo ruled out bleeding/);
|
||
assert.match(demo, /\[data-followup="tarryBlack"\]\[data-answer="yes"\]/);
|
||
assert.match(demo, /Say it was truly black and tarry and the same deterministic red-flag path fires instead/);
|
||
assert.match(demo, /Dark brown is not silently escalated, and the flag clears the moment the answer changes/);
|
||
});
|
||
|
||
test('release builder gates the hard-stool follow-up the same way CI does', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
assert.match(builder, /"test:followup"/);
|
||
assert.match(builder, /"hard_stool_followup": "passed"/);
|
||
assert.match(builder, /"src\/followup\.js"/);
|
||
assert.match(builder, /Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions/);
|
||
});
|
||
|
||
test('release demo states the declared release delta on camera and proves it with a receipt', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
// The delta comes from the builder, so a documentation-only release cannot imply a new feature.
|
||
assert.match(demo, /TIMMY_RELEASE_DEMO_DELTA/);
|
||
assert.match(demo, /This release: \$\{releaseDelta\}/);
|
||
assert.match(demo, /no new feature is claimed/);
|
||
// A caption string alone is not evidence; each caption must be measured as a painted bar, and the
|
||
// receipt must carry that per-caption measurement.
|
||
assert.match(demo, /TIMMY_DEMO_CAPTIONS_OUT/);
|
||
assert.match(demo, /captionsShown\.push\(text\)/);
|
||
assert.match(demo, /captionsVerified: captionEvidence/);
|
||
assert.match(demo, /Caption is not an on-screen bar/);
|
||
assert.match(demo, /Caption painted nothing into the caption band/);
|
||
assert.match(demo, /writeFile\(captionsOut/);
|
||
});
|
||
|
||
test('release demo shows the byte-bearing controls the threat model governs, without inventing UI', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /\[data-view="privacy"\]/);
|
||
assert.match(demo, /Every byte-bearing control the model names/);
|
||
assert.match(demo, /indicate\('#export'\)/);
|
||
assert.match(demo, /indicate\('#delete-all'\)/);
|
||
assert.match(demo, /Delete all local data clears the ledger and every saved photo in this browser; nothing is kept on a server/);
|
||
// Real controls only: the selectors must exist in the app shell, not be demo-only scaffolding.
|
||
const app = await readFile(new URL('../app.js', import.meta.url), 'utf8');
|
||
assert.match(app, /id="export"/);
|
||
assert.match(app, /id="delete-all"/);
|
||
});
|
||
|
||
test('release demo exercises the photo lifecycle on camera without showing an image by default', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
// Presence first: a label in the journal, with no thumbnail rendered.
|
||
assert.match(demo, /Photo presence is a label, not a thumbnail/);
|
||
assert.match(demo, /\.entries-card \[data-photo-indicator="present"\]/);
|
||
assert.match(demo, /exactly one log must be marked as holding a photo/);
|
||
assert.match(demo, /the journal must not show a thumbnail by default/);
|
||
// Revealing the image is an explicit request, asserted before the caption that claims it.
|
||
assert.match(demo, /opening a log must not reveal its photo/);
|
||
assert.match(demo, /#show-entry-photo/);
|
||
assert.match(demo, /Showing the photo is your explicit request/);
|
||
// Removing the photo keeps every confirmed field of the log.
|
||
assert.match(demo, /#remove-entry-photo/);
|
||
assert.match(demo, /Remove just the photo: the confirmed log, form, and note are all still here/);
|
||
assert.match(demo, /removing a photo must keep the confirmed log and clear only its image bytes/);
|
||
// Delete-all is actually performed, not just pointed at.
|
||
assert.match(demo, /tap\('#delete-all'/);
|
||
assert.match(demo, /no photo bytes may survive delete-all|JSON\.stringify\(localStorage\)\.includes\('data:image'\)/);
|
||
// Synthetic bytes only: a committed test-card PNG constant, never a real photo. A blank 1x1 pixel
|
||
// rendered as an empty white box on camera, which read as a broken image instead of a saved photo.
|
||
assert.match(demo, /const SYNTHETIC_PHOTO = 'data:image\/png;base64,/);
|
||
assert.match(demo, /synthetic test card/);
|
||
});
|
||
|
||
test('release builder refuses to publish without the on-camera delta receipt', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
assert.match(builder, /TIMMY_RELEASE_DELTA/);
|
||
assert.match(builder, /TIMMY_DEMO_CAPTIONS_OUT/);
|
||
assert.match(builder, /Release demo produced no on-camera caption receipt/);
|
||
assert.match(builder, /Release demo did not state the declared release delta on camera/);
|
||
assert.match(builder, /Release demo is missing a required on-camera caption/);
|
||
assert.match(builder, /Release demo caption never painted on screen/);
|
||
assert.match(builder, /recorded\.get\("captionsVerified"\)/);
|
||
assert.match(builder, /"release_delta_visible_on_camera": "passed"/);
|
||
assert.match(builder, /"release_delta": release_delta/);
|
||
});
|
||
|
||
test('caption styling survives the demo reloads instead of dying with the first document', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
// A stylesheet belongs to one document; the demo reloads the page, which stripped caption and touch
|
||
// styling and left later captions as clipped body text. Both now carry inline styles per node.
|
||
assert.doesNotMatch(demo, /page\.addStyleTag/);
|
||
assert.doesNotMatch(demo, /addInitScript/);
|
||
assert.match(demo, /In a stylesheet belongs to one document|a stylesheet belongs to one document/);
|
||
assert.match(demo, /node\.style\.cssText = 'position:fixed;left:16px;right:16px;top:14px;z-index:20000;'/);
|
||
assert.match(demo, /ring\.style\.cssText = 'position:fixed;z-index:20001/);
|
||
});
|
||
|
||
test('release demo captions expire and the contact sheet spans the whole recording', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// A caption that outlives its stated duration hangs the previous view's claim over the next screen.
|
||
assert.match(demo, /creates via removal after duration|A caption must not outlive its stated duration/);
|
||
assert.match(demo, /setTimeout\(\(\) => node\.remove\(\), 220\)/);
|
||
// The sheet must cover the entire demo, not just its opening.
|
||
assert.match(builder, /12 \/ demo_duration/);
|
||
assert.match(builder, /Release demo contact sheet was not produced/);
|
||
});
|
||
|
||
test('release demo visibly exercises the corpus refusal paths instead of only the happy path', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// The refusal responses must come from the recorded corpus and the real analysis code path,
|
||
// never from copy pasted into the recording script.
|
||
assert.match(demo, /loadVisionCorpus/);
|
||
assert.match(demo, /visionFailureResponse/);
|
||
assert.match(demo, /corpusAnalyzeResult/);
|
||
assert.match(demo, /tests\/fixtures\/vision-corpus\/non-stool-scene\.png/);
|
||
assert.match(demo, /tests\/fixtures\/vision-corpus\/out-of-range-fields\.png/);
|
||
assert.match(demo, /getByText\(\/No confident match\/i\)/);
|
||
assert.match(demo, /couldn’t analyze that safely/i);
|
||
|
||
assert.match(demo, /A non-stool photo gets a refusal, never a guess/);
|
||
assert.match(demo, /Unreadable visual fields abstain as well/);
|
||
assert.match(demo, /An out-of-range answer fails closed/);
|
||
assert.match(demo, /no real photos in the corpus|no real photos in the recording/);
|
||
// The refusal copy the user sees is gated on camera, so a release cannot drop it silently.
|
||
assert.match(builder, /An out-of-range answer fails closed/);
|
||
});
|
||
|
||
test('release demo shows a filled-in out-of-range field failing closed even beside a blank one', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #73's escape class, replayed on camera from the committed corpus rather than described.
|
||
assert.match(demo, /corpusCase\('blank-plus-out-of-range'\)/);
|
||
assert.match(demo, /tests\/fixtures\/vision-corpus\/blank-plus-out-of-range\.png/);
|
||
assert.match(demo, /A filled-in field that is out of range still fails closed, even when the other field came back blank/);
|
||
assert.match(demo, /no longer told the photo was unreadable when the model returned an impossible value/);
|
||
// The extra fail-closed scene is a real 503, so the demo's error budget must count it.
|
||
assert.match(demo, /expectedFailClosed\.length !== 2/);
|
||
// The caption is a required gate, so a later release cannot silently drop the scene.
|
||
assert.match(builder, /A filled-in field that is out of range still fails closed/);
|
||
});
|
||
|
||
test('release demo shows readiness waiting for the loaded pinned model before AI is offered', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #15: reachable is not loaded, and the recording must prove the app withholds AI.
|
||
assert.match(demo, /modelLoaded = false/);
|
||
assert.match(demo, /getByText\(\/model is not loaded yet\/i\)/);
|
||
assert.match(demo, /model-status\.loading/);
|
||
assert.match(demo, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/);
|
||
assert.match(demo, /No AI control may exist before the model is loaded/);
|
||
assert.match(demo, /Once the pinned model is loaded, the same screen offers the AI analysis again/);
|
||
// The scene is asserted in the browser, not narrated: the demo throws if the state is wrong.
|
||
assert.match(demo, /must never be reported ready/);
|
||
// The caption is a required release gate, so the scene cannot be dropped silently.
|
||
assert.match(builder, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/);
|
||
});
|
||
|
||
test('release demo shows the training contribution as a separate opt-in receipt that can be withdrawn', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
const app = await readFile(new URL('../app.js', import.meta.url), 'utf8');
|
||
|
||
// Issue #24: the recording must show the opt-in section, the consent gate, and the withdrawal.
|
||
assert.match(demo, /\[data-training-consent\]/);
|
||
assert.match(demo, /\[data-contribute-entry="demo-wipe"\]/);
|
||
assert.match(demo, /#training-consent-check/);
|
||
assert.match(demo, /#contribute-confirm/);
|
||
assert.match(demo, /\[data-contribution-state="queued-local"\]/);
|
||
assert.match(demo, /data-withdraw=/);
|
||
assert.match(demo, /\[data-withdrawn-artifacts\]/);
|
||
assert.match(demo, /training-consent-v1/);
|
||
// Every claim is measured in the browser before the caption that makes it.
|
||
assert.match(demo, /a journal save must not queue a training contribution/);
|
||
assert.match(demo, /the contribute control must stay disabled until the terms are accepted/);
|
||
assert.match(demo, /no receipt may exist before explicit consent/);
|
||
assert.match(demo, /a receipt must never copy the photo bytes/);
|
||
assert.match(demo, /every derived artifact must be addressed by contribution id/);
|
||
assert.match(demo, /delete-all must clear the contribution receipts as well/);
|
||
// The controls are the app's own, not demo-only scaffolding.
|
||
assert.match(app, /data-training-consent/);
|
||
assert.match(app, /data-contribution-queued-count/);
|
||
assert.match(app, /id="training-consent-check"/);
|
||
assert.match(app, /id="contribute-confirm" disabled/);
|
||
assert.match(app, /data-withdraw=/);
|
||
// The captions and the gate key are required, so a later release cannot drop the scenes silently.
|
||
assert.match(builder, /Training contribution is off by default: a saved log is never training data/);
|
||
assert.match(builder, /The consent version and the receipt are on screen before the contribute button works/);
|
||
assert.match(builder, /Withdrawing keeps a tombstone naming every derived artifact by contribution ID/);
|
||
assert.match(builder, /"training_contribution_consent": "passed"/);
|
||
assert.match(builder, /"test:training-contribution"/);
|
||
});
|