Files
timmy-talking-turd/tests/release-demo.test.js
T
timmy bc64055196
Quality gates / quality (pull_request) Canceled after 0s
Merge main into the #16 ingress branch and close what the RC gate caught
Brings #82 up to today's `main` (1bbcc10) and resolves the two conflicts as ordered
unions: `package.json` `test`/`check:syntax` and the release builder's syntax allow-list
and evidence keys. Both sides only appended, so nothing was dropped.

Verifying the merge against main's RC attack suite (#40) found two real defects, each
fixed test-first:

- An oversized body destroyed the socket mid-upload, so the caller saw `fetch failed`
  instead of the 413 it had already earned, and `security_attack_suite.mjs` could not
  complete at all. New test in `tests/analyze-ingress.acceptance.test.js` failed with
  `fetch failed` before the fix; `src/http-body.js` now keeps draining without buffering
  and refuses on `end`, so the ceiling still holds and the refusal is readable.
- #16's per-caller budget turned the suite's own load phase into 8/8 429s, which would
  have published refusals as analysis latency. The load phase now runs against a second
  permissive target and the suite refuses to publish a load figure if any sample was
  refused. `buildReport` also publishes `evidenceVerified`, so the verdict is
  re-derivable from the report — asserted by #40's acceptance test, which only trips once
  the verdict is GO.

Evidence on the merged tree: `npm test` 277/277, `test:security-attack` 8/8, the RC attack
suite returns VERDICT: GO with both P0 ingress cases passing (main is BLOCKED on exactly
those two), 14 browser acceptance suites green, and the release demo records end to end
(8:13, 97 captions) including the on-camera refusal scene.
2026-10-31 11:17:13 +00:00

561 lines
35 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import test from 'node:test';
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';
const demoPath = new URL('../scripts/record_release_demo.mjs', import.meta.url);
const builderPath = new URL('../scripts/build_release.py', import.meta.url);
test('release demo visibly explains the CI-protected browser path without overstating safety', async () => {
const demo = await readFile(demoPath, 'utf8');
assert.match(demo, /Automated checks replay this synthetic path before review/);
assert.match(demo, /One clear photo action\. Manual logging stays one tap away\./);
assert.match(demo, /Camera closed cleanly — gallery and manual logging are still available/);
assert.match(demo, /#camera-photo.*dispatchEvent\('cancel'\)/s);
assert.match(demo, /#gallery-photo.*synthetic-type4\.jpg/s);
assert.match(demo, /tests\/fixtures\/synthetic-type4\.jpg/);
assert.match(demo, /The pinned bootstrap verifies both model files before starting on private loopback/);
assert.match(demo, /AI may suggest visible form, broad color, and image quality — never symptoms or diagnosis/);
assert.match(demo, /Hermes Agent connected/);
assert.match(demo, /Photos stay out of chat/);
assert.match(demo, /I barfed/);
assert.match(demo, /Urgent language is intercepted deterministically before Hermes/);
assert.match(demo, /SLEEK\. SIMPLE\.<br>HERMES-POWERED\./);
});
test('release demo exercises the vision-offline manual path and the retained draft', async () => {
const demo = await readFile(demoPath, 'utf8');
assert.match(demo, /Vision worker offline — the AI and consent steps disappear instead of failing after upload/);
assert.match(demo, /No consent prompt and no upload when analysis cannot run/);
assert.match(demo, /One tap still reaches manual Bristol entry/);
assert.match(demo, /Closing the log keeps the draft in this session instead of throwing it away/);
assert.match(demo, /Start over clears the kept draft — nothing lingers silently/);
assert.match(demo, /#manual-from-scan/);
assert.match(demo, /#discard-draft/);
assert.match(demo, /\.draft-resume/);
// The offline pass must be a real state change driven by the app's own readiness contract.
assert.match(demo, /providerReady: false|visionOffline/);
});
test('release demo shows the provisional, confirmed, and corrected suggestion states on camera', async () => {
const demo = await readFile(demoPath, 'utf8');
assert.match(demo, /\[data-provenance="pending"\]/);
assert.match(demo, /The suggestion stays provisional — nothing is applied or saved until you confirm it/);
assert.match(demo, /Confirming is explicit, and every suggested value is still yours to overrule/);
assert.match(demo, /Change a suggested value and Timmy clears the AI-confirmed state/);
assert.match(demo, /\[data-provenance="corrected"\]/);
assert.match(demo, /Only visible form and color were ever suggested — urgency, discomfort, and symptoms stay yours/);
assert.match(demo, /Type 2/, 'the retained draft in the demo must show the value the user corrected to');
});
test('release demo shows the explainable pattern summary and follows a citation to its own records', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// Issue #34: the recording must exercise the summary, its citation sheet, and the sparse case.
assert.match(demo, /\[data-pattern-summary\]/);
assert.match(demo, /\[data-statement-link="typical"\]/);
assert.match(demo, /\[data-summary-record\]/);
assert.match(demo, /\[data-summary-insufficient\]/);
assert.match(demo, /The summary says which confirmed logs it read, and a citation opens exactly those records/);
assert.match(demo, /Every pattern line names the confirmed logs behind it, and following one shows exactly those records/);
assert.match(demo, /Fictional sample logs are seeded into this browser's own storage — no real health data appears here/);
assert.match(demo, /Two logs is not a pattern: the summary refuses to describe a trend it cannot support/);
// The cited records are asserted in the browser, not narrated.
assert.match(demo, /must list exactly the records it cites/);
// Both captions are required release gates, so a later release cannot drop the scene silently.
assert.match(builder, /Every pattern line names the confirmed logs behind it, and following one shows exactly those records/);
assert.match(builder, /Two logs is not a pattern: the summary refuses to describe a trend it cannot support/);
assert.match(builder, /"longitudinal_pattern_summary": "passed"/);
});
test('release demo shows the chat answering from cited records and refusing what it cannot cite', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// Issue #37: the recording must exercise the grounded local answer, its citation sheet, the
// refusal, and the Hermes reply whose refs resolve to the same records.
assert.match(demo, /\[data-chat-citation\]/);
assert.match(demo, /\[data-chat-record\]/);
assert.match(demo, /\[data-chat-record-sheet\]/);
assert.match(demo, /A grounded chat answer must cite every record it counted/);
assert.match(demo, /A chat citation must open exactly the one record it names/);
assert.match(demo, /A refusal must not borrow journal citations/);
assert.match(demo, /The answer states the count and cites the four confirmed logs it counted/);
assert.match(demo, /Following a citation opens exactly the confirmed log behind that answer/);
assert.match(demo, /Ask something the journal cannot answer and Timmy refuses instead of reassuring/);
// The local surface is a real state toggle, not narrated: the agent route is flipped off on camera.
assert.match(demo, /agentConnected = false/);
assert.match(demo, /entry:L1/);
assert.match(builder, /cites the four confirmed logs it counted/);
assert.match(builder, /"grounded_chat_citations": "passed"/);
});
test('release demo visibly exercises the red-flag language coverage instead of one canned word', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// Issue #21: the escalation has to fire for real phrasing, on both surfaces the detector serves.
assert.match(demo, /const expectEscalation = async \(text, label\)/);
assert.match(demo, /expectEscalation\('There is blood when I wipe', 'bleeding reported as wiping'\)/);
assert.match(demo, /expectEscalation\('I have tarry stools', 'tarry stool'\)/);
assert.match(demo, /note: 'stool was tarry yesterday'/);
assert.match(demo, /a tarry note in the journal must escalate the next answer/);
// Each scene is asserted in the browser, not narrated: one new bubble, the app's own copy, no provider call.
assert.match(demo, /expected exactly one escalation bubble/);
assert.match(demo, /the escalation copy must be the app's own fixed string/);
assert.match(demo, /an escalated turn must not reach Hermes/);
// The new captions are required release gates, so a later release cannot drop the scenes silently.
assert.match(builder, /Bleeding reported as wiping still escalates, in the app’s own fixed words/);
assert.match(builder, /Tarry and black stool wording escalates too: one red flag, one unchanging message/);
assert.match(builder, /A tarry note already saved in the journal escalates the same way, before any provider call/);
});
test('release demo shows a file that only claims to be a photo being refused before upload', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// Issue #16: the recording must show the device-side refusal of a non-image, prove nothing was
// posted, and keep manual entry reachable — all asserted in the browser, not narrated.
assert.match(demo, /tests\/fixtures\/not-a-photo\.jpg/);
assert.match(demo, /\[data-capture-error="unreadable"\]/);
assert.match(demo, /Nothing was uploaded/);
assert.match(demo, /An unreadable file was posted to \/api\/analyze/);
assert.match(demo, /A file that only claims to be a photo is refused on the device — nothing is uploaded/);
assert.match(demo, /Manual logging stays one tap away, and the photo picker is one tap back/);
// The caption is a required release gate, so a later release cannot drop the scene silently.
assert.match(builder, /A file that only claims to be a photo is refused on the device — nothing is uploaded/);
assert.match(builder, /"image_ingress_refusal": "passed"/);
});
test('release builder gates the sleek shell, Hermes chat, and bootstrap syntax', async () => {
const builder = await readFile(builderPath, 'utf8');
assert.match(builder, /"test:sleek"/);
assert.match(builder, /"test:mobile-capture"/);
assert.match(builder, /"test:manual-fallback"/);
assert.match(builder, /Manual fallback: vision-offline logging stays one action away with an in-session draft/);
assert.match(builder, /"manual_fallback_draft_recovery": "passed"/);
assert.match(builder, /"src\/draft\.js"/);
assert.match(builder, /"src\/vision-readiness\.js"/);
assert.match(builder, /offline vision status/);
assert.match(builder, /"sleek_hermes_chat_acceptance": "passed"/);
assert.match(builder, /Sleek three-destination shell/);
assert.match(builder, /"bash", "-n", "scripts\/bootstrap_selfhost_smolvlm\.sh"/);
});
test('release builder gates the provisional suggestion states the same way CI does', async () => {
const builder = await readFile(builderPath, 'utf8');
assert.match(builder, /"test:provisional"/);
assert.match(builder, /"provisional_suggestion_state": "passed"/);
assert.match(builder, /"src\/suggestion\.js"/);
assert.match(builder, /AI suggestions stay provisional/);
});
test('release demo visibly exercises the user-reported hard-stool follow-up and its red-flag handoff', async () => {
const demo = await readFile(demoPath, 'utf8');
assert.match(demo, /\[data-followup-block\]/);
assert.match(demo, /Type 2 is hard or lumpy, so Timmy asks about soreness, hydration, blood, and black-or-tarry — never guessed from the photo/);
assert.match(demo, /\[data-followup="bloodVisible"\]\[data-answer="no"\]/);
assert.match(demo, /Every answer is yours to give, and a skipped question stays unanswered/);
assert.match(demo, /#followup-guidance/);
assert.match(demo, /Mild report: steady water, no straining, gentle care — general self-care, not a diagnosis/);
assert.match(demo, /And it never claims the photo ruled out bleeding/);
assert.match(demo, /\[data-followup="tarryBlack"\]\[data-answer="yes"\]/);
assert.match(demo, /Say it was truly black and tarry and the same deterministic red-flag path fires instead/);
assert.match(demo, /Dark brown is not silently escalated, and the flag clears the moment the answer changes/);
});
test('release builder gates the hard-stool follow-up the same way CI does', async () => {
const builder = await readFile(builderPath, 'utf8');
assert.match(builder, /"test:followup"/);
assert.match(builder, /"hard_stool_followup": "passed"/);
assert.match(builder, /"src\/followup\.js"/);
assert.match(builder, /Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions/);
});
test('release demo states the declared release delta on camera and proves it with a receipt', async () => {
const demo = await readFile(demoPath, 'utf8');
// The delta comes from the builder, so a documentation-only release cannot imply a new feature.
assert.match(demo, /TIMMY_RELEASE_DEMO_DELTA/);
assert.match(demo, /This release: \$\{releaseDelta\}/);
assert.match(demo, /no new feature is claimed/);
// A caption string alone is not evidence; each caption must be measured as a painted bar, and the
// receipt must carry that per-caption measurement.
assert.match(demo, /TIMMY_DEMO_CAPTIONS_OUT/);
assert.match(demo, /captionsShown\.push\(text\)/);
assert.match(demo, /captionsVerified: captionEvidence/);
assert.match(demo, /Caption is not an on-screen bar/);
assert.match(demo, /Caption painted nothing into the caption band/);
assert.match(demo, /writeFile\(captionsOut/);
});
test('release demo shows the byte-bearing controls the threat model governs, without inventing UI', async () => {
const demo = await readFile(demoPath, 'utf8');
assert.match(demo, /\[data-view="privacy"\]/);
assert.match(demo, /Every byte-bearing control the model names/);
assert.match(demo, /indicate\('#export'\)/);
assert.match(demo, /indicate\('#delete-all'\)/);
assert.match(demo, /Delete all local data clears the ledger and every saved photo in this browser; nothing is kept on a server/);
// Real controls only: the selectors must exist in the app shell, not be demo-only scaffolding.
const app = await readFile(new URL('../app.js', import.meta.url), 'utf8');
assert.match(app, /id="export"/);
assert.match(app, /id="delete-all"/);
});
test('release demo exercises the photo lifecycle on camera without showing an image by default', async () => {
const demo = await readFile(demoPath, 'utf8');
// Presence first: a label in the journal, with no thumbnail rendered.
assert.match(demo, /Photo presence is a label, not a thumbnail/);
assert.match(demo, /\.entries-card \[data-photo-indicator="present"\]/);
assert.match(demo, /exactly one log must be marked as holding a photo/);
assert.match(demo, /the journal must not show a thumbnail by default/);
// Revealing the image is an explicit request, asserted before the caption that claims it.
assert.match(demo, /opening a log must not reveal its photo/);
assert.match(demo, /#show-entry-photo/);
assert.match(demo, /Showing the photo is your explicit request/);
// Removing the photo keeps every confirmed field of the log.
assert.match(demo, /#remove-entry-photo/);
assert.match(demo, /Remove just the photo: the confirmed log, form, and note are all still here/);
assert.match(demo, /removing a photo must keep the confirmed log and clear only its image bytes/);
// Delete-all is actually performed, not just pointed at.
assert.match(demo, /tap\('#delete-all'/);
assert.match(demo, /no photo bytes may survive delete-all|JSON\.stringify\(localStorage\)\.includes\('data:image'\)/);
// Synthetic bytes only: a committed test-card PNG constant, never a real photo. A blank 1x1 pixel
// rendered as an empty white box on camera, which read as a broken image instead of a saved photo.
assert.match(demo, /const SYNTHETIC_PHOTO = 'data:image\/png;base64,/);
assert.match(demo, /synthetic test card/);
});
test('release builder refuses to publish without the on-camera delta receipt', async () => {
const builder = await readFile(builderPath, 'utf8');
assert.match(builder, /TIMMY_RELEASE_DELTA/);
assert.match(builder, /TIMMY_DEMO_CAPTIONS_OUT/);
assert.match(builder, /Release demo produced no on-camera caption receipt/);
assert.match(builder, /Release demo did not state the declared release delta on camera/);
assert.match(builder, /Release demo is missing a required on-camera caption/);
assert.match(builder, /Release demo caption never painted on screen/);
assert.match(builder, /recorded\.get\("captionsVerified"\)/);
assert.match(builder, /"release_delta_visible_on_camera": "passed"/);
assert.match(builder, /"release_delta": release_delta/);
});
test('caption styling survives the demo reloads instead of dying with the first document', async () => {
const demo = await readFile(demoPath, 'utf8');
// A stylesheet belongs to one document; the demo reloads the page, which stripped caption and touch
// styling and left later captions as clipped body text. Both now carry inline styles per node.
assert.doesNotMatch(demo, /page\.addStyleTag/);
assert.doesNotMatch(demo, /addInitScript/);
assert.match(demo, /In a stylesheet belongs to one document|a stylesheet belongs to one document/);
assert.match(demo, /node\.style\.cssText = 'position:fixed;left:16px;right:16px;top:14px;z-index:20000;'/);
assert.match(demo, /ring\.style\.cssText = 'position:fixed;z-index:20001/);
});
test('release demo captions expire and the contact sheet spans the whole recording', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// A caption that outlives its stated duration hangs the previous view's claim over the next screen.
assert.match(demo, /creates via removal after duration|A caption must not outlive its stated duration/);
assert.match(demo, /setTimeout\(\(\) => node\.remove\(\), 220\)/);
// The sheet must cover the entire demo, not just its opening.
assert.match(builder, /12 \/ demo_duration/);
assert.match(builder, /Release demo contact sheet was not produced/);
});
test('release demo visibly exercises the corpus refusal paths instead of only the happy path', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// The refusal responses must come from the recorded corpus and the real analysis code path,
// never from copy pasted into the recording script.
assert.match(demo, /loadVisionCorpus/);
assert.match(demo, /visionFailureResponse/);
assert.match(demo, /corpusAnalyzeResult/);
assert.match(demo, /tests\/fixtures\/vision-corpus\/non-stool-scene\.png/);
assert.match(demo, /tests\/fixtures\/vision-corpus\/out-of-range-fields\.png/);
assert.match(demo, /getByText\(\/No confident match\/i\)/);
assert.match(demo, /couldn’t analyze that safely/i);
assert.match(demo, /A non-stool photo gets a refusal, never a guess/);
assert.match(demo, /Unreadable visual fields abstain as well/);
assert.match(demo, /An out-of-range answer fails closed/);
// Issue #32's quality gate: a confident answer about a low-quality photo must abstain on camera.
assert.match(demo, /corpusCase\('poor-quality-confident'\)/);
assert.match(demo, /tests\/fixtures\/vision-corpus\/blur-low-detail\.png/);
assert.match(demo, /too blurry or poorly lit/);
assert.match(demo, /must not prefill a form, however confident the reply looked/);
assert.match(demo, /A confident reply about a blurry photo still abstains/);
assert.match(builder, /A confident reply about a blurry photo still abstains — photo quality is part of the policy, not just confidence/);
assert.match(builder, /"abstention_quality_gate": "passed"/);
assert.match(demo, /no real photos in the corpus|no real photos in the recording/);
// The refusal copy the user sees is gated on camera, so a release cannot drop it silently.
assert.match(builder, /An out-of-range answer fails closed/);
});
test('release demo shows a filled-in out-of-range field failing closed even beside a blank one', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// Issue #73's escape class, replayed on camera from the committed corpus rather than described.
assert.match(demo, /corpusCase\('blank-plus-out-of-range'\)/);
assert.match(demo, /tests\/fixtures\/vision-corpus\/blank-plus-out-of-range\.png/);
assert.match(demo, /A filled-in field that is out of range still fails closed, even when the other field came back blank/);
assert.match(demo, /no longer told the photo was unreadable when the model returned an impossible value/);
// The extra fail-closed scene is a real 503, so the demo's error budget must count it.
assert.match(demo, /expectedFailClosed\.length !== 2/);
// The caption is a required gate, so a later release cannot silently drop the scene.
assert.match(builder, /A filled-in field that is out of range still fails closed/);
});
test('release demo shows readiness waiting for the loaded pinned model before AI is offered', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// Issue #15: reachable is not loaded, and the recording must prove the app withholds AI.
assert.match(demo, /modelLoaded = false/);
assert.match(demo, /getByText\(\/model is not loaded yet\/i\)/);
assert.match(demo, /model-status\.loading/);
assert.match(demo, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/);
assert.match(demo, /No AI control may exist before the model is loaded/);
assert.match(demo, /Once the pinned model is loaded, the same screen offers the AI analysis again/);
// The scene is asserted in the browser, not narrated: the demo throws if the state is wrong.
assert.match(demo, /must never be reported ready/);
// The caption is a required release gate, so the scene cannot be dropped silently.
assert.match(builder, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/);
});
test('release demo shows the training contribution as a separate opt-in receipt that can be withdrawn', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
const app = await readFile(new URL('../app.js', import.meta.url), 'utf8');
// Issue #24: the recording must show the opt-in section, the consent gate, and the withdrawal.
assert.match(demo, /\[data-training-consent\]/);
assert.match(demo, /\[data-contribute-entry="demo-wipe"\]/);
assert.match(demo, /#training-consent-check/);
assert.match(demo, /#contribute-confirm/);
assert.match(demo, /\[data-contribution-state="queued-local"\]/);
assert.match(demo, /data-withdraw=/);
assert.match(demo, /\[data-withdrawn-artifacts\]/);
assert.match(demo, /training-consent-v1/);
// Every claim is measured in the browser before the caption that makes it.
assert.match(demo, /a journal save must not queue a training contribution/);
assert.match(demo, /the contribute control must stay disabled until the terms are accepted/);
assert.match(demo, /no receipt may exist before explicit consent/);
assert.match(demo, /a receipt must never copy the photo bytes/);
assert.match(demo, /every derived artifact must be addressed by contribution id/);
assert.match(demo, /delete-all must clear the contribution receipts as well/);
// The controls are the app's own, not demo-only scaffolding.
assert.match(app, /data-training-consent/);
assert.match(app, /data-contribution-queued-count/);
assert.match(app, /id="training-consent-check"/);
assert.match(app, /id="contribute-confirm" disabled/);
assert.match(app, /data-withdraw=/);
// The captions and the gate key are required, so a later release cannot drop the scenes silently.
assert.match(builder, /Training contribution is off by default: a saved log is never training data/);
assert.match(builder, /The consent version and the receipt are on screen before the contribute button works/);
assert.match(builder, /Withdrawing keeps a tombstone naming every derived artifact by contribution ID/);
assert.match(builder, /"training_contribution_consent": "passed"/);
assert.match(builder, /"test:training-contribution"/);
});
test('release demo shows a versioned backup, an upgraded older file, and a refused newer one', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
const app = await readFile(new URL('../app.js', import.meta.url), 'utf8');
// Issue #35: the recording must export the real file, import an older version, refuse a newer one,
// and reopen a restored log so its provenance is on screen — not merely described.
assert.match(demo, /waitForEvent\('download'\)/);
assert.match(demo, /exportedBackup\.schemaVersion !== 2/);
assert.match(demo, /the backup must carry the withdrawn receipt as a tombstone/);
assert.match(demo, /\[data-portability-notice\]\[data-portability-version="1"\]/);
assert.match(demo, /\[data-portability-error\]/);
assert.match(demo, /\[data-entry-provenance="ai-corrected"\]/);
// Every claim is measured before the caption that makes it.
assert.match(demo, /an older backup must be reported as upgraded/);
assert.match(demo, /a refused import must not change the ledger it would have replaced/);
assert.match(demo, /a restored log must still show that it began as a corrected AI suggestion/);
// The controls belong to the app, and the provenance line is its own rendering.
assert.match(app, /id="export"/);
assert.match(app, /data-portability-notice/);
assert.match(app, /data-entry-provenance=/);
// Captions and the gate key are required, so a later release cannot drop the scenes silently.
assert.match(builder, /A backup is versioned, and it carries the withdrawn receipt as a tombstone rather than silence/);
assert.match(builder, /An older backup is upgraded on import, and the notice says which version the file came from/);
assert.match(builder, /A file from a newer Timmy is refused with an instruction, and the ledger it would have replaced is untouched/);
assert.match(builder, /A restored log still says it began as an AI suggestion you corrected — provenance is part of the record/);
assert.match(builder, /"portable_backup_restore": "passed"/);
assert.match(builder, /"test:portability"/);
});
test('the builder refuses to gate or record a release against a foreign server', async () => {
const builder = await readFile(builderPath, 'utf8');
// A leftover checkout answering on the acceptance port once satisfied every flow and the demo,
// so the release described a tree that was not the commit under test. The builder must refuse a
// live listener before it starts, then prove the served bytes are this tree's before any gate.
assert.match(builder, /def assert_port_free/);
assert.match(builder, /assert_port_free\("127\.0\.0\.1", 4173\)/);
assert.match(builder, /def assert_served_tree/);
assert.match(builder, /assert_served_tree\(tree, "127\.0\.0\.1", 4173\)/);
const identityCheck = builder.indexOf('assert_served_tree(tree, "127.0.0.1", 4173)');
const firstFlow = builder.indexOf('run(["npm", "run", "test:ui"]');
assert.ok(identityCheck > 0, 'the served-tree check must be present');
assert.ok(firstFlow > identityCheck, 'the served-tree check must run before the first acceptance flow');
});
test('release demo replays the calibration harness against synthetic predictions instead of narrating the gates', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// Issue #29: the gate math is only proven by running the real harness on camera.
assert.match(demo, /scripts\/calibration_report\.py/);
assert.match(demo, /tests\/fixtures\/calibration\/predictions\.jsonl/);
assert.match(demo, /tests\/fixtures\/calibration\/predictions-go\.jsonl/);
assert.match(demo, /tests\/fixtures\/calibration\/predictions-leaky\.jsonl/);
// All three verdict paths are exercised: blocked, passing, and refused before scoring.
assert.match(demo, /the synthetic candidate fixtures must block the release/);
assert.match(demo, /the engineered fixture must prove the passing branch/);
assert.match(demo, /a leaked corpus must be refused before scoring/);
// The card is drawn from the report the run wrote, and gated for readability like the corpus card.
assert.match(demo, /id: 'calibration-report'/);
assert.match(demo, /Calibration card is not fully readable on screen/);
assert.match(demo, /the card must show the macro-F1 the harness measured/);
assert.match(demo, /the card must show the refusal the leaked corpus earned/);
// The scores are labelled as invented: a synthetic fixture is not a measurement of any model.
assert.match(demo, /invented scores, not measurements of any model/);
assert.match(demo, /synthetic predictions · no model was measured/);
// The scene cannot be dropped silently: its captions are release gates.
assert.match(builder, /The calibration harness is a command-line release gate, not a screen in the app/);
assert.match(builder, /Every score on this card is invented to exercise the arithmetic/);
assert.match(builder, /Each candidate is scored on test at a threshold chosen on validation only/);
assert.match(builder, /One contributor spread across two splits is refused before anything is scored/);
});
test('release demo runs the blind review tool in its enforced order instead of describing it', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// Issue #26: the review tool is a CLI. The scene runs the real commands and shows their output.
assert.match(demo, /scripts\/review_corpus\.py/);
assert.match(demo, /tests\/fixtures\/synthetic-type4\.jpg/);
// Blindness is proven by the refusal, not by a caption: reveal is attempted before any commit.
assert.match(demo, /stays hidden until a review is committed/);
// The reviewer's labels are committed before the model answer is shown, and both are recorded.
assert.match(demo, /--reviewer'?,? ?'reviewer-a'/);
assert.match(demo, /disagreeFields/);
// A second reader adjudicates; the review it resolves must still be in the ledger afterwards.
assert.match(demo, /'adjudicate'/);
assert.match(demo, /resolvesRecord/);
// A withdrawn contribution leaves the queue, the training set, and the review path entirely.
assert.match(demo, /was withdrawn and cannot be reviewed/);
assert.match(demo, /id: 'review-audit'/);
assert.match(demo, /Review card is not fully readable on screen/);
assert.match(demo, /the card must show the model answer the run revealed/);
assert.match(demo, /the card must show the withdrawn refusal the run earned/);
// The card must not claim a model measured anything: the answer file is a synthetic fixture.
assert.match(demo, /synthetic answer file · no model was measured/);
// The scene is a release gate, so a later release cannot quietly drop it.
assert.match(builder, /tests\/corpus-review\.test\.py/);
assert.match(builder, /"reviewer_adjudication_blind": "passed"/);
assert.match(builder, /The review tool is a command line, not a screen in the app/);
assert.match(builder, /The queue shows the proposal and hides the model answer; revealing it is refused until a reviewer commits/);
assert.match(builder, /The model said Type 6 and the reviewer said Type 5: the disagreement is recorded, not smoothed over/);
assert.match(builder, /Withdrawing the contribution empties the queue and the training set, and the tool refuses to review it at all/);
});
test('release demo publishes the dataset card from a manifest it builds on camera', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// Issue #27: the card is a generated artifact. The scene builds a real corpus with the real ingest
// commands, publishes the card, and re-checks the committed page, instead of describing a document.
assert.match(demo, /scripts\/dataset_card\.py/);
assert.match(demo, /id: 'dataset-card'/);
assert.match(demo, /Dataset card is not fully readable on screen/);
// The split policy is proven by the refusal, not by a caption: a leaked contributor is attempted.
assert.match(demo, /leaks across/);
assert.match(demo, /the generator must refuse a corpus that leaks one contributor across splits/);
// The pending facts must be visibly pending, and the committed page must be the generator's output.
assert.match(demo, /pending/);
assert.match(demo, /the committed dataset card must be the generator's own output/);
assert.match(demo, /docs\/DATASET_CARD\.md/);
// Nothing on the card may claim a model measured or labelled anything here.
assert.match(demo, /no model labelled anything here/);
// The scene and its guide are release gates, so a later release cannot drop them silently.
assert.match(builder, /The dataset card is generated from the corpus manifest, so no count on it is typed by hand/);
assert.match(builder, /One contributor's photos stay in one split: a corpus that leaks a person across splits is refused/);
assert.match(builder, /Devices, contexts, and licenses stay pending until the pipeline collects them/);
assert.match(builder, /tests\/dataset-card\.test\.py/);
assert.match(builder, /"dataset_card_generated": "passed"/);
assert.match(builder, /scripts\/dataset_card\.py/);
});
test('release demo runs the operator backup and restore on synthetic records instead of describing it', async () => {
const demo = await readFile(demoPath, 'utf8');
const builder = await readFile(builderPath, 'utf8');
// Issue #39: the backup is a CLI, so the recording runs the real tool and shows what it stored,
// what it refused, and what the restore returned.
assert.match(demo, /scripts\/operator_backup\.py/);
assert.match(demo, /'backup', '--source'/);
assert.match(demo, /'verify', '--archive'/);
assert.match(demo, /'restore', '--archive'/);
assert.match(demo, /id: 'production-backup'/);
assert.match(demo, /Production backup card is not fully readable on screen/);
// The exclusions are proven by the run, not asserted by a caption: the decoys are real files.
assert.match(demo, /model-weights/);
assert.match(demo, /transient-inference/);
assert.match(demo, /must not archive the weight shard or a transient inference upload/);
// A restore must return the append-only manifest byte for byte, tombstone included.
assert.match(demo, /must return the same records, tombstone included/);
assert.match(demo, /The operator backup is a command line, not a screen in the app/);
assert.match(demo, /Model weights and transient inference images are refused by name, and every skipped path is listed with its reason/);
assert.match(demo, /Verify re-hashes every stored member against the manifest, and the restore only starts once that check passes/);
assert.match(demo, /A restore returns the same records, including the withdrawal tombstone a trainer must never lose/);
// The scene is a release gate, and the tool's own suite runs before the release exists.
assert.match(demo, /process\.env\.TIMMY_DEMO_URL \|\| 'http:\/\/127\.0\.0\.1:4173'/);
assert.match(builder, /tests\/backup-restore\.test\.py/);
assert.match(builder, /tests\/production-runbook\.test\.py/);
assert.match(builder, /"production_backup_restore": "passed"/);
assert.match(builder, /"production_runbook_commands": "passed"/);
assert.match(builder, /The operator backup is a command line, not a screen in the app/);
assert.match(builder, /A restore returns the same records, including the withdrawal tombstone a trainer must never lose/);
});
test('release demo runs the RC attack suite on camera and prints the verdict it derived', async () => {
const demo = await readFile(demoPath, 'utf8');
// Issue #40 turns hostile uploads into a release gate. The scene must run the real suite, not
// describe it, and the card must show the verdict the suite derived rather than a flattering one.
assert.match(demo, /scripts\/security_attack_suite\.mjs', '--json'/);
assert.match(demo, /SECURITY ATTACK SUITE/);
assert.match(demo, /Every hostile upload on screen went at the real server, and the worker behind it counted every call it received/);
assert.match(demo, /A failing case is a finding the release job has to see, so the card prints the verdict the suite derived/);
assert.match(demo, /The gate blocks RC approval on any P0 case that reached the worker/);
assert.match(demo, /securityReport\.verdict/);
assert.match(demo, /securityCardProof/);
// The finding must name where the fix lives instead of implying the boundary is already safe.
assert.match(demo, /Refusing bytes by their content is the fix in review, not a claim this video makes/);
// Each case row prints the fields it actually observed. A guarantee that the card never renders
// `http undefined` (a case with no status) or the bare status again (a case with no provider calls).
assert.match(demo, /'refused' in observed/);
assert.match(demo, /'providerTextLeaked' in observed/);
assert.doesNotMatch(demo, /http \$\{observed\.status\} · worker \$\{observed\.providerCalls\}`/);
});