Timmy
46f961668f
Quality gates / quality (pull_request) Successful in 3m29s
A worker that answers /v1/models has not necessarily loaded its weights, so the app was offering an AI control that could only fail after the upload. Readiness now passes only once the pinned model is loaded, and the not-loaded state reaches manual Bristol entry in one tap. Verified against the real pinned llama.cpp build: it serves 503 while loading and the gate reports the distinct not-loaded state. Adds the reproducible container recipe (digest-pinned bases, pinned llama.cpp commit, hash-verified read-only weights, unprivileged, loopback-only publish), one shared post-load readiness gate used by the host bootstrap and the image healthcheck, and a target-GPU benchmark that refuses to write a receipt it did not measure. Refs #15
205 lines
12 KiB
JavaScript
205 lines
12 KiB
JavaScript
import test from 'node:test';
|
||
import assert from 'node:assert/strict';
|
||
import { readFile } from 'node:fs/promises';
|
||
|
||
const demoPath = new URL('../scripts/record_release_demo.mjs', import.meta.url);
|
||
const builderPath = new URL('../scripts/build_release.py', import.meta.url);
|
||
|
||
test('release demo visibly explains the CI-protected browser path without overstating safety', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /Automated checks replay this synthetic path before review/);
|
||
assert.match(demo, /One clear photo action\. Manual logging stays one tap away\./);
|
||
assert.match(demo, /Camera closed cleanly — gallery and manual logging are still available/);
|
||
assert.match(demo, /#camera-photo.*dispatchEvent\('cancel'\)/s);
|
||
assert.match(demo, /#gallery-photo.*synthetic-type4\.jpg/s);
|
||
assert.match(demo, /tests\/fixtures\/synthetic-type4\.jpg/);
|
||
assert.match(demo, /The pinned bootstrap verifies both model files before starting on private loopback/);
|
||
assert.match(demo, /AI may suggest visible form, broad color, and image quality — never symptoms or diagnosis/);
|
||
assert.match(demo, /Hermes Agent connected/);
|
||
assert.match(demo, /Photos stay out of chat/);
|
||
assert.match(demo, /I barfed/);
|
||
assert.match(demo, /Urgent language is intercepted deterministically before Hermes/);
|
||
assert.match(demo, /SLEEK\. SIMPLE\.<br>HERMES-POWERED\./);
|
||
});
|
||
|
||
test('release demo exercises the vision-offline manual path and the retained draft', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /Vision worker offline — the AI and consent steps disappear instead of failing after upload/);
|
||
assert.match(demo, /No consent prompt and no upload when analysis cannot run/);
|
||
assert.match(demo, /One tap still reaches manual Bristol entry/);
|
||
assert.match(demo, /Closing the log keeps the draft in this session instead of throwing it away/);
|
||
assert.match(demo, /Start over clears the kept draft — nothing lingers silently/);
|
||
assert.match(demo, /#manual-from-scan/);
|
||
assert.match(demo, /#discard-draft/);
|
||
assert.match(demo, /\.draft-resume/);
|
||
// The offline pass must be a real state change driven by the app's own readiness contract.
|
||
assert.match(demo, /providerReady: false|visionOffline/);
|
||
});
|
||
|
||
test('release demo shows the provisional, confirmed, and corrected suggestion states on camera', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /\[data-provenance="pending"\]/);
|
||
assert.match(demo, /The suggestion stays provisional — nothing is applied or saved until you confirm it/);
|
||
assert.match(demo, /Confirming is explicit, and every suggested value is still yours to overrule/);
|
||
assert.match(demo, /Change a suggested value and Timmy clears the AI-confirmed state/);
|
||
assert.match(demo, /\[data-provenance="corrected"\]/);
|
||
assert.match(demo, /Only visible form and color were ever suggested — urgency, discomfort, and symptoms stay yours/);
|
||
assert.match(demo, /Type 2/, 'the retained draft in the demo must show the value the user corrected to');
|
||
});
|
||
|
||
test('release builder gates the sleek shell, Hermes chat, and bootstrap syntax', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
assert.match(builder, /"test:sleek"/);
|
||
assert.match(builder, /"test:mobile-capture"/);
|
||
assert.match(builder, /"test:manual-fallback"/);
|
||
assert.match(builder, /Manual fallback: vision-offline logging stays one action away with an in-session draft/);
|
||
assert.match(builder, /"manual_fallback_draft_recovery": "passed"/);
|
||
assert.match(builder, /"src\/draft\.js"/);
|
||
assert.match(builder, /"src\/vision-readiness\.js"/);
|
||
assert.match(builder, /offline vision status/);
|
||
assert.match(builder, /"sleek_hermes_chat_acceptance": "passed"/);
|
||
assert.match(builder, /Sleek three-destination shell/);
|
||
assert.match(builder, /"bash", "-n", "scripts\/bootstrap_selfhost_smolvlm\.sh"/);
|
||
});
|
||
|
||
test('release builder gates the provisional suggestion states the same way CI does', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
assert.match(builder, /"test:provisional"/);
|
||
assert.match(builder, /"provisional_suggestion_state": "passed"/);
|
||
assert.match(builder, /"src\/suggestion\.js"/);
|
||
assert.match(builder, /AI suggestions stay provisional/);
|
||
});
|
||
|
||
test('release demo visibly exercises the user-reported hard-stool follow-up and its red-flag handoff', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /\[data-followup-block\]/);
|
||
assert.match(demo, /Type 2 is hard or lumpy, so Timmy asks about soreness, hydration, blood, and black-or-tarry — never guessed from the photo/);
|
||
assert.match(demo, /\[data-followup="bloodVisible"\]\[data-answer="no"\]/);
|
||
assert.match(demo, /Every answer is yours to give, and a skipped question stays unanswered/);
|
||
assert.match(demo, /#followup-guidance/);
|
||
assert.match(demo, /Mild report: steady water, no straining, gentle care — general self-care, not a diagnosis/);
|
||
assert.match(demo, /And it never claims the photo ruled out bleeding/);
|
||
assert.match(demo, /\[data-followup="tarryBlack"\]\[data-answer="yes"\]/);
|
||
assert.match(demo, /Say it was truly black and tarry and the same deterministic red-flag path fires instead/);
|
||
assert.match(demo, /Dark brown is not silently escalated, and the flag clears the moment the answer changes/);
|
||
});
|
||
|
||
test('release builder gates the hard-stool follow-up the same way CI does', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
assert.match(builder, /"test:followup"/);
|
||
assert.match(builder, /"hard_stool_followup": "passed"/);
|
||
assert.match(builder, /"src\/followup\.js"/);
|
||
assert.match(builder, /Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions/);
|
||
});
|
||
|
||
test('release demo states the declared release delta on camera and proves it with a receipt', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
// The delta comes from the builder, so a documentation-only release cannot imply a new feature.
|
||
assert.match(demo, /TIMMY_RELEASE_DEMO_DELTA/);
|
||
assert.match(demo, /This release: \$\{releaseDelta\}/);
|
||
assert.match(demo, /no new feature is claimed/);
|
||
// A caption alone is not evidence; the demo must emit a machine-readable receipt of what rendered.
|
||
assert.match(demo, /TIMMY_DEMO_CAPTIONS_OUT/);
|
||
assert.match(demo, /captionsShown\.push\(text\)/);
|
||
assert.match(demo, /writeFile\(captionsOut/);
|
||
});
|
||
|
||
test('release demo shows the byte-bearing controls the threat model governs, without inventing UI', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
|
||
assert.match(demo, /\[data-view="privacy"\]/);
|
||
assert.match(demo, /Every byte-bearing control the model names/);
|
||
assert.match(demo, /indicate\('#export'\)/);
|
||
assert.match(demo, /indicate\('#delete-all'\)/);
|
||
assert.match(demo, /Delete all local data clears the ledger in this browser; there is no server copy of it to keep/);
|
||
// Real controls only: the selectors must exist in the app shell, not be demo-only scaffolding.
|
||
const app = await readFile(new URL('../app.js', import.meta.url), 'utf8');
|
||
assert.match(app, /id="export"/);
|
||
assert.match(app, /id="delete-all"/);
|
||
});
|
||
|
||
test('release builder refuses to publish without the on-camera delta receipt', async () => {
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
assert.match(builder, /TIMMY_RELEASE_DELTA/);
|
||
assert.match(builder, /TIMMY_DEMO_CAPTIONS_OUT/);
|
||
assert.match(builder, /Release demo produced no on-camera caption receipt/);
|
||
assert.match(builder, /Release demo did not state the declared release delta on camera/);
|
||
assert.match(builder, /Release demo is missing a required on-camera caption/);
|
||
assert.match(builder, /"release_delta_visible_on_camera": "passed"/);
|
||
assert.match(builder, /"release_delta": release_delta/);
|
||
});
|
||
|
||
test('release demo captions expire and the contact sheet spans the whole recording', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// A caption that outlives its stated duration hangs the previous view's claim over the next screen.
|
||
assert.match(demo, /creates via removal after duration|A caption must not outlive its stated duration/);
|
||
assert.match(demo, /setTimeout\(\(\) => node\.remove\(\), 220\)/);
|
||
// The sheet must cover the entire demo, not just its opening.
|
||
assert.match(builder, /12 \/ demo_duration/);
|
||
assert.match(builder, /Release demo contact sheet was not produced/);
|
||
});
|
||
|
||
test('release demo visibly exercises the corpus refusal paths instead of only the happy path', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// The refusal responses must come from the recorded corpus and the real analysis code path,
|
||
// never from copy pasted into the recording script.
|
||
assert.match(demo, /loadVisionCorpus/);
|
||
assert.match(demo, /visionFailureResponse/);
|
||
assert.match(demo, /corpusAnalyzeResult/);
|
||
assert.match(demo, /tests\/fixtures\/vision-corpus\/non-stool-scene\.png/);
|
||
assert.match(demo, /tests\/fixtures\/vision-corpus\/out-of-range-fields\.png/);
|
||
assert.match(demo, /getByText\(\/No confident match\/i\)/);
|
||
assert.match(demo, /couldn’t analyze that safely/i);
|
||
|
||
assert.match(demo, /A non-stool photo gets a refusal, never a guess/);
|
||
assert.match(demo, /Unreadable visual fields abstain as well/);
|
||
assert.match(demo, /An out-of-range answer fails closed/);
|
||
assert.match(demo, /no real photos in the corpus|no real photos in the recording/);
|
||
// The refusal copy the user sees is gated on camera, so a release cannot drop it silently.
|
||
assert.match(builder, /An out-of-range answer fails closed/);
|
||
});
|
||
|
||
test('release demo shows a filled-in out-of-range field failing closed even beside a blank one', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #73's escape class, replayed on camera from the committed corpus rather than described.
|
||
assert.match(demo, /corpusCase\('blank-plus-out-of-range'\)/);
|
||
assert.match(demo, /tests\/fixtures\/vision-corpus\/blank-plus-out-of-range\.png/);
|
||
assert.match(demo, /A filled-in field that is out of range still fails closed, even when the other field came back blank/);
|
||
assert.match(demo, /no longer told the photo was unreadable when the model returned an impossible value/);
|
||
// The extra fail-closed scene is a real 503, so the demo's error budget must count it.
|
||
assert.match(demo, /expectedFailClosed\.length !== 2/);
|
||
// The caption is a required gate, so a later release cannot silently drop the scene.
|
||
assert.match(builder, /A filled-in field that is out of range still fails closed/);
|
||
});
|
||
|
||
test('release demo shows readiness waiting for the loaded pinned model before AI is offered', async () => {
|
||
const demo = await readFile(demoPath, 'utf8');
|
||
const builder = await readFile(builderPath, 'utf8');
|
||
|
||
// Issue #15: reachable is not loaded, and the recording must prove the app withholds AI.
|
||
assert.match(demo, /modelLoaded = false/);
|
||
assert.match(demo, /getByText\(\/model is not loaded yet\/i\)/);
|
||
assert.match(demo, /model-status\.loading/);
|
||
assert.match(demo, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/);
|
||
assert.match(demo, /No AI control may exist before the model is loaded/);
|
||
assert.match(demo, /Once the pinned model is loaded, the same screen offers the AI analysis again/);
|
||
// The scene is asserted in the browser, not narrated: the demo throws if the state is wrong.
|
||
assert.match(demo, /must never be reported ready/);
|
||
// The caption is a required release gate, so the scene cannot be dropped silently.
|
||
assert.match(builder, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/);
|
||
});
|