import { chromium } from 'playwright'; import { execFile } from 'node:child_process'; import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { createHash } from 'node:crypto'; import os from 'node:os'; import { promisify } from 'node:util'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { analyzePhoto, visionFailureResponse } from '../src/vision-service.js'; import { loadVisionCorpus } from '../src/vision-corpus.js'; const execFileAsync = promisify(execFile); const output = path.resolve(process.argv[2] || 'release-demo.webm'); const version = process.env.TIMMY_RELEASE_VERSION || 'review build'; // The release delta is supplied by the builder so the recording states, on camera, what actually // changed in this release instead of implying a feature that did not ship. const releaseDelta = process.env.TIMMY_RELEASE_DEMO_DELTA || 'Verified rebuild of the current user-visible feature path — no new feature is claimed.'; // The recording targets the release builder's own acceptance server by default. A caller that // names another origin is explicit about it, so a local run can never silently record a different // tree than the one the gates measured. const demoUrl = process.env.TIMMY_DEMO_URL || 'http://127.0.0.1:4173'; const captionsOut = process.env.TIMMY_DEMO_CAPTIONS_OUT || ''; const captionsShown = []; const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); await mkdir(path.dirname(output), { recursive: true }); // The refusal scene replays committed synthetic corpus cases through the real analysis code path, // so the recording shows the status and copy the server would actually send rather than copy // written for the video. No real photo is used anywhere in the corpus. const corpus = await loadVisionCorpus(fileURLToPath(new URL('../tests/fixtures/vision-corpus/', import.meta.url))); const corpusCase = id => corpus.cases.find(entry => entry.id === id); async function corpusAnalyzeResult(entry) { const fetchImpl = async () => ({ ok: true, json: async () => ({ choices: [{ message: { content: entry.providerContent } }] }) }); try { const result = await analyzePhoto({ payload: { imageDataUrl: 'data:image/png;base64,YQ==', consent: true }, fetchImpl, config: { baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'corpus-replay', model: 'corpus-model' }, }); return { status: 200, body: result }; } catch (error) { const failure = visionFailureResponse(error); return { status: failure.status, body: failure.body }; } } const browser = await chromium.launch({ headless: true }); const context = await browser.newContext({ viewport: { width: 405, height: 720 }, deviceScaleFactor: 1, serviceWorkers: 'block', recordVideo: { dir: path.dirname(output), size: { width: 405, height: 720 } }, colorScheme: 'light', }); const page = await context.newPage(); const errors = []; // Delete-all asks for confirmation; the recording accepts it so the resulting state is shown, not implied. page.on('dialog', dialog => dialog.accept()); page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); page.on('pageerror', error => errors.push(error.message)); let visionOffline = false; // Reachable is not loaded: the worker can answer /v1/models before its pinned model is resident, // and the recording shows the app withholding AI in exactly that state (issue #15). let modelLoaded = true; await page.route('**/api/vision-status', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ enabled: true, profile: 'selfhost', processor: 'self-hosted', model: 'SmolVLM2-2.2B-Instruct', // The second pass flips the app's own readiness contract so no AI control is offered at all. providerReady: !visionOffline, modelSeen: !visionOffline && modelLoaded, }), })); // The happy path keeps one deterministic suggestion. The refusal scene swaps this for a real // corpus replay, so both the usable and the refused answers come from the same code path. const suggestionBody = { status: 'suggestion', isStool: true, bristolType: 4, color: 'brown', confidence: 0.83, imageQuality: 'good', observations: 'Smooth, formed appearance.', warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.', }; let analyzeOverride = null; let analyzeCalls = 0; await page.route('**/api/analyze', route => { analyzeCalls += 1; const response = analyzeOverride || { status: 200, body: suggestionBody }; return route.fulfill({ status: response.status, contentType: 'application/json', body: JSON.stringify(response.body) }); }); let hermesCalls = 0; // Issue #37 turns the agent on and off on camera: local Timmy mode is a real app state, and the // grounded local answer has to be shown as its own surface rather than implied. let agentConnected = true; await page.route('**/api/agent/status', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ enabled: true, // Local Timmy mode is the no-backend state the app really uses: nothing configured, nothing sent. configured: agentConnected, authenticated: agentConnected, mode: agentConnected ? 'hermes-agent' : 'local-fallback', }), })); await page.route('**/api/agent/chat', route => { hermesCalls += 1; return route.fulfill({ status: 200, contentType: 'application/json', // The refs are the positional handles the server gives the agent; the browser resolves them // against the same ledger it sent, so this reply is a real citation contract, not a prop. body: JSON.stringify({ connected: true, reply: 'You have 4 confirmed logs in the last 28 days [entry:L1] [entry:L2] [entry:L3] [entry:L4]. Your confirmed logs are mostly Type 4. I can explain that pattern, but I cannot diagnose a cause.' }), }); }); await page.goto(demoUrl, { waitUntil: 'networkidle' }); await page.evaluate(() => localStorage.clear()); await page.reload({ waitUntil: 'networkidle' }); const captionEvidence = []; // Pixel proof that a caption actually painted: if the caption band looks identical with and without // the node, the caption did not reach the screen even though the string was pushed. async function captionBand() { return page.screenshot({ clip: { x: 0, y: 0, width: 405, height: 68 } }); } async function caption(text, duration = 1400) { captionsShown.push(text); const blankBand = await captionBand().catch(() => null); await page.evaluate(text => { document.querySelector('#demo-caption')?.remove(); const node = document.createElement('div'); node.id = 'demo-caption'; node.textContent = text; // Inline styles, not a stylesheet: a stylesheet belongs to one document and the demo reloads the // page, which silently stripped caption styling and left every later caption clipped body text. // The bar is opaque, not 94%: a translucent bar let the app's own text show through a long // release-delta caption and read as run-together garbage. Long copy also steps down a size so a // paragraph-length caption keeps its bottom padding instead of pressing the last line to the edge. const size = text.length > 380 ? 9.5 : text.length > 220 ? 10.5 : 12; node.style.cssText = 'position:fixed;left:16px;right:16px;top:14px;z-index:20000;' + 'background:#28211d;color:#fff;padding:12px 14px 14px;border-radius:14px;' + `font:800 ${size}px/1.32 system-ui;letter-spacing:.02em;text-align:center;` + 'box-shadow:0 8px 24px rgba(0,0,0,.2)'; document.body.append(node); node.animate([{ opacity: 0, transform: 'translateY(-8px)' }, { opacity: 1, transform: 'translateY(0)' }], { duration: 260, fill: 'forwards' }); }, text); await sleep(320); // Structural proof: a fixed bar at the top of the viewport, on top of whatever is beneath it, fully // inside the frame. A caption that lost its stylesheet sits unstyled at the end of the body instead. const painted = await page.evaluate(() => { const node = document.querySelector('#demo-caption'); if (!node) return { ok: false, reason: 'caption node missing' }; const rect = node.getBoundingClientRect(); const style = getComputedStyle(node); const topmost = document.elementFromPoint(Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)); // Proving the string was pushed is not proving it was readable. Measure the last line's own box, // so a paragraph caption pressed flat against the bar's bottom edge fails here instead of shipping. const range = document.createRange(); range.selectNodeContents(node); const lines = Array.from(range.getClientRects()).filter(box => box.height > 1); const lastBottom = lines.length ? Math.max(...lines.map(box => box.bottom)) : rect.bottom; const padBottom = Math.round(rect.bottom - lastBottom); return { ok: style.position === 'fixed' && style.zIndex === '20000' && rect.top <= 40 && rect.height >= 24 && rect.width > 200 && rect.bottom <= window.innerHeight && (topmost === node || (topmost && node.contains(topmost))) && lines.length > 0 && padBottom >= 3, top: Math.round(rect.top), height: Math.round(rect.height), position: style.position, zIndex: style.zIndex, background: style.backgroundColor, topmost: topmost === node, padBottom, lines: lines.length, }; }); const shownBand = await captionBand().catch(() => null); const bandChanged = Boolean(blankBand && shownBand && !blankBand.equals(shownBand)); if (!painted.ok) throw new Error(`Caption is not an on-screen bar: ${text} ${JSON.stringify(painted)}`); if (!bandChanged) throw new Error(`Caption painted nothing into the caption band: ${text}`); captionEvidence.push({ text, visible: true, position: painted.position, top: painted.top, height: painted.height, bandChanged, padBottom: painted.padBottom, lines: painted.lines }); await sleep(duration); // A caption must not outlive its stated duration. It used to stay on screen until the next one // replaced it, so a view change left the previous screen's claim hanging over a different screen — // e.g. a delete-data caption sitting on top of the home view. await page.evaluate(() => { const node = document.querySelector('#demo-caption'); if (!node) return; node.animate([{ opacity: 1 }, { opacity: 0 }], { duration: 180, fill: 'forwards' }); setTimeout(() => node.remove(), 220); }); await sleep(220); } async function tap(selector, after = 650) { const target = page.locator(selector).first(); await target.scrollIntoViewIfNeeded(); const box = await target.boundingBox(); if (!box) throw new Error(`Missing demo target: ${selector}`); await page.evaluate(({ x, y }) => { document.querySelector('#demo-touch')?.remove(); const ring = document.createElement('div'); ring.id = 'demo-touch'; ring.style.cssText = 'position:fixed;z-index:20001;width:48px;height:48px;border:4px solid #fff;' + 'background:rgba(245,201,91,.55);box-shadow:0 0 0 7px rgba(21,125,120,.28);border-radius:50%;' + 'pointer-events:none;transform:translate(-50%,-50%)'; ring.style.left = `${x}px`; ring.style.top = `${y}px`; document.body.append(ring); ring.animate([{ opacity: .2, transform: 'translate(-50%,-50%) scale(.55)' }, { opacity: 1, transform: 'translate(-50%,-50%) scale(1)' }], { duration: 400 }); setTimeout(() => ring.remove(), 550); }, { x: box.x + box.width / 2, y: box.y + box.height / 2 }); await sleep(220); await target.click(); await sleep(after); } async function indicate(selector) { const target = page.locator(selector).first(); await target.scrollIntoViewIfNeeded(); const box = await target.boundingBox(); if (!box) throw new Error(`Missing demo target: ${selector}`); await page.evaluate(({ x, y }) => { document.querySelector('#demo-touch')?.remove(); const ring = document.createElement('div'); ring.id = 'demo-touch'; ring.style.cssText = 'position:fixed;z-index:20001;width:48px;height:48px;border:4px solid #fff;' + 'background:rgba(245,201,91,.55);box-shadow:0 0 0 7px rgba(21,125,120,.28);border-radius:50%;' + 'pointer-events:none;transform:translate(-50%,-50%)'; ring.style.left = `${x}px`; ring.style.top = `${y}px`; document.body.append(ring); ring.animate([{ opacity: .2, transform: 'translate(-50%,-50%) scale(.55)' }, { opacity: 1, transform: 'translate(-50%,-50%) scale(1)' }], { duration: 400 }); setTimeout(() => ring.remove(), 550); }, { x: box.x + box.width / 2, y: box.y + box.height / 2 }); await sleep(650); } // Command-line evidence is shown on the same card chrome in three scenes (corpus audit, review // ledger, calibration gate). One renderer keeps the readability gate identical for all of them. async function evidenceCard({ id, title, subtitle, rows, font = '600 10.5px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace', keyWidth = 62, gap = '1px', rowGap = '7px' }) { return page.evaluate(params => { document.querySelector(`#${params.id}`)?.remove(); const panel = document.createElement('div'); panel.id = params.id; Object.assign(panel.style, { position: 'fixed', inset: '96px 14px 40px 14px', zIndex: '15000', background: '#181412', color: '#f6f1e6', borderRadius: '18px', border: '1px solid rgba(245,201,91,.45)', padding: '13px 13px 11px', font: params.font, display: 'flex', flexDirection: 'column', gap: params.gap, overflow: 'hidden', boxShadow: '0 18px 40px rgba(0,0,0,.42)', }); const head = document.createElement('div'); head.textContent = params.title; Object.assign(head.style, { font: '800 14px/1.2 system-ui', letterSpacing: '.14em', color: '#f5c95b', marginBottom: '2px' }); const sub = document.createElement('div'); sub.textContent = params.subtitle; Object.assign(sub.style, { font: '700 9.5px/1.3 system-ui', color: '#c9bfae', marginBottom: '9px' }); panel.append(head, sub); for (const [key, value] of params.rows) { const row = document.createElement('div'); if (key === '—') { row.style.height = params.rowGap; panel.append(row); continue; } Object.assign(row.style, { display: 'flex', gap: '7px', alignItems: 'baseline' }); const k = document.createElement('span'); k.textContent = key; k.style.cssText = `flex:0 0 ${params.keyWidth}px;color:#8fd7c8;text-transform:uppercase;font-size:9px;letter-spacing:.08em`; const v = document.createElement('span'); v.textContent = value; v.style.cssText = 'flex:1 1 auto;word-break:break-word;color:#f6f1e6'; row.append(k, v); panel.append(row); } document.body.append(panel); panel.animate([{ opacity: 0, transform: 'translateY(12px)' }, { opacity: 1, transform: 'translateY(0)' }], { duration: 320, fill: 'forwards' }); return { height: panel.clientHeight, scroll: panel.scrollHeight, text: panel.innerText }; }, { id, title, subtitle, rows, font, keyWidth, gap, rowGap }); } // A card is only evidence if it is actually legible: opaque, inside the frame, not covered, and not // clipped. A translucent or scrolled card reads as a sloppy overlay on the thinnest part of a film. async function evidenceCardProof(id) { return page.evaluate(cardId => { const panel = document.querySelector(`#${cardId}`); if (!panel) return { ok: false, reason: 'panel missing' }; const rect = panel.getBoundingClientRect(); const topmost = document.elementFromPoint(Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2)); const background = getComputedStyle(panel).backgroundColor; const opaque = !/^rgba\(.*,\s*(0?\.\d+|0)\)$/.test(background); return { ok: opaque && rect.top >= 60 && rect.bottom <= window.innerHeight && rect.width > 300 && rect.height > 300 && (topmost === panel || panel.contains(topmost)) && panel.scrollHeight <= panel.clientHeight + 1, top: Math.round(rect.top), height: Math.round(rect.height), width: Math.round(rect.width), scroll: panel.scrollHeight, client: panel.clientHeight, background, opaque, covered: topmost === panel || panel.contains(topmost), }; }, id); } await caption(`TIMMY ${version} • RELEASE DEMO`, 1200); await caption(`This release: ${releaseDelta}`, 2600); await caption('Automated checks replay this synthetic path before review', 1200); await caption('One clear photo action. Manual logging stays one tap away.', 1500); await tap('[data-scan]', 450); await page.getByText(/Self-hosted model ready/i).waitFor(); await caption('The pinned bootstrap verifies both model files before starting on private loopback', 1500); await indicate('label[for="camera-photo"]'); await page.locator('#camera-photo').dispatchEvent('cancel'); await page.getByText(/Camera or photo picker closed/i).waitFor(); await caption('Camera closed cleanly — gallery and manual logging are still available', 1600); // Issue #16, on camera: the file below is committed as text and only claims to be a photo. It must // be refused on the device, the refusal must say nothing was uploaded, and the app must never post // it — asserted here, not narrated. await indicate('label[for="gallery-photo"]'); await page.locator('#gallery-photo').setInputFiles('tests/fixtures/not-a-photo.jpg'); await page.locator('[data-capture-error="unreadable"]').waitFor(); const unreadableCopy = await page.locator('[data-capture-error="unreadable"]').innerText(); if (!/isn’t a photo Timmy can read/.test(unreadableCopy) || !/Nothing was uploaded/.test(unreadableCopy)) { throw new Error(`An unreadable file must be refused with the upload-free copy, saw: ${unreadableCopy}`); } if (await page.getByText(/couldn’t analyze that safely/i).count() !== 0) { throw new Error('A file that never left the device must not be reported as a failed analysis'); } if (analyzeCalls !== 0) throw new Error('An unreadable file was posted to /api/analyze'); await caption('A file that only claims to be a photo is refused on the device — nothing is uploaded', 2600); await indicate('[data-capture-error="unreadable"]'); await caption('Manual logging stays one tap away, and the photo picker is one tap back', 2000); await tap('#retake-photo', 450); await indicate('label[for="gallery-photo"]'); await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg'); await caption('Nothing uploads until explicit consent', 1100); await page.locator('#ai-consent').check(); await tap('#analyze-photo', 450); await page.getByText(/83% confidence/i).waitFor(); await caption('AI may suggest visible form, broad color, and image quality — never symptoms or diagnosis', 1700); await indicate('.scan-result[data-provenance="pending"]'); await caption('The suggestion stays provisional — nothing is applied or saved until you confirm it', 1700); await tap('#use-suggestion', 450); await page.locator('.ai-provenance[data-provenance="confirmed"]').waitFor(); await caption('Confirming is explicit, and every suggested value is still yours to overrule', 1600); await tap('[data-type="2"]', 500); await page.locator('[data-provenance="corrected"]').waitFor(); await caption('Change a suggested value and Timmy clears the AI-confirmed state', 1700); await indicate('.ai-provenance[data-provenance="corrected"] .provenance-note'); await tap('#next', 450); await page.getByText(/You changed it to Type 2/).waitFor(); await caption('Only visible form and color were ever suggested — urgency, discomfort, and symptoms stay yours', 1900); await tap('#next', 450); await page.locator('[data-followup-block]').waitFor(); await caption('Type 2 is hard or lumpy, so Timmy asks about soreness, hydration, blood, and black-or-tarry — never guessed from the photo', 2600); await page.locator('[data-question="bloodVisible"]').scrollIntoViewIfNeeded(); await indicate('[data-question="bloodVisible"]'); await tap('[data-followup="bloodVisible"][data-answer="no"]', 400); await caption('Every answer is yours to give, and a skipped question stays unanswered', 1900); await tap('[data-followup="soreness"][data-answer="yes"]', 350); await tap('[data-followup="dehydration"][data-answer="yes"]', 350); await page.locator('#followup-guidance').waitFor(); await caption('Mild report: steady water, no straining, gentle care — general self-care, not a diagnosis', 2600); await indicate('#followup-guidance'); await caption('And it never claims the photo ruled out bleeding', 1900); await tap('[data-followup="tarryBlack"][data-answer="yes"]', 600); await page.locator('#urgent-box .alert').waitFor(); await caption('Say it was truly black and tarry and the same deterministic red-flag path fires instead', 2600); await tap('[data-followup="tarryBlack"][data-answer="no"]', 500); await page.locator('#urgent-box .alert').waitFor({ state: 'detached' }); await caption('Dark brown is not silently escalated, and the flag clears the moment the answer changes', 2200); await tap('#close-sheet', 450); await caption('Closing the log keeps the draft in this session instead of throwing it away', 1600); await caption('Now the same photo flow while the vision worker is offline', 1500); visionOffline = true; await tap('[data-scan]', 550); await page.getByText(/Vision worker offline/i).waitFor(); await caption('Vision worker offline — the AI and consent steps disappear instead of failing after upload', 1900); await indicate('label[for="gallery-photo"]'); await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg'); await page.getByText(/Ready for your review/i).waitFor(); await caption('No consent prompt and no upload when analysis cannot run', 1600); await tap('#manual-from-scan', 500); const keptDraft = await page.locator('.draft-resume').innerText({ timeout: 5000 }); if (!/Type 2/.test(keptDraft)) throw new Error('The retained draft must show the value the user corrected to'); await caption('One tap still reaches manual Bristol entry, and the draft keeps your corrected value', 1800); await tap('#discard-draft', 500); await caption('Start over clears the kept draft — nothing lingers silently', 1500); await tap('#close-sheet', 450); // Post-load readiness pass (issue #15): the worker answers, but its pinned model is not resident // yet. The app must withhold AI instead of offering a control that fails after the upload. Every // claim below is asserted in the browser, so the recording cannot show a state the app does not have. modelLoaded = false; visionOffline = false; await caption('Now the same worker while its pinned model is still loading', 1700); await tap('[data-scan]', 550); await page.getByText(/model is not loaded yet/i).first().waitFor(); if (await page.getByText(/Self-hosted model ready/i).count() !== 0) { throw new Error('A reachable worker without its loaded model must never be reported ready'); } const loadingAmber = await page.locator('.model-status.loading').first().evaluate(el => getComputedStyle(el).backgroundColor); if (loadingAmber !== 'rgb(253, 243, 216)') throw new Error('The not-loaded state must be visually distinct from ready and offline'); await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg'); await page.getByText(/Ready for your review/i).waitFor(); if (await page.locator('#analyze-photo').count() !== 0) throw new Error('No AI control may exist before the model is loaded'); if (await page.locator('.consent-card').count() !== 0) throw new Error('No consent may be requested before the model is loaded'); await caption('Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident', 2900); await tap('#manual-from-scan', 500); await page.getByText('Pick the closest form').waitFor(); await caption('One tap still reaches manual Bristol entry while the model finishes loading', 2000); await tap('#close-sheet', 450); modelLoaded = true; await caption('Once the pinned model is loaded, the same screen offers the AI analysis again', 2200); await tap('[data-scan]', 550); await page.getByText(/Self-hosted model ready/i).waitFor(); await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg'); await page.getByText(/Ready for your review/i).waitFor(); if (await page.locator('.consent-card').isVisible() !== true) throw new Error('The AI path must return once the model is loaded'); if (await page.locator('#analyze-photo').count() !== 1) throw new Error('The analyze control must return once the model is loaded'); await tap('#close-sheet', 450); // Corpus refusal pass (issue #18): the same photo flow, fed the answers Timmy has to refuse. visionOffline = false; await caption('The corpus also replays the answers Timmy has to refuse', 1700); analyzeOverride = await corpusAnalyzeResult(corpusCase('non-stool-scene')); await tap('[data-scan]', 500); await page.getByText(/Self-hosted model ready/i).waitFor(); await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/non-stool-scene.png'); await page.locator('#ai-consent').check(); await tap('#analyze-photo', 600); await page.getByText(/No confident match/i).waitFor(); await caption('A non-stool photo gets a refusal, never a guess — manual entry stays one tap away', 2600); await indicate('.scan-result'); analyzeOverride = await corpusAnalyzeResult(corpusCase('glare-overexposed')); await tap('#retake-photo', 400); await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/glare-overexposed.png'); await page.locator('#ai-consent').check(); await tap('#analyze-photo', 600); await page.getByText(/does not show a form Timmy can read safely/i).waitFor(); await caption('Unreadable visual fields abstain as well: a valid reply with nothing to prefill is not an error', 2700); await indicate('.scan-result'); // Issue #32's quality gate: the model can be highly confident about a photo it also calls poor, and // that confident answer must still abstain. Replayed from the committed corpus, and asserted in the // browser so the scene is a check rather than a claim. analyzeOverride = await corpusAnalyzeResult(corpusCase('poor-quality-confident')); await tap('#retake-photo', 400); await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/blur-low-detail.png'); await page.locator('#ai-consent').check(); await tap('#analyze-photo', 600); await page.getByText(/too blurry or poorly lit/i).waitFor(); if (await page.getByText(/BRISTOL FORM/i).count() !== 0) { throw new Error('A low-quality photo must not prefill a form, however confident the reply looked'); } await caption('A confident reply about a blurry photo still abstains — photo quality is part of the policy, not just confidence', 3100); await indicate('.scan-result'); analyzeOverride = await corpusAnalyzeResult(corpusCase('out-of-range-fields')); await tap('#retake-photo', 400); await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/out-of-range-fields.png'); await page.locator('#ai-consent').check(); await tap('#analyze-photo', 600); await page.getByText(/couldn’t analyze that safely/i).waitFor(); await caption('An out-of-range answer fails closed: nothing is applied and no provider detail reaches you', 2700); await indicate('.scan-result'); // Issue #73's escape class: one field blank, the other filled in and impossible. The old blank-first // ordering abstained here, which told the user the photo was unreadable while the model had returned a // value Timmy must never apply. The scene is asserted in the browser, not just captioned. analyzeOverride = await corpusAnalyzeResult(corpusCase('blank-plus-out-of-range')); await tap('#retake-photo', 400); await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/blank-plus-out-of-range.png'); await page.locator('#ai-consent').check(); await tap('#analyze-photo', 600); await page.getByText(/couldn’t analyze that safely/i).waitFor(); if (await page.getByText(/No confident match/i).count() !== 0) { throw new Error('A filled-in out-of-range field must not be shown as an unreadable-photo abstention'); } await caption('A filled-in field that is out of range still fails closed, even when the other field came back blank', 2900); await indicate('.scan-result'); await caption('You are no longer told the photo was unreadable when the model returned an impossible value', 2400); await indicate('.scan-result'); await caption('Every refusal replays a committed synthetic fixture — there are no real photos in the corpus', 2500); analyzeOverride = null; await tap('#close-sheet', 450); // Explainable pattern summary (issue #34). The recording seeds fictional multi-day sample logs into // the app's own local ledger contract, then follows one summary line to the exact records behind it. // Every claim below is checked in the browser, and no real health data is involved anywhere. const seedSyntheticLedger = async entries => { const key = await page.evaluate(() => `timmy:${document.querySelector('meta[name="timmy-base-path"]')?.content || '/'}:ledger-v1`); await page.evaluate(({ storeKey, list }) => localStorage.setItem(storeKey, JSON.stringify(list)), { storeKey: key, list: entries }); await page.reload({ waitUntil: 'networkidle' }); }; const sampleDay = (id, daysAgo, bristolType) => ({ id, occurredAt: new Date(Date.now() - daysAgo * 86400000).toISOString(), bristolType, color: 'brown', urgency: 1, discomfort: 1, note: '', photoDataUrl: '', symptoms: {}, }); await caption('The journal explains its own pattern lines instead of just asserting a trend', 2200); await seedSyntheticLedger([ sampleDay('demo-firm', 2, 2), sampleDay('demo-typical-1', 5, 3), sampleDay('demo-typical-2', 9, 4), sampleDay('demo-typical-3', 12, 4), sampleDay('demo-old', 40, 6), ]); await caption("Fictional sample logs are seeded into this browser's own storage — no real health data appears here", 2600); await page.locator('[data-pattern-summary]').waitFor(); await indicate('[data-pattern-summary]'); await caption('The summary says which confirmed logs it read, and a citation opens exactly those records', 2400); await indicate('[data-statement="typical"]'); await tap('[data-statement-link="typical"]', 650); const citedRecords = await page.locator('[data-summary-record]').evaluateAll(nodes => nodes.map(node => node.getAttribute('data-entry-id')).sort()); if (JSON.stringify(citedRecords) !== JSON.stringify(['demo-typical-1', 'demo-typical-2', 'demo-typical-3'])) { throw new Error('A summary citation must list exactly the records it cites'); } await caption('Every pattern line names the confirmed logs behind it, and following one shows exactly those records', 2900); await indicate('[data-summary-records]'); await tap('#pattern-records-close', 450); await seedSyntheticLedger([sampleDay('demo-sparse-1', 1, 4), sampleDay('demo-sparse-2', 4, 6)]); await page.locator('[data-summary-insufficient]').waitFor(); await indicate('[data-summary-insufficient]'); await caption('Two logs is not a pattern: the summary refuses to describe a trend it cannot support', 2900); // Restore the empty ledger the earlier scenes started from, so the rest of the recording is unchanged. await page.evaluate(() => localStorage.clear()); await page.reload({ waitUntil: 'networkidle' }); // Issue #36 — a saved photo and its confirmed log are separate things a person owns. The journal // states that a photo exists without rendering it, the image appears only when asked for, removing // it keeps every confirmed field, and delete-all takes the ledger and every photo together. The // only image here is a synthetic 1x1 PNG constant written by this script; no real photo is used. // A 160x120 synthetic test card (checkerboard + 'SYNTHETIC DEMO PHOTO'), so the reveal is legibly // an image on camera. It carries no photograph of any person or specimen. const SYNTHETIC_PHOTO = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAKAAAAB4CAIAAAD6wG44AAAG0ElEQVR4nO2deVATZxTAXxKSIldVZqyiTEWucMgpyCGggEK96gFYgSKVenCJoYAy1DpaBYpBnEqt1WoVb2071fHCqi2IiFo70gJaT7COWKdYBZExCW7/2HFJm8OFok5f3m/yx+6X9739Jr98m+zb+RKB1MYaCLwIX/UAiBcLCUaOEbfVEh/33OjGtAT+qYeWlvEPpsy9m9ly+w52g2YwckgwckgwckgwckgwckgwckgwckgwckgwckgwckgwcgTc7cILNeXPjUZTqkWf2dsvgt2gGYwcEowcEowcEowcEowcEowcEowcEowcEowcEowcEowcEowcEowcEoycrqUrfG5Robmbhj6z5bMNmsHIIcHIIcHIIcHIIcHIIcHIIcHIIcHIIcHIIcHIIcHIIcHIIcHI6Vp8xv0Q2r3lBa9uPMR/YsBHuewG90NoWlYXmgxwevkjI3qFx/cusRu0utBQIMHIIcHIIcHIIcHIIcHIMXp+CAAANNTVFRcVqFQqkUiUXyRvf9QuS0/57nC5UCgEgKjJEwrkq6Pfnuju6bV15x62i6+7y7na+uPlR8u2bAaAn3867zXCBwDiEhKXLM4+V1vPJWcjAcDT2WG4uwfbGBo+dsgQaz199+3euWvHNlNTMxMT02Ur8wcOsuqdlwQZUhtr9tH+RwP7YLQxZXxEc/MdhmHKjxySpSUzDJO5IPXQgf0Mw/x48gTb4uPmHBc97eyZaraLj5uzegb1XV1P/atdT9/Tpypnxc7o6OhgGKbih5OJsTO0djQ0OImcVr6n6JaWPxVPngBAaPi4uIREAEhJX7j+s7VPnz7dsK40dYGMDUuTfbB2zeoX81b8B5s3fiHLyjE2NgaA4NFjrN8cqlKpXsJx/3fwFSzLWhQXM/3DRdkXzp/z9vEFAFs7O0ep09K83EFWg23t7dkwP/8AADh7pvoFDZfj2tUrTi6u3O7y/EIjI74fNwYF3xdlalR02Nhxx78vL/h4WXhEZFqGDABSFmRMHBd28NgJ9cj0hZmflhSP9A/Qk02pVCbMjFHf1WzPzF7k4eWtK0NnZyfPkRs4vATfv9/S1Njo6eU9LSpmdGjY5MixrGCbYbampqY2w2zVg339/EUiUY3eSSwWi8t27e3q4u6itV0PQ22GXW6od/PwBACGYXKzMwvlJXw6Ghq8TtECgUCWlny3+Q4APPjrgZXVYP3x6QszS0uKe2F0uomNT1hTvEqhUADA4YMH2A1CE14zuF+//svzCzNSk42NjYVC4coiuf74Eb4jxWJxD1509VO0h6dXZs5iXZFvTZzU2Hhz+qTx/S0tLS0tlyxf0d1jGQh0PxgVmveDqZKFHBKMHBKMHL7XwWyVWCAQKBWK6Hdip0ZFg0bpODFpjoeT/Ziw8JLSz9nGHFnGsaOHL166CgDffr13R9kWsViiVCrenTV7yvQozeQqpXLxkqXD3dy56jQLt6uZRGutu739ka5jGRp8BXNXqB2PH6fMmd3HxCRy/ATNy1aJRHLzxo3Ozk6RSMQwzK1bTRKJBACqKiu+2bN7y/bd5hYWba2t85MS3xg40D9w1L+SX/ntcl5O1r79B7WOQWuS8IjI8IhIAPB1d2GTVFVWlG3+UtexDI1un6L7mJhk5eZt+2qTrgBnV9e6X2oB4HJDg6NUyjZu2rA+OzfP3MICAMwtLLJy8zauX6fZ18FRevv277oy80zCM8xA6MlnsKPUqampUdezo4JCqiorAKDqVMWo4BC28cb1a+qlY2cX1+vXrmr2rak+LXVy1pWZZxKeYQZCTwr0KpVKbCQGHaXjwKDgndu2pmbIaqpPz4zX/msxDMMIBAJul83DMIy5ufmKwlWgu1itJ4kueIZhpSeCf6296CCVgo7S8et9+wqEQrauaWZmxjba2tk31Nd5Prt50FBfZ2fvwHXRzKO1WK0/CQfPMAOh26fo1ocP5Z8UJM2drycmKGR0ibwoQO17TdLc+fKClW1tbQDQ1tpaXJj//rzk7h6aZ5JeORYa+M5g9pzJXsnMmZfs6+cPukvHIWNC18iL9h85xnUPDAq+e7d5VmyMRPKaUqmIT3jPLyCwu2PlmaRXjoUGqkWjgmrRBgcJRg6tLkQFrS40OLR8i+beBZqg+bVdw8lMMxg5JBg5JBg5JBg5JBg5JBg5JBg5JBg5JBg5JBg5JBg5JBg59N+FODPTfxcaCiQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOVpWF+oBzd009JlpdaGhQIKRQ4KRQ4KRQ4KRQ4KRQ4KRQ4KRQ4KRQ4KRQ4KR01WLJlBCMxg5JBg5fwMX/GJFpYaDtAAAAABJRU5ErkJggg=='; const photoDay = (id, daysAgo, bristolType = 4) => ({ ...sampleDay(id, daysAgo, bristolType), photoDataUrl: SYNTHETIC_PHOTO }); const storedLedger = async () => { const key = await page.evaluate(() => `timmy:${document.querySelector('meta[name="timmy-base-path"]')?.content || '/'}:ledger-v1`); const raw = await page.evaluate(storeKey => localStorage.getItem(storeKey), key); return { key, raw }; }; await caption('A confirmed log can hold a private photo — the journal says so without showing it', 2300); await seedSyntheticLedger([photoDay('demo-photo', 1), sampleDay('demo-nophoto', 4, 6)]); await tap('[data-view="calendar"]', 550); await page.locator('.entries-card .entry').first().waitFor(); if (await page.locator('.entries-card [data-photo-indicator="present"]').count() !== 1) { throw new Error('exactly one log must be marked as holding a photo'); } if (await page.locator('.entries-card img').count() !== 0) { throw new Error('the journal must not show a thumbnail by default'); } await caption('Photo presence is a label, not a thumbnail — nothing sensitive is on screen until you ask', 2700); await indicate('[data-photo-indicator="present"]'); await tap('[data-entry-open="demo-photo"]', 550); await page.locator('[data-photo-lifecycle="present"]').waitFor(); if (await page.locator('#entry-photo').count() !== 0) { throw new Error('opening a log must not reveal its photo'); } await caption('Opening the log states the photo is saved here — and still does not render it', 2500); await indicate('[data-photo-lifecycle="present"]'); await tap('#show-entry-photo', 600); await page.locator('#entry-photo').waitFor(); const revealedPhoto = await page.locator('#entry-photo').getAttribute('src'); if (!/^data:image\/png;base64,/.test(revealedPhoto || '')) throw new Error('the revealed image must be the stored bytes'); await caption('Showing the photo is your explicit request, from this browser’s own storage', 2300); await indicate('#entry-photo'); await tap('#remove-entry-photo', 650); await page.locator('[data-photo-lifecycle="absent"]').waitFor(); const afterRemoval = JSON.parse((await storedLedger()).raw || '[]'); const keptLog = afterRemoval.find(entry => entry.id === 'demo-photo'); if (afterRemoval.length !== 2 || !keptLog || keptLog.photoDataUrl !== '') { throw new Error('removing a photo must keep the confirmed log and clear only its image bytes'); } if (keptLog.bristolType !== 4 || keptLog.color !== 'brown') throw new Error('removing a photo must not alter the confirmed fields'); await caption('Remove just the photo: the confirmed log, form, and note are all still here', 2900); await indicate('[data-photo-lifecycle="absent"]'); await tap('#entry-sheet-close', 450); await page.locator('.entries-card .entry').first().waitFor(); if (await page.locator('.entries-card [data-photo-indicator="present"]').count() !== 0) { throw new Error('the journal must drop the photo label once the bytes are gone'); } await caption('The label follows the bytes: the log stays, the photo is gone', 2300); // One photo-bearing log remains for the deletion pass below, so the wipe shows photos leaving with // the ledger rather than an empty promise. await seedSyntheticLedger([photoDay('demo-wipe', 2)]); await caption('The threat model documents this surface, so the recording shows it live rather than describing it', 2200); await tap('[data-view="calendar"]', 550); await page.getByText(/Your private journal/i).waitFor(); await tap('[data-view="privacy"]', 550); await page.getByText(/Your poop\. Your phone\./i).waitFor(); await caption('Every byte-bearing control the model names, in the running app: local ledger, consent-gated upload, portable export, full deletion', 2800); await indicate('#export'); await caption('Export writes a plain JSON file you hold — the model records it as a data-lifetime boundary, not a server sync', 2600); // Issue #24 — training contribution is a second, separate consent. The recording shows the section // reporting nothing queued after ordinary saves, the version and receipt on screen before the // button can be pressed, one receipt queued on the device only, and a withdrawal tombstone that // names each derived artifact by contribution id. await caption('Training contribution is off by default: a saved log is never training data', 2600); await page.locator('[data-training-consent]').waitFor(); const contributionKey = await page.evaluate(() => `timmy:${document.querySelector('meta[name="timmy-base-path"]')?.content || '/'}:training-contributions-v1`); const storedContributions = async () => JSON.parse(await page.evaluate(key => localStorage.getItem(key) || '[]', contributionKey)); if ((await page.locator('[data-contribution-queued-count]').innerText()).trim() !== '0') { throw new Error('a journal save must not queue a training contribution'); } await indicate('[data-contribute-entry="demo-wipe"]'); await tap('[data-contribute-entry="demo-wipe"]', 600); await page.locator('[data-contribution-sheet]').waitFor(); const receiptId = (await page.locator('[data-receipt-id]').innerText()).trim(); if (!receiptId) throw new Error('the consent sheet must name the receipt before consent'); const sheetCopy = await page.locator('[data-contribution-sheet]').innerText(); if (!sheetCopy.includes('training-consent-v1')) throw new Error('the consent sheet must state the consent version'); if (!(await page.locator('#contribute-confirm').isDisabled())) throw new Error('the contribute control must stay disabled until the terms are accepted'); if ((await storedContributions()).length !== 0) throw new Error('no receipt may exist before explicit consent'); await caption('The consent version and the receipt are on screen before the contribute button works', 3000); await page.locator('#training-consent-check').check(); await tap('#contribute-confirm', 700); await page.locator('[data-contribution-state="queued-local"]').waitFor(); const queuedReceipts = await storedContributions(); if (queuedReceipts.length !== 1 || queuedReceipts[0].consentVersion !== 'training-consent-v1') { throw new Error('one queued receipt must carry the consent version it was accepted under'); } if (JSON.stringify(queuedReceipts).includes('data:image')) throw new Error('a receipt must never copy the photo bytes'); await caption('One confirmed log is queued on this device with a receipt you can withdraw — nothing is uploaded', 3200); await indicate('[data-contribution-state="queued-local"]'); await tap(`[data-withdraw="${receiptId}"]`, 700); await page.locator('[data-contribution-state="withdrawn"]').waitFor(); const withdrawnReceipts = await storedContributions(); const tombstone = withdrawnReceipts[0]; if (!tombstone || tombstone.state !== 'withdrawn') throw new Error('withdrawal must keep the receipt as a tombstone'); if ((tombstone.artifacts || []).length !== 2) throw new Error('the tombstone must name every derived artifact'); for (const artifact of tombstone.artifacts) { if (artifact.contributionId !== receiptId) throw new Error('every derived artifact must be addressed by contribution id'); if (artifact.state !== 'withdrawn') throw new Error('every derived artifact must be marked withdrawn'); } await caption('Withdrawing keeps a tombstone naming every derived artifact by contribution ID', 3000); await indicate('[data-withdrawn-artifacts]'); // Issue #25 — the corpus pipeline is a command-line tool, not a screen in the app. Rather than // inventing a UI for it, this scene runs the real ingest and revoke commands against the committed // synthetic fixture in a throwaway dataset and shows their actual output. Every number, hash prefix, // and refusal line below is read back from those commands at record time. async function corpusCli(args) { return execFileAsync('python3', args, { cwd: process.cwd(), env: { ...process.env, TIMMY_DATASET_SALT: 'demo-only-salt-not-a-secret' }, }); } async function corpusAudit() { const dataset = await mkdtemp(path.join(os.tmpdir(), 'timmy-corpus-demo-')); const fixture = 'tests/fixtures/synthetic-type4.jpg'; const contributionId = 'demo-contrib-0001-abcd'; const manifestPath = path.join(dataset, 'manifest.jsonl'); const lines = async () => (await readFile(manifestPath, 'utf8')) .split('\n').filter(line => line.trim()).map(line => JSON.parse(line)); const images = async () => (await readdir(path.join(dataset, 'images')).catch(() => [])).sort(); try { const ingest = await corpusCli([ 'scripts/ingest_training_photo.py', '--input', fixture, '--output-dir', dataset, '--subject-id', 'demo-subject', '--consent-version', 'training-consent-v1', '--contribution-id', contributionId, '--bristol-type', '4', '--color', 'brown', '--quality', 'good', '--reviewer', 'user-confirmed', ]); const record = JSON.parse(ingest.stdout); const fixtureDigest = createHash('sha256').update(await readFile(fixture)).digest('hex'); if (record.recordType !== 'sample' || record.lifecycle?.state !== 'active') { throw new Error('ingest must write one active sample record'); } // The record names the source by digest only: the caller's path must never be stored. if (record.source?.inputSha256 !== fixtureDigest || record.hashes?.rawInput !== fixtureDigest) { throw new Error('the record must carry the raw input digest'); } if (JSON.stringify(record).includes(fixture) || 'inputPath' in record) { throw new Error('the record must never store the caller path'); } const afterIngestImages = await images(); if (afterIngestImages.length !== 1 || (await lines()).length !== 1) { throw new Error('ingest must write exactly one derived image and one manifest record'); } const revoke = await corpusCli([ 'scripts/ingest_training_photo.py', '--output-dir', dataset, '--revoke', contributionId, '--reason', 'participant withdrew consent', ]); const tombstone = JSON.parse(revoke.stdout); const afterRevokeImages = await images(); const history = await lines(); if (tombstone.recordType !== 'tombstone' || history.length !== 2) { throw new Error('revocation must append exactly one tombstone to the existing history'); } if (afterRevokeImages.length !== 0) throw new Error('revocation must delete the derived bytes'); if (history[0].recordType !== 'sample' || history[0].lifecycle?.state !== 'active') { throw new Error('the original record must survive its own withdrawal'); } if ('labels' in tombstone || 'subjectKey' in tombstone) { throw new Error('a tombstone must not carry the labels or subject key'); } const deleted = tombstone.deletedArtifacts || []; if (!deleted.some(item => item.relativePath === `images/${record.imageId}.jpg` && item.deleted === true)) { throw new Error('the tombstone must name the derived artifact it removed'); } const active = await corpusCli([ '-c', 'import sys;sys.path.insert(0,"scripts");import corpus_manifest as c;from pathlib import Path;' + 'print(len(c.active_records(c.load_manifest(Path(sys.argv[1])/"manifest.jsonl"))))', dataset, ]); if (active.stdout.trim() !== '0') throw new Error('a trainer reading the manifest must see zero active samples'); // A second revocation of the same contribution has to be refused, and refuse without writing. let refusal = ''; try { await corpusCli([ 'scripts/ingest_training_photo.py', '--output-dir', dataset, '--revoke', contributionId, '--reason', 'participant withdrew consent again', ]); throw new Error('a second revocation must be refused'); } catch (error) { if (error.message === 'a second revocation must be refused') throw error; refusal = String(error.stderr || error.stdout || '').trim().split('\n').filter(Boolean).pop() || ''; } if (!/already revoked/.test(refusal)) throw new Error(`second revocation refusal was not legible: ${refusal}`); if ((await lines()).length !== 2) throw new Error('a refused revocation must append nothing'); return { record, tombstone, deleted, refusal, derived: record.imageId, fixtureDigest }; } finally { await rm(dataset, { recursive: true, force: true }); } } const audit = await corpusAudit(); const short = value => String(value).slice(0, 12); const deletedName = audit.deleted[0].relativePath.split('/').pop().replace(/\.jpg$/, ''); await tap('[data-view="home"]', 500); // The card is drawn from the values above, so a run that stops writing provenance or stops deleting // bytes fails here instead of recording a caption that claims it. const auditRows = [ ['command', 'python3 scripts/ingest_training_photo.py --input '], ['record', `sample · schemaVersion ${audit.record.schemaVersion}`], ['consent', audit.record.consent.version], ['source', `sha256 ${short(audit.record.source.inputSha256)}… (path never stored)`], ['hashes', `raw ${short(audit.record.hashes.rawInput)}… · derived ${short(audit.record.hashes.derived)}…`], ['derived', audit.record.derivation.pipeline + ' · jpeg · ≤1200px · metadata stripped'], ['review', audit.record.review.status], ['lifecycle', audit.record.lifecycle.state], ['—', ''], ['command', `--revoke ${audit.record.contributionId} --reason "participant withdrew consent"`], ['deleted', `images/${short(deletedName)}….jpg → gone from disk`], ['tombstone', 'appended · keeps id + digests · drops labels + subject key'], ['history', '2 lines: the original record is still there'], ['trainer', 'active samples: 0'], ['—', ''], ['command', '--revoke again'], ['refused', audit.refusal], ['manifest', 'unchanged at 2 lines'], ]; const panelOk = await evidenceCard({ id: 'corpus-audit', title: 'CORPUS AUDIT', subtitle: 'command line, not app UI — real run, synthetic fixture', rows: auditRows, }); await sleep(420); const panelProof = await evidenceCardProof('corpus-audit'); if (!panelProof.ok) throw new Error(`Corpus audit card is not fully readable on screen: ${JSON.stringify(panelProof)}`); // The card is drawn from live command output; prove the values on it are the ones the run produced. if (!panelOk.text.includes(audit.record.imageId.slice(0, 12))) throw new Error('the card must show the derived digest it recorded'); if (!panelOk.text.includes('active samples: 0')) throw new Error('the card must show the post-revocation active count'); await caption('The corpus pipeline is a command-line tool, not a screen in the app — this runs the real ingest against the committed synthetic fixture', 3400); await caption('Every ingested sample records consent, source digest, both hashes, derivation, review, and lifecycle — the caller file path is never stored', 3600); await caption('Withdrawing deletes the derived image and any raw copy, keeps the original record, and appends a tombstone — the trainer sees zero active samples', 3800); await caption('Revoking the same contribution twice is refused and appends nothing: the manifest only ever grows', 3200); await page.evaluate(() => document.querySelector('#corpus-audit')?.remove()); // Issue #26 — the reviewer and adjudication tool is a command line, not a screen in the app. This // scene runs the real commands against the committed synthetic sample in a throwaway dataset and // shows the order the tool enforces: the queue hides the model answer, revealing it is refused // until a human commits, and a withdrawal removes the sample from the queue, the review path, and // the training set. Every value on the card is read back from those runs. The answer file is a // synthetic fixture; no model measured anything here. async function reviewCli(args) { try { const result = await execFileAsync('python3', ['scripts/review_corpus.py', ...args], { cwd: process.cwd(), env: { ...process.env, TIMMY_DATASET_SALT: 'demo-only-salt-not-a-secret' }, }); return { code: 0, stdout: result.stdout, stderr: result.stderr }; } catch (error) { return { code: typeof error.code === 'number' ? error.code : 1, stdout: error.stdout || '', stderr: error.stderr || '', }; } } function lastLine(output) { return String(output).trim().split('\n').filter(Boolean).pop() || ''; } async function reviewLedger() { const dataset = await mkdtemp(path.join(os.tmpdir(), 'timmy-review-demo-')); const contributionId = 'demo-review-0001-abcd'; const predictionsPath = path.join(dataset, 'synthetic-answers.json'); const manifestPath = path.join(dataset, 'manifest.jsonl'); const reviewPath = path.join(dataset, 'reviews.jsonl'); const lines = async file => (await readFile(file, 'utf8').catch(() => '')) .split('\n').filter(line => line.trim()).map(line => JSON.parse(line)); try { const ingest = await corpusCli([ 'scripts/ingest_training_photo.py', '--input', 'tests/fixtures/synthetic-type4.jpg', '--output-dir', dataset, '--subject-id', 'demo-subject', '--consent-version', 'training-consent-v1', '--contribution-id', contributionId, '--bristol-type', '4', '--color', 'brown', '--quality', 'good', '--reviewer', 'user-confirmed', ]); if (JSON.parse(ingest.stdout).lifecycle?.state !== 'active') { throw new Error('the review scene needs one active synthetic sample'); } const before = await reviewCli(['queue', '--dataset', dataset]); const queue = JSON.parse(before.stdout); if (queue.count !== 1) throw new Error('one unreviewed sample must queue exactly once'); if (queue.items[0].proposed.bristolType !== 4) throw new Error('the queue must show the proposal'); if ('confidence' in queue.items[0] || /"model"/.test(JSON.stringify(queue.items[0]))) { throw new Error('the queue must not carry the model answer'); } const early = await reviewCli(['reveal', '--dataset', dataset, '--contribution', contributionId]); if (early.code === 0) throw new Error('revealing before a review must be refused'); const blindRefusal = lastLine(early.stderr); if (!/stays hidden until a review is committed/.test(blindRefusal)) { throw new Error(`the blind refusal was not legible: ${blindRefusal}`); } await writeFile(predictionsPath, JSON.stringify({ model: 'synthetic answer file', answers: { [contributionId]: { bristolType: 6, color: 'brown', quality: 'good', confidence: 0.71 } }, }, null, 2)); const committed = await reviewCli([ 'review', '--dataset', dataset, '--contribution', contributionId, '--reviewer', 'reviewer-a', '--bristol', '5', '--color', 'green', '--quality', 'fair', ]); if (committed.code !== 0) throw new Error(`the review was refused: ${lastLine(committed.stderr)}`); const review = JSON.parse(committed.stdout); if (review.decision !== 'corrected' || review.labels.bristolType !== 5) { throw new Error('the committed review must record the corrected label'); } if (review.proposed.bristolType !== 4) throw new Error('the committed review must keep the proposal it judged'); const revealed = await reviewCli(['reveal', '--dataset', dataset, '--contribution', contributionId, '--predictions', predictionsPath]); if (revealed.code !== 0) throw new Error(`reveal failed after a committed review: ${lastLine(revealed.stderr)}`); const reveal = JSON.parse(revealed.stdout); if (reveal.model?.labels.bristolType !== 6) throw new Error('the revealed answer must be the one that was withheld'); if (!reveal.disagreement || reveal.disagreeFields.length !== 3) { throw new Error('a Type 6 answer against a Type 5 review must be recorded as a disagreement'); } const ruled = await reviewCli([ 'adjudicate', '--dataset', dataset, '--contribution', contributionId, '--reviewer', 'reviewer-b', '--upheld', ]); if (ruled.code !== 0) throw new Error(`adjudication was refused: ${lastLine(ruled.stderr)}`); const ruling = JSON.parse(ruled.stdout); if (ruling.resolvesRecord !== 1) throw new Error('the ruling must point at the record it resolves'); const ledgerAfterRuling = await lines(reviewPath); if (ledgerAfterRuling.length !== 2) throw new Error('adjudication must append, not replace'); if (ledgerAfterRuling[0].reviewer !== 'reviewer-a' || ledgerAfterRuling[0].decision !== 'corrected') { throw new Error('the review being adjudicated must survive untouched'); } const trainableBefore = JSON.parse((await reviewCli(['trainable', '--dataset', dataset])).stdout); if (trainableBefore.count !== 1) throw new Error('one reviewed sample must be trainable'); if (trainableBefore.items[0].labels.bristolType !== 5) { throw new Error('the trainer must read the reviewed label, not the proposal'); } const revoke = await corpusCli([ 'scripts/ingest_training_photo.py', '--output-dir', dataset, '--revoke', contributionId, '--reason', 'participant withdrew consent', ]); if (JSON.parse(revoke.stdout).recordType !== 'tombstone') throw new Error('the withdrawal must be recorded'); const afterRevoke = JSON.parse((await reviewCli(['queue', '--dataset', dataset])).stdout); const trainableAfter = JSON.parse((await reviewCli(['trainable', '--dataset', dataset])).stdout); if (afterRevoke.count !== 0 || trainableAfter.count !== 0) { throw new Error('a withdrawn sample must leave the queue and the training set'); } const refused = await reviewCli([ 'review', '--dataset', dataset, '--contribution', contributionId, '--reviewer', 'reviewer-c', '--agree', ]); if (refused.code === 0) throw new Error('reviewing a withdrawn contribution must be refused'); const withdrawnRefusal = lastLine(refused.stderr); if (!/was withdrawn and cannot be reviewed/.test(withdrawnRefusal)) { throw new Error(`the withdrawal refusal was not legible: ${withdrawnRefusal}`); } if ((await lines(reviewPath)).length !== 2) throw new Error('a refused review must append nothing'); if ((await lines(manifestPath)).length !== 2) throw new Error('the manifest must hold the sample and its tombstone'); return { queue, blindRefusal, review, reveal, ruling, trainableBefore, afterRevoke, trainableAfter, withdrawnRefusal }; } finally { await rm(dataset, { recursive: true, force: true }); } } const reviewEvidence = await reviewLedger(); await tap('[data-view="home"]', 450); const reviewRows = [ ['command', 'python3 scripts/review_corpus.py queue --dataset '], ['queue', `${reviewEvidence.queue.count} pending · proposal Type ${reviewEvidence.queue.items[0].proposed.bristolType} · ${reviewEvidence.queue.items[0].proposed.color} · ${reviewEvidence.queue.items[0].proposed.quality}`], ['blind', 'model answer withheld from the reviewer'], ['—', ''], ['command', 'reveal --contribution (before any review)'], ['refused', reviewEvidence.blindRefusal], ['—', ''], ['command', 'review --reviewer reviewer-a --bristol 5 --color green --quality fair'], ['committed', `${reviewEvidence.review.decision} · proposal ${reviewEvidence.review.proposed.bristolType}/${reviewEvidence.review.proposed.color}/${reviewEvidence.review.proposed.quality} kept beside it`], ['command', 'reveal --predictions '], ['model', `Type ${reviewEvidence.reveal.model.labels.bristolType} · ${reviewEvidence.reveal.model.labels.color} · ${reviewEvidence.reveal.model.labels.quality} · confidence ${reviewEvidence.reveal.model.confidence}`], ['disagree', reviewEvidence.reveal.disagreeFields.join(' · ')], ['—', ''], ['command', 'adjudicate --reviewer reviewer-b --upheld'], ['ledger', `2 lines · resolves record ${reviewEvidence.ruling.resolvesRecord} · reviewer-a's review untouched`], ['trainer', `trainable samples: ${reviewEvidence.trainableBefore.count}`], ['—', ''], ['command', 'ingest --revoke --reason "participant withdrew consent"'], ['after', `queue ${reviewEvidence.afterRevoke.count} · trainable ${reviewEvidence.trainableAfter.count}`], ['review', reviewEvidence.withdrawnRefusal], ['data', 'synthetic answer file · no model was measured'], ]; const reviewCard = await evidenceCard({ id: 'review-audit', title: 'REVIEW LEDGER', subtitle: 'command line, not app UI — real run, synthetic sample', rows: reviewRows, }); await sleep(420); const reviewProof = await evidenceCardProof('review-audit'); if (!reviewProof.ok) throw new Error(`Review card is not fully readable on screen: ${JSON.stringify(reviewProof)}`); if (!reviewCard.text.includes(`Type ${reviewEvidence.reveal.model.labels.bristolType}`)) { throw new Error('the card must show the model answer the run revealed'); } if (!reviewCard.text.includes('was withdrawn and cannot be reviewed')) { throw new Error('the card must show the withdrawn refusal the run earned'); } if (!reviewCard.text.includes(`trainable samples: ${reviewEvidence.trainableBefore.count}`)) { throw new Error('the card must show the trainer count the run produced'); } await caption('The review tool is a command line, not a screen in the app — this runs it against the committed synthetic sample', 3600); await caption('The queue shows the proposal and hides the model answer; revealing it is refused until a reviewer commits', 3800); await caption('The model said Type 6 and the reviewer said Type 5: the disagreement is recorded, not smoothed over', 3600); await caption('A second reviewer adjudicates, and the review being overruled is still in the ledger', 3400); await caption('Withdrawing the contribution empties the queue and the training set, and the tool refuses to review it at all', 3800); await page.evaluate(() => document.querySelector('#review-audit')?.remove()); // Issue #27 — the dataset card is a generated document, not a hand-written page. This scene builds a // throwaway corpus with the real ingest command from committed synthetic fixtures, publishes the card // with the real generator, and shows the refusal the split policy earns when one contributor is // planted across two splits. Every count, id, and refusal line on the card is read back from those // runs. No model labelled anything in this corpus, and the caption says so. async function cardCli(args) { try { const result = await execFileAsync('python3', ['scripts/dataset_card.py', ...args], { cwd: process.cwd() }); return { code: 0, stdout: result.stdout, stderr: result.stderr }; } catch (error) { return { code: typeof error.code === 'number' ? error.code : 1, stdout: error.stdout || '', stderr: error.stderr || '', }; } } async function publishDatasetCard() { const dataset = await mkdtemp(path.join(os.tmpdir(), 'timmy-card-demo-')); const leaked = await mkdtemp(path.join(os.tmpdir(), 'timmy-card-leak-')); const cardPath = path.join(dataset, 'DATASET_CARD.md'); // Three distinct committed synthetic fixtures: a formed sample, a well-lit stool-like sample, and a // glare frame that can only honestly be labelled poor quality. const contributions = [ { id: 'demo-card-0001-abcd', subject: 'demo-contributor-a', file: 'tests/fixtures/synthetic-type4.jpg', type: '4', color: 'brown', quality: 'good' }, { id: 'demo-card-0002-abcd', subject: 'demo-contributor-a', file: 'tests/fixtures/vision-corpus/stool-like-formed.png', type: '3', color: 'green', quality: 'fair' }, { id: 'demo-card-0003-abcd', subject: 'demo-contributor-b', file: 'tests/fixtures/vision-corpus/glare-overexposed.png', type: '1', color: 'brown', quality: 'poor' }, ]; try { for (const item of contributions) { await corpusCli([ 'scripts/ingest_training_photo.py', '--input', item.file, '--output-dir', dataset, '--subject-id', item.subject, '--consent-version', 'training-consent-v1', '--contribution-id', item.id, '--bristol-type', item.type, '--color', item.color, '--quality', item.quality, '--reviewer', 'user-confirmed', ]); } // Withdraw one of the first contributor's two samples, so both contributors still have an active // sample and the class table shows the withdrawn label dropping out of the count. const withdrawn = contributions[1].id; await corpusCli([ 'scripts/ingest_training_photo.py', '--output-dir', dataset, '--revoke', withdrawn, '--reason', 'participant withdrew consent', ]); const published = await cardCli(['--corpus', dataset, '--out', cardPath]); if (published.code !== 0) throw new Error(`the dataset card was refused: ${lastLine(published.stderr)}`); const summary = JSON.parse((await cardCli(['--corpus', dataset, '--json'])).stdout); if (summary.activeSamples !== 2) throw new Error('the card must count the two active samples the run ingested'); if (summary.contributors !== 2) throw new Error('the card must count both contributors'); if (summary.withdrawals !== 1) throw new Error('the card must count the withdrawal as a withdrawal'); const card = await readFile(cardPath, 'utf8'); for (const fact of ['| Devices | pending', '| Contexts | pending', '| Licenses | pending']) { if (!card.includes(fact)) throw new Error(`an uncollected fact must be printed as pending: ${fact}`); } if (!card.includes(withdrawn) || card.includes('subjectKey')) { throw new Error('the card must name the withdrawn contribution and never carry a subject key'); } if (summary.bristol['3'] !== 0) throw new Error('a withdrawn sample must not be counted in the class balance'); // Plant one contributor in a second split and prove the policy refuses to publish a card at all. const original = await readFile(path.join(dataset, 'manifest.jsonl'), 'utf8'); const records = original.split('\n').filter(line => line.trim()).map(line => JSON.parse(line)); const sample = records.find(record => record.recordType === 'sample' && record.contributionId === contributions[0].id); const planted = { ...sample, contributionId: 'demo-card-0009-leak', split: sample.split === 'train' ? 'test' : 'train' }; await writeFile(path.join(leaked, 'manifest.jsonl'), `${original}${JSON.stringify(planted)}\n`); const refused = await cardCli(['--corpus', leaked, '--out', path.join(leaked, 'DATASET_CARD.md')]); if (refused.code === 0) throw new Error('the generator must refuse a corpus that leaks one contributor across splits'); const leakRefusal = lastLine(refused.stderr); if (!/leaks across/.test(leakRefusal)) throw new Error(`the leak refusal was not legible: ${leakRefusal}`); // The committed page has to be this generator's own output, not a document someone typed. const committed = await cardCli(['--corpus', '/srv/timmy-corpus', '--check', 'docs/DATASET_CARD.md']); if (committed.code !== 0) throw new Error("the committed dataset card must be the generator's own output"); return { summary, leakRefusal, withdrawn }; } finally { await rm(dataset, { recursive: true, force: true }); await rm(leaked, { recursive: true, force: true }); } } const cardEvidence = await publishDatasetCard(); await tap('[data-view="home"]', 450); const splits = cardEvidence.summary.bySplit; const cardRows = [ ['command', 'python3 scripts/dataset_card.py --corpus --out DATASET_CARD.md'], ['corpus', `${cardEvidence.summary.activeSamples} active · ${cardEvidence.summary.contributors} contributors · ${cardEvidence.summary.withdrawals} withdrawn`], ['splits', `train ${splits.train.contributors}c/${splits.train.samples}s · validation ${splits.validation.contributors}c/${splits.validation.samples}s · test ${splits.test.contributors}c/${splits.test.samples}s`], ['class', `Type 1 ${cardEvidence.summary.bristol['1']} · Type 3 ${cardEvidence.summary.bristol['3']} · Type 4 ${cardEvidence.summary.bristol['4']} · withdrawn label excluded`], ['withdrawn', `${cardEvidence.withdrawn} · "participant withdrew consent" · no label survives in any table`], ['pending', 'devices · contexts · licenses — not collected, not estimated'], ['—', ''], ['command', 'publish a manifest that plants one contributor in train and test'], ['refused', cardEvidence.leakRefusal], ['card', 'not written: the policy is checked before a line is rendered'], ['—', ''], ['command', '--check docs/DATASET_CARD.md'], ['committed', 'the published page is this generator\'s own output'], ['data', 'synthetic fixtures · no model labelled anything here'], ]; const cardPanel = await evidenceCard({ id: 'dataset-card', title: 'DATASET CARD', subtitle: 'generated from the manifest — command line, not app UI', rows: cardRows, keyWidth: 70, }); await sleep(420); const cardProof = await evidenceCardProof('dataset-card'); if (!cardProof.ok) throw new Error(`Dataset card is not fully readable on screen: ${JSON.stringify(cardProof)}`); if (!cardPanel.text.includes(`${cardEvidence.summary.contributors} contributors`)) { throw new Error('the card must show the contributor count the run produced'); } if (!cardPanel.text.includes('leaks across')) throw new Error('the card must show the leak refusal the run earned'); if (!cardPanel.text.includes('not collected')) throw new Error('the card must show the facts that stay pending'); await caption('The dataset card is generated from the corpus manifest, so no count on it is typed by hand', 3600); await caption('One contributor\'s photos stay in one split: a corpus that leaks a person across splits is refused', 3800); await caption('Devices, contexts, and licenses stay pending until the pipeline collects them', 3400); await caption('The card is a page, not a screen in the app — it is published with the real generator against synthetic fixtures', 3400); await page.evaluate(() => document.querySelector('#dataset-card')?.remove()); // Issue #29 — the release gate is a command-line harness, not a screen in the app. This scene runs // the real harness over the committed synthetic prediction fixtures and shows the receipt it wrote: // a blocked comparison, a passing fixture, and a refused leaked corpus. Every number and every // refusal line on the card is read back from those runs; the scores are invented fixtures, and the // captions say so, because a fixture is not a measurement of any model. async function calibrationCli(args) { try { const result = await execFileAsync('python3', ['scripts/calibration_report.py', ...args], { cwd: process.cwd() }); return { code: 0, stdout: result.stdout, stderr: result.stderr }; } catch (error) { return { code: typeof error.code === 'number' ? error.code : 1, stdout: error.stdout || '', stderr: error.stderr || '', }; } } async function calibrationGate() { const directory = await mkdtemp(path.join(os.tmpdir(), 'timmy-calibration-demo-')); const reportPath = path.join(directory, 'report.json'); const source = 'invented scores, not measurements of any model'; try { const blocked = await calibrationCli([ '--predictions', 'tests/fixtures/calibration/predictions.jsonl', '--json', reportPath, '--source', source, ]); if (blocked.code !== 3) throw new Error('the synthetic candidate fixtures must block the release'); const report = JSON.parse(await readFile(reportPath, 'utf8')); if (report.verdict !== 'BLOCKED' || report.recommended !== null) { throw new Error('a blocked gate must recommend nobody'); } if (report.models.length !== 3) throw new Error('the comparison must score all three candidate fixtures'); for (const row of report.models) { if (!row.gate.failed.length) throw new Error(`${row.model}: a fixture candidate must fail at least one gate`); if (row.selectionSplit !== 'validation' || row.evaluatedSplit !== 'test') { throw new Error(`${row.model}: the threshold must come from validation and the score from test`); } } const go = await calibrationCli([ '--predictions', 'tests/fixtures/calibration/predictions-go.jsonl', '--source', 'engineered fixture — proves the passing branch, not a result', ]); if (go.code !== 0 || !/verdict: GO/.test(go.stdout)) throw new Error('the engineered fixture must prove the passing branch'); const leak = await calibrationCli(['--predictions', 'tests/fixtures/calibration/predictions-leaky.jsonl']); if (leak.code !== 2 || !/leaks across/.test(leak.stderr)) throw new Error('a leaked corpus must be refused before scoring'); return { report, source, blocked, go, leak }; } finally { await rm(directory, { recursive: true, force: true }); } } const gate = await calibrationGate(); await tap('[data-view="home"]', 450); const gateRows = [ ['command', 'python3 scripts/calibration_report.py --predictions '], ['split', 'threshold chosen on validation · scored on test'], ['floor', `coverage floor ${Math.round(gate.report.models[0].minCoverage * 100)}%`], ['—', ''], ]; for (const row of gate.report.models) { // The model id gets its own row: a 74px key column broke "specialist-candidate" at its hyphen, // which reads as a label with no value and a stray line under it. The name is the value here. gateRows.push(['model', row.model]); gateRows.push([ 'metrics', `thr ${row.threshold.toFixed(2)} · F1 ${row.macroF1.toFixed(3)} · P@C ${row.precisionAtCoverage.toFixed(3)} · cov ${row.coverage.toFixed(2)}`, ]); gateRows.push(['gate', `FAIL: ${row.gate.failed.join(', ')} · ECE ${row.ece.toFixed(3)}`]); } gateRows.push(['—', '']); gateRows.push(['verdict', `${gate.report.verdict} · no candidate clears every gate`]); gateRows.push(['gates', 'macroF1 ≥ 0.80 · precision@coverage ≥ 0.90 · ECE ≤ 0.05']); gateRows.push(['—', '']); gateRows.push(['engineered', 'GO — gate-plumbing-fixture clears every gate (fixture, not a result)']); gateRows.push(['leaked', gate.leak.stderr.trim().replace(/^calibration harness refused:\s*/, '')]); gateRows.push(['data', 'synthetic predictions · no model was measured']); const gateCard = await evidenceCard({ id: 'calibration-report', title: 'CALIBRATION HARNESS', subtitle: 'command line, not app UI — real run, synthetic predictions', rows: gateRows, font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace', keyWidth: 74, gap: '1px', rowGap: '6px', }); await sleep(420); const gateProof = await evidenceCardProof('calibration-report'); if (!gateProof.ok) throw new Error(`Calibration card is not fully readable on screen: ${JSON.stringify(gateProof)}`); // The card is drawn from the report; prove the values on it are the ones the harness measured. for (const row of gate.report.models) { if (!gateCard.text.includes(row.macroF1.toFixed(3))) { throw new Error(`the card must show the macro-F1 the harness measured for ${row.model}`); } } if (!gateCard.text.includes('leaks across')) throw new Error('the card must show the refusal the leaked corpus earned'); if (!gateCard.text.includes('GO — gate-plumbing-fixture')) throw new Error('the card must show the passing branch it proved'); await caption('The calibration harness is a command-line release gate, not a screen in the app — this runs it against committed synthetic predictions', 3400); await caption('Every score on this card is invented to exercise the arithmetic: no model produced them and none of them describes the real classifier', 3600); await caption('Each candidate is scored on test at a threshold chosen on validation only; picking a threshold on the split you report is refused', 3600); await caption('One contributor spread across two splits is refused before anything is scored, and a passing verdict is proven only by a fixture engineered to clear it', 3800); await page.evaluate(() => document.querySelector('#calibration-report')?.remove()); // Issue #40 — the RC attack suite is a command line too, and unlike the other cards this one is // expected to come back with a verdict that is not flattering. Hostile synthetic bytes go at the // real server with a counting stub worker behind it, so "the worker was called" is measured, not // inferred. The card prints whatever verdict the suite derived from what it observed; the captions // name where a failing fix lives instead of implying the boundary is already safe. async function securityAttackSuite() { const directory = await mkdtemp(path.join(os.tmpdir(), 'timmy-security-demo-')); const reportPath = path.join(directory, 'rc-attack-report.json'); try { const run = await execFileAsync( process.execPath, ['scripts/security_attack_suite.mjs', '--json', reportPath], { cwd: process.cwd() }, ).then(result => ({ code: 0, stdout: result.stdout, stderr: result.stderr })) .catch(error => ({ code: typeof error.code === 'number' ? error.code : 1, stdout: error.stdout || '', stderr: error.stderr || '' })); if (![0, 1].includes(run.code)) throw new Error(`the attack suite could not complete: ${run.stderr.trim()}`); const report = JSON.parse(await readFile(reportPath, 'utf8')); if (run.code !== (report.verdict === 'BLOCKED' ? 1 : 0)) { throw new Error('the attack suite exit code must match the verdict it published'); } return { report, run }; } finally { await rm(directory, { recursive: true, force: true }); } } const securityRun = await securityAttackSuite(); const securityReport = securityRun.report; if (securityReport.cases.length < 8) throw new Error('the RC attack suite must run every named abuse class'); await tap('[data-view="home"]', 450); const caseDetail = (item) => { const observed = item.observed === null || typeof item.observed !== 'object' ? {} : item.observed; // Print each observed field the case actually recorded. A case with no `status` used to render // `http undefined` on camera, and one with no `providerCalls` rendered its status twice — both read // as a broken demo instead of the measurement it is. const parts = []; if ('status' in observed) parts.push(`http ${observed.status}`); if ('refused' in observed) parts.push(`refused ${observed.refused}`); if ('providerCalls' in observed) parts.push(`worker ${observed.providerCalls}`); if ('providerTextLeaked' in observed) parts.push(`text leaked ${observed.providerTextLeaked}`); if ('repeatedAnswerIdentical' in observed) parts.push(`identical ${observed.repeatedAnswerIdentical}`); if ('imageBytesInLog' in observed) parts.push(`image bytes ${observed.imageBytesInLog}`); return parts.length ? parts.join(' · ') : item.status; }; const securityRows = [ ['command', 'node scripts/security_attack_suite.mjs'], ['target', 'real server · counting stub worker · synthetic bytes'], ['—', ''], ]; for (const item of securityReport.cases) securityRows.push([item.category, `${item.severity} ${item.status} · ${caseDetail(item)}`]); securityRows.push(['—', '']); securityRows.push(['load', `${securityReport.load.requests} req · p50 ${securityReport.load.p50}ms · p95 ${securityReport.load.p95}ms · abstain ${securityReport.load.abstentions}`]); securityRows.push(['verdict', `${securityReport.verdict} · ${securityReport.blocking.length} blocking · ${securityReport.holds.length} held`]); for (const item of securityReport.blocking) securityRows.push(['blocks', item.id]); const securityCard = await evidenceCard({ id: 'security-attack', title: 'SECURITY ATTACK SUITE', subtitle: 'command line, not app UI — real server, counted worker calls', rows: securityRows, font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace', keyWidth: 62, gap: '1px', rowGap: '5px', }); await sleep(420); const securityCardProof = await evidenceCardProof('security-attack'); if (!securityCardProof.ok) throw new Error(`Security card is not fully readable on screen: ${JSON.stringify(securityCardProof)}`); for (const item of securityReport.blocking) { if (!securityCard.text.includes(item.id)) throw new Error(`the card must name the blocking case ${item.id}`); } if (!securityCard.text.includes(securityReport.verdict)) throw new Error('the card must show the verdict the suite derived'); if (!securityCard.text.includes(`${securityReport.load.p95}ms`)) throw new Error('the card must show the p95 the load phase measured'); await caption('Every hostile upload on screen went at the real server, and the worker behind it counted every call it received', 3800); await caption('Refusing bytes by their content is the fix in review, not a claim this video makes: an HTML file dressed as a JPEG still reaches the worker today', 4200); await caption('A failing case is a finding the release job has to see, so the card prints the verdict the suite derived instead of a reassuring one', 3800); await caption('The gate blocks RC approval on any P0 case that reached the worker — the gate decides, not the caption', 3400); await page.evaluate(() => document.querySelector('#security-attack')?.remove()); // Issue #39 — the operator backup is a command line, not a screen in the app. The app itself keeps // no server-side journal, so what an operator can actually lose is the corpus: the append-only // manifest with its withdrawal tombstones. This scene builds a synthetic data root with the real // ingest and revoke commands, plants the four things a backup must refuse, then runs the real // backup, verify, and restore and reads every value on the card back out of those runs. async function operatorBackupAudit() { const root = await mkdtemp(path.join(os.tmpdir(), 'timmy-backup-demo-')); const dataRoot = path.join(root, 'srv-timmy-corpus'); const archive = path.join(root, 'timmy-nightly.tar.gz'); const manifestPath = `${archive}.manifest.json`; const restoreInto = path.join(root, 'restored'); const secondPhoto = path.join(root, 'synthetic-second.png'); const backupCli = args => execFileAsync('python3', ['scripts/operator_backup.py', ...args], { cwd: process.cwd() }); const ingest = (input, contributionId, subject, bristol) => corpusCli([ 'scripts/ingest_training_photo.py', '--input', input, '--output-dir', dataRoot, '--subject-id', subject, '--consent-version', 'training-consent-v1', '--contribution-id', contributionId, '--bristol-type', bristol, '--color', 'brown', '--quality', 'good', '--reviewer', 'user-confirmed', ]); try { // One contribution stays active and one is withdrawn, so the manifest ends up with a live sample // and a real tombstone, and the withdrawn derived image is already gone from disk. await ingest('tests/fixtures/synthetic-type4.jpg', 'demo-backup-0001', 'demo-backup-kept', '4'); await writeFile(secondPhoto, Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAFElEQVR4nGOsCDFhwAaYsIoOWgkAzUwBECP1Kh0AAAAASUVORK5CYII=', 'base64')); await ingest(secondPhoto, 'demo-backup-0002', 'demo-backup-withdrawn', '3'); await corpusCli([ 'scripts/ingest_training_photo.py', '--output-dir', dataRoot, '--revoke', 'demo-backup-0002', '--reason', 'participant withdrew consent', ]); // The four things that must never be archived, as real files with real bytes: a weight shard, a // transient inference upload, an operator environment file, and a log. An exclusion that quietly // stopped working fails here instead of in a caption. await mkdir(path.join(dataRoot, 'weights'), { recursive: true }); await writeFile(path.join(dataRoot, 'weights', 'model.gguf'), 'synthetic weight shard — never archived'); await mkdir(path.join(dataRoot, 'transient'), { recursive: true }); await writeFile(path.join(dataRoot, 'transient', 'inference-upload.jpg'), 'transient inference bytes — never archived'); await writeFile(path.join(dataRoot, 'secrets.env'), 'TIMMY_DATASET_SALT=demo-only-placeholder\n'); await writeFile(path.join(dataRoot, 'operator.log'), 'GET /api/healthz 200\n'); const backupRun = await backupCli(['backup', '--source', dataRoot, '--archive', archive, '--label', 'nightly']); const receipt = JSON.parse(backupRun.stdout); const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); const reasons = manifest.excluded.reduce((out, entry) => ({ ...out, [entry.reason]: (out[entry.reason] || 0) + 1 }), {}); if (manifest.files.some(entry => /model\.gguf|inference-upload|secrets\.env|operator\.log/.test(entry.path))) { throw new Error('the backup must not archive the weight shard or a transient inference upload'); } for (const reason of ['model-weights', 'transient-inference', 'secret', 'log']) { if (!reasons[reason]) throw new Error(`the backup must record a ${reason} exclusion`); } if (receipt.records.samples !== 2 || receipt.records.tombstones !== 1 || receipt.records.activeSamples !== 1) { throw new Error('the backup receipt must report the real record counts'); } const verified = JSON.parse((await backupCli(['verify', '--archive', archive, '--manifest', manifestPath])).stdout); if (verified.files !== manifest.files.length) throw new Error('verify must re-hash every stored member'); const restored = JSON.parse((await backupCli([ 'restore', '--archive', archive, '--manifest', manifestPath, '--target', restoreInto, ])).stdout); if (restored.records.tombstones !== 1 || restored.records.activeSamples !== 1) { throw new Error('the restore must return the same record counts'); } const restoredManifest = await readFile(path.join(restoreInto, 'manifest.jsonl')); if (!restoredManifest.equals(await readFile(path.join(dataRoot, 'manifest.jsonl')))) { throw new Error('the restore must return the same records, tombstone included'); } return { receipt, manifest, reasons, verified, restored }; } finally { await rm(root, { recursive: true, force: true }); } } const opBackup = await operatorBackupAudit(); await tap('[data-view="home"]', 400); const shortDigest = value => String(value).slice(0, 12); const backupRows = [ ['command', 'python3 scripts/operator_backup.py backup --source srv-timmy-corpus'], ['stored', `${opBackup.receipt.files} members · ${opBackup.manifest.files.map(entry => entry.path.split('/').pop()).join(' + ')}`], ['records', `samples ${opBackup.receipt.records.samples} · tombstones ${opBackup.receipt.records.tombstones} · active ${opBackup.receipt.records.activeSamples}`], ['—', ''], ['refused', `model-weights ${opBackup.reasons['model-weights']} · transient-inference ${opBackup.reasons['transient-inference']}`], ['refused', `secret ${opBackup.reasons['secret']} · log ${opBackup.reasons['log']}`], ['reason', 'every skipped path is named in the manifest, not silently dropped'], ['—', ''], ['digest', `sha256 ${shortDigest(opBackup.manifest.archive.sha256)}… · reproducible (sorted, mtime 0)`], ['verify', `ok · ${opBackup.verified.files} members re-hashed against the manifest`], ['restore', `ok · samples ${opBackup.restored.records.samples} · tombstones ${opBackup.restored.records.tombstones} · active ${opBackup.restored.records.activeSamples}`], ['manifest', 'returned byte-for-byte, tombstone included'], ['server', 'the app holds no journal of its own — there is no server copy to back up'], ]; const backupCard = await evidenceCard({ id: 'production-backup', title: 'OPERATOR BACKUP', subtitle: 'command line, not app UI — real run, synthetic corpus', rows: backupRows, font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace', keyWidth: 66, gap: '1px', rowGap: '6px', }); await sleep(420); const backupProof = await evidenceCardProof('production-backup'); if (!backupProof.ok) throw new Error(`Production backup card is not fully readable on screen: ${JSON.stringify(backupProof)}`); // The card is drawn from the run: prove the counts and the refusals on it are the ones it produced. if (!backupCard.text.includes(`samples ${opBackup.receipt.records.samples} · tombstones ${opBackup.receipt.records.tombstones}`)) { throw new Error('the card must show the record counts the backup actually reported'); } if (!backupCard.text.includes('model-weights 1')) throw new Error('the card must show the exclusion the run actually recorded'); if (!backupCard.text.includes(shortDigest(opBackup.manifest.archive.sha256))) { throw new Error('the card must show the archive digest the run actually wrote'); } await caption('The operator backup is a command line, not a screen in the app — this runs it against a synthetic corpus the real ingest and revoke commands built', 3800); await caption('Model weights and transient inference images are refused by name, and every skipped path is listed with its reason', 3600); await caption('Verify re-hashes every stored member against the manifest, and the restore only starts once that check passes', 3400); await caption('A restore returns the same records, including the withdrawal tombstone a trainer must never lose', 3600); await page.evaluate(() => document.querySelector('#production-backup')?.remove()); // Back to the privacy view for the portability scene: the privacy entry lives on the journal view. await tap('[data-view="calendar"]', 550); await tap('[data-view="privacy"]', 600); // Issue #35 — a backup is the person's own record, so it carries a version and everything it is // claimed to hold. The recording exports the real file, imports an older version and a newer one, // and reopens a restored log. Fictional sample logs are used throughout; nothing here is real data. const exportDownload = page.waitForEvent('download'); await tap('#export', 600); const exportedBackup = JSON.parse(await readFile(await (await exportDownload).path(), 'utf8')); if (exportedBackup.schemaVersion !== 2) throw new Error('the exported backup must declare the current schema version'); if (!Array.isArray(exportedBackup.entries) || !Array.isArray(exportedBackup.contributions)) { throw new Error('a backup must carry both the confirmed logs and the contribution receipts'); } const exportedReceipt = exportedBackup.contributions.find(receipt => receipt.contributionId === receiptId); if (!exportedReceipt || exportedReceipt.state !== 'withdrawn') { throw new Error('the backup must carry the withdrawn receipt as a tombstone'); } if (!(exportedReceipt.artifacts || []).length) throw new Error('the backup tombstone must still name its derived artifacts'); await page.locator('[data-portability-notice][data-portability-version="2"]').waitFor(); await caption('A backup is versioned, and it carries the withdrawn receipt as a tombstone rather than silence', 3200); const fictionalLegacyBackup = JSON.stringify({ product: 'Timmy the Talking Turd', schemaVersion: 1, exportedAt: '2026-08-01T09:00:00.000Z', entries: [{ id: 'demo-legacy-backup', occurredAt: new Date(Date.now() - 3 * 86400000).toISOString(), bristolType: 3, color: 'brown', urgency: 1, discomfort: 0, note: 'fictional version 1 backup', photoDataUrl: '', symptoms: {}, }], }); await page.locator('#import').setInputFiles({ name: 'timmy-backup-v1.json', mimeType: 'application/json', buffer: Buffer.from(fictionalLegacyBackup) }); await page.locator('[data-portability-notice][data-portability-version="1"]').waitFor(); const upgradeNotice = (await page.locator('[data-portability-notice]').innerText()).trim(); if (!/upgraded/i.test(upgradeNotice)) throw new Error('an older backup must be reported as upgraded, not silently rewritten'); await caption('An older backup is upgraded on import, and the notice says which version the file came from', 3400); const ledgerBeforeFuture = await storedLedger(); await page.locator('#import').setInputFiles({ name: 'timmy-backup-v3.json', mimeType: 'application/json', buffer: Buffer.from(JSON.stringify({ product: 'Timmy the Talking Turd', schemaVersion: 3, entries: [{ id: 'demo-future', bristolType: 7 }] })), }); await page.locator('[data-portability-error]').waitFor(); if (!/newer version of Timmy/i.test(await page.locator('[data-portability-error]').innerText())) { throw new Error('a backup from a newer Timmy must be refused with an instruction to update'); } if ((await storedLedger()).raw !== ledgerBeforeFuture.raw) { throw new Error('a refused import must not change the ledger it would have replaced'); } await caption('A file from a newer Timmy is refused with an instruction, and the ledger it would have replaced is untouched', 3400); const fictionalAssistedBackup = JSON.stringify({ product: 'Timmy the Talking Turd', schemaVersion: 2, exportedAt: new Date().toISOString(), entries: [{ id: 'demo-restored-provenance', occurredAt: new Date(Date.now() - 86400000).toISOString(), bristolType: 2, color: 'brown', urgency: 1, discomfort: 0, note: 'fictional restored log', photoDataUrl: '', symptoms: {}, provenance: { source: 'ai-corrected', model: 'SmolVLM2-2.2B-Instruct', suggestedBristolType: 4, suggestedColor: 'brown', confirmedAt: new Date(Date.now() - 86400000).toISOString(), }, }], contributions: [], }); await page.locator('#import').setInputFiles({ name: 'timmy-backup-v2.json', mimeType: 'application/json', buffer: Buffer.from(fictionalAssistedBackup) }); await page.locator('[data-portability-notice][data-portability-version="2"]').waitFor(); await tap('[data-view="calendar"]', 550); await page.locator('[data-entry-open="demo-restored-provenance"]').waitFor(); await tap('[data-entry-open="demo-restored-provenance"]', 550); await page.locator('[data-entry-provenance="ai-corrected"]').waitFor(); const provenanceLine = (await page.locator('[data-entry-provenance="ai-corrected"]').innerText()).trim(); if (!/corrected/i.test(provenanceLine) || !/SmolVLM2/.test(provenanceLine)) { throw new Error('a restored log must still show that it began as a corrected AI suggestion'); } await caption('A restored log still says it began as an AI suggestion you corrected — provenance is part of the record', 3400); await indicate('[data-entry-provenance="ai-corrected"]'); await tap('#entry-sheet-close', 450); await tap('[data-view="privacy"]', 550); await page.locator('#delete-all').waitFor(); await indicate('#delete-all'); await tap('#delete-all', 650); await page.getByText(/Local ledger and photos deleted/).waitFor(); const wiped = await storedLedger(); if (wiped.raw !== null) throw new Error('delete-all must remove the stored ledger'); if (await page.evaluate(() => JSON.stringify(localStorage).includes('data:image'))) { throw new Error('no photo bytes may survive delete-all'); } if (await page.evaluate(key => localStorage.getItem(key), contributionKey) !== null) { throw new Error('delete-all must clear the contribution receipts as well'); } await caption('Delete all local data clears the ledger and every saved photo in this browser; nothing is kept on a server', 2900); await tap('[data-view="home"]', 500); // Issue #37 — a chat answer about the journal must point at the confirmed records behind it. The // scenes below run against the app's own local ledger contract with fictional logs, and every claim // is asserted in the browser before the caption is shown. agentConnected = false; await seedSyntheticLedger([ sampleDay('demo-chat-1', 2, 4), sampleDay('demo-chat-2', 5, 3), sampleDay('demo-chat-3', 9, 4), sampleDay('demo-chat-4', 12, 4), ]); await tap('[data-view="timmy"]', 550); await page.getByText(/Local Timmy mode/i).waitFor(); await caption('Fictional logs are seeded into this browser only — the chat answers from them, not from model memory', 2600); await page.locator('#chat-message').fill('How many times have I logged?'); await tap('#send-chat', 500); const groundedAnswer = page.locator('.bubble.timmy', { hasText: '4 confirmed logs' }).last(); await groundedAnswer.locator('[data-chat-citation]').first().waitFor(); if (await groundedAnswer.locator('[data-chat-citation]').count() !== 4) { throw new Error('A grounded chat answer must cite every record it counted'); } await caption('The answer states the count and cites the four confirmed logs it counted', 2600); await indicate('[data-chat-citations]'); await tap('[data-chat-citation]', 700); await page.locator('[data-chat-record]').waitFor(); if (await page.locator('[data-chat-record]').count() !== 1) { throw new Error('A chat citation must open exactly the one record it names'); } await caption('Following a citation opens exactly the confirmed log behind that answer', 2600); await indicate('[data-chat-record-sheet]'); await tap('#chat-record-close', 450); await page.locator('#chat-message').fill('Is it safe to eat tacos?'); await tap('#send-chat', 500); await page.getByText(/cannot clear a food/i).waitFor(); if (await page.locator('.bubble.timmy').last().locator('[data-chat-citation]').count() !== 0) { throw new Error('A refusal must not borrow journal citations'); } await caption('Ask something the journal cannot answer and Timmy refuses instead of reassuring', 2800); agentConnected = true; await page.reload({ waitUntil: 'networkidle' }); await tap('[data-view="timmy"]', 550); await page.getByText(/Hermes Agent connected/i).waitFor(); await caption('Hermes Agent connected — free-text, contextual, and server-side', 1400); await page.locator('#chat-message').fill('What pattern do you see?'); await tap('#send-chat', 500); await page.getByText(/mostly Type 4/i).waitFor(); if (await page.locator('.bubble.timmy').last().locator('[data-chat-citation]').count() !== 4) { throw new Error('A Hermes answer must render the records it cites as sources'); } await caption('Hermes answers carry the same ledger citations, rendered as sources in the transcript', 2400); await caption('Hermes keeps credentials, tools, and session continuity server-side. Photos stay out of chat.', 1600); await page.locator('#chat-message').fill('I barfed'); await tap('#send-chat', 450); await page.getByText(/Pause and get medical help/i).waitFor(); if (hermesCalls !== 1) throw new Error('Urgent chat must be intercepted before Hermes'); await caption('Urgent language is intercepted deterministically before Hermes', 1800); // Issue #21 — the escalation has to fire for the phrasing people actually use, not only for the one // word the previous scene typed. Each line below is asserted in the browser: exactly one new Timmy // bubble, carrying the app's own fixed copy, with hermesCalls unmoved. const escalationBubbles = () => page.locator('.bubble.timmy', { hasText: 'Pause and get medical help' }).count(); const expectEscalation = async (text, label) => { const before = await escalationBubbles(); await page.locator('#chat-message').fill(text); await tap('#send-chat', 450); const bubble = page.locator('.bubble.timmy', { hasText: 'Pause and get medical help' }).last(); await bubble.waitFor(); if (await escalationBubbles() !== before + 1) throw new Error(`${label}: expected exactly one escalation bubble`); if (!/^Pause and get medical help\. These reported symptoms can need prompt medical care\./.test(await bubble.innerText())) { throw new Error(`${label}: the escalation copy must be the app's own fixed string`); } if (hermesCalls !== 1) throw new Error(`${label}: an escalated turn must not reach Hermes`); }; await expectEscalation('There is blood when I wipe', 'bleeding reported as wiping'); await caption('Bleeding reported as wiping still escalates, in the app’s own fixed words', 2600); await expectEscalation('I have tarry stools', 'tarry stool'); await caption('Tarry and black stool wording escalates too: one red flag, one unchanging message', 2600); // The same detector runs over the confirmed ledger, so a symptom already written down as a note // escalates even when the question itself is innocuous. The seeded entry is fictional and stays in // this browser, exactly like the summary scene above. await page.evaluate(() => localStorage.clear()); await seedSyntheticLedger([{ ...sampleDay('demo-tarry-note', 1, 4), note: 'stool was tarry yesterday' }]); await tap('[data-view="timmy"]', 550); const ledgerEscalations = await escalationBubbles(); await page.locator('#chat-message').fill('What do my recent logs show?'); await tap('#send-chat', 450); await page.locator('.bubble.timmy', { hasText: 'Pause and get medical help' }).last().waitFor(); if (await escalationBubbles() !== ledgerEscalations + 1) throw new Error('a tarry note in the journal must escalate the next answer'); if (hermesCalls !== 1) throw new Error('a ledger red flag must be intercepted before Hermes'); await caption('A tarry note already saved in the journal escalates the same way, before any provider call', 2800); await page.evaluate(() => localStorage.clear()); await page.reload({ waitUntil: 'networkidle' }); await tap('[data-view="timmy"]', 500); await sleep(400); await page.evaluate(() => { document.querySelector('#demo-caption')?.remove(); const outro = document.createElement('div'); outro.id = 'release-outro'; outro.innerHTML = 'SLEEK. SIMPLE.
HERMES-POWERED.
User-confirmed guidance — never a diagnosis.'; Object.assign(outro.style, { position:'fixed', inset:'0', zIndex:'30000', background:'linear-gradient(145deg,#f7f3ea,#f5c95b)', display:'flex', flexDirection:'column', alignItems:'center', justifyContent:'center', textAlign:'center', color:'#28211d', fontFamily:'system-ui', opacity:'0' }); outro.querySelector('img').style.cssText = 'width:150px;height:150px;filter:drop-shadow(0 14px 16px rgba(63,37,30,.18))'; outro.querySelector('strong').style.cssText = 'font-size:31px;line-height:1.05;margin:18px 0 12px;letter-spacing:-.04em'; outro.querySelector('span').style.cssText = 'font-size:13px;font-weight:800;color:#176957'; document.body.append(outro); outro.animate([{opacity:0},{opacity:1}], {duration:500,fill:'forwards'}); }); await sleep(2200); // The refusal pass deliberately produces two fail-closed 503s (an out-of-range reply, and one blank field // beside an out-of-range one), which the browser logs as failed resources. Requiring exactly those two // keeps the scene honest: if the boundary stops refusing, this fails instead of quietly recording a happy // path, and a dropped scene is caught rather than silently shrinking the evidence. const expectedFailClosed = errors.filter(message => /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/.test(message)); if (expectedFailClosed.length !== 2 || errors.length !== expectedFailClosed.length) { throw new Error(`Browser errors: ${errors.join(' | ') || 'unexpected error count'}`); } // Machine-readable receipt of exactly what was rendered on camera, so the release gate can prove // the recorded delta statement reached the screen instead of trusting the script source. if (captionsOut) { await writeFile(captionsOut, JSON.stringify({ version, releaseDelta, captions: captionsShown, captionsVerified: captionEvidence }, null, 2) + '\n', 'utf8'); } const recording = page.video(); await page.close(); await recording.saveAs(output); await context.close(); await browser.close(); console.log(output);