timmy
f0a9a734db
Quality gates / quality (pull_request) Canceled after 0s
Add an executable abuse suite that drives the real server with synthetic hostile bytes while a counting stub worker records every call that reaches the model boundary, plus the verdict rule that turns those observations into a release decision. - src/release-security-gate.js: nearest-rank latency percentiles, per-case pass/fail derivation that names the field that broke, and the rule that any failing P0 case blocks RC approval while a required class with no verified evidence blocks it too. Reports are redacted and fail closed. - scripts/security_attack_suite.mjs: oversized body, HTML polyglot as JPEG, undecodable bytes, missing consent, non-conforming worker answer, byte-identical replay, concurrent burst, captured log scan, and an out-of-surface deletion case counted only against a real test file. Writes the report, prints the verdict, exits 1 when blocked. - docs/RC-SECURITY-EVIDENCE.md: procedure, rule, and the current finding stated plainly. - Demo: a real run of the suite on camera, printing the verdict it derived instead of a flattering one; CI regenerates the same evidence on every change. The suite currently returns VERDICT: BLOCKED: bytes are trusted from the data-URL label, so an HTML polyglot still reaches the worker and only fails afterwards on schema. That is a finding against #16, not a passing check, and it is why the gate treats a failing case as a block.
1592 lines
98 KiB
JavaScript
1592 lines
98 KiB
JavaScript
import { chromium } from 'playwright';
|
||
import { execFile } from 'node:child_process';
|
||
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
||
import { createHash } from 'node:crypto';
|
||
import os from 'node:os';
|
||
import { promisify } from 'node:util';
|
||
import path from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { analyzePhoto, visionFailureResponse } from '../src/vision-service.js';
|
||
import { loadVisionCorpus } from '../src/vision-corpus.js';
|
||
|
||
const execFileAsync = promisify(execFile);
|
||
|
||
const output = path.resolve(process.argv[2] || 'release-demo.webm');
|
||
const version = process.env.TIMMY_RELEASE_VERSION || 'review build';
|
||
// The release delta is supplied by the builder so the recording states, on camera, what actually
|
||
// changed in this release instead of implying a feature that did not ship.
|
||
const releaseDelta = process.env.TIMMY_RELEASE_DEMO_DELTA
|
||
|| 'Verified rebuild of the current user-visible feature path — no new feature is claimed.';
|
||
// The recording targets the release builder's own acceptance server by default. A caller that
|
||
// names another origin is explicit about it, so a local run can never silently record a different
|
||
// tree than the one the gates measured.
|
||
const demoUrl = process.env.TIMMY_DEMO_URL || 'http://127.0.0.1:4173';
|
||
const captionsOut = process.env.TIMMY_DEMO_CAPTIONS_OUT || '';
|
||
const captionsShown = [];
|
||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||
await mkdir(path.dirname(output), { recursive: true });
|
||
|
||
// The refusal scene replays committed synthetic corpus cases through the real analysis code path,
|
||
// so the recording shows the status and copy the server would actually send rather than copy
|
||
// written for the video. No real photo is used anywhere in the corpus.
|
||
const corpus = await loadVisionCorpus(fileURLToPath(new URL('../tests/fixtures/vision-corpus/', import.meta.url)));
|
||
const corpusCase = id => corpus.cases.find(entry => entry.id === id);
|
||
async function corpusAnalyzeResult(entry) {
|
||
const fetchImpl = async () => ({ ok: true, json: async () => ({ choices: [{ message: { content: entry.providerContent } }] }) });
|
||
try {
|
||
const result = await analyzePhoto({
|
||
payload: { imageDataUrl: 'data:image/png;base64,YQ==', consent: true },
|
||
fetchImpl,
|
||
config: { baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'corpus-replay', model: 'corpus-model' },
|
||
});
|
||
return { status: 200, body: result };
|
||
} catch (error) {
|
||
const failure = visionFailureResponse(error);
|
||
return { status: failure.status, body: failure.body };
|
||
}
|
||
}
|
||
|
||
const browser = await chromium.launch({ headless: true });
|
||
const context = await browser.newContext({
|
||
viewport: { width: 405, height: 720 },
|
||
deviceScaleFactor: 1,
|
||
serviceWorkers: 'block',
|
||
recordVideo: { dir: path.dirname(output), size: { width: 405, height: 720 } },
|
||
colorScheme: 'light',
|
||
});
|
||
const page = await context.newPage();
|
||
const errors = [];
|
||
// Delete-all asks for confirmation; the recording accepts it so the resulting state is shown, not implied.
|
||
page.on('dialog', dialog => dialog.accept());
|
||
page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); });
|
||
page.on('pageerror', error => errors.push(error.message));
|
||
let visionOffline = false;
|
||
// Reachable is not loaded: the worker can answer /v1/models before its pinned model is resident,
|
||
// and the recording shows the app withholding AI in exactly that state (issue #15).
|
||
let modelLoaded = true;
|
||
await page.route('**/api/vision-status', route => route.fulfill({
|
||
status: 200,
|
||
contentType: 'application/json',
|
||
body: JSON.stringify({
|
||
enabled: true,
|
||
profile: 'selfhost',
|
||
processor: 'self-hosted',
|
||
model: 'SmolVLM2-2.2B-Instruct',
|
||
// The second pass flips the app's own readiness contract so no AI control is offered at all.
|
||
providerReady: !visionOffline,
|
||
modelSeen: !visionOffline && modelLoaded,
|
||
}),
|
||
}));
|
||
// The happy path keeps one deterministic suggestion. The refusal scene swaps this for a real
|
||
// corpus replay, so both the usable and the refused answers come from the same code path.
|
||
const suggestionBody = {
|
||
status: 'suggestion',
|
||
isStool: true,
|
||
bristolType: 4,
|
||
color: 'brown',
|
||
confidence: 0.83,
|
||
imageQuality: 'good',
|
||
observations: 'Smooth, formed appearance.',
|
||
warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
|
||
};
|
||
let analyzeOverride = null;
|
||
await page.route('**/api/analyze', route => {
|
||
const response = analyzeOverride || { status: 200, body: suggestionBody };
|
||
return route.fulfill({ status: response.status, contentType: 'application/json', body: JSON.stringify(response.body) });
|
||
});
|
||
let hermesCalls = 0;
|
||
// Issue #37 turns the agent on and off on camera: local Timmy mode is a real app state, and the
|
||
// grounded local answer has to be shown as its own surface rather than implied.
|
||
let agentConnected = true;
|
||
await page.route('**/api/agent/status', route => route.fulfill({
|
||
status: 200,
|
||
contentType: 'application/json',
|
||
body: JSON.stringify({
|
||
enabled: true,
|
||
// Local Timmy mode is the no-backend state the app really uses: nothing configured, nothing sent.
|
||
configured: agentConnected,
|
||
authenticated: agentConnected,
|
||
mode: agentConnected ? 'hermes-agent' : 'local-fallback',
|
||
}),
|
||
}));
|
||
await page.route('**/api/agent/chat', route => {
|
||
hermesCalls += 1;
|
||
return route.fulfill({
|
||
status: 200,
|
||
contentType: 'application/json',
|
||
// The refs are the positional handles the server gives the agent; the browser resolves them
|
||
// against the same ledger it sent, so this reply is a real citation contract, not a prop.
|
||
body: JSON.stringify({ connected: true, reply: 'You have 4 confirmed logs in the last 28 days [entry:L1] [entry:L2] [entry:L3] [entry:L4]. Your confirmed logs are mostly Type 4. I can explain that pattern, but I cannot diagnose a cause.' }),
|
||
});
|
||
});
|
||
|
||
await page.goto(demoUrl, { waitUntil: 'networkidle' });
|
||
await page.evaluate(() => localStorage.clear());
|
||
await page.reload({ waitUntil: 'networkidle' });
|
||
|
||
const captionEvidence = [];
|
||
// Pixel proof that a caption actually painted: if the caption band looks identical with and without
|
||
// the node, the caption did not reach the screen even though the string was pushed.
|
||
async function captionBand() {
|
||
return page.screenshot({ clip: { x: 0, y: 0, width: 405, height: 68 } });
|
||
}
|
||
async function caption(text, duration = 1400) {
|
||
captionsShown.push(text);
|
||
const blankBand = await captionBand().catch(() => null);
|
||
await page.evaluate(text => {
|
||
document.querySelector('#demo-caption')?.remove();
|
||
const node = document.createElement('div');
|
||
node.id = 'demo-caption';
|
||
node.textContent = text;
|
||
// Inline styles, not a stylesheet: a stylesheet belongs to one document and the demo reloads the
|
||
// page, which silently stripped caption styling and left every later caption clipped body text.
|
||
// The bar is opaque, not 94%: a translucent bar let the app's own text show through a long
|
||
// release-delta caption and read as run-together garbage. Long copy also steps down a size so a
|
||
// paragraph-length caption keeps its bottom padding instead of pressing the last line to the edge.
|
||
const size = text.length > 380 ? 9.5 : text.length > 220 ? 10.5 : 12;
|
||
node.style.cssText = 'position:fixed;left:16px;right:16px;top:14px;z-index:20000;'
|
||
+ 'background:#28211d;color:#fff;padding:12px 14px 14px;border-radius:14px;'
|
||
+ `font:800 ${size}px/1.32 system-ui;letter-spacing:.02em;text-align:center;`
|
||
+ 'box-shadow:0 8px 24px rgba(0,0,0,.2)';
|
||
document.body.append(node);
|
||
node.animate([{ opacity: 0, transform: 'translateY(-8px)' }, { opacity: 1, transform: 'translateY(0)' }], { duration: 260, fill: 'forwards' });
|
||
}, text);
|
||
await sleep(320);
|
||
// Structural proof: a fixed bar at the top of the viewport, on top of whatever is beneath it, fully
|
||
// inside the frame. A caption that lost its stylesheet sits unstyled at the end of the body instead.
|
||
const painted = await page.evaluate(() => {
|
||
const node = document.querySelector('#demo-caption');
|
||
if (!node) return { ok: false, reason: 'caption node missing' };
|
||
const rect = node.getBoundingClientRect();
|
||
const style = getComputedStyle(node);
|
||
const topmost = document.elementFromPoint(Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2));
|
||
// Proving the string was pushed is not proving it was readable. Measure the last line's own box,
|
||
// so a paragraph caption pressed flat against the bar's bottom edge fails here instead of shipping.
|
||
const range = document.createRange();
|
||
range.selectNodeContents(node);
|
||
const lines = Array.from(range.getClientRects()).filter(box => box.height > 1);
|
||
const lastBottom = lines.length ? Math.max(...lines.map(box => box.bottom)) : rect.bottom;
|
||
const padBottom = Math.round(rect.bottom - lastBottom);
|
||
return {
|
||
ok: style.position === 'fixed' && style.zIndex === '20000' && rect.top <= 40 && rect.height >= 24
|
||
&& rect.width > 200 && rect.bottom <= window.innerHeight && (topmost === node || (topmost && node.contains(topmost)))
|
||
&& lines.length > 0 && padBottom >= 3,
|
||
top: Math.round(rect.top), height: Math.round(rect.height), position: style.position,
|
||
zIndex: style.zIndex, background: style.backgroundColor, topmost: topmost === node,
|
||
padBottom, lines: lines.length,
|
||
};
|
||
});
|
||
const shownBand = await captionBand().catch(() => null);
|
||
const bandChanged = Boolean(blankBand && shownBand && !blankBand.equals(shownBand));
|
||
if (!painted.ok) throw new Error(`Caption is not an on-screen bar: ${text} ${JSON.stringify(painted)}`);
|
||
if (!bandChanged) throw new Error(`Caption painted nothing into the caption band: ${text}`);
|
||
captionEvidence.push({ text, visible: true, position: painted.position, top: painted.top, height: painted.height, bandChanged, padBottom: painted.padBottom, lines: painted.lines });
|
||
await sleep(duration);
|
||
// A caption must not outlive its stated duration. It used to stay on screen until the next one
|
||
// replaced it, so a view change left the previous screen's claim hanging over a different screen —
|
||
// e.g. a delete-data caption sitting on top of the home view.
|
||
await page.evaluate(() => {
|
||
const node = document.querySelector('#demo-caption');
|
||
if (!node) return;
|
||
node.animate([{ opacity: 1 }, { opacity: 0 }], { duration: 180, fill: 'forwards' });
|
||
setTimeout(() => node.remove(), 220);
|
||
});
|
||
await sleep(220);
|
||
}
|
||
|
||
async function tap(selector, after = 650) {
|
||
const target = page.locator(selector).first();
|
||
await target.scrollIntoViewIfNeeded();
|
||
const box = await target.boundingBox();
|
||
if (!box) throw new Error(`Missing demo target: ${selector}`);
|
||
await page.evaluate(({ x, y }) => {
|
||
document.querySelector('#demo-touch')?.remove();
|
||
const ring = document.createElement('div');
|
||
ring.id = 'demo-touch';
|
||
ring.style.cssText = 'position:fixed;z-index:20001;width:48px;height:48px;border:4px solid #fff;'
|
||
+ 'background:rgba(245,201,91,.55);box-shadow:0 0 0 7px rgba(21,125,120,.28);border-radius:50%;'
|
||
+ 'pointer-events:none;transform:translate(-50%,-50%)';
|
||
ring.style.left = `${x}px`;
|
||
ring.style.top = `${y}px`;
|
||
document.body.append(ring);
|
||
ring.animate([{ opacity: .2, transform: 'translate(-50%,-50%) scale(.55)' }, { opacity: 1, transform: 'translate(-50%,-50%) scale(1)' }], { duration: 400 });
|
||
setTimeout(() => ring.remove(), 550);
|
||
}, { x: box.x + box.width / 2, y: box.y + box.height / 2 });
|
||
await sleep(220);
|
||
await target.click();
|
||
await sleep(after);
|
||
}
|
||
|
||
async function indicate(selector) {
|
||
const target = page.locator(selector).first();
|
||
await target.scrollIntoViewIfNeeded();
|
||
const box = await target.boundingBox();
|
||
if (!box) throw new Error(`Missing demo target: ${selector}`);
|
||
await page.evaluate(({ x, y }) => {
|
||
document.querySelector('#demo-touch')?.remove();
|
||
const ring = document.createElement('div');
|
||
ring.id = 'demo-touch';
|
||
ring.style.cssText = 'position:fixed;z-index:20001;width:48px;height:48px;border:4px solid #fff;'
|
||
+ 'background:rgba(245,201,91,.55);box-shadow:0 0 0 7px rgba(21,125,120,.28);border-radius:50%;'
|
||
+ 'pointer-events:none;transform:translate(-50%,-50%)';
|
||
ring.style.left = `${x}px`;
|
||
ring.style.top = `${y}px`;
|
||
document.body.append(ring);
|
||
ring.animate([{ opacity: .2, transform: 'translate(-50%,-50%) scale(.55)' }, { opacity: 1, transform: 'translate(-50%,-50%) scale(1)' }], { duration: 400 });
|
||
setTimeout(() => ring.remove(), 550);
|
||
}, { x: box.x + box.width / 2, y: box.y + box.height / 2 });
|
||
await sleep(650);
|
||
}
|
||
|
||
// Command-line evidence is shown on the same card chrome in three scenes (corpus audit, review
|
||
// ledger, calibration gate). One renderer keeps the readability gate identical for all of them.
|
||
async function evidenceCard({ id, title, subtitle, rows, font = '600 10.5px/1.5 ui-monospace, SFMono-Regular, Menlo, monospace', keyWidth = 62, gap = '1px', rowGap = '7px' }) {
|
||
return page.evaluate(params => {
|
||
document.querySelector(`#${params.id}`)?.remove();
|
||
const panel = document.createElement('div');
|
||
panel.id = params.id;
|
||
Object.assign(panel.style, {
|
||
position: 'fixed', inset: '96px 14px 40px 14px', zIndex: '15000',
|
||
background: '#181412', color: '#f6f1e6', borderRadius: '18px',
|
||
border: '1px solid rgba(245,201,91,.45)', padding: '13px 13px 11px',
|
||
font: params.font,
|
||
display: 'flex', flexDirection: 'column', gap: params.gap, overflow: 'hidden',
|
||
boxShadow: '0 18px 40px rgba(0,0,0,.42)',
|
||
});
|
||
const head = document.createElement('div');
|
||
head.textContent = params.title;
|
||
Object.assign(head.style, { font: '800 14px/1.2 system-ui', letterSpacing: '.14em', color: '#f5c95b', marginBottom: '2px' });
|
||
const sub = document.createElement('div');
|
||
sub.textContent = params.subtitle;
|
||
Object.assign(sub.style, { font: '700 9.5px/1.3 system-ui', color: '#c9bfae', marginBottom: '9px' });
|
||
panel.append(head, sub);
|
||
for (const [key, value] of params.rows) {
|
||
const row = document.createElement('div');
|
||
if (key === '—') { row.style.height = params.rowGap; panel.append(row); continue; }
|
||
Object.assign(row.style, { display: 'flex', gap: '7px', alignItems: 'baseline' });
|
||
const k = document.createElement('span');
|
||
k.textContent = key;
|
||
k.style.cssText = `flex:0 0 ${params.keyWidth}px;color:#8fd7c8;text-transform:uppercase;font-size:9px;letter-spacing:.08em`;
|
||
const v = document.createElement('span');
|
||
v.textContent = value;
|
||
v.style.cssText = 'flex:1 1 auto;word-break:break-word;color:#f6f1e6';
|
||
row.append(k, v);
|
||
panel.append(row);
|
||
}
|
||
document.body.append(panel);
|
||
panel.animate([{ opacity: 0, transform: 'translateY(12px)' }, { opacity: 1, transform: 'translateY(0)' }], { duration: 320, fill: 'forwards' });
|
||
return { height: panel.clientHeight, scroll: panel.scrollHeight, text: panel.innerText };
|
||
}, { id, title, subtitle, rows, font, keyWidth, gap, rowGap });
|
||
}
|
||
|
||
// A card is only evidence if it is actually legible: opaque, inside the frame, not covered, and not
|
||
// clipped. A translucent or scrolled card reads as a sloppy overlay on the thinnest part of a film.
|
||
async function evidenceCardProof(id) {
|
||
return page.evaluate(cardId => {
|
||
const panel = document.querySelector(`#${cardId}`);
|
||
if (!panel) return { ok: false, reason: 'panel missing' };
|
||
const rect = panel.getBoundingClientRect();
|
||
const topmost = document.elementFromPoint(Math.round(rect.left + rect.width / 2), Math.round(rect.top + rect.height / 2));
|
||
const background = getComputedStyle(panel).backgroundColor;
|
||
const opaque = !/^rgba\(.*,\s*(0?\.\d+|0)\)$/.test(background);
|
||
return {
|
||
ok: opaque && rect.top >= 60 && rect.bottom <= window.innerHeight && rect.width > 300 && rect.height > 300
|
||
&& (topmost === panel || panel.contains(topmost)) && panel.scrollHeight <= panel.clientHeight + 1,
|
||
top: Math.round(rect.top), height: Math.round(rect.height), width: Math.round(rect.width),
|
||
scroll: panel.scrollHeight, client: panel.clientHeight, background, opaque,
|
||
covered: topmost === panel || panel.contains(topmost),
|
||
};
|
||
}, id);
|
||
}
|
||
|
||
await caption(`TIMMY ${version} • RELEASE DEMO`, 1200);
|
||
await caption(`This release: ${releaseDelta}`, 2600);
|
||
await caption('Automated checks replay this synthetic path before review', 1200);
|
||
await caption('One clear photo action. Manual logging stays one tap away.', 1500);
|
||
await tap('[data-scan]', 450);
|
||
await page.getByText(/Self-hosted model ready/i).waitFor();
|
||
await caption('The pinned bootstrap verifies both model files before starting on private loopback', 1500);
|
||
await indicate('label[for="camera-photo"]');
|
||
await page.locator('#camera-photo').dispatchEvent('cancel');
|
||
await page.getByText(/Camera or photo picker closed/i).waitFor();
|
||
await caption('Camera closed cleanly — gallery and manual logging are still available', 1600);
|
||
await indicate('label[for="gallery-photo"]');
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||
await caption('Nothing uploads until explicit consent', 1100);
|
||
await page.locator('#ai-consent').check();
|
||
await tap('#analyze-photo', 450);
|
||
await page.getByText(/83% confidence/i).waitFor();
|
||
await caption('AI may suggest visible form, broad color, and image quality — never symptoms or diagnosis', 1700);
|
||
await indicate('.scan-result[data-provenance="pending"]');
|
||
await caption('The suggestion stays provisional — nothing is applied or saved until you confirm it', 1700);
|
||
await tap('#use-suggestion', 450);
|
||
await page.locator('.ai-provenance[data-provenance="confirmed"]').waitFor();
|
||
await caption('Confirming is explicit, and every suggested value is still yours to overrule', 1600);
|
||
await tap('[data-type="2"]', 500);
|
||
await page.locator('[data-provenance="corrected"]').waitFor();
|
||
await caption('Change a suggested value and Timmy clears the AI-confirmed state', 1700);
|
||
await indicate('.ai-provenance[data-provenance="corrected"] .provenance-note');
|
||
await tap('#next', 450);
|
||
await page.getByText(/You changed it to Type 2/).waitFor();
|
||
await caption('Only visible form and color were ever suggested — urgency, discomfort, and symptoms stay yours', 1900);
|
||
await tap('#next', 450);
|
||
await page.locator('[data-followup-block]').waitFor();
|
||
await caption('Type 2 is hard or lumpy, so Timmy asks about soreness, hydration, blood, and black-or-tarry — never guessed from the photo', 2600);
|
||
await page.locator('[data-question="bloodVisible"]').scrollIntoViewIfNeeded();
|
||
await indicate('[data-question="bloodVisible"]');
|
||
await tap('[data-followup="bloodVisible"][data-answer="no"]', 400);
|
||
await caption('Every answer is yours to give, and a skipped question stays unanswered', 1900);
|
||
await tap('[data-followup="soreness"][data-answer="yes"]', 350);
|
||
await tap('[data-followup="dehydration"][data-answer="yes"]', 350);
|
||
await page.locator('#followup-guidance').waitFor();
|
||
await caption('Mild report: steady water, no straining, gentle care — general self-care, not a diagnosis', 2600);
|
||
await indicate('#followup-guidance');
|
||
await caption('And it never claims the photo ruled out bleeding', 1900);
|
||
await tap('[data-followup="tarryBlack"][data-answer="yes"]', 600);
|
||
await page.locator('#urgent-box .alert').waitFor();
|
||
await caption('Say it was truly black and tarry and the same deterministic red-flag path fires instead', 2600);
|
||
await tap('[data-followup="tarryBlack"][data-answer="no"]', 500);
|
||
await page.locator('#urgent-box .alert').waitFor({ state: 'detached' });
|
||
await caption('Dark brown is not silently escalated, and the flag clears the moment the answer changes', 2200);
|
||
await tap('#close-sheet', 450);
|
||
await caption('Closing the log keeps the draft in this session instead of throwing it away', 1600);
|
||
await caption('Now the same photo flow while the vision worker is offline', 1500);
|
||
visionOffline = true;
|
||
await tap('[data-scan]', 550);
|
||
await page.getByText(/Vision worker offline/i).waitFor();
|
||
await caption('Vision worker offline — the AI and consent steps disappear instead of failing after upload', 1900);
|
||
await indicate('label[for="gallery-photo"]');
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||
await page.getByText(/Ready for your review/i).waitFor();
|
||
await caption('No consent prompt and no upload when analysis cannot run', 1600);
|
||
await tap('#manual-from-scan', 500);
|
||
const keptDraft = await page.locator('.draft-resume').innerText({ timeout: 5000 });
|
||
if (!/Type 2/.test(keptDraft)) throw new Error('The retained draft must show the value the user corrected to');
|
||
await caption('One tap still reaches manual Bristol entry, and the draft keeps your corrected value', 1800);
|
||
await tap('#discard-draft', 500);
|
||
await caption('Start over clears the kept draft — nothing lingers silently', 1500);
|
||
await tap('#close-sheet', 450);
|
||
|
||
// Post-load readiness pass (issue #15): the worker answers, but its pinned model is not resident
|
||
// yet. The app must withhold AI instead of offering a control that fails after the upload. Every
|
||
// claim below is asserted in the browser, so the recording cannot show a state the app does not have.
|
||
modelLoaded = false;
|
||
visionOffline = false;
|
||
await caption('Now the same worker while its pinned model is still loading', 1700);
|
||
await tap('[data-scan]', 550);
|
||
await page.getByText(/model is not loaded yet/i).first().waitFor();
|
||
if (await page.getByText(/Self-hosted model ready/i).count() !== 0) {
|
||
throw new Error('A reachable worker without its loaded model must never be reported ready');
|
||
}
|
||
const loadingAmber = await page.locator('.model-status.loading').first().evaluate(el => getComputedStyle(el).backgroundColor);
|
||
if (loadingAmber !== 'rgb(253, 243, 216)') throw new Error('The not-loaded state must be visually distinct from ready and offline');
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||
await page.getByText(/Ready for your review/i).waitFor();
|
||
if (await page.locator('#analyze-photo').count() !== 0) throw new Error('No AI control may exist before the model is loaded');
|
||
if (await page.locator('.consent-card').count() !== 0) throw new Error('No consent may be requested before the model is loaded');
|
||
await caption('Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident', 2900);
|
||
await tap('#manual-from-scan', 500);
|
||
await page.getByText('Pick the closest form').waitFor();
|
||
await caption('One tap still reaches manual Bristol entry while the model finishes loading', 2000);
|
||
await tap('#close-sheet', 450);
|
||
modelLoaded = true;
|
||
await caption('Once the pinned model is loaded, the same screen offers the AI analysis again', 2200);
|
||
await tap('[data-scan]', 550);
|
||
await page.getByText(/Self-hosted model ready/i).waitFor();
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||
await page.getByText(/Ready for your review/i).waitFor();
|
||
if (await page.locator('.consent-card').isVisible() !== true) throw new Error('The AI path must return once the model is loaded');
|
||
if (await page.locator('#analyze-photo').count() !== 1) throw new Error('The analyze control must return once the model is loaded');
|
||
await tap('#close-sheet', 450);
|
||
|
||
// Corpus refusal pass (issue #18): the same photo flow, fed the answers Timmy has to refuse.
|
||
visionOffline = false;
|
||
await caption('The corpus also replays the answers Timmy has to refuse', 1700);
|
||
analyzeOverride = await corpusAnalyzeResult(corpusCase('non-stool-scene'));
|
||
await tap('[data-scan]', 500);
|
||
await page.getByText(/Self-hosted model ready/i).waitFor();
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/non-stool-scene.png');
|
||
await page.locator('#ai-consent').check();
|
||
await tap('#analyze-photo', 600);
|
||
await page.getByText(/No confident match/i).waitFor();
|
||
await caption('A non-stool photo gets a refusal, never a guess — manual entry stays one tap away', 2600);
|
||
await indicate('.scan-result');
|
||
analyzeOverride = await corpusAnalyzeResult(corpusCase('glare-overexposed'));
|
||
await tap('#retake-photo', 400);
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/glare-overexposed.png');
|
||
await page.locator('#ai-consent').check();
|
||
await tap('#analyze-photo', 600);
|
||
await page.getByText(/does not show a form Timmy can read safely/i).waitFor();
|
||
await caption('Unreadable visual fields abstain as well: a valid reply with nothing to prefill is not an error', 2700);
|
||
await indicate('.scan-result');
|
||
// Issue #32's quality gate: the model can be highly confident about a photo it also calls poor, and
|
||
// that confident answer must still abstain. Replayed from the committed corpus, and asserted in the
|
||
// browser so the scene is a check rather than a claim.
|
||
analyzeOverride = await corpusAnalyzeResult(corpusCase('poor-quality-confident'));
|
||
await tap('#retake-photo', 400);
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/blur-low-detail.png');
|
||
await page.locator('#ai-consent').check();
|
||
await tap('#analyze-photo', 600);
|
||
await page.getByText(/too blurry or poorly lit/i).waitFor();
|
||
if (await page.getByText(/BRISTOL FORM/i).count() !== 0) {
|
||
throw new Error('A low-quality photo must not prefill a form, however confident the reply looked');
|
||
}
|
||
await caption('A confident reply about a blurry photo still abstains — photo quality is part of the policy, not just confidence', 3100);
|
||
await indicate('.scan-result');
|
||
analyzeOverride = await corpusAnalyzeResult(corpusCase('out-of-range-fields'));
|
||
await tap('#retake-photo', 400);
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/out-of-range-fields.png');
|
||
await page.locator('#ai-consent').check();
|
||
await tap('#analyze-photo', 600);
|
||
await page.getByText(/couldn’t analyze that safely/i).waitFor();
|
||
await caption('An out-of-range answer fails closed: nothing is applied and no provider detail reaches you', 2700);
|
||
await indicate('.scan-result');
|
||
// Issue #73's escape class: one field blank, the other filled in and impossible. The old blank-first
|
||
// ordering abstained here, which told the user the photo was unreadable while the model had returned a
|
||
// value Timmy must never apply. The scene is asserted in the browser, not just captioned.
|
||
analyzeOverride = await corpusAnalyzeResult(corpusCase('blank-plus-out-of-range'));
|
||
await tap('#retake-photo', 400);
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/blank-plus-out-of-range.png');
|
||
await page.locator('#ai-consent').check();
|
||
await tap('#analyze-photo', 600);
|
||
await page.getByText(/couldn’t analyze that safely/i).waitFor();
|
||
if (await page.getByText(/No confident match/i).count() !== 0) {
|
||
throw new Error('A filled-in out-of-range field must not be shown as an unreadable-photo abstention');
|
||
}
|
||
await caption('A filled-in field that is out of range still fails closed, even when the other field came back blank', 2900);
|
||
await indicate('.scan-result');
|
||
await caption('You are no longer told the photo was unreadable when the model returned an impossible value', 2400);
|
||
await indicate('.scan-result');
|
||
await caption('Every refusal replays a committed synthetic fixture — there are no real photos in the corpus', 2500);
|
||
analyzeOverride = null;
|
||
await tap('#close-sheet', 450);
|
||
|
||
// Explainable pattern summary (issue #34). The recording seeds fictional multi-day sample logs into
|
||
// the app's own local ledger contract, then follows one summary line to the exact records behind it.
|
||
// Every claim below is checked in the browser, and no real health data is involved anywhere.
|
||
const seedSyntheticLedger = async entries => {
|
||
const key = await page.evaluate(() => `timmy:${document.querySelector('meta[name="timmy-base-path"]')?.content || '/'}:ledger-v1`);
|
||
await page.evaluate(({ storeKey, list }) => localStorage.setItem(storeKey, JSON.stringify(list)), { storeKey: key, list: entries });
|
||
await page.reload({ waitUntil: 'networkidle' });
|
||
};
|
||
const sampleDay = (id, daysAgo, bristolType) => ({
|
||
id,
|
||
occurredAt: new Date(Date.now() - daysAgo * 86400000).toISOString(),
|
||
bristolType,
|
||
color: 'brown',
|
||
urgency: 1,
|
||
discomfort: 1,
|
||
note: '',
|
||
photoDataUrl: '',
|
||
symptoms: {},
|
||
});
|
||
await caption('The journal explains its own pattern lines instead of just asserting a trend', 2200);
|
||
await seedSyntheticLedger([
|
||
sampleDay('demo-firm', 2, 2),
|
||
sampleDay('demo-typical-1', 5, 3),
|
||
sampleDay('demo-typical-2', 9, 4),
|
||
sampleDay('demo-typical-3', 12, 4),
|
||
sampleDay('demo-old', 40, 6),
|
||
]);
|
||
await caption("Fictional sample logs are seeded into this browser's own storage — no real health data appears here", 2600);
|
||
await page.locator('[data-pattern-summary]').waitFor();
|
||
await indicate('[data-pattern-summary]');
|
||
await caption('The summary says which confirmed logs it read, and a citation opens exactly those records', 2400);
|
||
await indicate('[data-statement="typical"]');
|
||
await tap('[data-statement-link="typical"]', 650);
|
||
const citedRecords = await page.locator('[data-summary-record]').evaluateAll(nodes => nodes.map(node => node.getAttribute('data-entry-id')).sort());
|
||
if (JSON.stringify(citedRecords) !== JSON.stringify(['demo-typical-1', 'demo-typical-2', 'demo-typical-3'])) {
|
||
throw new Error('A summary citation must list exactly the records it cites');
|
||
}
|
||
await caption('Every pattern line names the confirmed logs behind it, and following one shows exactly those records', 2900);
|
||
await indicate('[data-summary-records]');
|
||
await tap('#pattern-records-close', 450);
|
||
await seedSyntheticLedger([sampleDay('demo-sparse-1', 1, 4), sampleDay('demo-sparse-2', 4, 6)]);
|
||
await page.locator('[data-summary-insufficient]').waitFor();
|
||
await indicate('[data-summary-insufficient]');
|
||
await caption('Two logs is not a pattern: the summary refuses to describe a trend it cannot support', 2900);
|
||
// Restore the empty ledger the earlier scenes started from, so the rest of the recording is unchanged.
|
||
await page.evaluate(() => localStorage.clear());
|
||
await page.reload({ waitUntil: 'networkidle' });
|
||
|
||
// Issue #36 — a saved photo and its confirmed log are separate things a person owns. The journal
|
||
// states that a photo exists without rendering it, the image appears only when asked for, removing
|
||
// it keeps every confirmed field, and delete-all takes the ledger and every photo together. The
|
||
// only image here is a synthetic 1x1 PNG constant written by this script; no real photo is used.
|
||
// A 160x120 synthetic test card (checkerboard + 'SYNTHETIC DEMO PHOTO'), so the reveal is legibly
|
||
// an image on camera. It carries no photograph of any person or specimen.
|
||
const SYNTHETIC_PHOTO = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAKAAAAB4CAIAAAD6wG44AAAG0ElEQVR4nO2deVATZxTAXxKSIldVZqyiTEWucMgpyCGggEK96gFYgSKVenCJoYAy1DpaBYpBnEqt1WoVb2071fHCqi2IiFo70gJaT7COWKdYBZExCW7/2HFJm8OFok5f3m/yx+6X9739Jr98m+zb+RKB1MYaCLwIX/UAiBcLCUaOEbfVEh/33OjGtAT+qYeWlvEPpsy9m9ly+w52g2YwckgwckgwckgwckgwckgwckgwckgwckgwckgwckgwcgTc7cILNeXPjUZTqkWf2dsvgt2gGYwcEowcEowcEowcEowcEowcEowcEowcEowcEowcEowcEowcEowcEoycrqUrfG5Robmbhj6z5bMNmsHIIcHIIcHIIcHIIcHIIcHIIcHIIcHIIcHIIcHIIcHIIcHIIcHI6Vp8xv0Q2r3lBa9uPMR/YsBHuewG90NoWlYXmgxwevkjI3qFx/cusRu0utBQIMHIIcHIIcHIIcHIIcHIMXp+CAAANNTVFRcVqFQqkUiUXyRvf9QuS0/57nC5UCgEgKjJEwrkq6Pfnuju6bV15x62i6+7y7na+uPlR8u2bAaAn3867zXCBwDiEhKXLM4+V1vPJWcjAcDT2WG4uwfbGBo+dsgQaz199+3euWvHNlNTMxMT02Ur8wcOsuqdlwQZUhtr9tH+RwP7YLQxZXxEc/MdhmHKjxySpSUzDJO5IPXQgf0Mw/x48gTb4uPmHBc97eyZaraLj5uzegb1XV1P/atdT9/Tpypnxc7o6OhgGKbih5OJsTO0djQ0OImcVr6n6JaWPxVPngBAaPi4uIREAEhJX7j+s7VPnz7dsK40dYGMDUuTfbB2zeoX81b8B5s3fiHLyjE2NgaA4NFjrN8cqlKpXsJx/3fwFSzLWhQXM/3DRdkXzp/z9vEFAFs7O0ep09K83EFWg23t7dkwP/8AADh7pvoFDZfj2tUrTi6u3O7y/EIjI74fNwYF3xdlalR02Nhxx78vL/h4WXhEZFqGDABSFmRMHBd28NgJ9cj0hZmflhSP9A/Qk02pVCbMjFHf1WzPzF7k4eWtK0NnZyfPkRs4vATfv9/S1Njo6eU9LSpmdGjY5MixrGCbYbampqY2w2zVg339/EUiUY3eSSwWi8t27e3q4u6itV0PQ22GXW6od/PwBACGYXKzMwvlJXw6Ghq8TtECgUCWlny3+Q4APPjrgZXVYP3x6QszS0uKe2F0uomNT1hTvEqhUADA4YMH2A1CE14zuF+//svzCzNSk42NjYVC4coiuf74Eb4jxWJxD1509VO0h6dXZs5iXZFvTZzU2Hhz+qTx/S0tLS0tlyxf0d1jGQh0PxgVmveDqZKFHBKMHBKMHL7XwWyVWCAQKBWK6Hdip0ZFg0bpODFpjoeT/Ziw8JLSz9nGHFnGsaOHL166CgDffr13R9kWsViiVCrenTV7yvQozeQqpXLxkqXD3dy56jQLt6uZRGutu739ka5jGRp8BXNXqB2PH6fMmd3HxCRy/ATNy1aJRHLzxo3Ozk6RSMQwzK1bTRKJBACqKiu+2bN7y/bd5hYWba2t85MS3xg40D9w1L+SX/ntcl5O1r79B7WOQWuS8IjI8IhIAPB1d2GTVFVWlG3+UtexDI1un6L7mJhk5eZt+2qTrgBnV9e6X2oB4HJDg6NUyjZu2rA+OzfP3MICAMwtLLJy8zauX6fZ18FRevv277oy80zCM8xA6MlnsKPUqampUdezo4JCqiorAKDqVMWo4BC28cb1a+qlY2cX1+vXrmr2rak+LXVy1pWZZxKeYQZCTwr0KpVKbCQGHaXjwKDgndu2pmbIaqpPz4zX/msxDMMIBAJul83DMIy5ufmKwlWgu1itJ4kueIZhpSeCf6296CCVgo7S8et9+wqEQrauaWZmxjba2tk31Nd5Prt50FBfZ2fvwHXRzKO1WK0/CQfPMAOh26fo1ocP5Z8UJM2drycmKGR0ibwoQO17TdLc+fKClW1tbQDQ1tpaXJj//rzk7h6aZ5JeORYa+M5g9pzJXsnMmZfs6+cPukvHIWNC18iL9h85xnUPDAq+e7d5VmyMRPKaUqmIT3jPLyCwu2PlmaRXjoUGqkWjgmrRBgcJRg6tLkQFrS40OLR8i+beBZqg+bVdw8lMMxg5JBg5JBg5JBg5JBg5JBg5JBg5JBg5JBg5JBg5JBg5JBg59N+FODPTfxcaCiQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOSQYOVpWF+oBzd009JlpdaGhQIKRQ4KRQ4KRQ4KRQ4KRQ4KRQ4KRQ4KRQ4KRQ4KR01WLJlBCMxg5JBg5fwMX/GJFpYaDtAAAAABJRU5ErkJggg==';
|
||
const photoDay = (id, daysAgo, bristolType = 4) => ({ ...sampleDay(id, daysAgo, bristolType), photoDataUrl: SYNTHETIC_PHOTO });
|
||
const storedLedger = async () => {
|
||
const key = await page.evaluate(() => `timmy:${document.querySelector('meta[name="timmy-base-path"]')?.content || '/'}:ledger-v1`);
|
||
const raw = await page.evaluate(storeKey => localStorage.getItem(storeKey), key);
|
||
return { key, raw };
|
||
};
|
||
await caption('A confirmed log can hold a private photo — the journal says so without showing it', 2300);
|
||
await seedSyntheticLedger([photoDay('demo-photo', 1), sampleDay('demo-nophoto', 4, 6)]);
|
||
await tap('[data-view="calendar"]', 550);
|
||
await page.locator('.entries-card .entry').first().waitFor();
|
||
if (await page.locator('.entries-card [data-photo-indicator="present"]').count() !== 1) {
|
||
throw new Error('exactly one log must be marked as holding a photo');
|
||
}
|
||
if (await page.locator('.entries-card img').count() !== 0) {
|
||
throw new Error('the journal must not show a thumbnail by default');
|
||
}
|
||
await caption('Photo presence is a label, not a thumbnail — nothing sensitive is on screen until you ask', 2700);
|
||
await indicate('[data-photo-indicator="present"]');
|
||
await tap('[data-entry-open="demo-photo"]', 550);
|
||
await page.locator('[data-photo-lifecycle="present"]').waitFor();
|
||
if (await page.locator('#entry-photo').count() !== 0) {
|
||
throw new Error('opening a log must not reveal its photo');
|
||
}
|
||
await caption('Opening the log states the photo is saved here — and still does not render it', 2500);
|
||
await indicate('[data-photo-lifecycle="present"]');
|
||
await tap('#show-entry-photo', 600);
|
||
await page.locator('#entry-photo').waitFor();
|
||
const revealedPhoto = await page.locator('#entry-photo').getAttribute('src');
|
||
if (!/^data:image\/png;base64,/.test(revealedPhoto || '')) throw new Error('the revealed image must be the stored bytes');
|
||
await caption('Showing the photo is your explicit request, from this browser’s own storage', 2300);
|
||
await indicate('#entry-photo');
|
||
await tap('#remove-entry-photo', 650);
|
||
await page.locator('[data-photo-lifecycle="absent"]').waitFor();
|
||
const afterRemoval = JSON.parse((await storedLedger()).raw || '[]');
|
||
const keptLog = afterRemoval.find(entry => entry.id === 'demo-photo');
|
||
if (afterRemoval.length !== 2 || !keptLog || keptLog.photoDataUrl !== '') {
|
||
throw new Error('removing a photo must keep the confirmed log and clear only its image bytes');
|
||
}
|
||
if (keptLog.bristolType !== 4 || keptLog.color !== 'brown') throw new Error('removing a photo must not alter the confirmed fields');
|
||
await caption('Remove just the photo: the confirmed log, form, and note are all still here', 2900);
|
||
await indicate('[data-photo-lifecycle="absent"]');
|
||
await tap('#entry-sheet-close', 450);
|
||
await page.locator('.entries-card .entry').first().waitFor();
|
||
if (await page.locator('.entries-card [data-photo-indicator="present"]').count() !== 0) {
|
||
throw new Error('the journal must drop the photo label once the bytes are gone');
|
||
}
|
||
await caption('The label follows the bytes: the log stays, the photo is gone', 2300);
|
||
|
||
// One photo-bearing log remains for the deletion pass below, so the wipe shows photos leaving with
|
||
// the ledger rather than an empty promise.
|
||
await seedSyntheticLedger([photoDay('demo-wipe', 2)]);
|
||
await caption('The threat model documents this surface, so the recording shows it live rather than describing it', 2200);
|
||
await tap('[data-view="calendar"]', 550);
|
||
await page.getByText(/Your private journal/i).waitFor();
|
||
await tap('[data-view="privacy"]', 550);
|
||
await page.getByText(/Your poop\. Your phone\./i).waitFor();
|
||
await caption('Every byte-bearing control the model names, in the running app: local ledger, consent-gated upload, portable export, full deletion', 2800);
|
||
await indicate('#export');
|
||
await caption('Export writes a plain JSON file you hold — the model records it as a data-lifetime boundary, not a server sync', 2600);
|
||
// Issue #24 — training contribution is a second, separate consent. The recording shows the section
|
||
// reporting nothing queued after ordinary saves, the version and receipt on screen before the
|
||
// button can be pressed, one receipt queued on the device only, and a withdrawal tombstone that
|
||
// names each derived artifact by contribution id.
|
||
await caption('Training contribution is off by default: a saved log is never training data', 2600);
|
||
await page.locator('[data-training-consent]').waitFor();
|
||
const contributionKey = await page.evaluate(() => `timmy:${document.querySelector('meta[name="timmy-base-path"]')?.content || '/'}:training-contributions-v1`);
|
||
const storedContributions = async () => JSON.parse(await page.evaluate(key => localStorage.getItem(key) || '[]', contributionKey));
|
||
if ((await page.locator('[data-contribution-queued-count]').innerText()).trim() !== '0') {
|
||
throw new Error('a journal save must not queue a training contribution');
|
||
}
|
||
await indicate('[data-contribute-entry="demo-wipe"]');
|
||
await tap('[data-contribute-entry="demo-wipe"]', 600);
|
||
await page.locator('[data-contribution-sheet]').waitFor();
|
||
const receiptId = (await page.locator('[data-receipt-id]').innerText()).trim();
|
||
if (!receiptId) throw new Error('the consent sheet must name the receipt before consent');
|
||
const sheetCopy = await page.locator('[data-contribution-sheet]').innerText();
|
||
if (!sheetCopy.includes('training-consent-v1')) throw new Error('the consent sheet must state the consent version');
|
||
if (!(await page.locator('#contribute-confirm').isDisabled())) throw new Error('the contribute control must stay disabled until the terms are accepted');
|
||
if ((await storedContributions()).length !== 0) throw new Error('no receipt may exist before explicit consent');
|
||
await caption('The consent version and the receipt are on screen before the contribute button works', 3000);
|
||
await page.locator('#training-consent-check').check();
|
||
await tap('#contribute-confirm', 700);
|
||
await page.locator('[data-contribution-state="queued-local"]').waitFor();
|
||
const queuedReceipts = await storedContributions();
|
||
if (queuedReceipts.length !== 1 || queuedReceipts[0].consentVersion !== 'training-consent-v1') {
|
||
throw new Error('one queued receipt must carry the consent version it was accepted under');
|
||
}
|
||
if (JSON.stringify(queuedReceipts).includes('data:image')) throw new Error('a receipt must never copy the photo bytes');
|
||
await caption('One confirmed log is queued on this device with a receipt you can withdraw — nothing is uploaded', 3200);
|
||
await indicate('[data-contribution-state="queued-local"]');
|
||
await tap(`[data-withdraw="${receiptId}"]`, 700);
|
||
await page.locator('[data-contribution-state="withdrawn"]').waitFor();
|
||
const withdrawnReceipts = await storedContributions();
|
||
const tombstone = withdrawnReceipts[0];
|
||
if (!tombstone || tombstone.state !== 'withdrawn') throw new Error('withdrawal must keep the receipt as a tombstone');
|
||
if ((tombstone.artifacts || []).length !== 2) throw new Error('the tombstone must name every derived artifact');
|
||
for (const artifact of tombstone.artifacts) {
|
||
if (artifact.contributionId !== receiptId) throw new Error('every derived artifact must be addressed by contribution id');
|
||
if (artifact.state !== 'withdrawn') throw new Error('every derived artifact must be marked withdrawn');
|
||
}
|
||
await caption('Withdrawing keeps a tombstone naming every derived artifact by contribution ID', 3000);
|
||
await indicate('[data-withdrawn-artifacts]');
|
||
|
||
// Issue #25 — the corpus pipeline is a command-line tool, not a screen in the app. Rather than
|
||
// inventing a UI for it, this scene runs the real ingest and revoke commands against the committed
|
||
// synthetic fixture in a throwaway dataset and shows their actual output. Every number, hash prefix,
|
||
// and refusal line below is read back from those commands at record time.
|
||
async function corpusCli(args) {
|
||
return execFileAsync('python3', args, {
|
||
cwd: process.cwd(),
|
||
env: { ...process.env, TIMMY_DATASET_SALT: 'demo-only-salt-not-a-secret' },
|
||
});
|
||
}
|
||
async function corpusAudit() {
|
||
const dataset = await mkdtemp(path.join(os.tmpdir(), 'timmy-corpus-demo-'));
|
||
const fixture = 'tests/fixtures/synthetic-type4.jpg';
|
||
const contributionId = 'demo-contrib-0001-abcd';
|
||
const manifestPath = path.join(dataset, 'manifest.jsonl');
|
||
const lines = async () => (await readFile(manifestPath, 'utf8'))
|
||
.split('\n').filter(line => line.trim()).map(line => JSON.parse(line));
|
||
const images = async () => (await readdir(path.join(dataset, 'images')).catch(() => [])).sort();
|
||
try {
|
||
const ingest = await corpusCli([
|
||
'scripts/ingest_training_photo.py',
|
||
'--input', fixture,
|
||
'--output-dir', dataset,
|
||
'--subject-id', 'demo-subject',
|
||
'--consent-version', 'training-consent-v1',
|
||
'--contribution-id', contributionId,
|
||
'--bristol-type', '4', '--color', 'brown', '--quality', 'good',
|
||
'--reviewer', 'user-confirmed',
|
||
]);
|
||
const record = JSON.parse(ingest.stdout);
|
||
const fixtureDigest = createHash('sha256').update(await readFile(fixture)).digest('hex');
|
||
if (record.recordType !== 'sample' || record.lifecycle?.state !== 'active') {
|
||
throw new Error('ingest must write one active sample record');
|
||
}
|
||
// The record names the source by digest only: the caller's path must never be stored.
|
||
if (record.source?.inputSha256 !== fixtureDigest || record.hashes?.rawInput !== fixtureDigest) {
|
||
throw new Error('the record must carry the raw input digest');
|
||
}
|
||
if (JSON.stringify(record).includes(fixture) || 'inputPath' in record) {
|
||
throw new Error('the record must never store the caller path');
|
||
}
|
||
const afterIngestImages = await images();
|
||
if (afterIngestImages.length !== 1 || (await lines()).length !== 1) {
|
||
throw new Error('ingest must write exactly one derived image and one manifest record');
|
||
}
|
||
|
||
const revoke = await corpusCli([
|
||
'scripts/ingest_training_photo.py', '--output-dir', dataset,
|
||
'--revoke', contributionId, '--reason', 'participant withdrew consent',
|
||
]);
|
||
const tombstone = JSON.parse(revoke.stdout);
|
||
const afterRevokeImages = await images();
|
||
const history = await lines();
|
||
if (tombstone.recordType !== 'tombstone' || history.length !== 2) {
|
||
throw new Error('revocation must append exactly one tombstone to the existing history');
|
||
}
|
||
if (afterRevokeImages.length !== 0) throw new Error('revocation must delete the derived bytes');
|
||
if (history[0].recordType !== 'sample' || history[0].lifecycle?.state !== 'active') {
|
||
throw new Error('the original record must survive its own withdrawal');
|
||
}
|
||
if ('labels' in tombstone || 'subjectKey' in tombstone) {
|
||
throw new Error('a tombstone must not carry the labels or subject key');
|
||
}
|
||
const deleted = tombstone.deletedArtifacts || [];
|
||
if (!deleted.some(item => item.relativePath === `images/${record.imageId}.jpg` && item.deleted === true)) {
|
||
throw new Error('the tombstone must name the derived artifact it removed');
|
||
}
|
||
const active = await corpusCli([
|
||
'-c',
|
||
'import sys;sys.path.insert(0,"scripts");import corpus_manifest as c;from pathlib import Path;'
|
||
+ 'print(len(c.active_records(c.load_manifest(Path(sys.argv[1])/"manifest.jsonl"))))',
|
||
dataset,
|
||
]);
|
||
if (active.stdout.trim() !== '0') throw new Error('a trainer reading the manifest must see zero active samples');
|
||
|
||
// A second revocation of the same contribution has to be refused, and refuse without writing.
|
||
let refusal = '';
|
||
try {
|
||
await corpusCli([
|
||
'scripts/ingest_training_photo.py', '--output-dir', dataset,
|
||
'--revoke', contributionId, '--reason', 'participant withdrew consent again',
|
||
]);
|
||
throw new Error('a second revocation must be refused');
|
||
} catch (error) {
|
||
if (error.message === 'a second revocation must be refused') throw error;
|
||
refusal = String(error.stderr || error.stdout || '').trim().split('\n').filter(Boolean).pop() || '';
|
||
}
|
||
if (!/already revoked/.test(refusal)) throw new Error(`second revocation refusal was not legible: ${refusal}`);
|
||
if ((await lines()).length !== 2) throw new Error('a refused revocation must append nothing');
|
||
return { record, tombstone, deleted, refusal, derived: record.imageId, fixtureDigest };
|
||
} finally {
|
||
await rm(dataset, { recursive: true, force: true });
|
||
}
|
||
}
|
||
const audit = await corpusAudit();
|
||
const short = value => String(value).slice(0, 12);
|
||
const deletedName = audit.deleted[0].relativePath.split('/').pop().replace(/\.jpg$/, '');
|
||
await tap('[data-view="home"]', 500);
|
||
// The card is drawn from the values above, so a run that stops writing provenance or stops deleting
|
||
// bytes fails here instead of recording a caption that claims it.
|
||
const auditRows = [
|
||
['command', 'python3 scripts/ingest_training_photo.py --input <synthetic fixture>'],
|
||
['record', `sample · schemaVersion ${audit.record.schemaVersion}`],
|
||
['consent', audit.record.consent.version],
|
||
['source', `sha256 ${short(audit.record.source.inputSha256)}… (path never stored)`],
|
||
['hashes', `raw ${short(audit.record.hashes.rawInput)}… · derived ${short(audit.record.hashes.derived)}…`],
|
||
['derived', audit.record.derivation.pipeline + ' · jpeg · ≤1200px · metadata stripped'],
|
||
['review', audit.record.review.status],
|
||
['lifecycle', audit.record.lifecycle.state],
|
||
['—', ''],
|
||
['command', `--revoke ${audit.record.contributionId} --reason "participant withdrew consent"`],
|
||
['deleted', `images/${short(deletedName)}….jpg → gone from disk`],
|
||
['tombstone', 'appended · keeps id + digests · drops labels + subject key'],
|
||
['history', '2 lines: the original record is still there'],
|
||
['trainer', 'active samples: 0'],
|
||
['—', ''],
|
||
['command', '--revoke again'],
|
||
['refused', audit.refusal],
|
||
['manifest', 'unchanged at 2 lines'],
|
||
];
|
||
const panelOk = await evidenceCard({
|
||
id: 'corpus-audit',
|
||
title: 'CORPUS AUDIT',
|
||
subtitle: 'command line, not app UI — real run, synthetic fixture',
|
||
rows: auditRows,
|
||
});
|
||
await sleep(420);
|
||
const panelProof = await evidenceCardProof('corpus-audit');
|
||
if (!panelProof.ok) throw new Error(`Corpus audit card is not fully readable on screen: ${JSON.stringify(panelProof)}`);
|
||
// The card is drawn from live command output; prove the values on it are the ones the run produced.
|
||
if (!panelOk.text.includes(audit.record.imageId.slice(0, 12))) throw new Error('the card must show the derived digest it recorded');
|
||
if (!panelOk.text.includes('active samples: 0')) throw new Error('the card must show the post-revocation active count');
|
||
await caption('The corpus pipeline is a command-line tool, not a screen in the app — this runs the real ingest against the committed synthetic fixture', 3400);
|
||
await caption('Every ingested sample records consent, source digest, both hashes, derivation, review, and lifecycle — the caller file path is never stored', 3600);
|
||
await caption('Withdrawing deletes the derived image and any raw copy, keeps the original record, and appends a tombstone — the trainer sees zero active samples', 3800);
|
||
await caption('Revoking the same contribution twice is refused and appends nothing: the manifest only ever grows', 3200);
|
||
await page.evaluate(() => document.querySelector('#corpus-audit')?.remove());
|
||
|
||
// Issue #26 — the reviewer and adjudication tool is a command line, not a screen in the app. This
|
||
// scene runs the real commands against the committed synthetic sample in a throwaway dataset and
|
||
// shows the order the tool enforces: the queue hides the model answer, revealing it is refused
|
||
// until a human commits, and a withdrawal removes the sample from the queue, the review path, and
|
||
// the training set. Every value on the card is read back from those runs. The answer file is a
|
||
// synthetic fixture; no model measured anything here.
|
||
async function reviewCli(args) {
|
||
try {
|
||
const result = await execFileAsync('python3', ['scripts/review_corpus.py', ...args], {
|
||
cwd: process.cwd(),
|
||
env: { ...process.env, TIMMY_DATASET_SALT: 'demo-only-salt-not-a-secret' },
|
||
});
|
||
return { code: 0, stdout: result.stdout, stderr: result.stderr };
|
||
} catch (error) {
|
||
return {
|
||
code: typeof error.code === 'number' ? error.code : 1,
|
||
stdout: error.stdout || '',
|
||
stderr: error.stderr || '',
|
||
};
|
||
}
|
||
}
|
||
function lastLine(output) {
|
||
return String(output).trim().split('\n').filter(Boolean).pop() || '';
|
||
}
|
||
async function reviewLedger() {
|
||
const dataset = await mkdtemp(path.join(os.tmpdir(), 'timmy-review-demo-'));
|
||
const contributionId = 'demo-review-0001-abcd';
|
||
const predictionsPath = path.join(dataset, 'synthetic-answers.json');
|
||
const manifestPath = path.join(dataset, 'manifest.jsonl');
|
||
const reviewPath = path.join(dataset, 'reviews.jsonl');
|
||
const lines = async file => (await readFile(file, 'utf8').catch(() => ''))
|
||
.split('\n').filter(line => line.trim()).map(line => JSON.parse(line));
|
||
try {
|
||
const ingest = await corpusCli([
|
||
'scripts/ingest_training_photo.py',
|
||
'--input', 'tests/fixtures/synthetic-type4.jpg',
|
||
'--output-dir', dataset,
|
||
'--subject-id', 'demo-subject',
|
||
'--consent-version', 'training-consent-v1',
|
||
'--contribution-id', contributionId,
|
||
'--bristol-type', '4', '--color', 'brown', '--quality', 'good',
|
||
'--reviewer', 'user-confirmed',
|
||
]);
|
||
if (JSON.parse(ingest.stdout).lifecycle?.state !== 'active') {
|
||
throw new Error('the review scene needs one active synthetic sample');
|
||
}
|
||
|
||
const before = await reviewCli(['queue', '--dataset', dataset]);
|
||
const queue = JSON.parse(before.stdout);
|
||
if (queue.count !== 1) throw new Error('one unreviewed sample must queue exactly once');
|
||
if (queue.items[0].proposed.bristolType !== 4) throw new Error('the queue must show the proposal');
|
||
if ('confidence' in queue.items[0] || /"model"/.test(JSON.stringify(queue.items[0]))) {
|
||
throw new Error('the queue must not carry the model answer');
|
||
}
|
||
|
||
const early = await reviewCli(['reveal', '--dataset', dataset, '--contribution', contributionId]);
|
||
if (early.code === 0) throw new Error('revealing before a review must be refused');
|
||
const blindRefusal = lastLine(early.stderr);
|
||
if (!/stays hidden until a review is committed/.test(blindRefusal)) {
|
||
throw new Error(`the blind refusal was not legible: ${blindRefusal}`);
|
||
}
|
||
|
||
await writeFile(predictionsPath, JSON.stringify({
|
||
model: 'synthetic answer file',
|
||
answers: { [contributionId]: { bristolType: 6, color: 'brown', quality: 'good', confidence: 0.71 } },
|
||
}, null, 2));
|
||
|
||
const committed = await reviewCli([
|
||
'review', '--dataset', dataset, '--contribution', contributionId,
|
||
'--reviewer', 'reviewer-a', '--bristol', '5', '--color', 'green', '--quality', 'fair',
|
||
]);
|
||
if (committed.code !== 0) throw new Error(`the review was refused: ${lastLine(committed.stderr)}`);
|
||
const review = JSON.parse(committed.stdout);
|
||
if (review.decision !== 'corrected' || review.labels.bristolType !== 5) {
|
||
throw new Error('the committed review must record the corrected label');
|
||
}
|
||
if (review.proposed.bristolType !== 4) throw new Error('the committed review must keep the proposal it judged');
|
||
|
||
const revealed = await reviewCli(['reveal', '--dataset', dataset, '--contribution', contributionId, '--predictions', predictionsPath]);
|
||
if (revealed.code !== 0) throw new Error(`reveal failed after a committed review: ${lastLine(revealed.stderr)}`);
|
||
const reveal = JSON.parse(revealed.stdout);
|
||
if (reveal.model?.labels.bristolType !== 6) throw new Error('the revealed answer must be the one that was withheld');
|
||
if (!reveal.disagreement || reveal.disagreeFields.length !== 3) {
|
||
throw new Error('a Type 6 answer against a Type 5 review must be recorded as a disagreement');
|
||
}
|
||
|
||
const ruled = await reviewCli([
|
||
'adjudicate', '--dataset', dataset, '--contribution', contributionId,
|
||
'--reviewer', 'reviewer-b', '--upheld',
|
||
]);
|
||
if (ruled.code !== 0) throw new Error(`adjudication was refused: ${lastLine(ruled.stderr)}`);
|
||
const ruling = JSON.parse(ruled.stdout);
|
||
if (ruling.resolvesRecord !== 1) throw new Error('the ruling must point at the record it resolves');
|
||
const ledgerAfterRuling = await lines(reviewPath);
|
||
if (ledgerAfterRuling.length !== 2) throw new Error('adjudication must append, not replace');
|
||
if (ledgerAfterRuling[0].reviewer !== 'reviewer-a' || ledgerAfterRuling[0].decision !== 'corrected') {
|
||
throw new Error('the review being adjudicated must survive untouched');
|
||
}
|
||
|
||
const trainableBefore = JSON.parse((await reviewCli(['trainable', '--dataset', dataset])).stdout);
|
||
if (trainableBefore.count !== 1) throw new Error('one reviewed sample must be trainable');
|
||
if (trainableBefore.items[0].labels.bristolType !== 5) {
|
||
throw new Error('the trainer must read the reviewed label, not the proposal');
|
||
}
|
||
|
||
const revoke = await corpusCli([
|
||
'scripts/ingest_training_photo.py', '--output-dir', dataset,
|
||
'--revoke', contributionId, '--reason', 'participant withdrew consent',
|
||
]);
|
||
if (JSON.parse(revoke.stdout).recordType !== 'tombstone') throw new Error('the withdrawal must be recorded');
|
||
|
||
const afterRevoke = JSON.parse((await reviewCli(['queue', '--dataset', dataset])).stdout);
|
||
const trainableAfter = JSON.parse((await reviewCli(['trainable', '--dataset', dataset])).stdout);
|
||
if (afterRevoke.count !== 0 || trainableAfter.count !== 0) {
|
||
throw new Error('a withdrawn sample must leave the queue and the training set');
|
||
}
|
||
const refused = await reviewCli([
|
||
'review', '--dataset', dataset, '--contribution', contributionId, '--reviewer', 'reviewer-c', '--agree',
|
||
]);
|
||
if (refused.code === 0) throw new Error('reviewing a withdrawn contribution must be refused');
|
||
const withdrawnRefusal = lastLine(refused.stderr);
|
||
if (!/was withdrawn and cannot be reviewed/.test(withdrawnRefusal)) {
|
||
throw new Error(`the withdrawal refusal was not legible: ${withdrawnRefusal}`);
|
||
}
|
||
if ((await lines(reviewPath)).length !== 2) throw new Error('a refused review must append nothing');
|
||
if ((await lines(manifestPath)).length !== 2) throw new Error('the manifest must hold the sample and its tombstone');
|
||
return { queue, blindRefusal, review, reveal, ruling, trainableBefore, afterRevoke, trainableAfter, withdrawnRefusal };
|
||
} finally {
|
||
await rm(dataset, { recursive: true, force: true });
|
||
}
|
||
}
|
||
const reviewEvidence = await reviewLedger();
|
||
await tap('[data-view="home"]', 450);
|
||
const reviewRows = [
|
||
['command', 'python3 scripts/review_corpus.py queue --dataset <tmp>'],
|
||
['queue', `${reviewEvidence.queue.count} pending · proposal Type ${reviewEvidence.queue.items[0].proposed.bristolType} · ${reviewEvidence.queue.items[0].proposed.color} · ${reviewEvidence.queue.items[0].proposed.quality}`],
|
||
['blind', 'model answer withheld from the reviewer'],
|
||
['—', ''],
|
||
['command', 'reveal --contribution <id> (before any review)'],
|
||
['refused', reviewEvidence.blindRefusal],
|
||
['—', ''],
|
||
['command', 'review --reviewer reviewer-a --bristol 5 --color green --quality fair'],
|
||
['committed', `${reviewEvidence.review.decision} · proposal ${reviewEvidence.review.proposed.bristolType}/${reviewEvidence.review.proposed.color}/${reviewEvidence.review.proposed.quality} kept beside it`],
|
||
['command', 'reveal --predictions <synthetic answer file>'],
|
||
['model', `Type ${reviewEvidence.reveal.model.labels.bristolType} · ${reviewEvidence.reveal.model.labels.color} · ${reviewEvidence.reveal.model.labels.quality} · confidence ${reviewEvidence.reveal.model.confidence}`],
|
||
['disagree', reviewEvidence.reveal.disagreeFields.join(' · ')],
|
||
['—', ''],
|
||
['command', 'adjudicate --reviewer reviewer-b --upheld'],
|
||
['ledger', `2 lines · resolves record ${reviewEvidence.ruling.resolvesRecord} · reviewer-a's review untouched`],
|
||
['trainer', `trainable samples: ${reviewEvidence.trainableBefore.count}`],
|
||
['—', ''],
|
||
['command', 'ingest --revoke <id> --reason "participant withdrew consent"'],
|
||
['after', `queue ${reviewEvidence.afterRevoke.count} · trainable ${reviewEvidence.trainableAfter.count}`],
|
||
['review', reviewEvidence.withdrawnRefusal],
|
||
['data', 'synthetic answer file · no model was measured'],
|
||
];
|
||
const reviewCard = await evidenceCard({
|
||
id: 'review-audit',
|
||
title: 'REVIEW LEDGER',
|
||
subtitle: 'command line, not app UI — real run, synthetic sample',
|
||
rows: reviewRows,
|
||
});
|
||
await sleep(420);
|
||
const reviewProof = await evidenceCardProof('review-audit');
|
||
if (!reviewProof.ok) throw new Error(`Review card is not fully readable on screen: ${JSON.stringify(reviewProof)}`);
|
||
if (!reviewCard.text.includes(`Type ${reviewEvidence.reveal.model.labels.bristolType}`)) {
|
||
throw new Error('the card must show the model answer the run revealed');
|
||
}
|
||
if (!reviewCard.text.includes('was withdrawn and cannot be reviewed')) {
|
||
throw new Error('the card must show the withdrawn refusal the run earned');
|
||
}
|
||
if (!reviewCard.text.includes(`trainable samples: ${reviewEvidence.trainableBefore.count}`)) {
|
||
throw new Error('the card must show the trainer count the run produced');
|
||
}
|
||
await caption('The review tool is a command line, not a screen in the app — this runs it against the committed synthetic sample', 3600);
|
||
await caption('The queue shows the proposal and hides the model answer; revealing it is refused until a reviewer commits', 3800);
|
||
await caption('The model said Type 6 and the reviewer said Type 5: the disagreement is recorded, not smoothed over', 3600);
|
||
await caption('A second reviewer adjudicates, and the review being overruled is still in the ledger', 3400);
|
||
await caption('Withdrawing the contribution empties the queue and the training set, and the tool refuses to review it at all', 3800);
|
||
await page.evaluate(() => document.querySelector('#review-audit')?.remove());
|
||
|
||
// Issue #27 — the dataset card is a generated document, not a hand-written page. This scene builds a
|
||
// throwaway corpus with the real ingest command from committed synthetic fixtures, publishes the card
|
||
// with the real generator, and shows the refusal the split policy earns when one contributor is
|
||
// planted across two splits. Every count, id, and refusal line on the card is read back from those
|
||
// runs. No model labelled anything in this corpus, and the caption says so.
|
||
async function cardCli(args) {
|
||
try {
|
||
const result = await execFileAsync('python3', ['scripts/dataset_card.py', ...args], { cwd: process.cwd() });
|
||
return { code: 0, stdout: result.stdout, stderr: result.stderr };
|
||
} catch (error) {
|
||
return {
|
||
code: typeof error.code === 'number' ? error.code : 1,
|
||
stdout: error.stdout || '',
|
||
stderr: error.stderr || '',
|
||
};
|
||
}
|
||
}
|
||
async function publishDatasetCard() {
|
||
const dataset = await mkdtemp(path.join(os.tmpdir(), 'timmy-card-demo-'));
|
||
const leaked = await mkdtemp(path.join(os.tmpdir(), 'timmy-card-leak-'));
|
||
const cardPath = path.join(dataset, 'DATASET_CARD.md');
|
||
// Three distinct committed synthetic fixtures: a formed sample, a well-lit stool-like sample, and a
|
||
// glare frame that can only honestly be labelled poor quality.
|
||
const contributions = [
|
||
{ id: 'demo-card-0001-abcd', subject: 'demo-contributor-a', file: 'tests/fixtures/synthetic-type4.jpg', type: '4', color: 'brown', quality: 'good' },
|
||
{ id: 'demo-card-0002-abcd', subject: 'demo-contributor-a', file: 'tests/fixtures/vision-corpus/stool-like-formed.png', type: '3', color: 'green', quality: 'fair' },
|
||
{ id: 'demo-card-0003-abcd', subject: 'demo-contributor-b', file: 'tests/fixtures/vision-corpus/glare-overexposed.png', type: '1', color: 'brown', quality: 'poor' },
|
||
];
|
||
try {
|
||
for (const item of contributions) {
|
||
await corpusCli([
|
||
'scripts/ingest_training_photo.py',
|
||
'--input', item.file,
|
||
'--output-dir', dataset,
|
||
'--subject-id', item.subject,
|
||
'--consent-version', 'training-consent-v1',
|
||
'--contribution-id', item.id,
|
||
'--bristol-type', item.type, '--color', item.color, '--quality', item.quality,
|
||
'--reviewer', 'user-confirmed',
|
||
]);
|
||
}
|
||
// Withdraw one of the first contributor's two samples, so both contributors still have an active
|
||
// sample and the class table shows the withdrawn label dropping out of the count.
|
||
const withdrawn = contributions[1].id;
|
||
await corpusCli([
|
||
'scripts/ingest_training_photo.py', '--output-dir', dataset,
|
||
'--revoke', withdrawn, '--reason', 'participant withdrew consent',
|
||
]);
|
||
|
||
const published = await cardCli(['--corpus', dataset, '--out', cardPath]);
|
||
if (published.code !== 0) throw new Error(`the dataset card was refused: ${lastLine(published.stderr)}`);
|
||
const summary = JSON.parse((await cardCli(['--corpus', dataset, '--json'])).stdout);
|
||
if (summary.activeSamples !== 2) throw new Error('the card must count the two active samples the run ingested');
|
||
if (summary.contributors !== 2) throw new Error('the card must count both contributors');
|
||
if (summary.withdrawals !== 1) throw new Error('the card must count the withdrawal as a withdrawal');
|
||
const card = await readFile(cardPath, 'utf8');
|
||
for (const fact of ['| Devices | pending', '| Contexts | pending', '| Licenses | pending']) {
|
||
if (!card.includes(fact)) throw new Error(`an uncollected fact must be printed as pending: ${fact}`);
|
||
}
|
||
if (!card.includes(withdrawn) || card.includes('subjectKey')) {
|
||
throw new Error('the card must name the withdrawn contribution and never carry a subject key');
|
||
}
|
||
if (summary.bristol['3'] !== 0) throw new Error('a withdrawn sample must not be counted in the class balance');
|
||
|
||
// Plant one contributor in a second split and prove the policy refuses to publish a card at all.
|
||
const original = await readFile(path.join(dataset, 'manifest.jsonl'), 'utf8');
|
||
const records = original.split('\n').filter(line => line.trim()).map(line => JSON.parse(line));
|
||
const sample = records.find(record => record.recordType === 'sample' && record.contributionId === contributions[0].id);
|
||
const planted = { ...sample, contributionId: 'demo-card-0009-leak', split: sample.split === 'train' ? 'test' : 'train' };
|
||
await writeFile(path.join(leaked, 'manifest.jsonl'), `${original}${JSON.stringify(planted)}\n`);
|
||
const refused = await cardCli(['--corpus', leaked, '--out', path.join(leaked, 'DATASET_CARD.md')]);
|
||
if (refused.code === 0) throw new Error('the generator must refuse a corpus that leaks one contributor across splits');
|
||
const leakRefusal = lastLine(refused.stderr);
|
||
if (!/leaks across/.test(leakRefusal)) throw new Error(`the leak refusal was not legible: ${leakRefusal}`);
|
||
|
||
// The committed page has to be this generator's own output, not a document someone typed.
|
||
const committed = await cardCli(['--corpus', '/srv/timmy-corpus', '--check', 'docs/DATASET_CARD.md']);
|
||
if (committed.code !== 0) throw new Error("the committed dataset card must be the generator's own output");
|
||
|
||
return { summary, leakRefusal, withdrawn };
|
||
} finally {
|
||
await rm(dataset, { recursive: true, force: true });
|
||
await rm(leaked, { recursive: true, force: true });
|
||
}
|
||
}
|
||
const cardEvidence = await publishDatasetCard();
|
||
await tap('[data-view="home"]', 450);
|
||
const splits = cardEvidence.summary.bySplit;
|
||
const cardRows = [
|
||
['command', 'python3 scripts/dataset_card.py --corpus <tmp> --out DATASET_CARD.md'],
|
||
['corpus', `${cardEvidence.summary.activeSamples} active · ${cardEvidence.summary.contributors} contributors · ${cardEvidence.summary.withdrawals} withdrawn`],
|
||
['splits', `train ${splits.train.contributors}c/${splits.train.samples}s · validation ${splits.validation.contributors}c/${splits.validation.samples}s · test ${splits.test.contributors}c/${splits.test.samples}s`],
|
||
['class', `Type 1 ${cardEvidence.summary.bristol['1']} · Type 3 ${cardEvidence.summary.bristol['3']} · Type 4 ${cardEvidence.summary.bristol['4']} · withdrawn label excluded`],
|
||
['withdrawn', `${cardEvidence.withdrawn} · "participant withdrew consent" · no label survives in any table`],
|
||
['pending', 'devices · contexts · licenses — not collected, not estimated'],
|
||
['—', ''],
|
||
['command', 'publish a manifest that plants one contributor in train and test'],
|
||
['refused', cardEvidence.leakRefusal],
|
||
['card', 'not written: the policy is checked before a line is rendered'],
|
||
['—', ''],
|
||
['command', '--check docs/DATASET_CARD.md'],
|
||
['committed', 'the published page is this generator\'s own output'],
|
||
['data', 'synthetic fixtures · no model labelled anything here'],
|
||
];
|
||
const cardPanel = await evidenceCard({
|
||
id: 'dataset-card',
|
||
title: 'DATASET CARD',
|
||
subtitle: 'generated from the manifest — command line, not app UI',
|
||
rows: cardRows,
|
||
keyWidth: 70,
|
||
});
|
||
await sleep(420);
|
||
const cardProof = await evidenceCardProof('dataset-card');
|
||
if (!cardProof.ok) throw new Error(`Dataset card is not fully readable on screen: ${JSON.stringify(cardProof)}`);
|
||
if (!cardPanel.text.includes(`${cardEvidence.summary.contributors} contributors`)) {
|
||
throw new Error('the card must show the contributor count the run produced');
|
||
}
|
||
if (!cardPanel.text.includes('leaks across')) throw new Error('the card must show the leak refusal the run earned');
|
||
if (!cardPanel.text.includes('not collected')) throw new Error('the card must show the facts that stay pending');
|
||
await caption('The dataset card is generated from the corpus manifest, so no count on it is typed by hand', 3600);
|
||
await caption('One contributor\'s photos stay in one split: a corpus that leaks a person across splits is refused', 3800);
|
||
await caption('Devices, contexts, and licenses stay pending until the pipeline collects them', 3400);
|
||
await caption('The card is a page, not a screen in the app — it is published with the real generator against synthetic fixtures', 3400);
|
||
await page.evaluate(() => document.querySelector('#dataset-card')?.remove());
|
||
|
||
// Issue #29 — the release gate is a command-line harness, not a screen in the app. This scene runs
|
||
// the real harness over the committed synthetic prediction fixtures and shows the receipt it wrote:
|
||
// a blocked comparison, a passing fixture, and a refused leaked corpus. Every number and every
|
||
// refusal line on the card is read back from those runs; the scores are invented fixtures, and the
|
||
// captions say so, because a fixture is not a measurement of any model.
|
||
async function calibrationCli(args) {
|
||
try {
|
||
const result = await execFileAsync('python3', ['scripts/calibration_report.py', ...args], { cwd: process.cwd() });
|
||
return { code: 0, stdout: result.stdout, stderr: result.stderr };
|
||
} catch (error) {
|
||
return {
|
||
code: typeof error.code === 'number' ? error.code : 1,
|
||
stdout: error.stdout || '',
|
||
stderr: error.stderr || '',
|
||
};
|
||
}
|
||
}
|
||
async function calibrationGate() {
|
||
const directory = await mkdtemp(path.join(os.tmpdir(), 'timmy-calibration-demo-'));
|
||
const reportPath = path.join(directory, 'report.json');
|
||
const source = 'invented scores, not measurements of any model';
|
||
try {
|
||
const blocked = await calibrationCli([
|
||
'--predictions', 'tests/fixtures/calibration/predictions.jsonl',
|
||
'--json', reportPath, '--source', source,
|
||
]);
|
||
if (blocked.code !== 3) throw new Error('the synthetic candidate fixtures must block the release');
|
||
const report = JSON.parse(await readFile(reportPath, 'utf8'));
|
||
if (report.verdict !== 'BLOCKED' || report.recommended !== null) {
|
||
throw new Error('a blocked gate must recommend nobody');
|
||
}
|
||
if (report.models.length !== 3) throw new Error('the comparison must score all three candidate fixtures');
|
||
for (const row of report.models) {
|
||
if (!row.gate.failed.length) throw new Error(`${row.model}: a fixture candidate must fail at least one gate`);
|
||
if (row.selectionSplit !== 'validation' || row.evaluatedSplit !== 'test') {
|
||
throw new Error(`${row.model}: the threshold must come from validation and the score from test`);
|
||
}
|
||
}
|
||
const go = await calibrationCli([
|
||
'--predictions', 'tests/fixtures/calibration/predictions-go.jsonl',
|
||
'--source', 'engineered fixture — proves the passing branch, not a result',
|
||
]);
|
||
if (go.code !== 0 || !/verdict: GO/.test(go.stdout)) throw new Error('the engineered fixture must prove the passing branch');
|
||
const leak = await calibrationCli(['--predictions', 'tests/fixtures/calibration/predictions-leaky.jsonl']);
|
||
if (leak.code !== 2 || !/leaks across/.test(leak.stderr)) throw new Error('a leaked corpus must be refused before scoring');
|
||
return { report, source, blocked, go, leak };
|
||
} finally {
|
||
await rm(directory, { recursive: true, force: true });
|
||
}
|
||
}
|
||
const gate = await calibrationGate();
|
||
await tap('[data-view="home"]', 450);
|
||
const gateRows = [
|
||
['command', 'python3 scripts/calibration_report.py --predictions <fixtures>'],
|
||
['split', 'threshold chosen on validation · scored on test'],
|
||
['floor', `coverage floor ${Math.round(gate.report.models[0].minCoverage * 100)}%`],
|
||
['—', ''],
|
||
];
|
||
for (const row of gate.report.models) {
|
||
// The model id gets its own row: a 74px key column broke "specialist-candidate" at its hyphen,
|
||
// which reads as a label with no value and a stray line under it. The name is the value here.
|
||
gateRows.push(['model', row.model]);
|
||
gateRows.push([
|
||
'metrics',
|
||
`thr ${row.threshold.toFixed(2)} · F1 ${row.macroF1.toFixed(3)} · P@C ${row.precisionAtCoverage.toFixed(3)} · cov ${row.coverage.toFixed(2)}`,
|
||
]);
|
||
gateRows.push(['gate', `FAIL: ${row.gate.failed.join(', ')} · ECE ${row.ece.toFixed(3)}`]);
|
||
}
|
||
gateRows.push(['—', '']);
|
||
gateRows.push(['verdict', `${gate.report.verdict} · no candidate clears every gate`]);
|
||
gateRows.push(['gates', 'macroF1 ≥ 0.80 · precision@coverage ≥ 0.90 · ECE ≤ 0.05']);
|
||
gateRows.push(['—', '']);
|
||
gateRows.push(['engineered', 'GO — gate-plumbing-fixture clears every gate (fixture, not a result)']);
|
||
gateRows.push(['leaked', gate.leak.stderr.trim().replace(/^calibration harness refused:\s*/, '')]);
|
||
gateRows.push(['data', 'synthetic predictions · no model was measured']);
|
||
const gateCard = await evidenceCard({
|
||
id: 'calibration-report',
|
||
title: 'CALIBRATION HARNESS',
|
||
subtitle: 'command line, not app UI — real run, synthetic predictions',
|
||
rows: gateRows,
|
||
font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace',
|
||
keyWidth: 74,
|
||
gap: '1px',
|
||
rowGap: '6px',
|
||
});
|
||
await sleep(420);
|
||
const gateProof = await evidenceCardProof('calibration-report');
|
||
if (!gateProof.ok) throw new Error(`Calibration card is not fully readable on screen: ${JSON.stringify(gateProof)}`);
|
||
// The card is drawn from the report; prove the values on it are the ones the harness measured.
|
||
for (const row of gate.report.models) {
|
||
if (!gateCard.text.includes(row.macroF1.toFixed(3))) {
|
||
throw new Error(`the card must show the macro-F1 the harness measured for ${row.model}`);
|
||
}
|
||
}
|
||
if (!gateCard.text.includes('leaks across')) throw new Error('the card must show the refusal the leaked corpus earned');
|
||
if (!gateCard.text.includes('GO — gate-plumbing-fixture')) throw new Error('the card must show the passing branch it proved');
|
||
await caption('The calibration harness is a command-line release gate, not a screen in the app — this runs it against committed synthetic predictions', 3400);
|
||
await caption('Every score on this card is invented to exercise the arithmetic: no model produced them and none of them describes the real classifier', 3600);
|
||
await caption('Each candidate is scored on test at a threshold chosen on validation only; picking a threshold on the split you report is refused', 3600);
|
||
await caption('One contributor spread across two splits is refused before anything is scored, and a passing verdict is proven only by a fixture engineered to clear it', 3800);
|
||
await page.evaluate(() => document.querySelector('#calibration-report')?.remove());
|
||
|
||
// Issue #40 — the RC attack suite is a command line too, and unlike the other cards this one is
|
||
// expected to come back with a verdict that is not flattering. Hostile synthetic bytes go at the
|
||
// real server with a counting stub worker behind it, so "the worker was called" is measured, not
|
||
// inferred. The card prints whatever verdict the suite derived from what it observed; the captions
|
||
// name where a failing fix lives instead of implying the boundary is already safe.
|
||
async function securityAttackSuite() {
|
||
const directory = await mkdtemp(path.join(os.tmpdir(), 'timmy-security-demo-'));
|
||
const reportPath = path.join(directory, 'rc-attack-report.json');
|
||
try {
|
||
const run = await execFileAsync(
|
||
process.execPath,
|
||
['scripts/security_attack_suite.mjs', '--json', reportPath],
|
||
{ cwd: process.cwd() },
|
||
).then(result => ({ code: 0, stdout: result.stdout, stderr: result.stderr }))
|
||
.catch(error => ({ code: typeof error.code === 'number' ? error.code : 1, stdout: error.stdout || '', stderr: error.stderr || '' }));
|
||
if (![0, 1].includes(run.code)) throw new Error(`the attack suite could not complete: ${run.stderr.trim()}`);
|
||
const report = JSON.parse(await readFile(reportPath, 'utf8'));
|
||
if (run.code !== (report.verdict === 'BLOCKED' ? 1 : 0)) {
|
||
throw new Error('the attack suite exit code must match the verdict it published');
|
||
}
|
||
return { report, run };
|
||
} finally {
|
||
await rm(directory, { recursive: true, force: true });
|
||
}
|
||
}
|
||
const securityRun = await securityAttackSuite();
|
||
const securityReport = securityRun.report;
|
||
if (securityReport.cases.length < 8) throw new Error('the RC attack suite must run every named abuse class');
|
||
await tap('[data-view="home"]', 450);
|
||
const caseDetail = (item) => {
|
||
const observed = item.observed || {};
|
||
if ('providerCalls' in observed) return `http ${observed.status} · worker ${observed.providerCalls}`;
|
||
if ('refused' in observed) return `refused ${observed.refused}`;
|
||
if ('imageBytesInLog' in observed) return `image bytes ${observed.imageBytesInLog}`;
|
||
return item.status;
|
||
};
|
||
const securityRows = [
|
||
['command', 'node scripts/security_attack_suite.mjs'],
|
||
['target', 'real server · counting stub worker · synthetic bytes'],
|
||
['—', ''],
|
||
];
|
||
for (const item of securityReport.cases) securityRows.push([item.category, `${item.severity} ${item.status} · ${caseDetail(item)}`]);
|
||
securityRows.push(['—', '']);
|
||
securityRows.push(['load', `${securityReport.load.requests} req · p50 ${securityReport.load.p50}ms · p95 ${securityReport.load.p95}ms · abstain ${securityReport.load.abstentions}`]);
|
||
securityRows.push(['verdict', `${securityReport.verdict} · ${securityReport.blocking.length} blocking · ${securityReport.holds.length} held`]);
|
||
for (const item of securityReport.blocking) securityRows.push(['blocks', item.id]);
|
||
const securityCard = await evidenceCard({
|
||
id: 'security-attack',
|
||
title: 'SECURITY ATTACK SUITE',
|
||
subtitle: 'command line, not app UI — real server, counted worker calls',
|
||
rows: securityRows,
|
||
font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace',
|
||
keyWidth: 62,
|
||
gap: '1px',
|
||
rowGap: '5px',
|
||
});
|
||
await sleep(420);
|
||
const securityCardProof = await evidenceCardProof('security-attack');
|
||
if (!securityCardProof.ok) throw new Error(`Security card is not fully readable on screen: ${JSON.stringify(securityCardProof)}`);
|
||
for (const item of securityReport.blocking) {
|
||
if (!securityCard.text.includes(item.id)) throw new Error(`the card must name the blocking case ${item.id}`);
|
||
}
|
||
if (!securityCard.text.includes(securityReport.verdict)) throw new Error('the card must show the verdict the suite derived');
|
||
if (!securityCard.text.includes(`${securityReport.load.p95}ms`)) throw new Error('the card must show the p95 the load phase measured');
|
||
await caption('Every hostile upload on screen went at the real server, and the worker behind it counted every call it received', 3800);
|
||
await caption('Refusing bytes by their content is the fix in review, not a claim this video makes: an HTML file dressed as a JPEG still reaches the worker today', 4200);
|
||
await caption('A failing case is a finding the release job has to see, so the card prints the verdict the suite derived instead of a reassuring one', 3800);
|
||
await caption('The gate blocks RC approval on any P0 case that reached the worker — the gate decides, not the caption', 3400);
|
||
await page.evaluate(() => document.querySelector('#security-attack')?.remove());
|
||
|
||
// Issue #39 — the operator backup is a command line, not a screen in the app. The app itself keeps
|
||
// no server-side journal, so what an operator can actually lose is the corpus: the append-only
|
||
// manifest with its withdrawal tombstones. This scene builds a synthetic data root with the real
|
||
// ingest and revoke commands, plants the four things a backup must refuse, then runs the real
|
||
// backup, verify, and restore and reads every value on the card back out of those runs.
|
||
async function operatorBackupAudit() {
|
||
const root = await mkdtemp(path.join(os.tmpdir(), 'timmy-backup-demo-'));
|
||
const dataRoot = path.join(root, 'srv-timmy-corpus');
|
||
const archive = path.join(root, 'timmy-nightly.tar.gz');
|
||
const manifestPath = `${archive}.manifest.json`;
|
||
const restoreInto = path.join(root, 'restored');
|
||
const secondPhoto = path.join(root, 'synthetic-second.png');
|
||
const backupCli = args => execFileAsync('python3', ['scripts/operator_backup.py', ...args], { cwd: process.cwd() });
|
||
const ingest = (input, contributionId, subject, bristol) => corpusCli([
|
||
'scripts/ingest_training_photo.py',
|
||
'--input', input, '--output-dir', dataRoot, '--subject-id', subject,
|
||
'--consent-version', 'training-consent-v1', '--contribution-id', contributionId,
|
||
'--bristol-type', bristol, '--color', 'brown', '--quality', 'good', '--reviewer', 'user-confirmed',
|
||
]);
|
||
try {
|
||
// One contribution stays active and one is withdrawn, so the manifest ends up with a live sample
|
||
// and a real tombstone, and the withdrawn derived image is already gone from disk.
|
||
await ingest('tests/fixtures/synthetic-type4.jpg', 'demo-backup-0001', 'demo-backup-kept', '4');
|
||
await writeFile(secondPhoto, Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAFElEQVR4nGOsCDFhwAaYsIoOWgkAzUwBECP1Kh0AAAAASUVORK5CYII=', 'base64'));
|
||
await ingest(secondPhoto, 'demo-backup-0002', 'demo-backup-withdrawn', '3');
|
||
await corpusCli([
|
||
'scripts/ingest_training_photo.py', '--output-dir', dataRoot,
|
||
'--revoke', 'demo-backup-0002', '--reason', 'participant withdrew consent',
|
||
]);
|
||
|
||
// The four things that must never be archived, as real files with real bytes: a weight shard, a
|
||
// transient inference upload, an operator environment file, and a log. An exclusion that quietly
|
||
// stopped working fails here instead of in a caption.
|
||
await mkdir(path.join(dataRoot, 'weights'), { recursive: true });
|
||
await writeFile(path.join(dataRoot, 'weights', 'model.gguf'), 'synthetic weight shard — never archived');
|
||
await mkdir(path.join(dataRoot, 'transient'), { recursive: true });
|
||
await writeFile(path.join(dataRoot, 'transient', 'inference-upload.jpg'), 'transient inference bytes — never archived');
|
||
await writeFile(path.join(dataRoot, 'secrets.env'), 'TIMMY_DATASET_SALT=demo-only-placeholder\n');
|
||
await writeFile(path.join(dataRoot, 'operator.log'), 'GET /api/healthz 200\n');
|
||
|
||
const backupRun = await backupCli(['backup', '--source', dataRoot, '--archive', archive, '--label', 'nightly']);
|
||
const receipt = JSON.parse(backupRun.stdout);
|
||
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
||
const reasons = manifest.excluded.reduce((out, entry) => ({ ...out, [entry.reason]: (out[entry.reason] || 0) + 1 }), {});
|
||
if (manifest.files.some(entry => /model\.gguf|inference-upload|secrets\.env|operator\.log/.test(entry.path))) {
|
||
throw new Error('the backup must not archive the weight shard or a transient inference upload');
|
||
}
|
||
for (const reason of ['model-weights', 'transient-inference', 'secret', 'log']) {
|
||
if (!reasons[reason]) throw new Error(`the backup must record a ${reason} exclusion`);
|
||
}
|
||
if (receipt.records.samples !== 2 || receipt.records.tombstones !== 1 || receipt.records.activeSamples !== 1) {
|
||
throw new Error('the backup receipt must report the real record counts');
|
||
}
|
||
const verified = JSON.parse((await backupCli(['verify', '--archive', archive, '--manifest', manifestPath])).stdout);
|
||
if (verified.files !== manifest.files.length) throw new Error('verify must re-hash every stored member');
|
||
const restored = JSON.parse((await backupCli([
|
||
'restore', '--archive', archive, '--manifest', manifestPath, '--target', restoreInto,
|
||
])).stdout);
|
||
if (restored.records.tombstones !== 1 || restored.records.activeSamples !== 1) {
|
||
throw new Error('the restore must return the same record counts');
|
||
}
|
||
const restoredManifest = await readFile(path.join(restoreInto, 'manifest.jsonl'));
|
||
if (!restoredManifest.equals(await readFile(path.join(dataRoot, 'manifest.jsonl')))) {
|
||
throw new Error('the restore must return the same records, tombstone included');
|
||
}
|
||
return { receipt, manifest, reasons, verified, restored };
|
||
} finally {
|
||
await rm(root, { recursive: true, force: true });
|
||
}
|
||
}
|
||
const opBackup = await operatorBackupAudit();
|
||
await tap('[data-view="home"]', 400);
|
||
const shortDigest = value => String(value).slice(0, 12);
|
||
const backupRows = [
|
||
['command', 'python3 scripts/operator_backup.py backup --source srv-timmy-corpus'],
|
||
['stored', `${opBackup.receipt.files} members · ${opBackup.manifest.files.map(entry => entry.path.split('/').pop()).join(' + ')}`],
|
||
['records', `samples ${opBackup.receipt.records.samples} · tombstones ${opBackup.receipt.records.tombstones} · active ${opBackup.receipt.records.activeSamples}`],
|
||
['—', ''],
|
||
['refused', `model-weights ${opBackup.reasons['model-weights']} · transient-inference ${opBackup.reasons['transient-inference']}`],
|
||
['refused', `secret ${opBackup.reasons['secret']} · log ${opBackup.reasons['log']}`],
|
||
['reason', 'every skipped path is named in the manifest, not silently dropped'],
|
||
['—', ''],
|
||
['digest', `sha256 ${shortDigest(opBackup.manifest.archive.sha256)}… · reproducible (sorted, mtime 0)`],
|
||
['verify', `ok · ${opBackup.verified.files} members re-hashed against the manifest`],
|
||
['restore', `ok · samples ${opBackup.restored.records.samples} · tombstones ${opBackup.restored.records.tombstones} · active ${opBackup.restored.records.activeSamples}`],
|
||
['manifest', 'returned byte-for-byte, tombstone included'],
|
||
['server', 'the app holds no journal of its own — there is no server copy to back up'],
|
||
];
|
||
const backupCard = await evidenceCard({
|
||
id: 'production-backup',
|
||
title: 'OPERATOR BACKUP',
|
||
subtitle: 'command line, not app UI — real run, synthetic corpus',
|
||
rows: backupRows,
|
||
font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace',
|
||
keyWidth: 66,
|
||
gap: '1px',
|
||
rowGap: '6px',
|
||
});
|
||
await sleep(420);
|
||
const backupProof = await evidenceCardProof('production-backup');
|
||
if (!backupProof.ok) throw new Error(`Production backup card is not fully readable on screen: ${JSON.stringify(backupProof)}`);
|
||
// The card is drawn from the run: prove the counts and the refusals on it are the ones it produced.
|
||
if (!backupCard.text.includes(`samples ${opBackup.receipt.records.samples} · tombstones ${opBackup.receipt.records.tombstones}`)) {
|
||
throw new Error('the card must show the record counts the backup actually reported');
|
||
}
|
||
if (!backupCard.text.includes('model-weights 1')) throw new Error('the card must show the exclusion the run actually recorded');
|
||
if (!backupCard.text.includes(shortDigest(opBackup.manifest.archive.sha256))) {
|
||
throw new Error('the card must show the archive digest the run actually wrote');
|
||
}
|
||
await caption('The operator backup is a command line, not a screen in the app — this runs it against a synthetic corpus the real ingest and revoke commands built', 3800);
|
||
await caption('Model weights and transient inference images are refused by name, and every skipped path is listed with its reason', 3600);
|
||
await caption('Verify re-hashes every stored member against the manifest, and the restore only starts once that check passes', 3400);
|
||
await caption('A restore returns the same records, including the withdrawal tombstone a trainer must never lose', 3600);
|
||
await page.evaluate(() => document.querySelector('#production-backup')?.remove());
|
||
|
||
// Back to the privacy view for the portability scene: the privacy entry lives on the journal view.
|
||
await tap('[data-view="calendar"]', 550);
|
||
await tap('[data-view="privacy"]', 600);
|
||
|
||
// Issue #35 — a backup is the person's own record, so it carries a version and everything it is
|
||
// claimed to hold. The recording exports the real file, imports an older version and a newer one,
|
||
// and reopens a restored log. Fictional sample logs are used throughout; nothing here is real data.
|
||
const exportDownload = page.waitForEvent('download');
|
||
await tap('#export', 600);
|
||
const exportedBackup = JSON.parse(await readFile(await (await exportDownload).path(), 'utf8'));
|
||
if (exportedBackup.schemaVersion !== 2) throw new Error('the exported backup must declare the current schema version');
|
||
if (!Array.isArray(exportedBackup.entries) || !Array.isArray(exportedBackup.contributions)) {
|
||
throw new Error('a backup must carry both the confirmed logs and the contribution receipts');
|
||
}
|
||
const exportedReceipt = exportedBackup.contributions.find(receipt => receipt.contributionId === receiptId);
|
||
if (!exportedReceipt || exportedReceipt.state !== 'withdrawn') {
|
||
throw new Error('the backup must carry the withdrawn receipt as a tombstone');
|
||
}
|
||
if (!(exportedReceipt.artifacts || []).length) throw new Error('the backup tombstone must still name its derived artifacts');
|
||
await page.locator('[data-portability-notice][data-portability-version="2"]').waitFor();
|
||
await caption('A backup is versioned, and it carries the withdrawn receipt as a tombstone rather than silence', 3200);
|
||
|
||
const fictionalLegacyBackup = JSON.stringify({
|
||
product: 'Timmy the Talking Turd',
|
||
schemaVersion: 1,
|
||
exportedAt: '2026-08-01T09:00:00.000Z',
|
||
entries: [{
|
||
id: 'demo-legacy-backup',
|
||
occurredAt: new Date(Date.now() - 3 * 86400000).toISOString(),
|
||
bristolType: 3,
|
||
color: 'brown',
|
||
urgency: 1,
|
||
discomfort: 0,
|
||
note: 'fictional version 1 backup',
|
||
photoDataUrl: '',
|
||
symptoms: {},
|
||
}],
|
||
});
|
||
await page.locator('#import').setInputFiles({ name: 'timmy-backup-v1.json', mimeType: 'application/json', buffer: Buffer.from(fictionalLegacyBackup) });
|
||
await page.locator('[data-portability-notice][data-portability-version="1"]').waitFor();
|
||
const upgradeNotice = (await page.locator('[data-portability-notice]').innerText()).trim();
|
||
if (!/upgraded/i.test(upgradeNotice)) throw new Error('an older backup must be reported as upgraded, not silently rewritten');
|
||
await caption('An older backup is upgraded on import, and the notice says which version the file came from', 3400);
|
||
|
||
const ledgerBeforeFuture = await storedLedger();
|
||
await page.locator('#import').setInputFiles({
|
||
name: 'timmy-backup-v3.json', mimeType: 'application/json',
|
||
buffer: Buffer.from(JSON.stringify({ product: 'Timmy the Talking Turd', schemaVersion: 3, entries: [{ id: 'demo-future', bristolType: 7 }] })),
|
||
});
|
||
await page.locator('[data-portability-error]').waitFor();
|
||
if (!/newer version of Timmy/i.test(await page.locator('[data-portability-error]').innerText())) {
|
||
throw new Error('a backup from a newer Timmy must be refused with an instruction to update');
|
||
}
|
||
if ((await storedLedger()).raw !== ledgerBeforeFuture.raw) {
|
||
throw new Error('a refused import must not change the ledger it would have replaced');
|
||
}
|
||
await caption('A file from a newer Timmy is refused with an instruction, and the ledger it would have replaced is untouched', 3400);
|
||
|
||
const fictionalAssistedBackup = JSON.stringify({
|
||
product: 'Timmy the Talking Turd',
|
||
schemaVersion: 2,
|
||
exportedAt: new Date().toISOString(),
|
||
entries: [{
|
||
id: 'demo-restored-provenance',
|
||
occurredAt: new Date(Date.now() - 86400000).toISOString(),
|
||
bristolType: 2,
|
||
color: 'brown',
|
||
urgency: 1,
|
||
discomfort: 0,
|
||
note: 'fictional restored log',
|
||
photoDataUrl: '',
|
||
symptoms: {},
|
||
provenance: {
|
||
source: 'ai-corrected',
|
||
model: 'SmolVLM2-2.2B-Instruct',
|
||
suggestedBristolType: 4,
|
||
suggestedColor: 'brown',
|
||
confirmedAt: new Date(Date.now() - 86400000).toISOString(),
|
||
},
|
||
}],
|
||
contributions: [],
|
||
});
|
||
await page.locator('#import').setInputFiles({ name: 'timmy-backup-v2.json', mimeType: 'application/json', buffer: Buffer.from(fictionalAssistedBackup) });
|
||
await page.locator('[data-portability-notice][data-portability-version="2"]').waitFor();
|
||
await tap('[data-view="calendar"]', 550);
|
||
await page.locator('[data-entry-open="demo-restored-provenance"]').waitFor();
|
||
await tap('[data-entry-open="demo-restored-provenance"]', 550);
|
||
await page.locator('[data-entry-provenance="ai-corrected"]').waitFor();
|
||
const provenanceLine = (await page.locator('[data-entry-provenance="ai-corrected"]').innerText()).trim();
|
||
if (!/corrected/i.test(provenanceLine) || !/SmolVLM2/.test(provenanceLine)) {
|
||
throw new Error('a restored log must still show that it began as a corrected AI suggestion');
|
||
}
|
||
await caption('A restored log still says it began as an AI suggestion you corrected — provenance is part of the record', 3400);
|
||
await indicate('[data-entry-provenance="ai-corrected"]');
|
||
await tap('#entry-sheet-close', 450);
|
||
await tap('[data-view="privacy"]', 550);
|
||
await page.locator('#delete-all').waitFor();
|
||
|
||
await indicate('#delete-all');
|
||
await tap('#delete-all', 650);
|
||
await page.getByText(/Local ledger and photos deleted/).waitFor();
|
||
const wiped = await storedLedger();
|
||
if (wiped.raw !== null) throw new Error('delete-all must remove the stored ledger');
|
||
if (await page.evaluate(() => JSON.stringify(localStorage).includes('data:image'))) {
|
||
throw new Error('no photo bytes may survive delete-all');
|
||
}
|
||
if (await page.evaluate(key => localStorage.getItem(key), contributionKey) !== null) {
|
||
throw new Error('delete-all must clear the contribution receipts as well');
|
||
}
|
||
await caption('Delete all local data clears the ledger and every saved photo in this browser; nothing is kept on a server', 2900);
|
||
await tap('[data-view="home"]', 500);
|
||
// Issue #37 — a chat answer about the journal must point at the confirmed records behind it. The
|
||
// scenes below run against the app's own local ledger contract with fictional logs, and every claim
|
||
// is asserted in the browser before the caption is shown.
|
||
agentConnected = false;
|
||
await seedSyntheticLedger([
|
||
sampleDay('demo-chat-1', 2, 4),
|
||
sampleDay('demo-chat-2', 5, 3),
|
||
sampleDay('demo-chat-3', 9, 4),
|
||
sampleDay('demo-chat-4', 12, 4),
|
||
]);
|
||
await tap('[data-view="timmy"]', 550);
|
||
await page.getByText(/Local Timmy mode/i).waitFor();
|
||
await caption('Fictional logs are seeded into this browser only — the chat answers from them, not from model memory', 2600);
|
||
await page.locator('#chat-message').fill('How many times have I logged?');
|
||
await tap('#send-chat', 500);
|
||
const groundedAnswer = page.locator('.bubble.timmy', { hasText: '4 confirmed logs' }).last();
|
||
await groundedAnswer.locator('[data-chat-citation]').first().waitFor();
|
||
if (await groundedAnswer.locator('[data-chat-citation]').count() !== 4) {
|
||
throw new Error('A grounded chat answer must cite every record it counted');
|
||
}
|
||
await caption('The answer states the count and cites the four confirmed logs it counted', 2600);
|
||
await indicate('[data-chat-citations]');
|
||
await tap('[data-chat-citation]', 700);
|
||
await page.locator('[data-chat-record]').waitFor();
|
||
if (await page.locator('[data-chat-record]').count() !== 1) {
|
||
throw new Error('A chat citation must open exactly the one record it names');
|
||
}
|
||
await caption('Following a citation opens exactly the confirmed log behind that answer', 2600);
|
||
await indicate('[data-chat-record-sheet]');
|
||
await tap('#chat-record-close', 450);
|
||
await page.locator('#chat-message').fill('Is it safe to eat tacos?');
|
||
await tap('#send-chat', 500);
|
||
await page.getByText(/cannot clear a food/i).waitFor();
|
||
if (await page.locator('.bubble.timmy').last().locator('[data-chat-citation]').count() !== 0) {
|
||
throw new Error('A refusal must not borrow journal citations');
|
||
}
|
||
await caption('Ask something the journal cannot answer and Timmy refuses instead of reassuring', 2800);
|
||
agentConnected = true;
|
||
await page.reload({ waitUntil: 'networkidle' });
|
||
await tap('[data-view="timmy"]', 550);
|
||
await page.getByText(/Hermes Agent connected/i).waitFor();
|
||
await caption('Hermes Agent connected — free-text, contextual, and server-side', 1400);
|
||
await page.locator('#chat-message').fill('What pattern do you see?');
|
||
await tap('#send-chat', 500);
|
||
await page.getByText(/mostly Type 4/i).waitFor();
|
||
if (await page.locator('.bubble.timmy').last().locator('[data-chat-citation]').count() !== 4) {
|
||
throw new Error('A Hermes answer must render the records it cites as sources');
|
||
}
|
||
await caption('Hermes answers carry the same ledger citations, rendered as sources in the transcript', 2400);
|
||
await caption('Hermes keeps credentials, tools, and session continuity server-side. Photos stay out of chat.', 1600);
|
||
await page.locator('#chat-message').fill('I barfed');
|
||
await tap('#send-chat', 450);
|
||
await page.getByText(/Pause and get medical help/i).waitFor();
|
||
if (hermesCalls !== 1) throw new Error('Urgent chat must be intercepted before Hermes');
|
||
await caption('Urgent language is intercepted deterministically before Hermes', 1800);
|
||
|
||
// Issue #21 — the escalation has to fire for the phrasing people actually use, not only for the one
|
||
// word the previous scene typed. Each line below is asserted in the browser: exactly one new Timmy
|
||
// bubble, carrying the app's own fixed copy, with hermesCalls unmoved.
|
||
const escalationBubbles = () => page.locator('.bubble.timmy', { hasText: 'Pause and get medical help' }).count();
|
||
const expectEscalation = async (text, label) => {
|
||
const before = await escalationBubbles();
|
||
await page.locator('#chat-message').fill(text);
|
||
await tap('#send-chat', 450);
|
||
const bubble = page.locator('.bubble.timmy', { hasText: 'Pause and get medical help' }).last();
|
||
await bubble.waitFor();
|
||
if (await escalationBubbles() !== before + 1) throw new Error(`${label}: expected exactly one escalation bubble`);
|
||
if (!/^Pause and get medical help\. These reported symptoms can need prompt medical care\./.test(await bubble.innerText())) {
|
||
throw new Error(`${label}: the escalation copy must be the app's own fixed string`);
|
||
}
|
||
if (hermesCalls !== 1) throw new Error(`${label}: an escalated turn must not reach Hermes`);
|
||
};
|
||
await expectEscalation('There is blood when I wipe', 'bleeding reported as wiping');
|
||
await caption('Bleeding reported as wiping still escalates, in the app’s own fixed words', 2600);
|
||
await expectEscalation('I have tarry stools', 'tarry stool');
|
||
await caption('Tarry and black stool wording escalates too: one red flag, one unchanging message', 2600);
|
||
|
||
// The same detector runs over the confirmed ledger, so a symptom already written down as a note
|
||
// escalates even when the question itself is innocuous. The seeded entry is fictional and stays in
|
||
// this browser, exactly like the summary scene above.
|
||
await page.evaluate(() => localStorage.clear());
|
||
await seedSyntheticLedger([{ ...sampleDay('demo-tarry-note', 1, 4), note: 'stool was tarry yesterday' }]);
|
||
await tap('[data-view="timmy"]', 550);
|
||
const ledgerEscalations = await escalationBubbles();
|
||
await page.locator('#chat-message').fill('What do my recent logs show?');
|
||
await tap('#send-chat', 450);
|
||
await page.locator('.bubble.timmy', { hasText: 'Pause and get medical help' }).last().waitFor();
|
||
if (await escalationBubbles() !== ledgerEscalations + 1) throw new Error('a tarry note in the journal must escalate the next answer');
|
||
if (hermesCalls !== 1) throw new Error('a ledger red flag must be intercepted before Hermes');
|
||
await caption('A tarry note already saved in the journal escalates the same way, before any provider call', 2800);
|
||
await page.evaluate(() => localStorage.clear());
|
||
await page.reload({ waitUntil: 'networkidle' });
|
||
await tap('[data-view="timmy"]', 500);
|
||
await sleep(400);
|
||
await page.evaluate(() => {
|
||
document.querySelector('#demo-caption')?.remove();
|
||
const outro = document.createElement('div');
|
||
outro.id = 'release-outro';
|
||
outro.innerHTML = '<img src="/assets/timmy.svg"><strong>SLEEK. SIMPLE.<br>HERMES-POWERED.</strong><span>User-confirmed guidance — never a diagnosis.</span>';
|
||
Object.assign(outro.style, { position:'fixed', inset:'0', zIndex:'30000', background:'linear-gradient(145deg,#f7f3ea,#f5c95b)', display:'flex', flexDirection:'column', alignItems:'center', justifyContent:'center', textAlign:'center', color:'#28211d', fontFamily:'system-ui', opacity:'0' });
|
||
outro.querySelector('img').style.cssText = 'width:150px;height:150px;filter:drop-shadow(0 14px 16px rgba(63,37,30,.18))';
|
||
outro.querySelector('strong').style.cssText = 'font-size:31px;line-height:1.05;margin:18px 0 12px;letter-spacing:-.04em';
|
||
outro.querySelector('span').style.cssText = 'font-size:13px;font-weight:800;color:#176957';
|
||
document.body.append(outro);
|
||
outro.animate([{opacity:0},{opacity:1}], {duration:500,fill:'forwards'});
|
||
});
|
||
await sleep(2200);
|
||
|
||
// The refusal pass deliberately produces two fail-closed 503s (an out-of-range reply, and one blank field
|
||
// beside an out-of-range one), which the browser logs as failed resources. Requiring exactly those two
|
||
// keeps the scene honest: if the boundary stops refusing, this fails instead of quietly recording a happy
|
||
// path, and a dropped scene is caught rather than silently shrinking the evidence.
|
||
const expectedFailClosed = errors.filter(message => /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/.test(message));
|
||
if (expectedFailClosed.length !== 2 || errors.length !== expectedFailClosed.length) {
|
||
throw new Error(`Browser errors: ${errors.join(' | ') || 'unexpected error count'}`);
|
||
}
|
||
// Machine-readable receipt of exactly what was rendered on camera, so the release gate can prove
|
||
// the recorded delta statement reached the screen instead of trusting the script source.
|
||
if (captionsOut) {
|
||
await writeFile(captionsOut, JSON.stringify({ version, releaseDelta, captions: captionsShown,
|
||
captionsVerified: captionEvidence }, null, 2) + '\n', 'utf8');
|
||
}
|
||
const recording = page.video();
|
||
await page.close();
|
||
await recording.saveAs(output);
|
||
await context.close();
|
||
await browser.close();
|
||
console.log(output);
|