timmy
d9f82299d0
Quality gates / quality (pull_request) Successful in 2m26s
The worker contract was only proven by one happy-path fixture and a handful of inline assertions, so nothing continuously checked that a reply Timmy cannot use is refused instead of guessed at. Corpus: a CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) generated deterministically by scripts/build_vision_corpus.py. Every fixture records its sha256, provenance, licence, case class, the provider reply it replays, and the outcome that reply must produce. loadVisionCorpus refuses a fixture whose bytes no longer match the manifest, so the corpus cannot be edited into a different test. No real medical imagery, camera, or third-party content is involved. Contract fixes the corpus exposed: - A schema-valid reply with a null form and colour is now an abstention (abstainedFor: unusable_visual_fields) instead of a hard provider failure. The declared response schema permits those nulls, so treating the reply as an error was wrong; it now abstains calmly and still prefill nothing. - Fail-closed replies carry a machine-readable code (invalid_json, invalid_fields) instead of prose matching, and visionFailureResponse() maps them to plain, user-safe copy with no provider text, image bytes, or credentials echoed. A running server now returns that copy for unparsable and out-of-range replies. Evidence: unit corpus contract and hash integrity, corpus-wide conformance through the real analysis path, the same corpus through a running server.mjs with a stub worker (200 abstain vs 503 fail-closed, no leaked bytes), and a real browser acceptance run for the refusal UI. The release demo replays three corpus cases through the real code path so the video shows the status the server would actually send, with the on-camera caption gated by the release builder.
322 lines
18 KiB
JavaScript
322 lines
18 KiB
JavaScript
import { chromium } from 'playwright';
|
||
import { mkdir, writeFile } from 'node:fs/promises';
|
||
import path from 'node:path';
|
||
import { fileURLToPath } from 'node:url';
|
||
import { analyzePhoto, visionFailureResponse } from '../src/vision-service.js';
|
||
import { loadVisionCorpus } from '../src/vision-corpus.js';
|
||
|
||
const output = path.resolve(process.argv[2] || 'release-demo.webm');
|
||
const version = process.env.TIMMY_RELEASE_VERSION || 'review build';
|
||
// The release delta is supplied by the builder so the recording states, on camera, what actually
|
||
// changed in this release instead of implying a feature that did not ship.
|
||
const releaseDelta = process.env.TIMMY_RELEASE_DEMO_DELTA
|
||
|| 'Verified rebuild of the current user-visible feature path — no new feature is claimed.';
|
||
const captionsOut = process.env.TIMMY_DEMO_CAPTIONS_OUT || '';
|
||
const captionsShown = [];
|
||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||
await mkdir(path.dirname(output), { recursive: true });
|
||
|
||
// The refusal scene replays committed synthetic corpus cases through the real analysis code path,
|
||
// so the recording shows the status and copy the server would actually send rather than copy
|
||
// written for the video. No real photo is used anywhere in the corpus.
|
||
const corpus = await loadVisionCorpus(fileURLToPath(new URL('../tests/fixtures/vision-corpus/', import.meta.url)));
|
||
const corpusCase = id => corpus.cases.find(entry => entry.id === id);
|
||
async function corpusAnalyzeResult(entry) {
|
||
const fetchImpl = async () => ({ ok: true, json: async () => ({ choices: [{ message: { content: entry.providerContent } }] }) });
|
||
try {
|
||
const result = await analyzePhoto({
|
||
payload: { imageDataUrl: 'data:image/png;base64,YQ==', consent: true },
|
||
fetchImpl,
|
||
config: { baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'corpus-replay', model: 'corpus-model' },
|
||
});
|
||
return { status: 200, body: result };
|
||
} catch (error) {
|
||
const failure = visionFailureResponse(error);
|
||
return { status: failure.status, body: failure.body };
|
||
}
|
||
}
|
||
|
||
const browser = await chromium.launch({ headless: true });
|
||
const context = await browser.newContext({
|
||
viewport: { width: 405, height: 720 },
|
||
deviceScaleFactor: 1,
|
||
serviceWorkers: 'block',
|
||
recordVideo: { dir: path.dirname(output), size: { width: 405, height: 720 } },
|
||
colorScheme: 'light',
|
||
});
|
||
const page = await context.newPage();
|
||
const errors = [];
|
||
page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); });
|
||
page.on('pageerror', error => errors.push(error.message));
|
||
let visionOffline = false;
|
||
await page.route('**/api/vision-status', route => route.fulfill({
|
||
status: 200,
|
||
contentType: 'application/json',
|
||
body: JSON.stringify({
|
||
enabled: true,
|
||
profile: 'selfhost',
|
||
processor: 'self-hosted',
|
||
model: 'SmolVLM2-2.2B-Instruct',
|
||
// The second pass flips the app's own readiness contract so no AI control is offered at all.
|
||
providerReady: !visionOffline,
|
||
modelSeen: !visionOffline,
|
||
}),
|
||
}));
|
||
// The happy path keeps one deterministic suggestion. The refusal scene swaps this for a real
|
||
// corpus replay, so both the usable and the refused answers come from the same code path.
|
||
const suggestionBody = {
|
||
status: 'suggestion',
|
||
isStool: true,
|
||
bristolType: 4,
|
||
color: 'brown',
|
||
confidence: 0.83,
|
||
imageQuality: 'good',
|
||
observations: 'Smooth, formed appearance.',
|
||
warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
|
||
};
|
||
let analyzeOverride = null;
|
||
await page.route('**/api/analyze', route => {
|
||
const response = analyzeOverride || { status: 200, body: suggestionBody };
|
||
return route.fulfill({ status: response.status, contentType: 'application/json', body: JSON.stringify(response.body) });
|
||
});
|
||
await page.route('**/api/agent/status', route => route.fulfill({
|
||
status: 200,
|
||
contentType: 'application/json',
|
||
body: JSON.stringify({ enabled: true, configured: true, authenticated: true, mode: 'hermes-agent' }),
|
||
}));
|
||
let hermesCalls = 0;
|
||
await page.route('**/api/agent/chat', route => {
|
||
hermesCalls += 1;
|
||
return route.fulfill({
|
||
status: 200,
|
||
contentType: 'application/json',
|
||
body: JSON.stringify({ connected: true, reply: 'Your confirmed logs are mostly Type 4. I can explain that pattern, but I cannot diagnose a cause.' }),
|
||
});
|
||
});
|
||
|
||
await page.goto('http://127.0.0.1:4173', { waitUntil: 'networkidle' });
|
||
await page.evaluate(() => localStorage.clear());
|
||
await page.reload({ waitUntil: 'networkidle' });
|
||
await page.addStyleTag({ content: `
|
||
#demo-caption{position:fixed;left:16px;right:16px;top:14px;z-index:20000;background:rgba(40,33,29,.94);color:#fff;padding:11px 14px;border-radius:14px;font:800 12px/1.25 system-ui;letter-spacing:.02em;text-align:center;box-shadow:0 8px 24px rgba(0,0,0,.2)}
|
||
#demo-touch{position:fixed;z-index:20001;width:48px;height:48px;border:4px solid #fff;background:rgba(245,201,91,.55);box-shadow:0 0 0 7px rgba(21,125,120,.28);border-radius:50%;pointer-events:none;transform:translate(-50%,-50%)}
|
||
` });
|
||
|
||
async function caption(text, duration = 1400) {
|
||
captionsShown.push(text);
|
||
await page.evaluate(text => {
|
||
document.querySelector('#demo-caption')?.remove();
|
||
const node = document.createElement('div');
|
||
node.id = 'demo-caption';
|
||
node.textContent = text;
|
||
document.body.append(node);
|
||
node.animate([{ opacity: 0, transform: 'translateY(-8px)' }, { opacity: 1, transform: 'translateY(0)' }], { duration: 260, fill: 'forwards' });
|
||
}, text);
|
||
await sleep(duration);
|
||
// A caption must not outlive its stated duration. It used to stay on screen until the next one
|
||
// replaced it, so a view change left the previous screen's claim hanging over a different screen —
|
||
// e.g. a delete-data caption sitting on top of the home view.
|
||
await page.evaluate(() => {
|
||
const node = document.querySelector('#demo-caption');
|
||
if (!node) return;
|
||
node.animate([{ opacity: 1 }, { opacity: 0 }], { duration: 180, fill: 'forwards' });
|
||
setTimeout(() => node.remove(), 220);
|
||
});
|
||
await sleep(220);
|
||
}
|
||
|
||
async function tap(selector, after = 650) {
|
||
const target = page.locator(selector).first();
|
||
await target.scrollIntoViewIfNeeded();
|
||
const box = await target.boundingBox();
|
||
if (!box) throw new Error(`Missing demo target: ${selector}`);
|
||
await page.evaluate(({ x, y }) => {
|
||
document.querySelector('#demo-touch')?.remove();
|
||
const ring = document.createElement('div');
|
||
ring.id = 'demo-touch';
|
||
ring.style.left = `${x}px`;
|
||
ring.style.top = `${y}px`;
|
||
document.body.append(ring);
|
||
ring.animate([{ opacity: .2, transform: 'translate(-50%,-50%) scale(.55)' }, { opacity: 1, transform: 'translate(-50%,-50%) scale(1)' }], { duration: 400 });
|
||
setTimeout(() => ring.remove(), 550);
|
||
}, { x: box.x + box.width / 2, y: box.y + box.height / 2 });
|
||
await sleep(220);
|
||
await target.click();
|
||
await sleep(after);
|
||
}
|
||
|
||
async function indicate(selector) {
|
||
const target = page.locator(selector).first();
|
||
await target.scrollIntoViewIfNeeded();
|
||
const box = await target.boundingBox();
|
||
if (!box) throw new Error(`Missing demo target: ${selector}`);
|
||
await page.evaluate(({ x, y }) => {
|
||
document.querySelector('#demo-touch')?.remove();
|
||
const ring = document.createElement('div');
|
||
ring.id = 'demo-touch';
|
||
ring.style.left = `${x}px`;
|
||
ring.style.top = `${y}px`;
|
||
document.body.append(ring);
|
||
ring.animate([{ opacity: .2, transform: 'translate(-50%,-50%) scale(.55)' }, { opacity: 1, transform: 'translate(-50%,-50%) scale(1)' }], { duration: 400 });
|
||
setTimeout(() => ring.remove(), 550);
|
||
}, { x: box.x + box.width / 2, y: box.y + box.height / 2 });
|
||
await sleep(650);
|
||
}
|
||
|
||
await caption(`TIMMY ${version} • RELEASE DEMO`, 1200);
|
||
await caption(`This release: ${releaseDelta}`, 2600);
|
||
await caption('Automated checks replay this synthetic path before review', 1200);
|
||
await caption('One clear photo action. Manual logging stays one tap away.', 1500);
|
||
await tap('[data-scan]', 450);
|
||
await page.getByText(/Self-hosted model ready/i).waitFor();
|
||
await caption('The pinned bootstrap verifies both model files before starting on private loopback', 1500);
|
||
await indicate('label[for="camera-photo"]');
|
||
await page.locator('#camera-photo').dispatchEvent('cancel');
|
||
await page.getByText(/Camera or photo picker closed/i).waitFor();
|
||
await caption('Camera closed cleanly — gallery and manual logging are still available', 1600);
|
||
await indicate('label[for="gallery-photo"]');
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||
await caption('Nothing uploads until explicit consent', 1100);
|
||
await page.locator('#ai-consent').check();
|
||
await tap('#analyze-photo', 450);
|
||
await page.getByText(/83% confidence/i).waitFor();
|
||
await caption('AI may suggest visible form, broad color, and image quality — never symptoms or diagnosis', 1700);
|
||
await indicate('.scan-result[data-provenance="pending"]');
|
||
await caption('The suggestion stays provisional — nothing is applied or saved until you confirm it', 1700);
|
||
await tap('#use-suggestion', 450);
|
||
await page.locator('.ai-provenance[data-provenance="confirmed"]').waitFor();
|
||
await caption('Confirming is explicit, and every suggested value is still yours to overrule', 1600);
|
||
await tap('[data-type="2"]', 500);
|
||
await page.locator('[data-provenance="corrected"]').waitFor();
|
||
await caption('Change a suggested value and Timmy clears the AI-confirmed state', 1700);
|
||
await indicate('.ai-provenance[data-provenance="corrected"] .provenance-note');
|
||
await tap('#next', 450);
|
||
await page.getByText(/You changed it to Type 2/).waitFor();
|
||
await caption('Only visible form and color were ever suggested — urgency, discomfort, and symptoms stay yours', 1900);
|
||
await tap('#next', 450);
|
||
await page.locator('[data-followup-block]').waitFor();
|
||
await caption('Type 2 is hard or lumpy, so Timmy asks about soreness, hydration, blood, and black-or-tarry — never guessed from the photo', 2600);
|
||
await page.locator('[data-question="bloodVisible"]').scrollIntoViewIfNeeded();
|
||
await indicate('[data-question="bloodVisible"]');
|
||
await tap('[data-followup="bloodVisible"][data-answer="no"]', 400);
|
||
await caption('Every answer is yours to give, and a skipped question stays unanswered', 1900);
|
||
await tap('[data-followup="soreness"][data-answer="yes"]', 350);
|
||
await tap('[data-followup="dehydration"][data-answer="yes"]', 350);
|
||
await page.locator('#followup-guidance').waitFor();
|
||
await caption('Mild report: steady water, no straining, gentle care — general self-care, not a diagnosis', 2600);
|
||
await indicate('#followup-guidance');
|
||
await caption('And it never claims the photo ruled out bleeding', 1900);
|
||
await tap('[data-followup="tarryBlack"][data-answer="yes"]', 600);
|
||
await page.locator('#urgent-box .alert').waitFor();
|
||
await caption('Say it was truly black and tarry and the same deterministic red-flag path fires instead', 2600);
|
||
await tap('[data-followup="tarryBlack"][data-answer="no"]', 500);
|
||
await page.locator('#urgent-box .alert').waitFor({ state: 'detached' });
|
||
await caption('Dark brown is not silently escalated, and the flag clears the moment the answer changes', 2200);
|
||
await tap('#close-sheet', 450);
|
||
await caption('Closing the log keeps the draft in this session instead of throwing it away', 1600);
|
||
await caption('Now the same photo flow while the vision worker is offline', 1500);
|
||
visionOffline = true;
|
||
await tap('[data-scan]', 550);
|
||
await page.getByText(/Vision worker offline/i).waitFor();
|
||
await caption('Vision worker offline — the AI and consent steps disappear instead of failing after upload', 1900);
|
||
await indicate('label[for="gallery-photo"]');
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||
await page.getByText(/Ready for your review/i).waitFor();
|
||
await caption('No consent prompt and no upload when analysis cannot run', 1600);
|
||
await tap('#manual-from-scan', 500);
|
||
const keptDraft = await page.locator('.draft-resume').innerText({ timeout: 5000 });
|
||
if (!/Type 2/.test(keptDraft)) throw new Error('The retained draft must show the value the user corrected to');
|
||
await caption('One tap still reaches manual Bristol entry, and the draft keeps your corrected value', 1800);
|
||
await tap('#discard-draft', 500);
|
||
await caption('Start over clears the kept draft — nothing lingers silently', 1500);
|
||
await tap('#close-sheet', 450);
|
||
|
||
// Corpus refusal pass (issue #18): the same photo flow, fed the answers Timmy has to refuse.
|
||
visionOffline = false;
|
||
await caption('The corpus also replays the answers Timmy has to refuse', 1700);
|
||
analyzeOverride = await corpusAnalyzeResult(corpusCase('non-stool-scene'));
|
||
await tap('[data-scan]', 500);
|
||
await page.getByText(/Self-hosted model ready/i).waitFor();
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/non-stool-scene.png');
|
||
await page.locator('#ai-consent').check();
|
||
await tap('#analyze-photo', 600);
|
||
await page.getByText(/No confident match/i).waitFor();
|
||
await caption('A non-stool photo gets a refusal, never a guess — manual entry stays one tap away', 2600);
|
||
await indicate('.scan-result');
|
||
analyzeOverride = await corpusAnalyzeResult(corpusCase('glare-overexposed'));
|
||
await tap('#retake-photo', 400);
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/glare-overexposed.png');
|
||
await page.locator('#ai-consent').check();
|
||
await tap('#analyze-photo', 600);
|
||
await page.getByText(/does not show a form Timmy can read safely/i).waitFor();
|
||
await caption('Unreadable visual fields abstain as well: a valid reply with nothing to prefill is not an error', 2700);
|
||
await indicate('.scan-result');
|
||
analyzeOverride = await corpusAnalyzeResult(corpusCase('out-of-range-fields'));
|
||
await tap('#retake-photo', 400);
|
||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/out-of-range-fields.png');
|
||
await page.locator('#ai-consent').check();
|
||
await tap('#analyze-photo', 600);
|
||
await page.getByText(/couldn’t analyze that safely/i).waitFor();
|
||
await caption('An out-of-range answer fails closed: nothing is applied and no provider detail reaches you', 2700);
|
||
await indicate('.scan-result');
|
||
await caption('Every refusal replays a committed synthetic fixture — there are no real photos in the corpus', 2500);
|
||
analyzeOverride = null;
|
||
await tap('#close-sheet', 450);
|
||
|
||
await caption('The threat model documents this surface, so the recording shows it live rather than describing it', 2200);
|
||
await tap('[data-view="calendar"]', 550);
|
||
await page.getByText(/Your private journal/i).waitFor();
|
||
await tap('[data-view="privacy"]', 550);
|
||
await page.getByText(/Your poop\. Your phone\./i).waitFor();
|
||
await caption('Every byte-bearing control the model names, in the running app: local ledger, consent-gated upload, portable export, full deletion', 2800);
|
||
await indicate('#export');
|
||
await caption('Export writes a plain JSON file you hold — the model records it as a data-lifetime boundary, not a server sync', 2600);
|
||
await indicate('#delete-all');
|
||
await caption('Delete all local data clears the ledger in this browser; there is no server copy of it to keep', 2600);
|
||
await tap('[data-view="home"]', 500);
|
||
await tap('[data-view="timmy"]', 550);
|
||
await page.getByText(/Hermes Agent connected/i).waitFor();
|
||
await caption('Hermes Agent connected — free-text, contextual, and server-side', 1400);
|
||
await page.locator('#chat-message').fill('What pattern do you see?');
|
||
await tap('#send-chat', 500);
|
||
await page.getByText(/mostly Type 4/i).waitFor();
|
||
await caption('Hermes keeps credentials, tools, and session continuity server-side. Photos stay out of chat.', 1600);
|
||
await page.locator('#chat-message').fill('I barfed');
|
||
await tap('#send-chat', 450);
|
||
await page.getByText(/Pause and get medical help/i).waitFor();
|
||
if (hermesCalls !== 1) throw new Error('Urgent chat must be intercepted before Hermes');
|
||
await caption('Urgent language is intercepted deterministically before Hermes', 1800);
|
||
await sleep(400);
|
||
await page.evaluate(() => {
|
||
document.querySelector('#demo-caption')?.remove();
|
||
const outro = document.createElement('div');
|
||
outro.id = 'release-outro';
|
||
outro.innerHTML = '<img src="/assets/timmy.svg"><strong>SLEEK. SIMPLE.<br>HERMES-POWERED.</strong><span>User-confirmed guidance — never a diagnosis.</span>';
|
||
Object.assign(outro.style, { position:'fixed', inset:'0', zIndex:'30000', background:'linear-gradient(145deg,#f7f3ea,#f5c95b)', display:'flex', flexDirection:'column', alignItems:'center', justifyContent:'center', textAlign:'center', color:'#28211d', fontFamily:'system-ui', opacity:'0' });
|
||
outro.querySelector('img').style.cssText = 'width:150px;height:150px;filter:drop-shadow(0 14px 16px rgba(63,37,30,.18))';
|
||
outro.querySelector('strong').style.cssText = 'font-size:31px;line-height:1.05;margin:18px 0 12px;letter-spacing:-.04em';
|
||
outro.querySelector('span').style.cssText = 'font-size:13px;font-weight:800;color:#176957';
|
||
document.body.append(outro);
|
||
outro.animate([{opacity:0},{opacity:1}], {duration:500,fill:'forwards'});
|
||
});
|
||
await sleep(2200);
|
||
|
||
// The refusal pass deliberately produces one fail-closed 503, which the browser logs as a failed
|
||
// resource. Requiring exactly that one keeps the scene honest: if the boundary stops refusing, this
|
||
// fails instead of quietly recording a happy path.
|
||
const expectedFailClosed = errors.filter(message => /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/.test(message));
|
||
if (expectedFailClosed.length !== 1 || errors.length !== expectedFailClosed.length) {
|
||
throw new Error(`Browser errors: ${errors.join(' | ') || 'unexpected error count'}`);
|
||
}
|
||
// Machine-readable receipt of exactly what was rendered on camera, so the release gate can prove
|
||
// the recorded delta statement reached the screen instead of trusting the script source.
|
||
if (captionsOut) {
|
||
await writeFile(captionsOut, JSON.stringify({ version, releaseDelta, captions: captionsShown }, null, 2) + '\n', 'utf8');
|
||
}
|
||
const recording = page.video();
|
||
await page.close();
|
||
await recording.saveAs(output);
|
||
await context.close();
|
||
await browser.close();
|
||
console.log(output);
|