timmy
5c759078c4
Quality gates / quality (pull_request) Failing after 14s
The blank-first ordering let one blank field short-circuit the whole reply, so a response carrying an impossible value came back as `unusable_visual_fields`: the user was told the photo was unreadable when the model had in fact returned something Timmy must never apply. Blank fields still abstain and still prefill nothing. A filled-in field is a claim, and a claim outside the contract now fails closed even when the other field came back blank. The corpus earns its label too. `out-of-range-fields` drops from confidence 1.4 to 0.9 so the case reaches the field-range check instead of dying at the confidence gate, and a new `blank-plus-out-of-range` case covers the class the old ordering let through. The release demo replays that case on camera and the fail-closed 503 budget moves from one to two, so the scene cannot be dropped silently. Closes #73
105 lines
6.5 KiB
JavaScript
105 lines
6.5 KiB
JavaScript
// Browser acceptance for issue #18: the refusal paths the corpus proves must be visible in the app,
|
||
// not just in unit tests. The app talks to a real server.mjs with vision enabled, and the only thing
|
||
// the harness chooses is which recorded corpus response the stub worker returns.
|
||
import { chromium } from 'playwright';
|
||
import assert from 'node:assert/strict';
|
||
import { mkdir } from 'node:fs/promises';
|
||
import { loadVisionCorpus } from '../src/vision-corpus.js';
|
||
import { caseFor, corpusDir, startProvider, startServer, stopServer } from './helpers/vision-harness.mjs';
|
||
|
||
await mkdir('artifacts', { recursive: true });
|
||
const corpus = await loadVisionCorpus(corpusDir);
|
||
const provider = await startProvider();
|
||
const server = await startServer({ providerPort: provider.port, port: 4199 });
|
||
const browser = await chromium.launch({ headless: true });
|
||
const context = await browser.newContext({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 2, serviceWorkers: 'block' });
|
||
const page = await context.newPage();
|
||
const errors = [];
|
||
page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); });
|
||
page.on('pageerror', error => errors.push(error.message));
|
||
// A fail-closed 503 is the intended boundary response, so the browser logs it as a failed resource.
|
||
// It is expected exactly once; anything else is a real page error.
|
||
const EXPECTED_FAIL_CLOSED_LOG = /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/;
|
||
|
||
async function closeSheet() {
|
||
await page.locator('#close-sheet').click();
|
||
await page.locator('.scan-sheet').waitFor({ state: 'detached' });
|
||
}
|
||
|
||
async function analyzeCorpusCase(entry) {
|
||
provider.state.content = entry.providerContent;
|
||
await page.locator('[data-scan]').click();
|
||
await page.getByText(/Self-hosted model ready/i).waitFor();
|
||
await page.locator('#gallery-photo').setInputFiles(`tests/fixtures/vision-corpus/${entry.file}`);
|
||
await page.locator('#ai-consent').check();
|
||
await page.locator('#analyze-photo').click();
|
||
await page.locator('.scan-result').first().waitFor();
|
||
}
|
||
|
||
try {
|
||
await page.goto('http://127.0.0.1:4199', { waitUntil: 'networkidle' });
|
||
await page.evaluate(() => localStorage.clear());
|
||
await page.reload({ waitUntil: 'networkidle' });
|
||
|
||
// 1. Non-stool photo: no prefill, a plain refusal, and manual entry still one action away.
|
||
await analyzeCorpusCase(caseFor(corpus, 'non-stool-scene'));
|
||
assert.equal(await page.getByText(/No confident match/i).isVisible(), true, 'a non-stool photo must refuse');
|
||
assert.equal(await page.locator('.ai-badge').count(), 0, 'a refusal must not carry an AI suggestion badge');
|
||
assert.equal(await page.getByText('Type 4', { exact: true }).count(), 0, 'a refusal must not prefill a form');
|
||
assert.match(await page.locator('.scan-result').innerText(), /does not clearly show stool/i);
|
||
const manual = page.locator('#manual-from-scan');
|
||
assert.match(await manual.innerText(), /Choose the form yourself/i);
|
||
await page.screenshot({ path: 'artifacts/vision-refusal-non-stool-mobile.png' });
|
||
await manual.click();
|
||
await page.getByText('Pick the closest form').waitFor();
|
||
assert.equal(await page.locator('.ai-prefill').count(), 0, 'refused analysis must not leave AI provenance in the log form');
|
||
await closeSheet();
|
||
|
||
// 2. Glare with unreadable visual fields: schema-valid, so it abstains calmly instead of erroring.
|
||
await analyzeCorpusCase(caseFor(corpus, 'glare-overexposed'));
|
||
assert.equal(await page.getByText(/No confident match/i).isVisible(), true);
|
||
assert.match(await page.locator('.scan-result').innerText(), /does not show a form Timmy can read safely/i);
|
||
assert.equal(await page.getByText(/couldn’t analyze that safely/i).count(), 0, 'a schema-valid abstention is not a failure');
|
||
await page.screenshot({ path: 'artifacts/vision-refusal-unreadable-fields-mobile.png' });
|
||
await closeSheet();
|
||
|
||
// 3. Out-of-range model output: fails closed, and the copy says plainly that nothing was applied.
|
||
await analyzeCorpusCase(caseFor(corpus, 'out-of-range-fields'));
|
||
await page.getByText(/couldn’t analyze that safely/i).waitFor();
|
||
assert.match(await page.locator('.scan-result').innerText(), /No suggestion was applied/i);
|
||
assert.doesNotMatch(await page.locator('.scan-result').innerText(), /out of range|Invalid AI response/i, 'provider detail must not reach the user');
|
||
assert.equal(await page.locator('.ai-badge').count(), 0);
|
||
await page.screenshot({ path: 'artifacts/vision-refusal-fail-closed-mobile.png' });
|
||
await closeSheet();
|
||
|
||
// 4. Issue #73: a filled-in field that is out of range fails closed even when the other field is blank.
|
||
// The old blank-first ordering abstained here, so the user was told the photo was unreadable while the
|
||
// model had actually returned an impossible value. The abstention copy must not appear on this path.
|
||
await analyzeCorpusCase(caseFor(corpus, 'blank-plus-out-of-range'));
|
||
await page.getByText(/couldn’t analyze that safely/i).waitFor();
|
||
assert.match(await page.locator('.scan-result').innerText(), /No suggestion was applied/i);
|
||
assert.equal(await page.getByText(/No confident match/i).count(), 0, 'a contract breach must not be shown as an abstention');
|
||
assert.equal(await page.locator('.ai-badge').count(), 0);
|
||
assert.doesNotMatch(await page.locator('.scan-result').innerText(), /out of range|Invalid AI response/i, 'provider detail must not reach the user');
|
||
await page.screenshot({ path: 'artifacts/vision-refusal-blank-plus-out-of-range-mobile.png' });
|
||
await closeSheet();
|
||
|
||
// 5. Positive control: the same corpus still yields a real suggestion when the response is usable.
|
||
await analyzeCorpusCase(caseFor(corpus, 'stool-like-formed'));
|
||
await page.getByText(/83% confidence/i).waitFor();
|
||
assert.equal(await page.getByText('Type 4', { exact: true }).isVisible(), true);
|
||
await page.screenshot({ path: 'artifacts/vision-corpus-suggestion-mobile.png' });
|
||
|
||
// Two fail-closed scenes reach the browser as real 503s; the count is asserted so a scene cannot be
|
||
// dropped silently, and any other console error still fails the run.
|
||
const failClosedLogs = errors.filter(message => EXPECTED_FAIL_CLOSED_LOG.test(message));
|
||
assert.equal(failClosedLogs.length, 2, 'each fail-closed response must reach the browser as a 503');
|
||
assert.deepEqual(errors.filter(message => !EXPECTED_FAIL_CLOSED_LOG.test(message)), []);
|
||
console.log('PASS vision corpus refusal: non-stool, unreadable fields, out-of-range, and blank-plus-out-of-range replies never prefill; the usable reply still suggests');
|
||
} finally {
|
||
await context.close();
|
||
await browser.close();
|
||
await stopServer(server);
|
||
await provider.close();
|
||
}
|