// Browser acceptance for issue #18: the refusal paths the corpus proves must be visible in the app, // not just in unit tests. The app talks to a real server.mjs with vision enabled, and the only thing // the harness chooses is which recorded corpus response the stub worker returns. import { chromium } from 'playwright'; import assert from 'node:assert/strict'; import { mkdir } from 'node:fs/promises'; import { loadVisionCorpus } from '../src/vision-corpus.js'; import { caseFor, corpusDir, startProvider, startServer, stopServer } from './helpers/vision-harness.mjs'; await mkdir('artifacts', { recursive: true }); const corpus = await loadVisionCorpus(corpusDir); const provider = await startProvider(); const server = await startServer({ providerPort: provider.port, port: 4199 }); const browser = await chromium.launch({ headless: true }); const context = await browser.newContext({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 2, serviceWorkers: 'block' }); const page = await context.newPage(); const errors = []; page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); page.on('pageerror', error => errors.push(error.message)); // A fail-closed 503 is the intended boundary response, so the browser logs it as a failed resource. // It is expected exactly once; anything else is a real page error. const EXPECTED_FAIL_CLOSED_LOG = /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/; async function closeSheet() { await page.locator('#close-sheet').click(); await page.locator('.scan-sheet').waitFor({ state: 'detached' }); } async function analyzeCorpusCase(entry) { provider.state.content = entry.providerContent; await page.locator('[data-scan]').click(); await page.getByText(/Self-hosted model ready/i).waitFor(); await page.locator('#gallery-photo').setInputFiles(`tests/fixtures/vision-corpus/${entry.file}`); await page.locator('#ai-consent').check(); await page.locator('#analyze-photo').click(); await page.locator('.scan-result').first().waitFor(); } try { await page.goto('http://127.0.0.1:4199', { waitUntil: 'networkidle' }); await page.evaluate(() => localStorage.clear()); await page.reload({ waitUntil: 'networkidle' }); // 1. Non-stool photo: no prefill, a plain refusal, and manual entry still one action away. await analyzeCorpusCase(caseFor(corpus, 'non-stool-scene')); assert.equal(await page.getByText(/No confident match/i).isVisible(), true, 'a non-stool photo must refuse'); assert.equal(await page.locator('.ai-badge').count(), 0, 'a refusal must not carry an AI suggestion badge'); assert.equal(await page.getByText('Type 4', { exact: true }).count(), 0, 'a refusal must not prefill a form'); assert.match(await page.locator('.scan-result').innerText(), /does not clearly show stool/i); const manual = page.locator('#manual-from-scan'); assert.match(await manual.innerText(), /Choose the form yourself/i); await page.screenshot({ path: 'artifacts/vision-refusal-non-stool-mobile.png' }); await manual.click(); await page.getByText('Pick the closest form').waitFor(); assert.equal(await page.locator('.ai-prefill').count(), 0, 'refused analysis must not leave AI provenance in the log form'); await closeSheet(); // 2. Glare with unreadable visual fields: schema-valid, so it abstains calmly instead of erroring. await analyzeCorpusCase(caseFor(corpus, 'glare-overexposed')); assert.equal(await page.getByText(/No confident match/i).isVisible(), true); assert.match(await page.locator('.scan-result').innerText(), /does not show a form Timmy can read safely/i); assert.equal(await page.getByText(/couldn’t analyze that safely/i).count(), 0, 'a schema-valid abstention is not a failure'); await page.screenshot({ path: 'artifacts/vision-refusal-unreadable-fields-mobile.png' }); await closeSheet(); // 3. Out-of-range model output: fails closed, and the copy says plainly that nothing was applied. await analyzeCorpusCase(caseFor(corpus, 'out-of-range-fields')); await page.getByText(/couldn’t analyze that safely/i).waitFor(); assert.match(await page.locator('.scan-result').innerText(), /No suggestion was applied/i); assert.doesNotMatch(await page.locator('.scan-result').innerText(), /out of range|Invalid AI response/i, 'provider detail must not reach the user'); assert.equal(await page.locator('.ai-badge').count(), 0); await page.screenshot({ path: 'artifacts/vision-refusal-fail-closed-mobile.png' }); await closeSheet(); // 4. Issue #73: a filled-in field that is out of range fails closed even when the other field is blank. // The old blank-first ordering abstained here, so the user was told the photo was unreadable while the // model had actually returned an impossible value. The abstention copy must not appear on this path. await analyzeCorpusCase(caseFor(corpus, 'blank-plus-out-of-range')); await page.getByText(/couldn’t analyze that safely/i).waitFor(); assert.match(await page.locator('.scan-result').innerText(), /No suggestion was applied/i); assert.equal(await page.getByText(/No confident match/i).count(), 0, 'a contract breach must not be shown as an abstention'); assert.equal(await page.locator('.ai-badge').count(), 0); assert.doesNotMatch(await page.locator('.scan-result').innerText(), /out of range|Invalid AI response/i, 'provider detail must not reach the user'); await page.screenshot({ path: 'artifacts/vision-refusal-blank-plus-out-of-range-mobile.png' }); await closeSheet(); // 5. Positive control: the same corpus still yields a real suggestion when the response is usable. await analyzeCorpusCase(caseFor(corpus, 'stool-like-formed')); await page.getByText(/83% confidence/i).waitFor(); assert.equal(await page.getByText('Type 4', { exact: true }).isVisible(), true); await page.screenshot({ path: 'artifacts/vision-corpus-suggestion-mobile.png' }); // Two fail-closed scenes reach the browser as real 503s; the count is asserted so a scene cannot be // dropped silently, and any other console error still fails the run. const failClosedLogs = errors.filter(message => EXPECTED_FAIL_CLOSED_LOG.test(message)); assert.equal(failClosedLogs.length, 2, 'each fail-closed response must reach the browser as a 503'); assert.deepEqual(errors.filter(message => !EXPECTED_FAIL_CLOSED_LOG.test(message)), []); console.log('PASS vision corpus refusal: non-stool, unreadable fields, out-of-range, and blank-plus-out-of-range replies never prefill; the usable reply still suggests'); } finally { await context.close(); await browser.close(); await stopServer(server); await provider.close(); }