diff --git a/.gitea/workflows/quality.yml b/.gitea/workflows/quality.yml index 96c4c78..15dfdb5 100644 --- a/.gitea/workflows/quality.yml +++ b/.gitea/workflows/quality.yml @@ -55,6 +55,7 @@ jobs: npm run test:manual-fallback npm run test:provisional npm run test:followup + npm run test:vision-refusal - name: Dependency audit run: npm audit --audit-level=high - name: Syntax checks diff --git a/README.md b/README.md index 3c14257..ae458df 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ npm run test:ui # server required on port 4173 npm run test:photo # mocked positive suggestion through the real browser UX npm run test:sleek # simplified 390x844 shell + free-text Hermes chat npm run test:manual-fallback # vision-offline manual path + in-session draft recovery +npm run test:vision-refusal # corpus refusals through a real server and a real browser npm audit --audit-level=high ``` @@ -115,6 +116,7 @@ npm audit --audit-level=high - Red-flag symptom escalation - A short user-reported follow-up after a hard/lumpy (Bristol 1–2) log — difficulty passing, soreness, visible blood, truly black/tarry versus dark brown, and hydration. Mild reports get concise non-diagnostic self-care that never claims a photo ruled out bleeding; a confirmed red flag sets the same deterministic escalation the manual checkboxes use - Manual fallback that never depends on AI: when the vision worker is offline or disabled, no AI or consent control is offered and manual Bristol entry stays one action away +- A committed CC0 synthetic vision corpus that continuously proves every reply is schema-valid or fails closed. A schema-valid reply with no readable form or colour now **abstains** ("No confident match") instead of surfacing as a worker failure, and unparsable or out-of-range replies return plain copy — "No suggestion was applied. Continue manually, or try a clearer photo." — with no provider text, image bytes, or credentials echoed. See [tests/fixtures/vision-corpus](tests/fixtures/vision-corpus/README.md) - Unsaved logging drafts are retained in the session with an explicit start over, and the service worker never caches analysis requests or image bodies - Local browser ledger, calendar, pattern summary, JSON portability, and delete-all - Installable/offline PWA shell for the manual and saved-ledger paths diff --git a/artifacts/vision-corpus-suggestion-mobile.png b/artifacts/vision-corpus-suggestion-mobile.png new file mode 100644 index 0000000..3af6520 Binary files /dev/null and b/artifacts/vision-corpus-suggestion-mobile.png differ diff --git a/artifacts/vision-refusal-fail-closed-mobile.png b/artifacts/vision-refusal-fail-closed-mobile.png new file mode 100644 index 0000000..82476f5 Binary files /dev/null and b/artifacts/vision-refusal-fail-closed-mobile.png differ diff --git a/artifacts/vision-refusal-non-stool-mobile.png b/artifacts/vision-refusal-non-stool-mobile.png new file mode 100644 index 0000000..d059df8 Binary files /dev/null and b/artifacts/vision-refusal-non-stool-mobile.png differ diff --git a/artifacts/vision-refusal-unreadable-fields-mobile.png b/artifacts/vision-refusal-unreadable-fields-mobile.png new file mode 100644 index 0000000..7f1f303 Binary files /dev/null and b/artifacts/vision-refusal-unreadable-fields-mobile.png differ diff --git a/package.json b/package.json index 2715c91..faaa16a 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "private": true, "type": "module", "scripts": { - "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js", + "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js", "test:ui": "node tests/ui.acceptance.mjs", "test:photo": "node tests/photo-first.acceptance.mjs", "test:mobile-capture": "node tests/mobile-capture.acceptance.mjs", @@ -12,8 +12,9 @@ "test:manual-fallback": "node tests/manual-fallback.acceptance.mjs", "test:provisional": "node tests/provisional-suggestion.acceptance.mjs", "test:followup": "node tests/followup.acceptance.mjs", + "test:vision-refusal": "node tests/vision-refusal.acceptance.mjs", "test:staging-smoke": "node tests/staging.acceptance.mjs", - "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py", + "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py", "check:diff": "bash scripts/check_diff.sh", "start": "node server.mjs" }, diff --git a/scripts/build_release.py b/scripts/build_release.py index e2dfdfa..a94692c 100755 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -37,6 +37,7 @@ DEMO_REQUIRED_CAPTIONS = ( "Delete all local data clears the ledger in this browser; there is no server copy of it to keep", "Hermes Agent connected — free-text, contextual, and server-side", "Urgent language is intercepted deterministically before Hermes", + "An out-of-range answer fails closed: nothing is applied and no provider detail reaches you", ) @@ -108,6 +109,8 @@ def main() -> int: run(["git", "checkout", "--quiet", commit], tree) run(["npm", "ci", "--ignore-scripts"], tree) run(["npm", "test"], tree) + # The committed corpus hashes are an integrity receipt: verify them before anything is built. + run(["python3", "scripts/build_vision_corpus.py", "--verify"], tree) server_env = dict(os.environ) server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false"}) server_log = (release_dir / "acceptance-server.log").open("w", encoding="utf-8") @@ -131,6 +134,7 @@ def main() -> int: run(["npm", "run", "test:manual-fallback"], tree) run(["npm", "run", "test:provisional"], tree) run(["npm", "run", "test:followup"], tree) + run(["npm", "run", "test:vision-refusal"], tree) demo_raw = release_dir / f"timmy-talking-turd-{version}-demo.raw.webm" caption_receipt = release_dir / f"timmy-talking-turd-{version}-demo-captions.json" demo_env = dict(server_env) @@ -172,11 +176,11 @@ def main() -> int: if not contact_sheet.exists() or contact_sheet.stat().st_size == 0: raise SystemExit("Release demo contact sheet was not produced") run(["npm", "audit", "--audit-level=high"], tree) - for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "tests/staging.acceptance.mjs"): + for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs"): run(["node", "--check", file], tree) run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree) run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree) - run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py"], tree) + run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py", "scripts/build_vision_corpus.py"], tree) run(["git", "diff", "--check", commit], tree) names = tracked_files(tree) @@ -203,6 +207,7 @@ def main() -> int: "- Vision profiles: hosted and self-hosted SmolVLM2 bootstrap\n" "- AI suggestions stay provisional: suggested, confirmed, and corrected states are distinct and screen-reader described; correcting one clears the AI-confirmed state without losing user context\n" "- Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions (difficulty passing, soreness, visible blood, truly black/tarry versus dark brown, hydration); mild reports get non-diagnostic self-care, and a confirmed red-flag answer lands on the same deterministic escalation path as manual logging\n" + "- Vision corpus: a committed CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) proves every reply is schema-valid or fails closed; a schema-valid reply with unreadable fields abstains instead of erroring, and unparsable or out-of-range replies return plain copy with nothing applied\n" "- Safety: suggestions are observable-field assistance, never diagnosis; user confirmation is required\n" "- Known limitation: open-weight VLM acceptance is proven, Bristol accuracy is not clinically validated\n" "- Package excludes model weights, medical-image corpora, raw research pages, generated screenshots, and video production assets\n" @@ -248,7 +253,7 @@ def main() -> int: "codec": "h264", "pixel_format": "yuv420p", "recorded_from_working_app": True, - "fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, offline vision status, and deterministic Hermes chat response", + "fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, and an out-of-range fail-closed reply, offline vision status, and deterministic Hermes chat response", }, "gates": { "unit_security": "passed", @@ -258,6 +263,7 @@ def main() -> int: "manual_fallback_draft_recovery": "passed", "provisional_suggestion_state": "passed", "hard_stool_followup": "passed", + "vision_corpus_refusal": "passed", "dependency_audit_high": "passed", "syntax": "passed", "secret_scan": "passed", diff --git a/scripts/build_vision_corpus.py b/scripts/build_vision_corpus.py new file mode 100644 index 0000000..afeb232 --- /dev/null +++ b/scripts/build_vision_corpus.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Generate the deterministic synthetic vision corpus for issue #18. + +Every fixture is drawn from fixed integer math: no camera, no human subject, no +third-party content, and no randomness. That keeps the corpus redistributable and +keeps the recorded SHA-256 hashes meaningful as an integrity receipt. + +Usage: + python3 scripts/build_vision_corpus.py # (re)write fixtures and manifest.json + python3 scripts/build_vision_corpus.py --verify # verify committed fixtures against the manifest +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any, Callable + +from PIL import Image, ImageDraw, ImageFilter + +LICENSE = "CC0-1.0" +CORPUS_DIR = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "vision-corpus" +WIDTH, HEIGHT = 640, 480 +PROVENANCE = "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content." + +OBSERVATION_STRINGS = { + "smooth": "Smooth, formed appearance.", + "flatware": "Flatware on a tray.", + "unclear": "Form is unclear.", + "highlight": "Large specular highlight hides the surface.", + "container": "Empty container.", + "confident": "Confident.", +} + + +def background(color: tuple[int, int, int]) -> Image.Image: + """A flat canvas with a coarse, deterministic shade band.""" + image = Image.new("RGB", (WIDTH, HEIGHT), color) + draw = ImageDraw.Draw(image) + for index in range(4): + shade = tuple(min(255, channel + 6 * index) for channel in color) + draw.rectangle([0, index * 120, WIDTH, index * 120 + 119], fill=shade) + return image + + +def blob(image: Image.Image, box: tuple[int, int, int, int], tone: tuple[int, int, int]) -> None: + draw = ImageDraw.Draw(image) + draw.ellipse(box, fill=tone) + left, top, right, bottom = box + inset = max(4, (right - left) // 6) + draw.ellipse([left + inset, top + inset, right - inset, bottom - inset], + fill=tuple(min(255, channel + 10) for channel in tone)) + + +def stool_like() -> Image.Image: + image = background((226, 222, 212)) + blob(image, (170, 150, 470, 330), (108, 76, 48)) + return image + + +def non_stool_scene() -> Image.Image: + image = background((198, 202, 208)) + draw = ImageDraw.Draw(image) + draw.ellipse([120, 90, 520, 390], fill=(244, 244, 240)) + draw.rectangle([60, 200, 300, 224], fill=(120, 126, 134)) + draw.rectangle([340, 200, 580, 224], fill=(120, 126, 134)) + return image + + +def blur_low_detail() -> Image.Image: + return stool_like().filter(ImageFilter.GaussianBlur(5)) + + +def glare_overexposed() -> Image.Image: + image = background((222, 218, 208)) + blob(image, (150, 140, 430, 300), (104, 74, 46)) + draw = ImageDraw.Draw(image) + draw.polygon([(380, 200), (640, 120), (640, 360), (420, 330)], fill=(252, 252, 250)) + draw.ellipse([470, 210, 620, 320], fill=(255, 255, 255)) + return image + + +def empty_bowl() -> Image.Image: + image = background((214, 210, 202)) + draw = ImageDraw.Draw(image) + draw.ellipse([130, 90, 510, 400], fill=(238, 236, 230)) + draw.ellipse([175, 130, 465, 360], fill=(206, 204, 198)) + draw.ellipse([220, 170, 420, 320], fill=(232, 230, 224)) + return image + + +def unusable_tone() -> Image.Image: + image = background((202, 206, 198)) + blob(image, (190, 160, 450, 320), (128, 118, 92)) + return image + + +def shaded_tone() -> Image.Image: + image = background((216, 214, 206)) + blob(image, (180, 155, 460, 325), (96, 84, 52)) + draw = ImageDraw.Draw(image) + draw.rectangle([0, 300, WIDTH, 306], fill=(180, 178, 170)) + return image + + +def content(payload: dict[str, object] | str) -> str: + return payload if isinstance(payload, str) else json.dumps(payload, separators=(",", ":")) + + +def cases() -> list[dict[str, Any]]: + return [ + { + "id": "stool-like-formed", + "caseClass": "stool", + "image": stool_like, + "expectedOutcome": "suggestion", + "expectedReason": "schema_valid_suggestion", + "providerContent": content({ + "isStool": True, "bristolType": 4, "color": "brown", "confidence": 0.83, + "imageQuality": "good", "observations": OBSERVATION_STRINGS["smooth"], + }), + }, + { + "id": "non-stool-scene", + "caseClass": "non-stool", + "image": non_stool_scene, + "expectedOutcome": "abstained", + "expectedReason": "not_stool", + "providerContent": content({ + "isStool": False, "bristolType": None, "color": None, "confidence": 0.97, + "imageQuality": "good", "observations": OBSERVATION_STRINGS["flatware"], + }), + }, + { + "id": "blur-low-detail", + "caseClass": "blur", + "image": blur_low_detail, + "expectedOutcome": "abstained", + "expectedReason": "low_confidence", + "providerContent": content({ + "isStool": True, "bristolType": 4, "color": "brown", "confidence": 0.41, + "imageQuality": "poor", "observations": OBSERVATION_STRINGS["unclear"], + }), + }, + { + "id": "glare-overexposed", + "caseClass": "glare", + "image": glare_overexposed, + "expectedOutcome": "abstained", + "expectedReason": "unusable_visual_fields", + "providerContent": content({ + "isStool": True, "bristolType": None, "color": None, "confidence": 0.66, + "imageQuality": "poor", "observations": OBSERVATION_STRINGS["highlight"], + }), + }, + { + "id": "empty-bowl", + "caseClass": "empty-bowl", + "image": empty_bowl, + "expectedOutcome": "abstained", + "expectedReason": "not_stool", + "providerContent": content({ + "isStool": False, "bristolType": None, "color": None, "confidence": 0.93, + "imageQuality": "good", "observations": OBSERVATION_STRINGS["container"], + }), + }, + { + "id": "malformed-response", + "caseClass": "malformed-response", + "image": unusable_tone, + "expectedOutcome": "fail_closed", + "expectedReason": "unparsable_content", + "providerContent": "Sure! It looks like a Type 4 to me. Hope that helps.", + }, + { + "id": "out-of-range-fields", + "caseClass": "malformed-response", + "image": shaded_tone, + "expectedOutcome": "fail_closed", + "expectedReason": "out_of_range_or_missing_fields", + "providerContent": content({ + "isStool": True, "bristolType": 9, "color": "purple", "confidence": 1.4, + "imageQuality": "excellent", "observations": OBSERVATION_STRINGS["confident"], + }), + }, + ] + + +def write_fixture(definition: dict[str, Any], directory: Path) -> dict[str, Any]: + name = f"{definition['id']}.png" + target = directory / name + factory: Callable[[], Image.Image] = definition["image"] + factory().save(target, "PNG", optimize=True) + data = target.read_bytes() + return { + "id": definition["id"], + "caseClass": definition["caseClass"], + "file": name, + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + "width": WIDTH, + "height": HEIGHT, + "format": "png", + "synthetic": True, + "license": LICENSE, + "provenance": PROVENANCE, + "expectedOutcome": definition["expectedOutcome"], + "expectedReason": definition["expectedReason"], + "providerContent": definition["providerContent"], + } + + +def manifest(entries: list[dict[str, object]]) -> dict[str, object]: + return { + "schemaVersion": 1, + "generator": "scripts/build_vision_corpus.py", + "license": LICENSE, + "note": "Synthetic fixtures only. No real medical imagery, no personal data, and no third-party content.", + "cases": entries, + } + + +def write(directory: Path) -> int: + directory.mkdir(parents=True, exist_ok=True) + entries = [write_fixture(definition, directory) for definition in cases()] + (directory / "manifest.json").write_text(json.dumps(manifest(entries), indent=2) + "\n", encoding="utf-8") + print(f"wrote {len(entries)} fixtures to {directory}") + return 0 + + +def verify(directory: Path) -> int: + manifest_path = directory / "manifest.json" + recorded = json.loads(manifest_path.read_text(encoding="utf-8")) + problems: list[str] = [] + for entry in recorded.get("cases", []): + path = directory / str(entry.get("file")) + if not path.is_file(): + problems.append(f"{entry.get('id')}: missing fixture {entry.get('file')}") + continue + data = path.read_bytes() + if hashlib.sha256(data).hexdigest() != entry.get("sha256"): + problems.append(f"{entry.get('id')}: bytes do not match the recorded sha256") + if len(data) != entry.get("bytes"): + problems.append(f"{entry.get('id')}: byte length does not match the manifest") + if problems: + for problem in problems: + print(problem, file=sys.stderr) + return 1 + print(f"verified {len(recorded.get('cases', []))} corpus fixtures against manifest.json") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--verify", action="store_true", help="check committed fixtures against the manifest instead of rewriting") + parser.add_argument("--directory", default=str(CORPUS_DIR)) + args = parser.parse_args() + directory = Path(args.directory) + return verify(directory) if args.verify else write(directory) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/record_release_demo.mjs b/scripts/record_release_demo.mjs index b0fb5bd..3c74f2c 100644 --- a/scripts/record_release_demo.mjs +++ b/scripts/record_release_demo.mjs @@ -1,6 +1,9 @@ import { chromium } from 'playwright'; import { mkdir, writeFile } from 'node:fs/promises'; import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { analyzePhoto, visionFailureResponse } from '../src/vision-service.js'; +import { loadVisionCorpus } from '../src/vision-corpus.js'; const output = path.resolve(process.argv[2] || 'release-demo.webm'); const version = process.env.TIMMY_RELEASE_VERSION || 'review build'; @@ -13,6 +16,26 @@ const captionsShown = []; const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); await mkdir(path.dirname(output), { recursive: true }); +// The refusal scene replays committed synthetic corpus cases through the real analysis code path, +// so the recording shows the status and copy the server would actually send rather than copy +// written for the video. No real photo is used anywhere in the corpus. +const corpus = await loadVisionCorpus(fileURLToPath(new URL('../tests/fixtures/vision-corpus/', import.meta.url))); +const corpusCase = id => corpus.cases.find(entry => entry.id === id); +async function corpusAnalyzeResult(entry) { + const fetchImpl = async () => ({ ok: true, json: async () => ({ choices: [{ message: { content: entry.providerContent } }] }) }); + try { + const result = await analyzePhoto({ + payload: { imageDataUrl: 'data:image/png;base64,YQ==', consent: true }, + fetchImpl, + config: { baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'corpus-replay', model: 'corpus-model' }, + }); + return { status: 200, body: result }; + } catch (error) { + const failure = visionFailureResponse(error); + return { status: failure.status, body: failure.body }; + } +} + const browser = await chromium.launch({ headless: true }); const context = await browser.newContext({ viewport: { width: 405, height: 720 }, @@ -39,20 +62,23 @@ await page.route('**/api/vision-status', route => route.fulfill({ modelSeen: !visionOffline, }), })); -await page.route('**/api/analyze', route => route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ - status: 'suggestion', - isStool: true, - bristolType: 4, - color: 'brown', - confidence: 0.83, - imageQuality: 'good', - observations: 'Smooth, formed appearance.', - warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.', - }), -})); +// The happy path keeps one deterministic suggestion. The refusal scene swaps this for a real +// corpus replay, so both the usable and the refused answers come from the same code path. +const suggestionBody = { + status: 'suggestion', + isStool: true, + bristolType: 4, + color: 'brown', + confidence: 0.83, + imageQuality: 'good', + observations: 'Smooth, formed appearance.', + warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.', +}; +let analyzeOverride = null; +await page.route('**/api/analyze', route => { + const response = analyzeOverride || { status: 200, body: suggestionBody }; + return route.fulfill({ status: response.status, contentType: 'application/json', body: JSON.stringify(response.body) }); +}); await page.route('**/api/agent/status', route => route.fulfill({ status: 200, contentType: 'application/json', @@ -204,6 +230,39 @@ await caption('One tap still reaches manual Bristol entry, and the draft keeps y await tap('#discard-draft', 500); await caption('Start over clears the kept draft — nothing lingers silently', 1500); await tap('#close-sheet', 450); + +// Corpus refusal pass (issue #18): the same photo flow, fed the answers Timmy has to refuse. +visionOffline = false; +await caption('The corpus also replays the answers Timmy has to refuse', 1700); +analyzeOverride = await corpusAnalyzeResult(corpusCase('non-stool-scene')); +await tap('[data-scan]', 500); +await page.getByText(/Self-hosted model ready/i).waitFor(); +await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/non-stool-scene.png'); +await page.locator('#ai-consent').check(); +await tap('#analyze-photo', 600); +await page.getByText(/No confident match/i).waitFor(); +await caption('A non-stool photo gets a refusal, never a guess — manual entry stays one tap away', 2600); +await indicate('.scan-result'); +analyzeOverride = await corpusAnalyzeResult(corpusCase('glare-overexposed')); +await tap('#retake-photo', 400); +await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/glare-overexposed.png'); +await page.locator('#ai-consent').check(); +await tap('#analyze-photo', 600); +await page.getByText(/does not show a form Timmy can read safely/i).waitFor(); +await caption('Unreadable visual fields abstain as well: a valid reply with nothing to prefill is not an error', 2700); +await indicate('.scan-result'); +analyzeOverride = await corpusAnalyzeResult(corpusCase('out-of-range-fields')); +await tap('#retake-photo', 400); +await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/out-of-range-fields.png'); +await page.locator('#ai-consent').check(); +await tap('#analyze-photo', 600); +await page.getByText(/couldn’t analyze that safely/i).waitFor(); +await caption('An out-of-range answer fails closed: nothing is applied and no provider detail reaches you', 2700); +await indicate('.scan-result'); +await caption('Every refusal replays a committed synthetic fixture — there are no real photos in the corpus', 2500); +analyzeOverride = null; +await tap('#close-sheet', 450); + await caption('The threat model documents this surface, so the recording shows it live rather than describing it', 2200); await tap('[data-view="calendar"]', 550); await page.getByText(/Your private journal/i).waitFor(); @@ -242,7 +301,13 @@ await page.evaluate(() => { }); await sleep(2200); -if (errors.length) throw new Error(`Browser errors: ${errors.join(' | ')}`); +// The refusal pass deliberately produces one fail-closed 503, which the browser logs as a failed +// resource. Requiring exactly that one keeps the scene honest: if the boundary stops refusing, this +// fails instead of quietly recording a happy path. +const expectedFailClosed = errors.filter(message => /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/.test(message)); +if (expectedFailClosed.length !== 1 || errors.length !== expectedFailClosed.length) { + throw new Error(`Browser errors: ${errors.join(' | ') || 'unexpected error count'}`); +} // Machine-readable receipt of exactly what was rendered on camera, so the release gate can prove // the recorded delta statement reached the screen instead of trusting the script source. if (captionsOut) { diff --git a/server.mjs b/server.mjs index 82f2ba1..8ce96f8 100644 --- a/server.mjs +++ b/server.mjs @@ -3,7 +3,7 @@ import { isIP } from 'node:net'; import { readFile, stat } from 'node:fs/promises'; import { extname, join, normalize } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { analyzePhoto } from './src/vision-service.js'; +import { analyzePhoto, visionFailureResponse } from './src/vision-service.js'; import { probeVisionProvider, resolveVisionConfig } from './src/vision-config.js'; import { AgentGatewayError, createHermesAgentService, resolveHermesAgentConfig } from './src/hermes-agent-service.js'; @@ -55,7 +55,7 @@ http.createServer(async(req,res)=>{ if(!visionConfig.enabled)return sendJson(res,503,{error:'AI analysis is disabled. Continue manually.'}); const payload=await readJson(req); try{return sendJson(res,200,await analyzePhoto({payload,config:visionConfig}))} - catch(error){const safe=/consent|JPEG|PNG|WebP|empty|too large/i.test(error.message);return sendJson(res,safe?400:503,{error:error.message})} + catch(error){const failure=visionFailureResponse(error);return sendJson(res,failure.status,failure.body)} } if(appPath==='/api/agent/status'&&req.method==='GET')return sendJson(res,200,agentService.status(cookie(req,'timmy_agent'))); if(appPath==='/api/agent/unlock'&&req.method==='POST'){ diff --git a/src/analysis.js b/src/analysis.js index 8053cfc..fe625d1 100644 --- a/src/analysis.js +++ b/src/analysis.js @@ -2,32 +2,55 @@ const COLORS = new Set(['brown', 'green', 'yellow', 'pale', 'red', 'black']); const QUALITIES = new Set(['good', 'fair', 'poor']); const MAX_IMAGE_BYTES = 4 * 1024 * 1024; +// Fail-closed responses carry a code so the server boundary and the corpus can name the failure +// class without matching on prose. +export const VISION_FAILURE_CODES = Object.freeze({ invalidJson: 'invalid_json', invalidFields: 'invalid_fields' }); + +function visionResponseError(code, message) { + const error = new Error(message); + error.name = 'VisionResponseError'; + error.code = code; + return error; +} + function asObject(raw) { if (raw && typeof raw === 'object' && !Array.isArray(raw)) return raw; - if (typeof raw !== 'string') throw new Error('Invalid AI response.'); + if (typeof raw !== 'string') throw visionResponseError(VISION_FAILURE_CODES.invalidJson, 'Invalid AI response.'); const cleaned = raw.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, ''); try { const parsed = JSON.parse(cleaned); if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error(); return parsed; } catch { - throw new Error('Invalid AI response.'); + throw visionResponseError(VISION_FAILURE_CODES.invalidJson, 'Invalid AI response.'); } } export function parseVisionResponse(raw) { const value = asObject(raw); - if (typeof value.isStool !== 'boolean') throw new Error('Invalid AI response: isStool is required.'); + if (typeof value.isStool !== 'boolean') throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: isStool is required.'); const confidence = Number(value.confidence); - if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) throw new Error('Invalid AI response: confidence is required.'); - if (!value.isStool) return { status: 'needs_user_input', isStool: false, reason: 'The image does not clearly show stool.' }; - if (confidence < 0.55) return { status: 'needs_user_input', isStool: true, reason: 'The image is too uncertain to prefill safely.' }; - const bristolType = Number(value.bristolType); - const color = String(value.color || '').toLowerCase(); - const imageQuality = String(value.imageQuality || '').toLowerCase(); - if (!Number.isInteger(bristolType) || bristolType < 1 || bristolType > 7 || !COLORS.has(color) || !QUALITIES.has(imageQuality)) { - throw new Error('Invalid AI response: visual fields are out of range.'); + if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: confidence is required.'); + if (!value.isStool) return { status: 'needs_user_input', isStool: false, abstainedFor: 'not_stool', reason: 'The image does not clearly show stool.' }; + if (confidence < 0.55) return { status: 'needs_user_input', isStool: true, abstainedFor: 'low_confidence', reason: 'The image is too uncertain to prefill safely.' }; + // The request schema lets the model return a null form and color when it cannot read them. + // A schema-valid answer with nothing to prefill abstains; an out-of-range value still fails closed. + const blank = input => input === null || input === undefined || (typeof input === 'string' && !input.trim()); + if (blank(value.bristolType) || blank(value.color)) { + return { + status: 'needs_user_input', + isStool: true, + abstainedFor: 'unusable_visual_fields', + reason: 'The photo does not show a form Timmy can read safely.', + }; } + const bristolType = Number(value.bristolType); + const color = String(value.color).toLowerCase(); + if (!Number.isInteger(bristolType) || bristolType < 1 || bristolType > 7 || !COLORS.has(color)) { + throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.'); + } + const imageQuality = String(value.imageQuality || '').toLowerCase(); + if (!QUALITIES.has(imageQuality)) throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.'); return { status: 'suggestion', isStool: true, diff --git a/src/vision-corpus.js b/src/vision-corpus.js new file mode 100644 index 0000000..f9ecc94 --- /dev/null +++ b/src/vision-corpus.js @@ -0,0 +1,111 @@ +// Contract for the deterministic vision corpus under tests/fixtures/vision-corpus. +// +// The corpus exists to prove one claim continuously: every provider response Timmy +// receives is either a schema-valid visible-field suggestion or it fails closed. +// Fixtures are synthetic images generated by scripts/build_vision_corpus.py; no real +// medical imagery, personal data, or third-party content is redistributable here. +import { readFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { join } from 'node:path'; +import { parseVisionResponse, VISION_FAILURE_CODES } from './analysis.js'; + +export const VISION_CORPUS_LICENSE = 'CC0-1.0'; + +// Case classes the corpus must always cover, from issue #18. +export const VISION_CORPUS_CLASSES = Object.freeze([ + 'stool', + 'non-stool', + 'blur', + 'glare', + 'empty-bowl', + 'malformed-response', +]); + +export const VISION_CORPUS_OUTCOMES = Object.freeze(['suggestion', 'abstained', 'fail_closed']); + +const ALLOWED_CLASSES = new Set(VISION_CORPUS_CLASSES); + +export function validateCorpusManifest(manifest) { + if (!manifest || typeof manifest !== 'object') throw new Error('vision corpus manifest must be an object'); + if (manifest.schemaVersion !== 1) throw new Error('vision corpus manifest must declare schemaVersion 1'); + if (manifest.license !== VISION_CORPUS_LICENSE) { + throw new Error(`vision corpus must be redistributable under ${VISION_CORPUS_LICENSE}`); + } + if (!Array.isArray(manifest.cases) || manifest.cases.length === 0) throw new Error('vision corpus must list cases'); + const ids = new Set(); + for (const entry of manifest.cases) { + if (!entry?.id) throw new Error('every corpus case needs an id'); + if (ids.has(entry.id)) throw new Error(`duplicate corpus case: ${entry.id}`); + ids.add(entry.id); + if (!ALLOWED_CLASSES.has(entry.caseClass)) throw new Error(`${entry.id} has an unsupported caseClass: ${entry.caseClass}`); + if (entry.synthetic !== true) throw new Error(`${entry.id} must be a synthetic fixture`); + if (entry.license !== VISION_CORPUS_LICENSE) throw new Error(`${entry.id} must record its ${VISION_CORPUS_LICENSE} license`); + if (typeof entry.provenance !== 'string' || !entry.provenance.trim()) throw new Error(`${entry.id} must record provenance`); + if (!/^[a-f0-9]{64}$/.test(String(entry.sha256 || ''))) throw new Error(`${entry.id} must record a sha256`); + if (!VISION_CORPUS_OUTCOMES.includes(entry.expectedOutcome)) { + throw new Error(`${entry.id} must declare an expectedOutcome from ${VISION_CORPUS_OUTCOMES.join(', ')}`); + } + if (!('providerContent' in entry)) throw new Error(`${entry.id} must record the provider response it replays`); + } + for (const required of VISION_CORPUS_CLASSES) { + if (!manifest.cases.some(entry => entry.caseClass === required)) { + throw new Error(`vision corpus is missing the required ${required} case`); + } + } + return manifest; +} + +export function classifyVisionOutcome(providerContent) { + let parsed; + try { + parsed = parseVisionResponse(providerContent); + } catch (error) { + const reason = error?.code === VISION_FAILURE_CODES.invalidJson + ? 'unparsable_content' + : error?.code === VISION_FAILURE_CODES.invalidFields + ? 'out_of_range_or_missing_fields' + : 'unexpected_error'; + return { outcome: 'fail_closed', reason, detail: error.message }; + } + if (parsed.status === 'suggestion') { + return { + outcome: 'suggestion', + reason: 'schema_valid_suggestion', + bristolType: parsed.bristolType, + color: parsed.color, + confidence: parsed.confidence, + }; + } + if (parsed.status === 'needs_user_input') { + return { + outcome: 'abstained', + reason: parsed.abstainedFor, + detail: parsed.reason, + }; + } + return { outcome: 'fail_closed', reason: 'unparsable_content', detail: 'The response could not be used.' }; +} + +export async function loadVisionCorpus(directory) { + const manifestPath = join(directory, 'manifest.json'); + let manifest; + try { + manifest = JSON.parse(await readFile(manifestPath, 'utf8')); + } catch (error) { + throw new Error(`vision corpus manifest could not be read at ${manifestPath}: ${error.message}`); + } + validateCorpusManifest(manifest); + for (const entry of manifest.cases) { + let bytes; + try { + bytes = await readFile(join(directory, entry.file)); + } catch { + throw new Error(`${entry.id} is missing its fixture ${entry.file}`); + } + const digest = createHash('sha256').update(bytes).digest('hex'); + if (digest !== entry.sha256 || bytes.length !== entry.bytes) { + throw new Error(`${entry.id} fixture ${entry.file} does not match its recorded sha256; regenerate the corpus instead of editing it`); + } + } + return { ...manifest, directory }; +} diff --git a/src/vision-service.js b/src/vision-service.js index 6edb789..6453701 100644 --- a/src/vision-service.js +++ b/src/vision-service.js @@ -1,4 +1,17 @@ -import { buildVisionRequest, parseVisionResponse, validatePhotoPayload } from './analysis.js'; +import { buildVisionRequest, parseVisionResponse, validatePhotoPayload, VISION_FAILURE_CODES } from './analysis.js'; + +// One place decides what a failed analysis request returns, so the browser copy is testable and a +// provider's own words never reach the user. +const FAIL_CLOSED_COPY = 'No suggestion was applied. Continue manually, or try a clearer photo.'; +const INPUT_REJECTION = /consent|JPEG|PNG|WebP|empty|too large/i; + +export function visionFailureResponse(error) { + const message = String(error?.message || ''); + if (INPUT_REJECTION.test(message)) return { status: 400, body: { error: message } }; + const schemaFailure = error?.code === VISION_FAILURE_CODES.invalidJson || error?.code === VISION_FAILURE_CODES.invalidFields; + if (schemaFailure) return { status: 503, body: { error: FAIL_CLOSED_COPY } }; + return { status: 503, body: { error: message || 'AI analysis is temporarily unavailable. Continue manually.' } }; +} function providerEndpoint(baseUrl) { let url; diff --git a/tests/analysis.test.js b/tests/analysis.test.js index 56e130e..20a4d39 100644 --- a/tests/analysis.test.js +++ b/tests/analysis.test.js @@ -37,14 +37,17 @@ test('validates a confident visual suggestion without inventing nonvisual fields }); test('fails closed on a non-stool or low-confidence image', () => { + // abstainedFor names which abstention happened, so callers never have to parse prose. assert.deepEqual(parseVisionResponse({ isStool: false, confidence: 0.9 }), { status: 'needs_user_input', isStool: false, + abstainedFor: 'not_stool', reason: 'The image does not clearly show stool.', }); assert.deepEqual(parseVisionResponse({ isStool: true, bristolType: 4, color: 'brown', confidence: 0.31 }), { status: 'needs_user_input', isStool: true, + abstainedFor: 'low_confidence', reason: 'The image is too uncertain to prefill safely.', }); }); @@ -78,3 +81,21 @@ test('builds a structured multimodal request that forbids diagnosis and nonvisua assert.match(prompt, /not a diagnosis/i); assert.equal(body.messages[0].content.find(part => part.type === 'image_url').image_url.url, 'data:image/jpeg;base64,YQ=='); }); + +test('abstains instead of failing when a schema-valid response carries no readable visual fields', () => { + // The declared JSON schema allows a null form and color whenever the model cannot read them, + // so a schema-valid response must come back as an abstention — never as a provider failure. + assert.deepEqual(parseVisionResponse({ + isStool: true, + bristolType: null, + color: null, + confidence: 0.66, + imageQuality: 'poor', + observations: 'Large specular highlight hides the surface.', + }), { + status: 'needs_user_input', + isStool: true, + abstainedFor: 'unusable_visual_fields', + reason: 'The photo does not show a form Timmy can read safely.', + }); +}); diff --git a/tests/ci-workflow.test.js b/tests/ci-workflow.test.js index 9d346eb..9852962 100644 --- a/tests/ci-workflow.test.js +++ b/tests/ci-workflow.test.js @@ -98,3 +98,24 @@ test('diff hygiene checks only the pull request or latest commit range', async ( assert.match(script, /HEAD\^\.\.HEAD/); assert.doesNotMatch(script, /hash-object -t tree/); }); + +test('the vision corpus and its refusal boundary are unit-, browser-, and release-gated', async () => { + const [packageJsonText, workflow, builder] = await Promise.all([ + readFile(packagePath, 'utf8'), + readFile(workflowPath, 'utf8'), + readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'), + ]); + const packageJson = JSON.parse(packageJsonText); + + assert.match(packageJson.scripts.test, /tests\/vision-corpus\.test\.js/); + assert.match(packageJson.scripts.test, /tests\/vision-boundary\.test\.js/); + assert.equal(packageJson.scripts['test:vision-refusal'], 'node tests/vision-refusal.acceptance.mjs'); + assert.match(packageJson.scripts['check:syntax'], /node --check src\/vision-corpus\.js/); + assert.match(packageJson.scripts['check:syntax'], /node --check tests\/vision-refusal\.acceptance\.mjs/); + assert.match(packageJson.scripts['check:syntax'], /py_compile[^&]*scripts\/build_vision_corpus\.py/); + assert.match(workflow, /npm run test:vision-refusal/); + // The release builder must verify the committed corpus hashes and run the refusal suite itself. + assert.match(builder, /"test:vision-refusal"/); + assert.match(builder, /"vision_corpus_refusal": "passed"/); + assert.match(builder, /build_vision_corpus\.py", "--verify"/); +}); diff --git a/tests/fixtures/vision-corpus/README.md b/tests/fixtures/vision-corpus/README.md new file mode 100644 index 0000000..bb28b25 --- /dev/null +++ b/tests/fixtures/vision-corpus/README.md @@ -0,0 +1,48 @@ +# Vision corpus (issue #18) + +A committed, deterministic corpus that proves one claim on every run: **every reply Timmy receives is +either a schema-valid visible-field suggestion or it fails closed.** Nothing is guessed, and a +refusal never carries a form or a colour. + +## Licence and provenance + +- All fixtures are licensed **CC0-1.0**. +- Every fixture is a synthetic image drawn from fixed integer math by + [`scripts/build_vision_corpus.py`](../../../scripts/build_vision_corpus.py). There is no camera, no + human subject, no third-party content, and no real medical imagery anywhere in this directory. +- `manifest.json` records each fixture's `sha256`, byte length, provenance, licence, case class, the + provider reply it replays, and the outcome that reply must produce. +- The hashes are an integrity receipt: `loadVisionCorpus()` refuses a fixture whose bytes no longer + match the manifest, so the corpus cannot be quietly edited into a different test. + +## Coverage + +| Case | Class | Replayed provider reply | Required outcome | +|---|---|---|---| +| `stool-like-formed` | stool | valid suggestion, confidence 0.83 | `suggestion` | +| `non-stool-scene` | non-stool | `isStool: false` | `abstained` (`not_stool`) | +| `blur-low-detail` | blur | confidence 0.41 | `abstained` (`low_confidence`) | +| `glare-overexposed` | glare | schema-valid with null form and colour | `abstained` (`unusable_visual_fields`) | +| `empty-bowl` | empty-bowl | `isStool: false` | `abstained` (`not_stool`) | +| `malformed-response` | malformed-response | plain prose, not JSON | `fail_closed` (`unparsable_content`) | +| `out-of-range-fields` | malformed-response | Bristol 9, colour `purple`, confidence 1.4 | `fail_closed` (`out_of_range_or_missing_fields`) | + +## Where the corpus is enforced + +- `tests/vision-corpus.test.js` — corpus contract, hash integrity, and conformance of every reply. +- `tests/vision-boundary.test.js` — the corpus replayed through the real provider call path and + through a running `server.mjs`, asserting status codes and that no image bytes or credentials + appear in a fail-closed body. +- `tests/vision-refusal.acceptance.mjs` — the same refusals in a real browser. +- `scripts/record_release_demo.mjs` — the release demo replays corpus cases through the real analysis + code path, so the video shows the status the server would actually send. + +## Regenerating + +```bash +python3 scripts/build_vision_corpus.py # rewrite fixtures and manifest.json +python3 scripts/build_vision_corpus.py --verify # verify committed fixtures against the manifest +``` + +The release builder runs `--verify` before anything is packaged. Regenerate deliberately and commit +the new hashes; never hand-edit a fixture or a recorded reply. diff --git a/tests/fixtures/vision-corpus/blur-low-detail.png b/tests/fixtures/vision-corpus/blur-low-detail.png new file mode 100644 index 0000000..c0c1427 Binary files /dev/null and b/tests/fixtures/vision-corpus/blur-low-detail.png differ diff --git a/tests/fixtures/vision-corpus/empty-bowl.png b/tests/fixtures/vision-corpus/empty-bowl.png new file mode 100644 index 0000000..7a9bbc1 Binary files /dev/null and b/tests/fixtures/vision-corpus/empty-bowl.png differ diff --git a/tests/fixtures/vision-corpus/glare-overexposed.png b/tests/fixtures/vision-corpus/glare-overexposed.png new file mode 100644 index 0000000..2ad3dd8 Binary files /dev/null and b/tests/fixtures/vision-corpus/glare-overexposed.png differ diff --git a/tests/fixtures/vision-corpus/malformed-response.png b/tests/fixtures/vision-corpus/malformed-response.png new file mode 100644 index 0000000..d467190 Binary files /dev/null and b/tests/fixtures/vision-corpus/malformed-response.png differ diff --git a/tests/fixtures/vision-corpus/manifest.json b/tests/fixtures/vision-corpus/manifest.json new file mode 100644 index 0000000..8361c54 --- /dev/null +++ b/tests/fixtures/vision-corpus/manifest.json @@ -0,0 +1,120 @@ +{ + "schemaVersion": 1, + "generator": "scripts/build_vision_corpus.py", + "license": "CC0-1.0", + "note": "Synthetic fixtures only. No real medical imagery, no personal data, and no third-party content.", + "cases": [ + { + "id": "stool-like-formed", + "caseClass": "stool", + "file": "stool-like-formed.png", + "bytes": 2414, + "sha256": "4bb50ec6fc4ef7e9c83ed3b7ba69cdf5e73ef74b9f05e97898892bc852df970a", + "width": 640, + "height": 480, + "format": "png", + "synthetic": true, + "license": "CC0-1.0", + "provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.", + "expectedOutcome": "suggestion", + "expectedReason": "schema_valid_suggestion", + "providerContent": "{\"isStool\":true,\"bristolType\":4,\"color\":\"brown\",\"confidence\":0.83,\"imageQuality\":\"good\",\"observations\":\"Smooth, formed appearance.\"}" + }, + { + "id": "non-stool-scene", + "caseClass": "non-stool", + "file": "non-stool-scene.png", + "bytes": 2406, + "sha256": "4d12150073856c1d7dfb9332049182ee0145da6ad50f6964a7273c8c61f21f91", + "width": 640, + "height": 480, + "format": "png", + "synthetic": true, + "license": "CC0-1.0", + "provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.", + "expectedOutcome": "abstained", + "expectedReason": "not_stool", + "providerContent": "{\"isStool\":false,\"bristolType\":null,\"color\":null,\"confidence\":0.97,\"imageQuality\":\"good\",\"observations\":\"Flatware on a tray.\"}" + }, + { + "id": "blur-low-detail", + "caseClass": "blur", + "file": "blur-low-detail.png", + "bytes": 21598, + "sha256": "17000163662a42fd434bb2b2428d9e5a2156da08cbc82210e41dfc6fa7e24852", + "width": 640, + "height": 480, + "format": "png", + "synthetic": true, + "license": "CC0-1.0", + "provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.", + "expectedOutcome": "abstained", + "expectedReason": "low_confidence", + "providerContent": "{\"isStool\":true,\"bristolType\":4,\"color\":\"brown\",\"confidence\":0.41,\"imageQuality\":\"poor\",\"observations\":\"Form is unclear.\"}" + }, + { + "id": "glare-overexposed", + "caseClass": "glare", + "file": "glare-overexposed.png", + "bytes": 3140, + "sha256": "1dc87a80951dd244c9831cba8b352e24718cf5f103f4dfd0a30e8c8c1c599f95", + "width": 640, + "height": 480, + "format": "png", + "synthetic": true, + "license": "CC0-1.0", + "provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.", + "expectedOutcome": "abstained", + "expectedReason": "unusable_visual_fields", + "providerContent": "{\"isStool\":true,\"bristolType\":null,\"color\":null,\"confidence\":0.66,\"imageQuality\":\"poor\",\"observations\":\"Large specular highlight hides the surface.\"}" + }, + { + "id": "empty-bowl", + "caseClass": "empty-bowl", + "file": "empty-bowl.png", + "bytes": 3479, + "sha256": "ab443d1d9d28338e5c9f48054e7989cfce59ae25e2a9008be31cd9223be6abc4", + "width": 640, + "height": 480, + "format": "png", + "synthetic": true, + "license": "CC0-1.0", + "provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.", + "expectedOutcome": "abstained", + "expectedReason": "not_stool", + "providerContent": "{\"isStool\":false,\"bristolType\":null,\"color\":null,\"confidence\":0.93,\"imageQuality\":\"good\",\"observations\":\"Empty container.\"}" + }, + { + "id": "malformed-response", + "caseClass": "malformed-response", + "file": "malformed-response.png", + "bytes": 2341, + "sha256": "634d16337879783b307d0ecf84348cfdc07950f6893095267901383f2efc83c2", + "width": 640, + "height": 480, + "format": "png", + "synthetic": true, + "license": "CC0-1.0", + "provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.", + "expectedOutcome": "fail_closed", + "expectedReason": "unparsable_content", + "providerContent": "Sure! It looks like a Type 4 to me. Hope that helps." + }, + { + "id": "out-of-range-fields", + "caseClass": "malformed-response", + "file": "out-of-range-fields.png", + "bytes": 2375, + "sha256": "ac20ca9f3c9cbd118d3a2957110505d049a85861f97d361dcfbc4787960a5f8f", + "width": 640, + "height": 480, + "format": "png", + "synthetic": true, + "license": "CC0-1.0", + "provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.", + "expectedOutcome": "fail_closed", + "expectedReason": "out_of_range_or_missing_fields", + "providerContent": "{\"isStool\":true,\"bristolType\":9,\"color\":\"purple\",\"confidence\":1.4,\"imageQuality\":\"excellent\",\"observations\":\"Confident.\"}" + } + ] +} diff --git a/tests/fixtures/vision-corpus/non-stool-scene.png b/tests/fixtures/vision-corpus/non-stool-scene.png new file mode 100644 index 0000000..df123f5 Binary files /dev/null and b/tests/fixtures/vision-corpus/non-stool-scene.png differ diff --git a/tests/fixtures/vision-corpus/out-of-range-fields.png b/tests/fixtures/vision-corpus/out-of-range-fields.png new file mode 100644 index 0000000..56ff9c5 Binary files /dev/null and b/tests/fixtures/vision-corpus/out-of-range-fields.png differ diff --git a/tests/fixtures/vision-corpus/stool-like-formed.png b/tests/fixtures/vision-corpus/stool-like-formed.png new file mode 100644 index 0000000..fa970fe Binary files /dev/null and b/tests/fixtures/vision-corpus/stool-like-formed.png differ diff --git a/tests/helpers/vision-harness.mjs b/tests/helpers/vision-harness.mjs new file mode 100644 index 0000000..f0b000c --- /dev/null +++ b/tests/helpers/vision-harness.mjs @@ -0,0 +1,80 @@ +// Shared harness for driving the real /api/analyze boundary with recorded corpus responses. +// +// Nothing here fakes the application code: the test only decides which recorded provider +// response a stub OpenAI-compatible worker returns, and then talks to the real server. +import http from 'node:http'; +import { readFile } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { join } from 'node:path'; + +export const repoRoot = fileURLToPath(new URL('../..', import.meta.url)); +export const corpusDir = fileURLToPath(new URL('../fixtures/vision-corpus/', import.meta.url)); +export const CORPUS_MODEL = 'corpus-model'; + +export function caseFor(corpus, id) { + const entry = corpus.cases.find(item => item.id === id); + if (!entry) throw new Error(`corpus case ${id} is missing`); + return entry; +} + +export async function corpusPayload(entry) { + const bytes = await readFile(join(corpusDir, entry.file)); + return { imageDataUrl: `data:image/png;base64,${bytes.toString('base64')}`, consent: true }; +} + +/** Stub OpenAI-compatible worker whose answer the test selects per request. */ +export function startProvider() { + const state = { content: '', requests: [] }; + const server = http.createServer((req, res) => { + if (req.url.endsWith('/models')) { + res.writeHead(200, { 'content-type': 'application/json' }); + return res.end(JSON.stringify({ data: [{ id: CORPUS_MODEL }] })); + } + let body = ''; + req.on('data', chunk => { body += chunk; }); + req.on('end', () => { + state.requests.push(body); + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ choices: [{ message: { content: state.content } }] })); + }); + }); + return new Promise(resolve => server.listen(0, '127.0.0.1', () => resolve({ + state, + port: server.address().port, + close: () => new Promise(done => server.close(done)), + }))); +} + +/** Real server.mjs, vision enabled, pointed at the stub worker. */ +export async function startServer({ providerPort, port = 4199 }) { + const child = spawn(process.execPath, ['server.mjs'], { + cwd: repoRoot, + env: { + ...process.env, + HOST: '127.0.0.1', + PORT: String(port), + TIMMY_VISION_ENABLED: 'true', + TIMMY_VISION_PROFILE: 'selfhost', + TIMMY_VISION_BASE_URL: `http://127.0.0.1:${providerPort}/v1`, + TIMMY_VISION_MODEL: CORPUS_MODEL, + TIMMY_VISION_TIMEOUT_MS: '5000', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + for (let attempt = 0; attempt < 80; attempt += 1) { + try { + const probe = await fetch(`http://127.0.0.1:${port}/api/healthz`); + if (probe.ok) return child; + } catch {} + await new Promise(resolve => setTimeout(resolve, 100)); + } + child.kill(); + throw new Error('acceptance server did not become ready'); +} + +export async function stopServer(child) { + if (!child || child.exitCode !== null) return; + child.kill(); + await new Promise(resolve => child.once('exit', resolve)); +} diff --git a/tests/release-demo.test.js b/tests/release-demo.test.js index e09f694..1d71301 100644 --- a/tests/release-demo.test.js +++ b/tests/release-demo.test.js @@ -148,3 +148,25 @@ test('release demo captions expire and the contact sheet spans the whole recordi assert.match(builder, /12 \/ demo_duration/); assert.match(builder, /Release demo contact sheet was not produced/); }); + +test('release demo visibly exercises the corpus refusal paths instead of only the happy path', async () => { + const demo = await readFile(demoPath, 'utf8'); + const builder = await readFile(builderPath, 'utf8'); + + // The refusal responses must come from the recorded corpus and the real analysis code path, + // never from copy pasted into the recording script. + assert.match(demo, /loadVisionCorpus/); + assert.match(demo, /visionFailureResponse/); + assert.match(demo, /corpusAnalyzeResult/); + assert.match(demo, /tests\/fixtures\/vision-corpus\/non-stool-scene\.png/); + assert.match(demo, /tests\/fixtures\/vision-corpus\/out-of-range-fields\.png/); + assert.match(demo, /getByText\(\/No confident match\/i\)/); + assert.match(demo, /couldn’t analyze that safely/i); + + assert.match(demo, /A non-stool photo gets a refusal, never a guess/); + assert.match(demo, /Unreadable visual fields abstain as well/); + assert.match(demo, /An out-of-range answer fails closed/); + assert.match(demo, /no real photos in the corpus|no real photos in the recording/); + // The refusal copy the user sees is gated on camera, so a release cannot drop it silently. + assert.match(builder, /An out-of-range answer fails closed/); +}); diff --git a/tests/vision-boundary.test.js b/tests/vision-boundary.test.js new file mode 100644 index 0000000..3d78d9b --- /dev/null +++ b/tests/vision-boundary.test.js @@ -0,0 +1,100 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { analyzePhoto, visionFailureResponse } from '../src/vision-service.js'; +import { loadVisionCorpus } from '../src/vision-corpus.js'; +import { CORPUS_MODEL, caseFor, corpusDir, corpusPayload, startProvider, startServer, stopServer } from './helpers/vision-harness.mjs'; + +const FAIL_CLOSED_COPY = /no suggestion was applied/i; + +test('a malformed provider response fails closed with user-safe copy instead of echoing provider detail', () => { + const failure = visionFailureResponse(Object.assign(new Error('Invalid AI response: visual fields are out of range.'), { code: 'invalid_fields' })); + + assert.equal(failure.status, 503); + assert.match(failure.body.error, FAIL_CLOSED_COPY); + assert.doesNotMatch(failure.body.error, /invalid ai response|out of range/i); + + const consent = visionFailureResponse(new Error('Explicit consent is required before AI analysis.')); + assert.equal(consent.status, 400); + assert.match(consent.body.error, /consent/i); + + const offline = visionFailureResponse(new Error('AI analysis is temporarily unavailable. Continue manually.')); + assert.equal(offline.status, 503); + assert.match(offline.body.error, /temporarily unavailable/i); +}); + +test('the corpus replays through the real provider call path without inventing a suggestion', async () => { + const corpus = await loadVisionCorpus(corpusDir); + for (const entry of corpus.cases) { + const fetchImpl = async () => ({ ok: true, json: async () => ({ choices: [{ message: { content: entry.providerContent } }] }) }); + const options = { + payload: { imageDataUrl: 'data:image/png;base64,YQ==', consent: true }, + fetchImpl, + config: { baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'corpus-key', model: CORPUS_MODEL }, + }; + if (entry.expectedOutcome === 'suggestion') { + const result = await analyzePhoto(options); + assert.equal(result.status, 'suggestion', `${entry.id} must return a suggestion`); + assert.equal(result.imageQuality, 'good'); + } else if (entry.expectedOutcome === 'abstained') { + const result = await analyzePhoto(options); + assert.equal(result.status, 'needs_user_input', `${entry.id} must abstain`); + assert.equal(result.abstainedFor, entry.expectedReason, `${entry.id} must name its abstention`); + assert.equal('bristolType' in result, false, `${entry.id} must not prefill a form`); + } else { + await assert.rejects(() => analyzePhoto(options), (error) => { + assert.equal(visionFailureResponse(error).status, 503, `${entry.id} must fail closed`); + return true; + }); + } + } +}); + +test('the running server answers every corpus case with a fail-closed status and no leaked image bytes', async () => { + const corpus = await loadVisionCorpus(corpusDir); + const provider = await startProvider(); + const server = await startServer({ providerPort: provider.port }); + const postAnalyze = async (payload) => { + const response = await fetch('http://127.0.0.1:4199/api/analyze', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(payload), + }); + return { status: response.status, body: await response.json() }; + }; + try { + const suggestion = caseFor(corpus, 'stool-like-formed'); + provider.state.content = suggestion.providerContent; + const payload = await corpusPayload(suggestion); + const ok = await postAnalyze(payload); + assert.equal(ok.status, 200, 'a schema-valid suggestion is the only case that returns 200 with fields'); + assert.equal(ok.body.bristolType, 4); + + const glare = caseFor(corpus, 'glare-overexposed'); + provider.state.content = glare.providerContent; + const abstained = await postAnalyze(await corpusPayload(glare)); + assert.equal(abstained.status, 200, 'a schema-valid response with unreadable fields abstains, it is not a server failure'); + assert.equal(abstained.body.status, 'needs_user_input'); + assert.equal(abstained.body.abstainedFor, 'unusable_visual_fields'); + assert.equal('bristolType' in abstained.body, false); + + const malformed = caseFor(corpus, 'malformed-response'); + provider.state.content = malformed.providerContent; + const unparsable = await postAnalyze(await corpusPayload(malformed)); + assert.equal(unparsable.status, 503, 'unparsable provider content must not be served as a suggestion'); + assert.match(unparsable.body.error, FAIL_CLOSED_COPY); + assert.doesNotMatch(JSON.stringify(unparsable.body), /base64|data:image|corpus-key|Bearer/i); + + const outOfRangeCase = caseFor(corpus, 'out-of-range-fields'); + provider.state.content = outOfRangeCase.providerContent; + const outOfRange = await postAnalyze(await corpusPayload(outOfRangeCase)); + assert.equal(outOfRange.status, 503, 'out-of-range fields must not be served as a suggestion'); + assert.match(outOfRange.body.error, FAIL_CLOSED_COPY); + + const noConsent = await postAnalyze({ imageDataUrl: payload.imageDataUrl, consent: false }); + assert.equal(noConsent.status, 400, 'analysis without explicit consent is a client error'); + assert.doesNotMatch(JSON.stringify(noConsent.body), /base64|data:image/i); + } finally { + await stopServer(server); + await provider.close(); + } +}); diff --git a/tests/vision-corpus.test.js b/tests/vision-corpus.test.js new file mode 100644 index 0000000..1e4661e --- /dev/null +++ b/tests/vision-corpus.test.js @@ -0,0 +1,85 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { cp, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { classifyVisionOutcome, loadVisionCorpus, VISION_CORPUS_CLASSES, VISION_CORPUS_LICENSE, VISION_CORPUS_OUTCOMES } from '../src/vision-corpus.js'; + +const corpusDir = fileURLToPath(new URL('./fixtures/vision-corpus/', import.meta.url)); + +async function scratchCorpus() { + const directory = await mkdtemp(join(tmpdir(), 'timmy-corpus-')); + await cp(corpusDir, directory, { recursive: true }); + return directory; +} + +test('vision corpus ships licensed synthetic fixtures covering every required visual case class', async () => { + const corpus = await loadVisionCorpus(corpusDir); + + assert.equal(corpus.license, VISION_CORPUS_LICENSE); + const classes = new Set(corpus.cases.map(entry => entry.caseClass)); + for (const required of VISION_CORPUS_CLASSES) { + assert.ok(classes.has(required), `corpus must cover the ${required} case`); + } + for (const entry of corpus.cases) { + assert.equal(entry.synthetic, true, `${entry.id} must be synthetic; no real medical image is redistributable`); + assert.equal(entry.license, VISION_CORPUS_LICENSE); + assert.match(entry.provenance, /\S/, `${entry.id} must record where it came from`); + assert.match(entry.sha256, /^[a-f0-9]{64}$/, `${entry.id} must record its content hash`); + } +}); + +test('vision corpus rejects fixtures whose bytes no longer match the recorded hash', async () => { + const directory = await scratchCorpus(); + try { + const manifest = JSON.parse(await readFile(join(directory, 'manifest.json'), 'utf8')); + const target = join(directory, manifest.cases[0].file); + const bytes = await readFile(target); + await writeFile(target, Buffer.concat([bytes, Buffer.from([0])])); + + await assert.rejects(() => loadVisionCorpus(directory), new RegExp(manifest.cases[0].id)); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('vision corpus rejects an unlicensed, non-synthetic, or incomplete manifest', async () => { + const directory = await scratchCorpus(); + try { + const manifestPath = join(directory, 'manifest.json'); + const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); + + manifest.cases[0].synthetic = false; + await writeFile(manifestPath, JSON.stringify(manifest, null, 2)); + await assert.rejects(() => loadVisionCorpus(directory), /synthetic/i); + + manifest.cases[0].synthetic = true; + manifest.cases = manifest.cases.filter(entry => entry.caseClass !== 'non-stool'); + await writeFile(manifestPath, JSON.stringify(manifest, null, 2)); + await assert.rejects(() => loadVisionCorpus(directory), /non-stool/); + + manifest.cases[0].license = 'all-rights-reserved'; + await writeFile(manifestPath, JSON.stringify(manifest, null, 2)); + await assert.rejects(() => loadVisionCorpus(directory), /CC0-1\.0/); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); + +test('every recorded corpus response is schema-valid or fails closed, exactly as declared', async () => { + const corpus = await loadVisionCorpus(corpusDir); + + for (const entry of corpus.cases) { + const result = classifyVisionOutcome(entry.providerContent); + assert.ok(VISION_CORPUS_OUTCOMES.includes(result.outcome), `${entry.id} produced an unknown outcome ${result.outcome}`); + assert.equal(result.outcome, entry.expectedOutcome, `${entry.id} must resolve to ${entry.expectedOutcome}`); + assert.equal(result.reason, entry.expectedReason, `${entry.id} must report ${entry.expectedReason}`); + // A response may never smuggle a visual claim out of a non-suggestion outcome. + if (result.outcome !== 'suggestion') { + assert.equal('bristolType' in result, false, `${entry.id} must not carry a suggested form`); + assert.equal('color' in result, false, `${entry.id} must not carry a suggested color`); + } + assert.doesNotMatch(JSON.stringify(result), /base64|data:image|Bear\s?er|api[_-]?key/i, `${entry.id} must not echo image bytes or credentials`); + } +}); diff --git a/tests/vision-refusal.acceptance.mjs b/tests/vision-refusal.acceptance.mjs new file mode 100644 index 0000000..ffd5d2f --- /dev/null +++ b/tests/vision-refusal.acceptance.mjs @@ -0,0 +1,90 @@ +// Browser acceptance for issue #18: the refusal paths the corpus proves must be visible in the app, +// not just in unit tests. The app talks to a real server.mjs with vision enabled, and the only thing +// the harness chooses is which recorded corpus response the stub worker returns. +import { chromium } from 'playwright'; +import assert from 'node:assert/strict'; +import { mkdir } from 'node:fs/promises'; +import { loadVisionCorpus } from '../src/vision-corpus.js'; +import { caseFor, corpusDir, startProvider, startServer, stopServer } from './helpers/vision-harness.mjs'; + +await mkdir('artifacts', { recursive: true }); +const corpus = await loadVisionCorpus(corpusDir); +const provider = await startProvider(); +const server = await startServer({ providerPort: provider.port, port: 4199 }); +const browser = await chromium.launch({ headless: true }); +const context = await browser.newContext({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 2, serviceWorkers: 'block' }); +const page = await context.newPage(); +const errors = []; +page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); +page.on('pageerror', error => errors.push(error.message)); +// A fail-closed 503 is the intended boundary response, so the browser logs it as a failed resource. +// It is expected exactly once; anything else is a real page error. +const EXPECTED_FAIL_CLOSED_LOG = /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/; + +async function closeSheet() { + await page.locator('#close-sheet').click(); + await page.locator('.scan-sheet').waitFor({ state: 'detached' }); +} + +async function analyzeCorpusCase(entry) { + provider.state.content = entry.providerContent; + await page.locator('[data-scan]').click(); + await page.getByText(/Self-hosted model ready/i).waitFor(); + await page.locator('#gallery-photo').setInputFiles(`tests/fixtures/vision-corpus/${entry.file}`); + await page.locator('#ai-consent').check(); + await page.locator('#analyze-photo').click(); + await page.locator('.scan-result').first().waitFor(); +} + +try { + await page.goto('http://127.0.0.1:4199', { waitUntil: 'networkidle' }); + await page.evaluate(() => localStorage.clear()); + await page.reload({ waitUntil: 'networkidle' }); + + // 1. Non-stool photo: no prefill, a plain refusal, and manual entry still one action away. + await analyzeCorpusCase(caseFor(corpus, 'non-stool-scene')); + assert.equal(await page.getByText(/No confident match/i).isVisible(), true, 'a non-stool photo must refuse'); + assert.equal(await page.locator('.ai-badge').count(), 0, 'a refusal must not carry an AI suggestion badge'); + assert.equal(await page.getByText('Type 4', { exact: true }).count(), 0, 'a refusal must not prefill a form'); + assert.match(await page.locator('.scan-result').innerText(), /does not clearly show stool/i); + const manual = page.locator('#manual-from-scan'); + assert.match(await manual.innerText(), /Choose the form yourself/i); + await page.screenshot({ path: 'artifacts/vision-refusal-non-stool-mobile.png' }); + await manual.click(); + await page.getByText('Pick the closest form').waitFor(); + assert.equal(await page.locator('.ai-prefill').count(), 0, 'refused analysis must not leave AI provenance in the log form'); + await closeSheet(); + + // 2. Glare with unreadable visual fields: schema-valid, so it abstains calmly instead of erroring. + await analyzeCorpusCase(caseFor(corpus, 'glare-overexposed')); + assert.equal(await page.getByText(/No confident match/i).isVisible(), true); + assert.match(await page.locator('.scan-result').innerText(), /does not show a form Timmy can read safely/i); + assert.equal(await page.getByText(/couldn’t analyze that safely/i).count(), 0, 'a schema-valid abstention is not a failure'); + await page.screenshot({ path: 'artifacts/vision-refusal-unreadable-fields-mobile.png' }); + await closeSheet(); + + // 3. Out-of-range model output: fails closed, and the copy says plainly that nothing was applied. + await analyzeCorpusCase(caseFor(corpus, 'out-of-range-fields')); + await page.getByText(/couldn’t analyze that safely/i).waitFor(); + assert.match(await page.locator('.scan-result').innerText(), /No suggestion was applied/i); + assert.doesNotMatch(await page.locator('.scan-result').innerText(), /out of range|Invalid AI response/i, 'provider detail must not reach the user'); + assert.equal(await page.locator('.ai-badge').count(), 0); + await page.screenshot({ path: 'artifacts/vision-refusal-fail-closed-mobile.png' }); + await closeSheet(); + + // 4. Positive control: the same corpus still yields a real suggestion when the response is usable. + await analyzeCorpusCase(caseFor(corpus, 'stool-like-formed')); + await page.getByText(/83% confidence/i).waitFor(); + assert.equal(await page.getByText('Type 4', { exact: true }).isVisible(), true); + await page.screenshot({ path: 'artifacts/vision-corpus-suggestion-mobile.png' }); + + const failClosedLogs = errors.filter(message => EXPECTED_FAIL_CLOSED_LOG.test(message)); + assert.equal(failClosedLogs.length, 1, 'the fail-closed response must reach the browser as a 503'); + assert.deepEqual(errors.filter(message => !EXPECTED_FAIL_CLOSED_LOG.test(message)), []); + console.log('PASS vision corpus refusal: non-stool, unreadable fields, and out-of-range replies never prefill; the usable reply still suggests'); +} finally { + await context.close(); + await browser.close(); + await stopServer(server); + await provider.close(); +}