test: prove schema conformance and refusal with a committed vision corpus (#18)

The worker contract was only proven by one happy-path fixture and a handful of
inline assertions, so nothing continuously checked that a reply Timmy cannot use
is refused instead of guessed at.

Corpus: a CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl,
malformed response) generated deterministically by scripts/build_vision_corpus.py.
Every fixture records its sha256, provenance, licence, case class, the provider
reply it replays, and the outcome that reply must produce. loadVisionCorpus
refuses a fixture whose bytes no longer match the manifest, so the corpus cannot
be edited into a different test. No real medical imagery, camera, or third-party
content is involved.

Contract fixes the corpus exposed:
- A schema-valid reply with a null form and colour is now an abstention
  (abstainedFor: unusable_visual_fields) instead of a hard provider failure. The
  declared response schema permits those nulls, so treating the reply as an error
  was wrong; it now abstains calmly and still prefill nothing.
- Fail-closed replies carry a machine-readable code (invalid_json, invalid_fields)
  instead of prose matching, and visionFailureResponse() maps them to plain,
  user-safe copy with no provider text, image bytes, or credentials echoed. A
  running server now returns that copy for unparsable and out-of-range replies.

Evidence: unit corpus contract and hash integrity, corpus-wide conformance
through the real analysis path, the same corpus through a running server.mjs with
a stub worker (200 abstain vs 503 fail-closed, no leaked bytes), and a real
browser acceptance run for the refusal UI. The release demo replays three corpus
cases through the real code path so the video shows the status the server would
actually send, with the on-camera caption gated by the release builder.

scripts/build_release.py

@@ -37,6 +37,7 @@ DEMO_REQUIRED_CAPTIONS = (
     "Delete all local data clears the ledger in this browser; there is no server copy of it to keep",
     "Hermes Agent connected — free-text, contextual, and server-side",
     "Urgent language is intercepted deterministically before Hermes",
+    "An out-of-range answer fails closed: nothing is applied and no provider detail reaches you",
 )
 
 
@@ -108,6 +109,8 @@ def main() -> int:
         run(["git", "checkout", "--quiet", commit], tree)
         run(["npm", "ci", "--ignore-scripts"], tree)
         run(["npm", "test"], tree)
+        # The committed corpus hashes are an integrity receipt: verify them before anything is built.
+        run(["python3", "scripts/build_vision_corpus.py", "--verify"], tree)
         server_env = dict(os.environ)
         server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false"})
         server_log = (release_dir / "acceptance-server.log").open("w", encoding="utf-8")
@@ -131,6 +134,7 @@ def main() -> int:
             run(["npm", "run", "test:manual-fallback"], tree)
             run(["npm", "run", "test:provisional"], tree)
             run(["npm", "run", "test:followup"], tree)
+            run(["npm", "run", "test:vision-refusal"], tree)
             demo_raw = release_dir / f"timmy-talking-turd-{version}-demo.raw.webm"
             caption_receipt = release_dir / f"timmy-talking-turd-{version}-demo-captions.json"
             demo_env = dict(server_env)
@@ -172,11 +176,11 @@ def main() -> int:
         if not contact_sheet.exists() or contact_sheet.stat().st_size == 0:
             raise SystemExit("Release demo contact sheet was not produced")
         run(["npm", "audit", "--audit-level=high"], tree)
-        for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "tests/staging.acceptance.mjs"):
+        for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs"):
             run(["node", "--check", file], tree)
         run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree)
         run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree)
-        run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py"], tree)
+        run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py", "scripts/build_vision_corpus.py"], tree)
         run(["git", "diff", "--check", commit], tree)
 
         names = tracked_files(tree)
@@ -203,6 +207,7 @@ def main() -> int:
             "- Vision profiles: hosted and self-hosted SmolVLM2 bootstrap\n"
             "- AI suggestions stay provisional: suggested, confirmed, and corrected states are distinct and screen-reader described; correcting one clears the AI-confirmed state without losing user context\n"
             "- Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions (difficulty passing, soreness, visible blood, truly black/tarry versus dark brown, hydration); mild reports get non-diagnostic self-care, and a confirmed red-flag answer lands on the same deterministic escalation path as manual logging\n"
+            "- Vision corpus: a committed CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) proves every reply is schema-valid or fails closed; a schema-valid reply with unreadable fields abstains instead of erroring, and unparsable or out-of-range replies return plain copy with nothing applied\n"
             "- Safety: suggestions are observable-field assistance, never diagnosis; user confirmation is required\n"
             "- Known limitation: open-weight VLM acceptance is proven, Bristol accuracy is not clinically validated\n"
             "- Package excludes model weights, medical-image corpora, raw research pages, generated screenshots, and video production assets\n"
@@ -248,7 +253,7 @@ def main() -> int:
             "codec": "h264",
             "pixel_format": "yuv420p",
             "recorded_from_working_app": True,
-            "fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, offline vision status, and deterministic Hermes chat response",
+            "fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, and an out-of-range fail-closed reply, offline vision status, and deterministic Hermes chat response",
         },
         "gates": {
             "unit_security": "passed",
@@ -258,6 +263,7 @@ def main() -> int:
             "manual_fallback_draft_recovery": "passed",
             "provisional_suggestion_state": "passed",
             "hard_stool_followup": "passed",
+            "vision_corpus_refusal": "passed",
             "dependency_audit_high": "passed",
             "syntax": "passed",
             "secret_scan": "passed",

scripts/build_vision_corpus.py

@@ -0,0 +1,265 @@
+#!/usr/bin/env python3
+"""Generate the deterministic synthetic vision corpus for issue #18.
+
+Every fixture is drawn from fixed integer math: no camera, no human subject, no
+third-party content, and no randomness. That keeps the corpus redistributable and
+keeps the recorded SHA-256 hashes meaningful as an integrity receipt.
+
+Usage:
+  python3 scripts/build_vision_corpus.py            # (re)write fixtures and manifest.json
+  python3 scripts/build_vision_corpus.py --verify   # verify committed fixtures against the manifest
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path
+from typing import Any, Callable
+
+from PIL import Image, ImageDraw, ImageFilter
+
+LICENSE = "CC0-1.0"
+CORPUS_DIR = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "vision-corpus"
+WIDTH, HEIGHT = 640, 480
+PROVENANCE = "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content."
+
+OBSERVATION_STRINGS = {
+    "smooth": "Smooth, formed appearance.",
+    "flatware": "Flatware on a tray.",
+    "unclear": "Form is unclear.",
+    "highlight": "Large specular highlight hides the surface.",
+    "container": "Empty container.",
+    "confident": "Confident.",
+}
+
+
+def background(color: tuple[int, int, int]) -> Image.Image:
+    """A flat canvas with a coarse, deterministic shade band."""
+    image = Image.new("RGB", (WIDTH, HEIGHT), color)
+    draw = ImageDraw.Draw(image)
+    for index in range(4):
+        shade = tuple(min(255, channel + 6 * index) for channel in color)
+        draw.rectangle([0, index * 120, WIDTH, index * 120 + 119], fill=shade)
+    return image
+
+
+def blob(image: Image.Image, box: tuple[int, int, int, int], tone: tuple[int, int, int]) -> None:
+    draw = ImageDraw.Draw(image)
+    draw.ellipse(box, fill=tone)
+    left, top, right, bottom = box
+    inset = max(4, (right - left) // 6)
+    draw.ellipse([left + inset, top + inset, right - inset, bottom - inset],
+                 fill=tuple(min(255, channel + 10) for channel in tone))
+
+
+def stool_like() -> Image.Image:
+    image = background((226, 222, 212))
+    blob(image, (170, 150, 470, 330), (108, 76, 48))
+    return image
+
+
+def non_stool_scene() -> Image.Image:
+    image = background((198, 202, 208))
+    draw = ImageDraw.Draw(image)
+    draw.ellipse([120, 90, 520, 390], fill=(244, 244, 240))
+    draw.rectangle([60, 200, 300, 224], fill=(120, 126, 134))
+    draw.rectangle([340, 200, 580, 224], fill=(120, 126, 134))
+    return image
+
+
+def blur_low_detail() -> Image.Image:
+    return stool_like().filter(ImageFilter.GaussianBlur(5))
+
+
+def glare_overexposed() -> Image.Image:
+    image = background((222, 218, 208))
+    blob(image, (150, 140, 430, 300), (104, 74, 46))
+    draw = ImageDraw.Draw(image)
+    draw.polygon([(380, 200), (640, 120), (640, 360), (420, 330)], fill=(252, 252, 250))
+    draw.ellipse([470, 210, 620, 320], fill=(255, 255, 255))
+    return image
+
+
+def empty_bowl() -> Image.Image:
+    image = background((214, 210, 202))
+    draw = ImageDraw.Draw(image)
+    draw.ellipse([130, 90, 510, 400], fill=(238, 236, 230))
+    draw.ellipse([175, 130, 465, 360], fill=(206, 204, 198))
+    draw.ellipse([220, 170, 420, 320], fill=(232, 230, 224))
+    return image
+
+
+def unusable_tone() -> Image.Image:
+    image = background((202, 206, 198))
+    blob(image, (190, 160, 450, 320), (128, 118, 92))
+    return image
+
+
+def shaded_tone() -> Image.Image:
+    image = background((216, 214, 206))
+    blob(image, (180, 155, 460, 325), (96, 84, 52))
+    draw = ImageDraw.Draw(image)
+    draw.rectangle([0, 300, WIDTH, 306], fill=(180, 178, 170))
+    return image
+
+
+def content(payload: dict[str, object] | str) -> str:
+    return payload if isinstance(payload, str) else json.dumps(payload, separators=(",", ":"))
+
+
+def cases() -> list[dict[str, Any]]:
+    return [
+        {
+            "id": "stool-like-formed",
+            "caseClass": "stool",
+            "image": stool_like,
+            "expectedOutcome": "suggestion",
+            "expectedReason": "schema_valid_suggestion",
+            "providerContent": content({
+                "isStool": True, "bristolType": 4, "color": "brown", "confidence": 0.83,
+                "imageQuality": "good", "observations": OBSERVATION_STRINGS["smooth"],
+            }),
+        },
+        {
+            "id": "non-stool-scene",
+            "caseClass": "non-stool",
+            "image": non_stool_scene,
+            "expectedOutcome": "abstained",
+            "expectedReason": "not_stool",
+            "providerContent": content({
+                "isStool": False, "bristolType": None, "color": None, "confidence": 0.97,
+                "imageQuality": "good", "observations": OBSERVATION_STRINGS["flatware"],
+            }),
+        },
+        {
+            "id": "blur-low-detail",
+            "caseClass": "blur",
+            "image": blur_low_detail,
+            "expectedOutcome": "abstained",
+            "expectedReason": "low_confidence",
+            "providerContent": content({
+                "isStool": True, "bristolType": 4, "color": "brown", "confidence": 0.41,
+                "imageQuality": "poor", "observations": OBSERVATION_STRINGS["unclear"],
+            }),
+        },
+        {
+            "id": "glare-overexposed",
+            "caseClass": "glare",
+            "image": glare_overexposed,
+            "expectedOutcome": "abstained",
+            "expectedReason": "unusable_visual_fields",
+            "providerContent": content({
+                "isStool": True, "bristolType": None, "color": None, "confidence": 0.66,
+                "imageQuality": "poor", "observations": OBSERVATION_STRINGS["highlight"],
+            }),
+        },
+        {
+            "id": "empty-bowl",
+            "caseClass": "empty-bowl",
+            "image": empty_bowl,
+            "expectedOutcome": "abstained",
+            "expectedReason": "not_stool",
+            "providerContent": content({
+                "isStool": False, "bristolType": None, "color": None, "confidence": 0.93,
+                "imageQuality": "good", "observations": OBSERVATION_STRINGS["container"],
+            }),
+        },
+        {
+            "id": "malformed-response",
+            "caseClass": "malformed-response",
+            "image": unusable_tone,
+            "expectedOutcome": "fail_closed",
+            "expectedReason": "unparsable_content",
+            "providerContent": "Sure! It looks like a Type 4 to me. Hope that helps.",
+        },
+        {
+            "id": "out-of-range-fields",
+            "caseClass": "malformed-response",
+            "image": shaded_tone,
+            "expectedOutcome": "fail_closed",
+            "expectedReason": "out_of_range_or_missing_fields",
+            "providerContent": content({
+                "isStool": True, "bristolType": 9, "color": "purple", "confidence": 1.4,
+                "imageQuality": "excellent", "observations": OBSERVATION_STRINGS["confident"],
+            }),
+        },
+    ]
+
+
+def write_fixture(definition: dict[str, Any], directory: Path) -> dict[str, Any]:
+    name = f"{definition['id']}.png"
+    target = directory / name
+    factory: Callable[[], Image.Image] = definition["image"]
+    factory().save(target, "PNG", optimize=True)
+    data = target.read_bytes()
+    return {
+        "id": definition["id"],
+        "caseClass": definition["caseClass"],
+        "file": name,
+        "bytes": len(data),
+        "sha256": hashlib.sha256(data).hexdigest(),
+        "width": WIDTH,
+        "height": HEIGHT,
+        "format": "png",
+        "synthetic": True,
+        "license": LICENSE,
+        "provenance": PROVENANCE,
+        "expectedOutcome": definition["expectedOutcome"],
+        "expectedReason": definition["expectedReason"],
+        "providerContent": definition["providerContent"],
+    }
+
+
+def manifest(entries: list[dict[str, object]]) -> dict[str, object]:
+    return {
+        "schemaVersion": 1,
+        "generator": "scripts/build_vision_corpus.py",
+        "license": LICENSE,
+        "note": "Synthetic fixtures only. No real medical imagery, no personal data, and no third-party content.",
+        "cases": entries,
+    }
+
+
+def write(directory: Path) -> int:
+    directory.mkdir(parents=True, exist_ok=True)
+    entries = [write_fixture(definition, directory) for definition in cases()]
+    (directory / "manifest.json").write_text(json.dumps(manifest(entries), indent=2) + "\n", encoding="utf-8")
+    print(f"wrote {len(entries)} fixtures to {directory}")
+    return 0
+
+
+def verify(directory: Path) -> int:
+    manifest_path = directory / "manifest.json"
+    recorded = json.loads(manifest_path.read_text(encoding="utf-8"))
+    problems: list[str] = []
+    for entry in recorded.get("cases", []):
+        path = directory / str(entry.get("file"))
+        if not path.is_file():
+            problems.append(f"{entry.get('id')}: missing fixture {entry.get('file')}")
+            continue
+        data = path.read_bytes()
+        if hashlib.sha256(data).hexdigest() != entry.get("sha256"):
+            problems.append(f"{entry.get('id')}: bytes do not match the recorded sha256")
+        if len(data) != entry.get("bytes"):
+            problems.append(f"{entry.get('id')}: byte length does not match the manifest")
+    if problems:
+        for problem in problems:
+            print(problem, file=sys.stderr)
+        return 1
+    print(f"verified {len(recorded.get('cases', []))} corpus fixtures against manifest.json")
+    return 0
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--verify", action="store_true", help="check committed fixtures against the manifest instead of rewriting")
+    parser.add_argument("--directory", default=str(CORPUS_DIR))
+    args = parser.parse_args()
+    directory = Path(args.directory)
+    return verify(directory) if args.verify else write(directory)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/record_release_demo.mjs

@@ -1,6 +1,9 @@
 import { chromium } from 'playwright';
 import { mkdir, writeFile } from 'node:fs/promises';
 import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { analyzePhoto, visionFailureResponse } from '../src/vision-service.js';
+import { loadVisionCorpus } from '../src/vision-corpus.js';
 
 const output = path.resolve(process.argv[2] || 'release-demo.webm');
 const version = process.env.TIMMY_RELEASE_VERSION || 'review build';
@@ -13,6 +16,26 @@ const captionsShown = [];
 const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
 await mkdir(path.dirname(output), { recursive: true });
 
+// The refusal scene replays committed synthetic corpus cases through the real analysis code path,
+// so the recording shows the status and copy the server would actually send rather than copy
+// written for the video. No real photo is used anywhere in the corpus.
+const corpus = await loadVisionCorpus(fileURLToPath(new URL('../tests/fixtures/vision-corpus/', import.meta.url)));
+const corpusCase = id => corpus.cases.find(entry => entry.id === id);
+async function corpusAnalyzeResult(entry) {
+  const fetchImpl = async () => ({ ok: true, json: async () => ({ choices: [{ message: { content: entry.providerContent } }] }) });
+  try {
+    const result = await analyzePhoto({
+      payload: { imageDataUrl: 'data:image/png;base64,YQ==', consent: true },
+      fetchImpl,
+      config: { baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'corpus-replay', model: 'corpus-model' },
+    });
+    return { status: 200, body: result };
+  } catch (error) {
+    const failure = visionFailureResponse(error);
+    return { status: failure.status, body: failure.body };
+  }
+}
+
 const browser = await chromium.launch({ headless: true });
 const context = await browser.newContext({
   viewport: { width: 405, height: 720 },
@@ -39,20 +62,23 @@ await page.route('**/api/vision-status', route => route.fulfill({
     modelSeen: !visionOffline,
   }),
 }));
-await page.route('**/api/analyze', route => route.fulfill({
-  status: 200,
-  contentType: 'application/json',
-  body: JSON.stringify({
-    status: 'suggestion',
-    isStool: true,
-    bristolType: 4,
-    color: 'brown',
-    confidence: 0.83,
-    imageQuality: 'good',
-    observations: 'Smooth, formed appearance.',
-    warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
-  }),
-}));
+// The happy path keeps one deterministic suggestion. The refusal scene swaps this for a real
+// corpus replay, so both the usable and the refused answers come from the same code path.
+const suggestionBody = {
+  status: 'suggestion',
+  isStool: true,
+  bristolType: 4,
+  color: 'brown',
+  confidence: 0.83,
+  imageQuality: 'good',
+  observations: 'Smooth, formed appearance.',
+  warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
+};
+let analyzeOverride = null;
+await page.route('**/api/analyze', route => {
+  const response = analyzeOverride || { status: 200, body: suggestionBody };
+  return route.fulfill({ status: response.status, contentType: 'application/json', body: JSON.stringify(response.body) });
+});
 await page.route('**/api/agent/status', route => route.fulfill({
   status: 200,
   contentType: 'application/json',
@@ -204,6 +230,39 @@ await caption('One tap still reaches manual Bristol entry, and the draft keeps y
 await tap('#discard-draft', 500);
 await caption('Start over clears the kept draft — nothing lingers silently', 1500);
 await tap('#close-sheet', 450);
+
+// Corpus refusal pass (issue #18): the same photo flow, fed the answers Timmy has to refuse.
+visionOffline = false;
+await caption('The corpus also replays the answers Timmy has to refuse', 1700);
+analyzeOverride = await corpusAnalyzeResult(corpusCase('non-stool-scene'));
+await tap('[data-scan]', 500);
+await page.getByText(/Self-hosted model ready/i).waitFor();
+await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/non-stool-scene.png');
+await page.locator('#ai-consent').check();
+await tap('#analyze-photo', 600);
+await page.getByText(/No confident match/i).waitFor();
+await caption('A non-stool photo gets a refusal, never a guess — manual entry stays one tap away', 2600);
+await indicate('.scan-result');
+analyzeOverride = await corpusAnalyzeResult(corpusCase('glare-overexposed'));
+await tap('#retake-photo', 400);
+await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/glare-overexposed.png');
+await page.locator('#ai-consent').check();
+await tap('#analyze-photo', 600);
+await page.getByText(/does not show a form Timmy can read safely/i).waitFor();
+await caption('Unreadable visual fields abstain as well: a valid reply with nothing to prefill is not an error', 2700);
+await indicate('.scan-result');
+analyzeOverride = await corpusAnalyzeResult(corpusCase('out-of-range-fields'));
+await tap('#retake-photo', 400);
+await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/out-of-range-fields.png');
+await page.locator('#ai-consent').check();
+await tap('#analyze-photo', 600);
+await page.getByText(/couldn’t analyze that safely/i).waitFor();
+await caption('An out-of-range answer fails closed: nothing is applied and no provider detail reaches you', 2700);
+await indicate('.scan-result');
+await caption('Every refusal replays a committed synthetic fixture — there are no real photos in the corpus', 2500);
+analyzeOverride = null;
+await tap('#close-sheet', 450);
+
 await caption('The threat model documents this surface, so the recording shows it live rather than describing it', 2200);
 await tap('[data-view="calendar"]', 550);
 await page.getByText(/Your private journal/i).waitFor();
@@ -242,7 +301,13 @@ await page.evaluate(() => {
 });
 await sleep(2200);
 
-if (errors.length) throw new Error(`Browser errors: ${errors.join(' | ')}`);
+// The refusal pass deliberately produces one fail-closed 503, which the browser logs as a failed
+// resource. Requiring exactly that one keeps the scene honest: if the boundary stops refusing, this
+// fails instead of quietly recording a happy path.
+const expectedFailClosed = errors.filter(message => /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/.test(message));
+if (expectedFailClosed.length !== 1 || errors.length !== expectedFailClosed.length) {
+  throw new Error(`Browser errors: ${errors.join(' | ') || 'unexpected error count'}`);
+}
 // Machine-readable receipt of exactly what was rendered on camera, so the release gate can prove
 // the recorded delta statement reached the screen instead of trusting the script source.
 if (captionsOut) {