fix: fail closed when a filled-in visual field is out of range (#73)

The blank-first ordering let one blank field short-circuit the whole reply, so a
response carrying an impossible value came back as `unusable_visual_fields`:
the user was told the photo was unreadable when the model had in fact returned
something Timmy must never apply.

Blank fields still abstain and still prefill nothing. A filled-in field is a
claim, and a claim outside the contract now fails closed even when the other
field came back blank.

The corpus earns its label too. `out-of-range-fields` drops from confidence 1.4
to 0.9 so the case reaches the field-range check instead of dying at the
confidence gate, and a new `blank-plus-out-of-range` case covers the class the
old ordering let through. The release demo replays that case on camera and the
fail-closed 503 budget moves from one to two, so the scene cannot be dropped
silently.

Closes #73

scripts/build_release.py

@@ -38,6 +38,7 @@ DEMO_REQUIRED_CAPTIONS = (
     "Hermes Agent connected — free-text, contextual, and server-side",
     "Urgent language is intercepted deterministically before Hermes",
     "An out-of-range answer fails closed: nothing is applied and no provider detail reaches you",
+    "A filled-in field that is out of range still fails closed, even when the other field came back blank",
 )
 
 
@@ -207,7 +208,7 @@ def main() -> int:
             "- Vision profiles: hosted and self-hosted SmolVLM2 bootstrap\n"
             "- AI suggestions stay provisional: suggested, confirmed, and corrected states are distinct and screen-reader described; correcting one clears the AI-confirmed state without losing user context\n"
             "- Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions (difficulty passing, soreness, visible blood, truly black/tarry versus dark brown, hydration); mild reports get non-diagnostic self-care, and a confirmed red-flag answer lands on the same deterministic escalation path as manual logging\n"
-            "- Vision corpus: a committed CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) proves every reply is schema-valid or fails closed; a schema-valid reply with unreadable fields abstains instead of erroring, and unparsable or out-of-range replies return plain copy with nothing applied\n"
+            "- Vision corpus: a committed CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) proves every reply is schema-valid or fails closed; a schema-valid reply with unreadable fields abstains instead of erroring, and unparsable or out-of-range replies return plain copy with nothing applied — including a reply that fills in one field and leaves the other blank, where an out-of-range value still fails closed instead of being reported as an unreadable photo (issue #73)\n"
             "- Safety: suggestions are observable-field assistance, never diagnosis; user confirmation is required\n"
             "- Known limitation: open-weight VLM acceptance is proven, Bristol accuracy is not clinically validated\n"
             "- Package excludes model weights, medical-image corpora, raw research pages, generated screenshots, and video production assets\n"
@@ -253,7 +254,7 @@ def main() -> int:
             "codec": "h264",
             "pixel_format": "yuv420p",
             "recorded_from_working_app": True,
-            "fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, and an out-of-range fail-closed reply, offline vision status, and deterministic Hermes chat response",
+            "fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, an out-of-range fail-closed reply, and a blank-plus-out-of-range fail-closed reply, offline vision status, and deterministic Hermes chat response",
         },
         "gates": {
             "unit_security": "passed",

scripts/build_vision_corpus.py

@@ -32,6 +32,7 @@ OBSERVATION_STRINGS = {
     "highlight": "Large specular highlight hides the surface.",
     "container": "Empty container.",
     "confident": "Confident.",
+    "partial": "Form read, colour not.",
 }
 
 
@@ -105,6 +106,16 @@ def shaded_tone() -> Image.Image:
     return image
 
 
+def half_read_tone() -> Image.Image:
+    """A fixture for the reply shape that fills one field and blanks the other (issue #73)."""
+    image = background((208, 212, 204))
+    blob(image, (175, 150, 455, 322), (112, 96, 66))
+    draw = ImageDraw.Draw(image)
+    draw.rectangle([0, 200, WIDTH // 2, 206], fill=(196, 194, 186))
+    draw.rectangle([WIDTH // 2, 260, WIDTH, 266], fill=(170, 168, 160))
+    return image
+
+
 def content(payload: dict[str, object] | str) -> str:
     return payload if isinstance(payload, str) else json.dumps(payload, separators=(",", ":"))
 
@@ -180,11 +191,28 @@ def cases() -> list[dict[str, Any]]:
             "image": shaded_tone,
             "expectedOutcome": "fail_closed",
             "expectedReason": "out_of_range_or_missing_fields",
+            # Confidence stays inside the contract so this reply reaches the field-range check. At 1.4 it
+            # died at the confidence gate instead, which credited the `out_of_range_or_missing_fields`
+            # label to a guard the case never exercised (issue #73).
             "providerContent": content({
-                "isStool": True, "bristolType": 9, "color": "purple", "confidence": 1.4,
+                "isStool": True, "bristolType": 9, "color": "purple", "confidence": 0.9,
                 "imageQuality": "excellent", "observations": OBSERVATION_STRINGS["confident"],
             }),
         },
+        {
+            "id": "blank-plus-out-of-range",
+            "caseClass": "malformed-response",
+            "image": half_read_tone,
+            "expectedOutcome": "fail_closed",
+            "expectedReason": "out_of_range_or_missing_fields",
+            # The class the blank-first ordering used to let through (issue #73): one field is blank and
+            # the other is a filled-in, impossible value. A filled-in field is a claim, so this must fail
+            # closed rather than abstain as `unusable_visual_fields`.
+            "providerContent": content({
+                "isStool": True, "bristolType": 9, "color": None, "confidence": 0.9,
+                "imageQuality": "good", "observations": OBSERVATION_STRINGS["partial"],
+            }),
+        },
     ]
 
 

scripts/record_release_demo.mjs

@@ -259,6 +259,22 @@ await tap('#analyze-photo', 600);
 await page.getByText(/couldn’t analyze that safely/i).waitFor();
 await caption('An out-of-range answer fails closed: nothing is applied and no provider detail reaches you', 2700);
 await indicate('.scan-result');
+// Issue #73's escape class: one field blank, the other filled in and impossible. The old blank-first
+// ordering abstained here, which told the user the photo was unreadable while the model had returned a
+// value Timmy must never apply. The scene is asserted in the browser, not just captioned.
+analyzeOverride = await corpusAnalyzeResult(corpusCase('blank-plus-out-of-range'));
+await tap('#retake-photo', 400);
+await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/blank-plus-out-of-range.png');
+await page.locator('#ai-consent').check();
+await tap('#analyze-photo', 600);
+await page.getByText(/couldn’t analyze that safely/i).waitFor();
+if (await page.getByText(/No confident match/i).count() !== 0) {
+  throw new Error('A filled-in out-of-range field must not be shown as an unreadable-photo abstention');
+}
+await caption('A filled-in field that is out of range still fails closed, even when the other field came back blank', 2900);
+await indicate('.scan-result');
+await caption('You are no longer told the photo was unreadable when the model returned an impossible value', 2400);
+await indicate('.scan-result');
 await caption('Every refusal replays a committed synthetic fixture — there are no real photos in the corpus', 2500);
 analyzeOverride = null;
 await tap('#close-sheet', 450);
@@ -301,11 +317,12 @@ await page.evaluate(() => {
 });
 await sleep(2200);
 
-// The refusal pass deliberately produces one fail-closed 503, which the browser logs as a failed
-// resource. Requiring exactly that one keeps the scene honest: if the boundary stops refusing, this
-// fails instead of quietly recording a happy path.
+// The refusal pass deliberately produces two fail-closed 503s (an out-of-range reply, and one blank field
+// beside an out-of-range one), which the browser logs as failed resources. Requiring exactly those two
+// keeps the scene honest: if the boundary stops refusing, this fails instead of quietly recording a happy
+// path, and a dropped scene is caught rather than silently shrinking the evidence.
 const expectedFailClosed = errors.filter(message => /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/.test(message));
-if (expectedFailClosed.length !== 1 || errors.length !== expectedFailClosed.length) {
+if (expectedFailClosed.length !== 2 || errors.length !== expectedFailClosed.length) {
   throw new Error(`Browser errors: ${errors.join(' | ') || 'unexpected error count'}`);
 }
 // Machine-readable receipt of exactly what was rendered on camera, so the release gate can prove

src/analysis.js

@@ -34,9 +34,19 @@ export function parseVisionResponse(raw) {
   if (!value.isStool) return { status: 'needs_user_input', isStool: false, abstainedFor: 'not_stool', reason: 'The image does not clearly show stool.' };
   if (confidence < 0.55) return { status: 'needs_user_input', isStool: true, abstainedFor: 'low_confidence', reason: 'The image is too uncertain to prefill safely.' };
   // The request schema lets the model return a null form and color when it cannot read them.
-  // A schema-valid answer with nothing to prefill abstains; an out-of-range value still fails closed.
+  // A blank field means the model could not read it, so a schema-valid answer with nothing usable
+  // abstains and nothing is prefilled. A filled-in field is a claim, though: if it sits outside the
+  // contract the response fails closed, even when the other field is blank (#73), because a caller
+  // reading `unusable_visual_fields` would be told the photo was unreadable when the model had in
+  // fact returned an impossible value.
   const blank = input => input === null || input === undefined || (typeof input === 'string' && !input.trim());
-  if (blank(value.bristolType) || blank(value.color)) {
+  const bristolType = blank(value.bristolType) ? null : Number(value.bristolType);
+  const color = blank(value.color) ? null : String(value.color).toLowerCase();
+  if ((bristolType !== null && (!Number.isInteger(bristolType) || bristolType < 1 || bristolType > 7))
+    || (color !== null && !COLORS.has(color))) {
+    throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.');
+  }
+  if (bristolType === null || color === null) {
     return {
       status: 'needs_user_input',
       isStool: true,
@@ -44,11 +54,6 @@ export function parseVisionResponse(raw) {
       reason: 'The photo does not show a form Timmy can read safely.',
     };
   }
-  const bristolType = Number(value.bristolType);
-  const color = String(value.color).toLowerCase();
-  if (!Number.isInteger(bristolType) || bristolType < 1 || bristolType > 7 || !COLORS.has(color)) {
-    throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.');
-  }
   const imageQuality = String(value.imageQuality || '').toLowerCase();
   if (!QUALITIES.has(imageQuality)) throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.');
   return {

tests/analysis.test.js

@@ -82,6 +82,38 @@ test('builds a structured multimodal request that forbids diagnosis and nonvisua
   assert.equal(body.messages[0].content.find(part => part.type === 'image_url').image_url.url, 'data:image/jpeg;base64,YQ==');
 });
 
+test('fails closed when a field the model did fill in is out of range while another field is blank', () => {
+  // Issue #73: the blank-first ordering let a blank field short-circuit the whole response, so a reply
+  // that carried an impossible value came back as `unusable_visual_fields` — telling the user the photo
+  // was unreadable when the model had in fact returned something Timmy must never apply. A blank field
+  // means "could not read it"; a filled-in field is a claim, and a claim outside the contract fails closed.
+  assert.throws(
+    () => parseVisionResponse({ isStool: true, bristolType: 9, color: null, confidence: 0.9, imageQuality: 'good', observations: 'Form read, colour not.' }),
+    (error) => {
+      assert.equal(error.code, 'invalid_fields', 'an out-of-range form next to a blank colour must fail closed');
+      return true;
+    },
+  );
+  assert.throws(
+    () => parseVisionResponse({ isStool: true, bristolType: null, color: 'purple', confidence: 0.9, imageQuality: 'good', observations: 'Colour read, form not.' }),
+    (error) => {
+      assert.equal(error.code, 'invalid_fields', 'an out-of-range colour next to a blank form must fail closed');
+      return true;
+    },
+  );
+});
+
+test('still abstains when the only filled-in visual field is inside the contract', () => {
+  // The complement of the rule above: a partial read that is entirely valid is an abstention, not a
+  // failure, so the glare path keeps returning 200 with nothing prefilled.
+  assert.deepEqual(parseVisionResponse({ isStool: true, bristolType: 4, color: null, confidence: 0.9, imageQuality: 'good', observations: 'Form read, colour not.' }), {
+    status: 'needs_user_input',
+    isStool: true,
+    abstainedFor: 'unusable_visual_fields',
+    reason: 'The photo does not show a form Timmy can read safely.',
+  });
+});
+
 test('abstains instead of failing when a schema-valid response carries no readable visual fields', () => {
   // The declared JSON schema allows a null form and color whenever the model cannot read them,
   // so a schema-valid response must come back as an abstention — never as a provider failure.

tests/fixtures/vision-corpus/README.md

@@ -25,7 +25,14 @@ refusal never carries a form or a colour.
 | `glare-overexposed` | glare | schema-valid with null form and colour | `abstained` (`unusable_visual_fields`) |
 | `empty-bowl` | empty-bowl | `isStool: false` | `abstained` (`not_stool`) |
 | `malformed-response` | malformed-response | plain prose, not JSON | `fail_closed` (`unparsable_content`) |
-| `out-of-range-fields` | malformed-response | Bristol 9, colour `purple`, confidence 1.4 | `fail_closed` (`out_of_range_or_missing_fields`) |
+| `out-of-range-fields` | malformed-response | Bristol 9, colour `purple`, confidence 0.9 | `fail_closed` (`out_of_range_or_missing_fields`) |
+| `blank-plus-out-of-range` | malformed-response | Bristol 9 with a `null` colour, confidence 0.9 | `fail_closed` (`out_of_range_or_missing_fields`) |
+
+Both out-of-range cases keep `confidence` inside the contract on purpose: at `1.4` the reply died at the
+confidence gate, so the `out_of_range_or_missing_fields` label was credited to a guard the case never
+reached. `blank-plus-out-of-range` covers the reply shape that the blank-first ordering used to abstain on
+(issue #73) — a filled-in field is a claim, and a claim outside the contract fails closed even when the
+other field came back blank.
 
 ## Where the corpus is enforced
 

tests/fixtures/vision-corpus/manifest.json

@@ -114,7 +114,23 @@
       "provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
       "expectedOutcome": "fail_closed",
       "expectedReason": "out_of_range_or_missing_fields",
-      "providerContent": "{\"isStool\":true,\"bristolType\":9,\"color\":\"purple\",\"confidence\":1.4,\"imageQuality\":\"excellent\",\"observations\":\"Confident.\"}"
+      "providerContent": "{\"isStool\":true,\"bristolType\":9,\"color\":\"purple\",\"confidence\":0.9,\"imageQuality\":\"excellent\",\"observations\":\"Confident.\"}"
+    },
+    {
+      "id": "blank-plus-out-of-range",
+      "caseClass": "malformed-response",
+      "file": "blank-plus-out-of-range.png",
+      "bytes": 2403,
+      "sha256": "d45048cd1f5bfb1be1667238c9c21fca0cd1cae8b55e12d4f325203625eeafbd",
+      "width": 640,
+      "height": 480,
+      "format": "png",
+      "synthetic": true,
+      "license": "CC0-1.0",
+      "provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
+      "expectedOutcome": "fail_closed",
+      "expectedReason": "out_of_range_or_missing_fields",
+      "providerContent": "{\"isStool\":true,\"bristolType\":9,\"color\":null,\"confidence\":0.9,\"imageQuality\":\"good\",\"observations\":\"Form read, colour not.\"}"
     }
   ]
 }

tests/release-demo.test.js

@@ -170,3 +170,18 @@ test('release demo visibly exercises the corpus refusal paths instead of only th
   // The refusal copy the user sees is gated on camera, so a release cannot drop it silently.
   assert.match(builder, /An out-of-range answer fails closed/);
 });
+
+test('release demo shows a filled-in out-of-range field failing closed even beside a blank one', async () => {
+  const demo = await readFile(demoPath, 'utf8');
+  const builder = await readFile(builderPath, 'utf8');
+
+  // Issue #73's escape class, replayed on camera from the committed corpus rather than described.
+  assert.match(demo, /corpusCase\('blank-plus-out-of-range'\)/);
+  assert.match(demo, /tests\/fixtures\/vision-corpus\/blank-plus-out-of-range\.png/);
+  assert.match(demo, /A filled-in field that is out of range still fails closed, even when the other field came back blank/);
+  assert.match(demo, /no longer told the photo was unreadable when the model returned an impossible value/);
+  // The extra fail-closed scene is a real 503, so the demo's error budget must count it.
+  assert.match(demo, /expectedFailClosed\.length !== 2/);
+  // The caption is a required gate, so a later release cannot silently drop the scene.
+  assert.match(builder, /A filled-in field that is out of range still fails closed/);
+});

tests/vision-boundary.test.js

@@ -90,6 +90,16 @@ test('the running server answers every corpus case with a fail-closed status and
     assert.equal(outOfRange.status, 503, 'out-of-range fields must not be served as a suggestion');
     assert.match(outOfRange.body.error, FAIL_CLOSED_COPY);
 
+    // Issue #73: a filled-in field that is out of range fails closed even when the other field is blank,
+    // so the user is never told the photo was unreadable when the model returned an impossible value.
+    const blankAndOutOfRange = caseFor(corpus, 'blank-plus-out-of-range');
+    provider.state.content = blankAndOutOfRange.providerContent;
+    const blankOutOfRange = await postAnalyze(await corpusPayload(blankAndOutOfRange));
+    assert.equal(blankOutOfRange.status, 503, 'a filled-in out-of-range field must not be served as an abstention');
+    assert.match(blankOutOfRange.body.error, FAIL_CLOSED_COPY);
+    assert.equal('abstainedFor' in blankOutOfRange.body, false, 'a contract breach must not be reported as an abstention');
+    assert.doesNotMatch(JSON.stringify(blankOutOfRange.body), /base64|data:image|corpus-key|Bearer/i);
+
     const noConsent = await postAnalyze({ imageDataUrl: payload.imageDataUrl, consent: false });
     assert.equal(noConsent.status, 400, 'analysis without explicit consent is a client error');
     assert.doesNotMatch(JSON.stringify(noConsent.body), /base64|data:image/i);

tests/vision-corpus.test.js

@@ -7,6 +7,8 @@ import { fileURLToPath } from 'node:url';
 import { classifyVisionOutcome, loadVisionCorpus, VISION_CORPUS_CLASSES, VISION_CORPUS_LICENSE, VISION_CORPUS_OUTCOMES } from '../src/vision-corpus.js';
 
 const corpusDir = fileURLToPath(new URL('./fixtures/vision-corpus/', import.meta.url));
+// The colour contract `src/analysis.js` enforces, restated here so the corpus can be checked against it.
+const COLORS_IN_CONTRACT = ['brown', 'green', 'yellow', 'pale', 'red', 'black'];
 
 async function scratchCorpus() {
   const directory = await mkdtemp(join(tmpdir(), 'timmy-corpus-'));
@@ -67,6 +69,29 @@ test('vision corpus rejects an unlicensed, non-synthetic, or incomplete manifest
   }
 });
 
+test('the corpus earns its out-of-range label and covers the blank-plus-out-of-range class', async () => {
+  const corpus = await loadVisionCorpus(corpusDir);
+  const outOfRange = corpus.cases.filter(entry => entry.expectedReason === 'out_of_range_or_missing_fields');
+  assert.ok(outOfRange.length > 0, 'the corpus must cover the out-of-range failure class');
+
+  // Issue #73: a case whose confidence breaks the gate never reaches the field-range check, so its
+  // `out_of_range_or_missing_fields` label would be attributed to a guard it never exercised.
+  const reachesRangeCheck = outOfRange.filter(entry => Number(JSON.parse(entry.providerContent).confidence) <= 1);
+  assert.ok(reachesRangeCheck.length > 0, 'at least one out-of-range case must clear the confidence gate to exercise the range check');
+
+  // Issue #73's escape class: a blank field next to a non-blank, out-of-range field. This is the exact
+  // reply shape that used to abstain as `unusable_visual_fields` instead of failing closed.
+  const blank = value => value === null || value === undefined || (typeof value === 'string' && !value.trim());
+  const colourInContract = value => COLORS_IN_CONTRACT.includes(String(value).toLowerCase());
+  const formInContract = value => Number.isInteger(Number(value)) && Number(value) >= 1 && Number(value) <= 7;
+  const escapeCases = outOfRange.filter(entry => {
+    const value = JSON.parse(entry.providerContent);
+    const fields = [value.bristolType, value.color];
+    return fields.some(blank) && fields.some(field => !blank(field) && !(colourInContract(field) || formInContract(field)));
+  });
+  assert.ok(escapeCases.length > 0, 'the corpus must carry a blank-plus-out-of-range case so the fail-closed label is really earned');
+});
+
 test('every recorded corpus response is schema-valid or fails closed, exactly as declared', async () => {
   const corpus = await loadVisionCorpus(corpusDir);
 

tests/vision-refusal.acceptance.mjs

@@ -72,16 +72,30 @@ try {
   await page.screenshot({ path: 'artifacts/vision-refusal-fail-closed-mobile.png' });
   await closeSheet();
 
-  // 4. Positive control: the same corpus still yields a real suggestion when the response is usable.
+  // 4. Issue #73: a filled-in field that is out of range fails closed even when the other field is blank.
+  // The old blank-first ordering abstained here, so the user was told the photo was unreadable while the
+  // model had actually returned an impossible value. The abstention copy must not appear on this path.
+  await analyzeCorpusCase(caseFor(corpus, 'blank-plus-out-of-range'));
+  await page.getByText(/couldn’t analyze that safely/i).waitFor();
+  assert.match(await page.locator('.scan-result').innerText(), /No suggestion was applied/i);
+  assert.equal(await page.getByText(/No confident match/i).count(), 0, 'a contract breach must not be shown as an abstention');
+  assert.equal(await page.locator('.ai-badge').count(), 0);
+  assert.doesNotMatch(await page.locator('.scan-result').innerText(), /out of range|Invalid AI response/i, 'provider detail must not reach the user');
+  await page.screenshot({ path: 'artifacts/vision-refusal-blank-plus-out-of-range-mobile.png' });
+  await closeSheet();
+
+  // 5. Positive control: the same corpus still yields a real suggestion when the response is usable.
   await analyzeCorpusCase(caseFor(corpus, 'stool-like-formed'));
   await page.getByText(/83% confidence/i).waitFor();
   assert.equal(await page.getByText('Type 4', { exact: true }).isVisible(), true);
   await page.screenshot({ path: 'artifacts/vision-corpus-suggestion-mobile.png' });
 
+  // Two fail-closed scenes reach the browser as real 503s; the count is asserted so a scene cannot be
+  // dropped silently, and any other console error still fails the run.
   const failClosedLogs = errors.filter(message => EXPECTED_FAIL_CLOSED_LOG.test(message));
-  assert.equal(failClosedLogs.length, 1, 'the fail-closed response must reach the browser as a 503');
+  assert.equal(failClosedLogs.length, 2, 'each fail-closed response must reach the browser as a 503');
   assert.deepEqual(errors.filter(message => !EXPECTED_FAIL_CLOSED_LOG.test(message)), []);
-  console.log('PASS vision corpus refusal: non-stool, unreadable fields, and out-of-range replies never prefill; the usable reply still suggests');
+  console.log('PASS vision corpus refusal: non-stool, unreadable fields, out-of-range, and blank-plus-out-of-range replies never prefill; the usable reply still suggests');
 } finally {
   await context.close();
   await browser.close();