Files
timmy-talking-turd/src/analysis.js
T
timmy 209e21a951
Quality gates / quality (pull_request) Successful in 13m29s
Gate visual prefill on a versioned abstention policy (#32)
Confidence was the only thing standing between a model answer and a
prefilled journal entry, so a confident reading of a blurry photo could
still be offered as fact. Quality is now part of the policy: `poor`
abstains however confident the reply looked, with its own fixed copy.

The confidence floor, accepted quality levels, and refusal copy move into
src/abstention-policy.js so the parser, the corpus, and the browser cannot
drift into three explanations of the same refusal. parseVisionResponse
takes an optional policy, which is how the tests prove the parser carries
no hidden limits of its own.

evaluatePrefillGate turns the #29 calibration report into one release
decision: GO plus a validation-selected threshold scored on the held-out
test split, or prefill is blocked with each reason named. A missing report
is a block, never a default allow. scripts/abstention_policy_receipt.mjs
exposes that as a receipt (exit 0 / 3 / 2).

The committed synthetic predictions currently report BLOCKED and the
receipt says so. No candidate exists until #31, and the builder does not
hard-gate on the receipt yet; the on-camera quality gate is the evidence
this release carries.

The corpus gains a case that replays a 0.94-confidence answer about the
synthetic motion-blur fixture, and the release demo asserts in the browser
that nothing is prefilled before captioning it.

Verified: 226 unit tests, calibration harness (19), corpus manifest (8),
vision-refusal, photo-first, and provisional acceptance, check:syntax,
check:diff, and a full real-browser demo recording whose caption receipt
confirms the new scene painted on screen.
2026-10-26 10:58:38 +00:00

130 lines
6.5 KiB
JavaScript

import { ABSTENTION_POLICY, abstainFor, isLowQuality } from './abstention-policy.js';
const COLORS = new Set(['brown', 'green', 'yellow', 'pale', 'red', 'black']);
const QUALITIES = new Set(['good', 'fair', 'poor']);
const MAX_IMAGE_BYTES = 4 * 1024 * 1024;
// Fail-closed responses carry a code so the server boundary and the corpus can name the failure
// class without matching on prose.
export const VISION_FAILURE_CODES = Object.freeze({ invalidJson: 'invalid_json', invalidFields: 'invalid_fields' });
function visionResponseError(code, message) {
const error = new Error(message);
error.name = 'VisionResponseError';
error.code = code;
return error;
}
function asObject(raw) {
if (raw && typeof raw === 'object' && !Array.isArray(raw)) return raw;
if (typeof raw !== 'string') throw visionResponseError(VISION_FAILURE_CODES.invalidJson, 'Invalid AI response.');
const cleaned = raw.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
try {
const parsed = JSON.parse(cleaned);
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error();
return parsed;
} catch {
throw visionResponseError(VISION_FAILURE_CODES.invalidJson, 'Invalid AI response.');
}
}
export function parseVisionResponse(raw, policy = ABSTENTION_POLICY) {
const value = asObject(raw);
if (typeof value.isStool !== 'boolean') throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: isStool is required.');
const confidence = Number(value.confidence);
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: confidence is required.');
if (!value.isStool) return abstainFor('not_stool', false);
if (confidence < policy.minConfidence) return abstainFor('low_confidence');
// The request schema lets the model return a null form and color when it cannot read them.
// A blank field means the model could not read it, so a schema-valid answer with nothing usable
// abstains and nothing is prefilled. A filled-in field is a claim, though: if it sits outside the
// contract the response fails closed, even when the other field is blank (#73), because a caller
// reading `unusable_visual_fields` would be told the photo was unreadable when the model had in
// fact returned an impossible value.
const blank = input => input === null || input === undefined || (typeof input === 'string' && !input.trim());
const bristolType = blank(value.bristolType) ? null : Number(value.bristolType);
const color = blank(value.color) ? null : String(value.color).toLowerCase();
if ((bristolType !== null && (!Number.isInteger(bristolType) || bristolType < 1 || bristolType > 7))
|| (color !== null && !COLORS.has(color))) {
throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.');
}
if (bristolType === null || color === null) {
return abstainFor('unusable_visual_fields');
}
const imageQuality = String(value.imageQuality || '').toLowerCase();
if (!QUALITIES.has(imageQuality)) throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.');
// A confident reading of a photo the model itself calls poor is still not a safe prefill.
if (isLowQuality(imageQuality, policy)) return abstainFor('low_quality');
return {
status: 'suggestion',
isStool: true,
bristolType,
color,
confidence: Math.round(confidence * 100) / 100,
imageQuality,
observations: String(value.observations || '').trim().slice(0, 240),
warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
};
}
export function mergeVisualSuggestion(form, suggestion) {
if (suggestion?.status !== 'suggestion') return { ...form };
return { ...form, bristolType: suggestion.bristolType, color: suggestion.color };
}
export function validatePhotoPayload(payload = {}) {
if (payload.consent !== true) throw new Error('Explicit consent is required before AI analysis.');
const match = /^data:(image\/(?:jpeg|png|webp));base64,([A-Za-z0-9+/=]+)$/.exec(String(payload.imageDataUrl || ''));
if (!match) throw new Error('Upload a JPEG, PNG, or WebP photo.');
const padding = (match[2].match(/=*$/) || [''])[0].length;
const bytes = Math.floor(match[2].length * 3 / 4) - padding;
if (bytes <= 0) throw new Error('The photo is empty.');
if (bytes > MAX_IMAGE_BYTES) throw new Error('The photo is too large. Use an image under 4 MB.');
return { imageDataUrl: payload.imageDataUrl, mime: match[1], bytes };
}
export function buildVisionRequest({ imageDataUrl, model }) {
return {
model,
temperature: 0,
max_tokens: 500,
messages: [{
role: 'user',
content: [
{
type: 'text',
text: [
'You are a conservative visual form-suggestion tool for a bowel diary, not a clinician.',
'First decide whether the image clearly shows human stool. If it does not, set isStool=false.',
'If it does, suggest only the closest Bristol Stool Form Scale type (1-7), visible color (brown, green, yellow, pale, red, or black), image quality, confidence, and a short neutral visual observation.',
'Do not infer urgency, discomfort, pain, symptoms, bleeding, disease, diet safety, cause, or treatment from the image.',
'Red or black is a visible color description only and is not a diagnosis. Be uncertain when lighting or visibility is poor.',
'The user must confirm every suggestion. Return only the requested JSON.',
].join(' '),
},
{ type: 'image_url', image_url: { url: imageDataUrl } },
],
}],
response_format: {
type: 'json_schema',
json_schema: {
name: 'timmy_visual_suggestion',
strict: true,
schema: {
type: 'object',
additionalProperties: false,
required: ['isStool', 'bristolType', 'color', 'confidence', 'imageQuality', 'observations'],
properties: {
isStool: { type: 'boolean' },
bristolType: { anyOf: [{ type: 'integer', minimum: 1, maximum: 7 }, { type: 'null' }] },
color: { anyOf: [{ type: 'string', enum: [...COLORS] }, { type: 'null' }] },
confidence: { type: 'number', minimum: 0, maximum: 1 },
imageQuality: { type: 'string', enum: [...QUALITIES] },
observations: { type: 'string', maxLength: 240 },
},
},
},
},
};
}