Quality gates / quality (pull_request) Successful in 13m29s
Confidence was the only thing standing between a model answer and a prefilled journal entry, so a confident reading of a blurry photo could still be offered as fact. Quality is now part of the policy: `poor` abstains however confident the reply looked, with its own fixed copy. The confidence floor, accepted quality levels, and refusal copy move into src/abstention-policy.js so the parser, the corpus, and the browser cannot drift into three explanations of the same refusal. parseVisionResponse takes an optional policy, which is how the tests prove the parser carries no hidden limits of its own. evaluatePrefillGate turns the #29 calibration report into one release decision: GO plus a validation-selected threshold scored on the held-out test split, or prefill is blocked with each reason named. A missing report is a block, never a default allow. scripts/abstention_policy_receipt.mjs exposes that as a receipt (exit 0 / 3 / 2). The committed synthetic predictions currently report BLOCKED and the receipt says so. No candidate exists until #31, and the builder does not hard-gate on the receipt yet; the on-camera quality gate is the evidence this release carries. The corpus gains a case that replays a 0.94-confidence answer about the synthetic motion-blur fixture, and the release demo asserts in the browser that nothing is prefilled before captioning it. Verified: 226 unit tests, calibration harness (19), corpus manifest (8), vision-refusal, photo-first, and provisional acceptance, check:syntax, check:diff, and a full real-browser demo recording whose caption receipt confirms the new scene painted on screen.
130 lines
6.5 KiB
JavaScript
130 lines
6.5 KiB
JavaScript
import { ABSTENTION_POLICY, abstainFor, isLowQuality } from './abstention-policy.js';
|
|
|
|
const COLORS = new Set(['brown', 'green', 'yellow', 'pale', 'red', 'black']);
|
|
const QUALITIES = new Set(['good', 'fair', 'poor']);
|
|
const MAX_IMAGE_BYTES = 4 * 1024 * 1024;
|
|
|
|
// Fail-closed responses carry a code so the server boundary and the corpus can name the failure
|
|
// class without matching on prose.
|
|
export const VISION_FAILURE_CODES = Object.freeze({ invalidJson: 'invalid_json', invalidFields: 'invalid_fields' });
|
|
|
|
function visionResponseError(code, message) {
|
|
const error = new Error(message);
|
|
error.name = 'VisionResponseError';
|
|
error.code = code;
|
|
return error;
|
|
}
|
|
|
|
function asObject(raw) {
|
|
if (raw && typeof raw === 'object' && !Array.isArray(raw)) return raw;
|
|
if (typeof raw !== 'string') throw visionResponseError(VISION_FAILURE_CODES.invalidJson, 'Invalid AI response.');
|
|
const cleaned = raw.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
|
|
try {
|
|
const parsed = JSON.parse(cleaned);
|
|
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error();
|
|
return parsed;
|
|
} catch {
|
|
throw visionResponseError(VISION_FAILURE_CODES.invalidJson, 'Invalid AI response.');
|
|
}
|
|
}
|
|
|
|
export function parseVisionResponse(raw, policy = ABSTENTION_POLICY) {
|
|
const value = asObject(raw);
|
|
if (typeof value.isStool !== 'boolean') throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: isStool is required.');
|
|
const confidence = Number(value.confidence);
|
|
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: confidence is required.');
|
|
if (!value.isStool) return abstainFor('not_stool', false);
|
|
if (confidence < policy.minConfidence) return abstainFor('low_confidence');
|
|
// The request schema lets the model return a null form and color when it cannot read them.
|
|
// A blank field means the model could not read it, so a schema-valid answer with nothing usable
|
|
// abstains and nothing is prefilled. A filled-in field is a claim, though: if it sits outside the
|
|
// contract the response fails closed, even when the other field is blank (#73), because a caller
|
|
// reading `unusable_visual_fields` would be told the photo was unreadable when the model had in
|
|
// fact returned an impossible value.
|
|
const blank = input => input === null || input === undefined || (typeof input === 'string' && !input.trim());
|
|
const bristolType = blank(value.bristolType) ? null : Number(value.bristolType);
|
|
const color = blank(value.color) ? null : String(value.color).toLowerCase();
|
|
if ((bristolType !== null && (!Number.isInteger(bristolType) || bristolType < 1 || bristolType > 7))
|
|
|| (color !== null && !COLORS.has(color))) {
|
|
throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.');
|
|
}
|
|
if (bristolType === null || color === null) {
|
|
return abstainFor('unusable_visual_fields');
|
|
}
|
|
const imageQuality = String(value.imageQuality || '').toLowerCase();
|
|
if (!QUALITIES.has(imageQuality)) throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.');
|
|
// A confident reading of a photo the model itself calls poor is still not a safe prefill.
|
|
if (isLowQuality(imageQuality, policy)) return abstainFor('low_quality');
|
|
return {
|
|
status: 'suggestion',
|
|
isStool: true,
|
|
bristolType,
|
|
color,
|
|
confidence: Math.round(confidence * 100) / 100,
|
|
imageQuality,
|
|
observations: String(value.observations || '').trim().slice(0, 240),
|
|
warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
|
|
};
|
|
}
|
|
|
|
export function mergeVisualSuggestion(form, suggestion) {
|
|
if (suggestion?.status !== 'suggestion') return { ...form };
|
|
return { ...form, bristolType: suggestion.bristolType, color: suggestion.color };
|
|
}
|
|
|
|
export function validatePhotoPayload(payload = {}) {
|
|
if (payload.consent !== true) throw new Error('Explicit consent is required before AI analysis.');
|
|
const match = /^data:(image\/(?:jpeg|png|webp));base64,([A-Za-z0-9+/=]+)$/.exec(String(payload.imageDataUrl || ''));
|
|
if (!match) throw new Error('Upload a JPEG, PNG, or WebP photo.');
|
|
const padding = (match[2].match(/=*$/) || [''])[0].length;
|
|
const bytes = Math.floor(match[2].length * 3 / 4) - padding;
|
|
if (bytes <= 0) throw new Error('The photo is empty.');
|
|
if (bytes > MAX_IMAGE_BYTES) throw new Error('The photo is too large. Use an image under 4 MB.');
|
|
return { imageDataUrl: payload.imageDataUrl, mime: match[1], bytes };
|
|
}
|
|
|
|
export function buildVisionRequest({ imageDataUrl, model }) {
|
|
return {
|
|
model,
|
|
temperature: 0,
|
|
max_tokens: 500,
|
|
messages: [{
|
|
role: 'user',
|
|
content: [
|
|
{
|
|
type: 'text',
|
|
text: [
|
|
'You are a conservative visual form-suggestion tool for a bowel diary, not a clinician.',
|
|
'First decide whether the image clearly shows human stool. If it does not, set isStool=false.',
|
|
'If it does, suggest only the closest Bristol Stool Form Scale type (1-7), visible color (brown, green, yellow, pale, red, or black), image quality, confidence, and a short neutral visual observation.',
|
|
'Do not infer urgency, discomfort, pain, symptoms, bleeding, disease, diet safety, cause, or treatment from the image.',
|
|
'Red or black is a visible color description only and is not a diagnosis. Be uncertain when lighting or visibility is poor.',
|
|
'The user must confirm every suggestion. Return only the requested JSON.',
|
|
].join(' '),
|
|
},
|
|
{ type: 'image_url', image_url: { url: imageDataUrl } },
|
|
],
|
|
}],
|
|
response_format: {
|
|
type: 'json_schema',
|
|
json_schema: {
|
|
name: 'timmy_visual_suggestion',
|
|
strict: true,
|
|
schema: {
|
|
type: 'object',
|
|
additionalProperties: false,
|
|
required: ['isStool', 'bristolType', 'color', 'confidence', 'imageQuality', 'observations'],
|
|
properties: {
|
|
isStool: { type: 'boolean' },
|
|
bristolType: { anyOf: [{ type: 'integer', minimum: 1, maximum: 7 }, { type: 'null' }] },
|
|
color: { anyOf: [{ type: 'string', enum: [...COLORS] }, { type: 'null' }] },
|
|
confidence: { type: 'number', minimum: 0, maximum: 1 },
|
|
imageQuality: { type: 'string', enum: [...QUALITIES] },
|
|
observations: { type: 'string', maxLength: 240 },
|
|
},
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|