import test from 'node:test'; import assert from 'node:assert/strict'; import { cp, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { classifyVisionOutcome, loadVisionCorpus, VISION_CORPUS_CLASSES, VISION_CORPUS_LICENSE, VISION_CORPUS_OUTCOMES } from '../src/vision-corpus.js'; const corpusDir = fileURLToPath(new URL('./fixtures/vision-corpus/', import.meta.url)); // The colour contract `src/analysis.js` enforces, restated here so the corpus can be checked against it. const COLORS_IN_CONTRACT = ['brown', 'green', 'yellow', 'pale', 'red', 'black']; async function scratchCorpus() { const directory = await mkdtemp(join(tmpdir(), 'timmy-corpus-')); await cp(corpusDir, directory, { recursive: true }); return directory; } test('vision corpus ships licensed synthetic fixtures covering every required visual case class', async () => { const corpus = await loadVisionCorpus(corpusDir); assert.equal(corpus.license, VISION_CORPUS_LICENSE); const classes = new Set(corpus.cases.map(entry => entry.caseClass)); for (const required of VISION_CORPUS_CLASSES) { assert.ok(classes.has(required), `corpus must cover the ${required} case`); } for (const entry of corpus.cases) { assert.equal(entry.synthetic, true, `${entry.id} must be synthetic; no real medical image is redistributable`); assert.equal(entry.license, VISION_CORPUS_LICENSE); assert.match(entry.provenance, /\S/, `${entry.id} must record where it came from`); assert.match(entry.sha256, /^[a-f0-9]{64}$/, `${entry.id} must record its content hash`); } }); test('vision corpus rejects fixtures whose bytes no longer match the recorded hash', async () => { const directory = await scratchCorpus(); try { const manifest = JSON.parse(await readFile(join(directory, 'manifest.json'), 'utf8')); const target = join(directory, manifest.cases[0].file); const bytes = await readFile(target); await writeFile(target, Buffer.concat([bytes, Buffer.from([0])])); await assert.rejects(() => loadVisionCorpus(directory), new RegExp(manifest.cases[0].id)); } finally { await rm(directory, { recursive: true, force: true }); } }); test('vision corpus rejects an unlicensed, non-synthetic, or incomplete manifest', async () => { const directory = await scratchCorpus(); try { const manifestPath = join(directory, 'manifest.json'); const manifest = JSON.parse(await readFile(manifestPath, 'utf8')); manifest.cases[0].synthetic = false; await writeFile(manifestPath, JSON.stringify(manifest, null, 2)); await assert.rejects(() => loadVisionCorpus(directory), /synthetic/i); manifest.cases[0].synthetic = true; manifest.cases = manifest.cases.filter(entry => entry.caseClass !== 'non-stool'); await writeFile(manifestPath, JSON.stringify(manifest, null, 2)); await assert.rejects(() => loadVisionCorpus(directory), /non-stool/); manifest.cases[0].license = 'all-rights-reserved'; await writeFile(manifestPath, JSON.stringify(manifest, null, 2)); await assert.rejects(() => loadVisionCorpus(directory), /CC0-1\.0/); } finally { await rm(directory, { recursive: true, force: true }); } }); test('the corpus earns its out-of-range label and covers the blank-plus-out-of-range class', async () => { const corpus = await loadVisionCorpus(corpusDir); const outOfRange = corpus.cases.filter(entry => entry.expectedReason === 'out_of_range_or_missing_fields'); assert.ok(outOfRange.length > 0, 'the corpus must cover the out-of-range failure class'); // Issue #73: a case whose confidence breaks the gate never reaches the field-range check, so its // `out_of_range_or_missing_fields` label would be attributed to a guard it never exercised. const reachesRangeCheck = outOfRange.filter(entry => Number(JSON.parse(entry.providerContent).confidence) <= 1); assert.ok(reachesRangeCheck.length > 0, 'at least one out-of-range case must clear the confidence gate to exercise the range check'); // Issue #73's escape class: a blank field next to a non-blank, out-of-range field. This is the exact // reply shape that used to abstain as `unusable_visual_fields` instead of failing closed. const blank = value => value === null || value === undefined || (typeof value === 'string' && !value.trim()); const colourInContract = value => COLORS_IN_CONTRACT.includes(String(value).toLowerCase()); const formInContract = value => Number.isInteger(Number(value)) && Number(value) >= 1 && Number(value) <= 7; const escapeCases = outOfRange.filter(entry => { const value = JSON.parse(entry.providerContent); const fields = [value.bristolType, value.color]; return fields.some(blank) && fields.some(field => !blank(field) && !(colourInContract(field) || formInContract(field))); }); assert.ok(escapeCases.length > 0, 'the corpus must carry a blank-plus-out-of-range case so the fail-closed label is really earned'); }); test('every recorded corpus response is schema-valid or fails closed, exactly as declared', async () => { const corpus = await loadVisionCorpus(corpusDir); for (const entry of corpus.cases) { const result = classifyVisionOutcome(entry.providerContent); assert.ok(VISION_CORPUS_OUTCOMES.includes(result.outcome), `${entry.id} produced an unknown outcome ${result.outcome}`); assert.equal(result.outcome, entry.expectedOutcome, `${entry.id} must resolve to ${entry.expectedOutcome}`); assert.equal(result.reason, entry.expectedReason, `${entry.id} must report ${entry.expectedReason}`); // A response may never smuggle a visual claim out of a non-suggestion outcome. if (result.outcome !== 'suggestion') { assert.equal('bristolType' in result, false, `${entry.id} must not carry a suggested form`); assert.equal('color' in result, false, `${entry.id} must not carry a suggested color`); } assert.doesNotMatch(JSON.stringify(result), /base64|data:image|Bear\s?er|api[_-]?key/i, `${entry.id} must not echo image bytes or credentials`); } });