timmy-talking-turd/tests/vision-corpus.test.js

import test from 'node:test';
import assert from 'node:assert/strict';
import { cp, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { classifyVisionOutcome, loadVisionCorpus, VISION_CORPUS_CLASSES, VISION_CORPUS_LICENSE, VISION_CORPUS_OUTCOMES } from '../src/vision-corpus.js';

const corpusDir = fileURLToPath(new URL('./fixtures/vision-corpus/', import.meta.url));
// The colour contract `src/analysis.js` enforces, restated here so the corpus can be checked against it.
const COLORS_IN_CONTRACT = ['brown', 'green', 'yellow', 'pale', 'red', 'black'];

async function scratchCorpus() {
  const directory = await mkdtemp(join(tmpdir(), 'timmy-corpus-'));
  await cp(corpusDir, directory, { recursive: true });
  return directory;
}

test('vision corpus ships licensed synthetic fixtures covering every required visual case class', async () => {
  const corpus = await loadVisionCorpus(corpusDir);

  assert.equal(corpus.license, VISION_CORPUS_LICENSE);
  const classes = new Set(corpus.cases.map(entry => entry.caseClass));
  for (const required of VISION_CORPUS_CLASSES) {
    assert.ok(classes.has(required), `corpus must cover the ${required} case`);
  }
  for (const entry of corpus.cases) {
    assert.equal(entry.synthetic, true, `${entry.id} must be synthetic; no real medical image is redistributable`);
    assert.equal(entry.license, VISION_CORPUS_LICENSE);
    assert.match(entry.provenance, /\S/, `${entry.id} must record where it came from`);
    assert.match(entry.sha256, /^[a-f0-9]{64}$/, `${entry.id} must record its content hash`);
  }
});

test('vision corpus rejects fixtures whose bytes no longer match the recorded hash', async () => {
  const directory = await scratchCorpus();
  try {
    const manifest = JSON.parse(await readFile(join(directory, 'manifest.json'), 'utf8'));
    const target = join(directory, manifest.cases[0].file);
    const bytes = await readFile(target);
    await writeFile(target, Buffer.concat([bytes, Buffer.from([0])]));

    await assert.rejects(() => loadVisionCorpus(directory), new RegExp(manifest.cases[0].id));
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test('vision corpus rejects an unlicensed, non-synthetic, or incomplete manifest', async () => {
  const directory = await scratchCorpus();
  try {
    const manifestPath = join(directory, 'manifest.json');
    const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));

    manifest.cases[0].synthetic = false;
    await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
    await assert.rejects(() => loadVisionCorpus(directory), /synthetic/i);

    manifest.cases[0].synthetic = true;
    manifest.cases = manifest.cases.filter(entry => entry.caseClass !== 'non-stool');
    await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
    await assert.rejects(() => loadVisionCorpus(directory), /non-stool/);

    manifest.cases[0].license = 'all-rights-reserved';
    await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
    await assert.rejects(() => loadVisionCorpus(directory), /CC0-1\.0/);
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
});

test('the corpus earns its out-of-range label and covers the blank-plus-out-of-range class', async () => {
  const corpus = await loadVisionCorpus(corpusDir);
  const outOfRange = corpus.cases.filter(entry => entry.expectedReason === 'out_of_range_or_missing_fields');
  assert.ok(outOfRange.length > 0, 'the corpus must cover the out-of-range failure class');

  // Issue #73: a case whose confidence breaks the gate never reaches the field-range check, so its
  // `out_of_range_or_missing_fields` label would be attributed to a guard it never exercised.
  const reachesRangeCheck = outOfRange.filter(entry => Number(JSON.parse(entry.providerContent).confidence) <= 1);
  assert.ok(reachesRangeCheck.length > 0, 'at least one out-of-range case must clear the confidence gate to exercise the range check');

  // Issue #73's escape class: a blank field next to a non-blank, out-of-range field. This is the exact
  // reply shape that used to abstain as `unusable_visual_fields` instead of failing closed.
  const blank = value => value === null || value === undefined || (typeof value === 'string' && !value.trim());
  const colourInContract = value => COLORS_IN_CONTRACT.includes(String(value).toLowerCase());
  const formInContract = value => Number.isInteger(Number(value)) && Number(value) >= 1 && Number(value) <= 7;
  const escapeCases = outOfRange.filter(entry => {
    const value = JSON.parse(entry.providerContent);
    const fields = [value.bristolType, value.color];
    return fields.some(blank) && fields.some(field => !blank(field) && !(colourInContract(field) || formInContract(field)));
  });
  assert.ok(escapeCases.length > 0, 'the corpus must carry a blank-plus-out-of-range case so the fail-closed label is really earned');
});

test('every recorded corpus response is schema-valid or fails closed, exactly as declared', async () => {
  const corpus = await loadVisionCorpus(corpusDir);

  for (const entry of corpus.cases) {
    const result = classifyVisionOutcome(entry.providerContent);
    assert.ok(VISION_CORPUS_OUTCOMES.includes(result.outcome), `${entry.id} produced an unknown outcome ${result.outcome}`);
    assert.equal(result.outcome, entry.expectedOutcome, `${entry.id} must resolve to ${entry.expectedOutcome}`);
    assert.equal(result.reason, entry.expectedReason, `${entry.id} must report ${entry.expectedReason}`);
    // A response may never smuggle a visual claim out of a non-suggestion outcome.
    if (result.outcome !== 'suggestion') {
      assert.equal('bristolType' in result, false, `${entry.id} must not carry a suggested form`);
      assert.equal('color' in result, false, `${entry.id} must not carry a suggested color`);
    }
    assert.doesNotMatch(JSON.stringify(result), /base64|data:image|Bear\s?er|api[_-]?key/i, `${entry.id} must not echo image bytes or credentials`);
  }
});