timmy-talking-turd/tests/abstention-policy.test.js

import test from 'node:test';
import assert from 'node:assert/strict';
import { execFile } from 'node:child_process';
import { mkdtemp, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import { ABSTENTION_COPY, ABSTENTION_POLICY, abstainFor, evaluatePrefillGate } from '../src/abstention-policy.js';

test('every abstention reason has one fixed piece of copy the user sees', () => {
  assert.equal(abstainFor('not_stool').reason, 'The image does not clearly show stool.');
  assert.equal(abstainFor('low_confidence').reason, 'The image is too uncertain to prefill safely.');
  assert.equal(abstainFor('unusable_visual_fields').reason, 'The photo does not show a form Timmy can read safely.');
  assert.equal(abstainFor('low_quality').reason, 'The photo is too blurry or poorly lit to suggest a form safely.');
  assert.deepEqual(abstainFor('low_quality'), {
    status: 'needs_user_input',
    isStool: true,
    abstainedFor: 'low_quality',
    reason: ABSTENTION_COPY.low_quality,
  });
  assert.equal(abstainFor('not_stool', false).isStool, false);
  assert.throws(() => abstainFor('looked_wrong'), /unknown abstention reason/);
});

test('the policy states the confidence floor and the quality levels that may prefill', () => {
  assert.ok(Number.isInteger(ABSTENTION_POLICY.version));
  assert.equal(ABSTENTION_POLICY.minConfidence, 0.55);
  assert.deepEqual([...ABSTENTION_POLICY.acceptedImageQuality], ['good', 'fair']);
  assert.equal(ABSTENTION_POLICY.acceptedImageQuality.includes('poor'), false);
  assert.equal(Object.isFrozen(ABSTENTION_POLICY), true, 'the shipped policy must not be mutated at runtime');
});

test('prefill is authorised only by a GO report whose threshold was selected on validation', () => {
  const passing = {
    verdict: 'GO',
    recommended: 'specialist-candidate',
    models: [{
      model: 'specialist-candidate',
      selectionSplit: 'validation',
      evaluatedSplit: 'test',
      gate: { passed: true, failed: [] },
    }],
  };
  assert.deepEqual(evaluatePrefillGate(passing), {
    prefillAllowed: true,
    verdict: 'GO',
    calibrationModel: 'specialist-candidate',
    blockedBy: [],
  });

  const blocked = evaluatePrefillGate({
    verdict: 'BLOCKED',
    recommended: null,
    models: [{ model: 'specialist-candidate', selectionSplit: 'validation', evaluatedSplit: 'test', gate: { passed: false, failed: ['precisionAtCoverage'] } }],
  });
  assert.equal(blocked.prefillAllowed, false);
  assert.deepEqual(blocked.blockedBy, ['calibration-verdict-not-go', 'no-recommended-calibration']);

  const selectedOnTest = evaluatePrefillGate({
    verdict: 'GO',
    recommended: 'opportunist',
    models: [{ model: 'opportunist', selectionSplit: 'test', evaluatedSplit: 'test', gate: { passed: true, failed: [] } }],
  });
  assert.equal(selectedOnTest.prefillAllowed, false, 'a threshold chosen on the split it is scored on is not evidence');
  assert.deepEqual(selectedOnTest.blockedBy, ['threshold-not-selected-on-validation']);

  const noReport = evaluatePrefillGate(undefined);
  assert.equal(noReport.prefillAllowed, false);
  assert.match(noReport.blockedBy.join(','), /calibration-verdict-not-go/);
});

const execFileAsync = promisify(execFile);
const receiptScript = fileURLToPath(new URL('../scripts/abstention_policy_receipt.mjs', import.meta.url));

async function runReceipt(report) {
  const directory = await mkdtemp(join(tmpdir(), 'timmy-policy-'));
  const reportPath = join(directory, 'calibration-report.json');
  await writeFile(reportPath, JSON.stringify(report));
  try {
    const { stdout } = await execFileAsync(process.execPath, [receiptScript, '--report', reportPath]);
    return { code: 0, stdout };
  } catch (error) {
    return { code: error.code, stdout: error.stdout || '', stderr: error.stderr || '' };
  }
}

test('the receipt refuses to authorise prefill without a passing held-out calibration', async () => {
  const blocked = await runReceipt({
    verdict: 'BLOCKED',
    recommended: null,
    models: [{ model: 'specialist-candidate', selectionSplit: 'validation', evaluatedSplit: 'test', gate: { passed: false, failed: ['precisionAtCoverage'] } }],
  });
  assert.equal(blocked.code, 3, 'a blocked calibration must exit non-zero instead of shipping prefill');
  assert.match(blocked.stdout, /prefill: BLOCKED/);
  assert.match(blocked.stdout, /calibration-verdict-not-go/);
  assert.match(blocked.stdout, /policy: version 1 · confidence ≥ 0\.55 · quality good\|fair/);

  const passing = await runReceipt({
    verdict: 'GO',
    recommended: 'specialist-candidate',
    models: [{ model: 'specialist-candidate', selectionSplit: 'validation', evaluatedSplit: 'test', gate: { passed: true, failed: [] } }],
  });
  assert.equal(passing.code, 0);
  assert.match(passing.stdout, /prefill: AUTHORISED for specialist-candidate/);
});

test('the receipt treats a missing report as a block and never as a default allow', async () => {
  const directory = await mkdtemp(join(tmpdir(), 'timmy-policy-'));
  try {
    await execFileAsync(process.execPath, [receiptScript, '--report', join(directory, 'absent.json')]);
    assert.fail('a missing calibration report must not exit 0');
  } catch (error) {
    assert.equal(error.code, 2);
    assert.match(String(error.stderr), /could not read|absent/i);
  }
});