timmy-talking-turd/tests/vision-boundary.test.js

import test from 'node:test';
import assert from 'node:assert/strict';
import { analyzePhoto, visionFailureResponse } from '../src/vision-service.js';
import { loadVisionCorpus } from '../src/vision-corpus.js';
import { CORPUS_MODEL, caseFor, corpusDir, corpusPayload, startProvider, startServer, stopServer } from './helpers/vision-harness.mjs';

const FAIL_CLOSED_COPY = /no suggestion was applied/i;

test('a malformed provider response fails closed with user-safe copy instead of echoing provider detail', () => {
  const failure = visionFailureResponse(Object.assign(new Error('Invalid AI response: visual fields are out of range.'), { code: 'invalid_fields' }));

  assert.equal(failure.status, 503);
  assert.match(failure.body.error, FAIL_CLOSED_COPY);
  assert.doesNotMatch(failure.body.error, /invalid ai response|out of range/i);

  const consent = visionFailureResponse(new Error('Explicit consent is required before AI analysis.'));
  assert.equal(consent.status, 400);
  assert.match(consent.body.error, /consent/i);

  const offline = visionFailureResponse(new Error('AI analysis is temporarily unavailable. Continue manually.'));
  assert.equal(offline.status, 503);
  assert.match(offline.body.error, /temporarily unavailable/i);
});

test('the corpus replays through the real provider call path without inventing a suggestion', async () => {
  const corpus = await loadVisionCorpus(corpusDir);
  for (const entry of corpus.cases) {
    const fetchImpl = async () => ({ ok: true, json: async () => ({ choices: [{ message: { content: entry.providerContent } }] }) });
    const options = {
      payload: { imageDataUrl: 'data:image/png;base64,YQ==', consent: true },
      fetchImpl,
      config: { baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'corpus-key', model: CORPUS_MODEL },
    };
    if (entry.expectedOutcome === 'suggestion') {
      const result = await analyzePhoto(options);
      assert.equal(result.status, 'suggestion', `${entry.id} must return a suggestion`);
      assert.equal(result.imageQuality, 'good');
    } else if (entry.expectedOutcome === 'abstained') {
      const result = await analyzePhoto(options);
      assert.equal(result.status, 'needs_user_input', `${entry.id} must abstain`);
      assert.equal(result.abstainedFor, entry.expectedReason, `${entry.id} must name its abstention`);
      assert.equal('bristolType' in result, false, `${entry.id} must not prefill a form`);
    } else {
      await assert.rejects(() => analyzePhoto(options), (error) => {
        assert.equal(visionFailureResponse(error).status, 503, `${entry.id} must fail closed`);
        return true;
      });
    }
  }
});

test('the running server answers every corpus case with a fail-closed status and no leaked image bytes', async () => {
  const corpus = await loadVisionCorpus(corpusDir);
  const provider = await startProvider();
  const server = await startServer({ providerPort: provider.port });
  const postAnalyze = async (payload) => {
    const response = await fetch('http://127.0.0.1:4199/api/analyze', {
      method: 'POST',
      headers: { 'content-type': 'application/json' },
      body: JSON.stringify(payload),
    });
    return { status: response.status, body: await response.json() };
  };
  try {
    const suggestion = caseFor(corpus, 'stool-like-formed');
    provider.state.content = suggestion.providerContent;
    const payload = await corpusPayload(suggestion);
    const ok = await postAnalyze(payload);
    assert.equal(ok.status, 200, 'a schema-valid suggestion is the only case that returns 200 with fields');
    assert.equal(ok.body.bristolType, 4);

    const glare = caseFor(corpus, 'glare-overexposed');
    provider.state.content = glare.providerContent;
    const abstained = await postAnalyze(await corpusPayload(glare));
    assert.equal(abstained.status, 200, 'a schema-valid response with unreadable fields abstains, it is not a server failure');
    assert.equal(abstained.body.status, 'needs_user_input');
    assert.equal(abstained.body.abstainedFor, 'unusable_visual_fields');
    assert.equal('bristolType' in abstained.body, false);

    const malformed = caseFor(corpus, 'malformed-response');
    provider.state.content = malformed.providerContent;
    const unparsable = await postAnalyze(await corpusPayload(malformed));
    assert.equal(unparsable.status, 503, 'unparsable provider content must not be served as a suggestion');
    assert.match(unparsable.body.error, FAIL_CLOSED_COPY);
    assert.doesNotMatch(JSON.stringify(unparsable.body), /base64|data:image|corpus-key|Bearer/i);

    const outOfRangeCase = caseFor(corpus, 'out-of-range-fields');
    provider.state.content = outOfRangeCase.providerContent;
    const outOfRange = await postAnalyze(await corpusPayload(outOfRangeCase));
    assert.equal(outOfRange.status, 503, 'out-of-range fields must not be served as a suggestion');
    assert.match(outOfRange.body.error, FAIL_CLOSED_COPY);

    // Issue #73: a filled-in field that is out of range fails closed even when the other field is blank,
    // so the user is never told the photo was unreadable when the model returned an impossible value.
    const blankAndOutOfRange = caseFor(corpus, 'blank-plus-out-of-range');
    provider.state.content = blankAndOutOfRange.providerContent;
    const blankOutOfRange = await postAnalyze(await corpusPayload(blankAndOutOfRange));
    assert.equal(blankOutOfRange.status, 503, 'a filled-in out-of-range field must not be served as an abstention');
    assert.match(blankOutOfRange.body.error, FAIL_CLOSED_COPY);
    assert.equal('abstainedFor' in blankOutOfRange.body, false, 'a contract breach must not be reported as an abstention');
    assert.doesNotMatch(JSON.stringify(blankOutOfRange.body), /base64|data:image|corpus-key|Bearer/i);

    const noConsent = await postAnalyze({ imageDataUrl: payload.imageDataUrl, consent: false });
    assert.equal(noConsent.status, 400, 'analysis without explicit consent is a client error');
    assert.doesNotMatch(JSON.stringify(noConsent.body), /base64|data:image/i);
  } finally {
    await stopServer(server);
    await provider.close();
  }
});