timmy-talking-turd/tests/metrics-endpoint.acceptance.test.js

import test from 'node:test';
import assert from 'node:assert/strict';
import http from 'node:http';
import net from 'node:net';
import { spawn } from 'node:child_process';
import { once } from 'node:events';
import { fileURLToPath } from 'node:url';

// Issue #19 at the real boundary: the server that ships, behind a provider stub this test owns, so
// "the metric reflects a real request" and "the metric never contains the request" are both
// measurable instead of asserted from a mock.

const repoRoot = fileURLToPath(new URL('..', import.meta.url));
const OPERATOR_TOKEN = 'test-operator-token-9f2a';

function cleanJpeg({ width = 1200, height = 900 } = {}) {
  const sof = Buffer.alloc(17);
  sof.writeUInt16BE(0xffc0, 0);
  sof.writeUInt16BE(15, 2);
  sof[4] = 8;
  sof.writeUInt16BE(height, 5);
  sof.writeUInt16BE(width, 7);
  sof[9] = 3;
  for (let i = 0; i < 3; i += 1) {
    sof[10 + i * 3] = i + 1;
    sof[11 + i * 3] = 0x11;
  }
  return Buffer.concat([Buffer.from([0xff, 0xd8]), sof, Buffer.from([0xff, 0xd9])]);
}

const dataUrl = (mime, bytes) => `data:${mime};base64,${bytes.toString('base64')}`;

async function freePort() {
  const server = net.createServer();
  server.listen(0, '127.0.0.1');
  await once(server, 'listening');
  const { port } = server.address();
  server.close();
  return port;
}

async function startProvider() {
  const state = { calls: 0 };
  const server = http.createServer(async (req, res) => {
    if (req.url === '/v1/models') {
      res.writeHead(200, { 'content-type': 'application/json' });
      res.end(JSON.stringify({ data: [{ id: 'metrics-test-model' }] }));
      return;
    }
    state.calls += 1;
    for await (const _ of req) { /* drain */ }
    res.writeHead(200, { 'content-type': 'application/json' });
    res.end(JSON.stringify({
      choices: [{
        message: {
          content: JSON.stringify({
            isStool: true, bristolType: 4, color: 'brown', confidence: 0.8,
            imageQuality: 'good', observations: 'Synthetic test card.',
          }),
        },
      }],
    }));
  });
  server.listen(0, '127.0.0.1');
  await once(server, 'listening');
  return {
    state,
    port: server.address().port,
    close: () => new Promise(resolve => {
      server.closeAllConnections?.();
      server.close(resolve);
    }),
  };
}

async function withServer(env, run) {
  const provider = await startProvider();
  const port = await freePort();
  const child = spawn(process.execPath, ['server.mjs'], {
    cwd: repoRoot,
    env: {
      ...process.env,
      HOST: '127.0.0.1',
      PORT: String(port),
      TIMMY_VISION_ENABLED: 'true',
      TIMMY_VISION_PROFILE: 'selfhost',
      TIMMY_VISION_BASE_URL: `http://127.0.0.1:${provider.port}/v1`,
      TIMMY_VISION_MODEL: 'metrics-test-model',
      ...env,
    },
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  const logs = [];
  child.stdout.on('data', chunk => logs.push(chunk.toString()));
  child.stderr.on('data', chunk => logs.push(chunk.toString()));
  const base = `http://127.0.0.1:${port}`;
  try {
    const deadline = Date.now() + 15_000;
    for (;;) {
      try {
        const response = await fetch(`${base}/api/healthz`);
        if (response.ok) break;
      } catch { /* not up yet */ }
      if (Date.now() > deadline) throw new Error(`server did not start: ${logs.join('')}`);
      await new Promise(resolve => setTimeout(resolve, 100));
    }
    await run({ base, provider, logs });
  } finally {
    child.kill('SIGKILL');
    await provider.close();
  }
}

const postAnalyze = (base, payload) => fetch(`${base}/api/analyze`, {
  method: 'POST',
  headers: { 'content-type': 'application/json' },
  body: JSON.stringify(payload),
});
const getMetrics = (base, token = OPERATOR_TOKEN) => fetch(`${base}/api/metrics`, {
  headers: token ? { authorization: `Bearer ${token}` } : {},
});

test('the metrics snapshot reports the request the server actually analysed', async () => {
  await withServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN }, async ({ base, provider }) => {
    const analyzed = await postAnalyze(base, { consent: true, imageDataUrl: dataUrl('image/jpeg', cleanJpeg()) });
    assert.equal(analyzed.status, 200);
    assert.equal(provider.state.calls, 1);

    const response = await getMetrics(base);
    const snapshot = await response.json();

    assert.equal(response.status, 200);
    assert.equal(snapshot.app.analyzeRequests, 1);
    assert.equal(snapshot.queue.accepted, 1);
    assert.equal(snapshot.queue.inFlight, 0, 'a finished request must release its slot');
    assert.equal(snapshot.model.outcomes.suggestion, 1);
    assert.equal(snapshot.model.latency.count, 1);
    assert.equal(snapshot.model.abstentionRate, 0);
    assert.equal(snapshot.model.readiness.ready, true, 'readiness is probed from the configured worker');
  });
});

test('the metrics route is operator-only, and absent when no token is configured', async () => {
  await withServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN }, async ({ base }) => {
    const wrong = await getMetrics(base, 'not-the-operator-token');
    assert.equal(wrong.status, 403);
    assert.doesNotMatch(await wrong.text(), /analyzeRequests/, 'a rejected caller learns nothing about traffic');

    const anonymous = await getMetrics(base, '');
    assert.equal(anonymous.status, 403);
  });

  await withServer({}, async ({ base }) => {
    const response = await getMetrics(base, '');
    assert.equal(response.status, 404, 'an unconfigured deployment exposes no metrics route at all');
  });
});

test('the snapshot proves the privacy boundary from the outside', async () => {
  await withServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN }, async ({ base }) => {
    // A real synthetic photo goes through the real path, so any leak would have real bytes to leak.
    const bytes = cleanJpeg();
    const analyzed = await postAnalyze(base, {
      consent: true,
      note: 'fictional note about a very private symptom',
      imageDataUrl: dataUrl('image/jpeg', bytes),
    });
    assert.equal(analyzed.status, 200);

    const snapshot = await getMetrics(base).then(response => response.json());
    const serialized = JSON.stringify(snapshot);

    assert.doesNotMatch(serialized, /very private symptom/i, 'note text must never be a metric');
    const base64Run = bytes.toString('base64').slice(0, 24);
    assert.ok(base64Run.length === 24, 'sanity: the synthetic image has bytes to leak');
    assert.doesNotMatch(serialized, new RegExp(base64Run.replace(/[+/=]/g, '\\$&')), 'image bytes must never be a metric');
    assert.doesNotMatch(serialized, /data:image/, 'no data URL may appear in the snapshot');
    assert.doesNotMatch(serialized, /127\.0\.0\.1|::1/, 'no caller address may appear in the snapshot');
    assert.doesNotMatch(serialized, /image|note|photo/i, 'no request-derived key may appear in the snapshot');
  });
});