timmy-talking-turd/tests/model-metrics-report.acceptance.test.js

import test from 'node:test';
import assert from 'node:assert/strict';
import http from 'node:http';
import net from 'node:net';
import { spawn, execFile } from 'node:child_process';
import { once } from 'node:events';
import { promisify } from 'node:util';
import { fileURLToPath } from 'node:url';

const execFileAsync = promisify(execFile);
const repoRoot = fileURLToPath(new URL('..', import.meta.url));
const OPERATOR_TOKEN = 'report-operator-token-4b1c';

async function freePort() {
  const server = net.createServer();
  server.listen(0, '127.0.0.1');
  await once(server, 'listening');
  const { port } = server.address();
  server.close();
  return port;
}

async function startProvider() {
  const server = http.createServer(async (req, res) => {
    if (req.url === '/v1/models') {
      res.writeHead(200, { 'content-type': 'application/json' });
      res.end(JSON.stringify({ data: [{ id: 'report-model' }] }));
      return;
    }
    for await (const _ of req) { /* drain */ }
    res.writeHead(200, { 'content-type': 'application/json' });
    res.end(JSON.stringify({
      choices: [{ message: { content: JSON.stringify({
        isStool: true, bristolType: 4, color: 'brown', confidence: 0.8,
        imageQuality: 'good', observations: 'Synthetic test card.',
      }) } }],
    }));
  });
  server.listen(0, '127.0.0.1');
  await once(server, 'listening');
  return { port: server.address().port, close: () => new Promise(resolve => { server.closeAllConnections?.(); server.close(resolve); }) };
}

test('the operator report renders the app, queue, and model boundaries from a live snapshot', async () => {
  const provider = await startProvider();
  const port = await freePort();
  const child = spawn(process.execPath, ['server.mjs'], {
    cwd: repoRoot,
    env: {
      ...process.env,
      HOST: '127.0.0.1',
      PORT: String(port),
      TIMMY_VISION_ENABLED: 'true',
      TIMMY_VISION_PROFILE: 'selfhost',
      TIMMY_VISION_BASE_URL: `http://127.0.0.1:${provider.port}/v1`,
      TIMMY_VISION_MODEL: 'report-model',
      TIMMY_METRICS_TOKEN: OPERATOR_TOKEN,
    },
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  const logs = [];
  child.stdout.on('data', c => logs.push(c.toString()));
  child.stderr.on('data', c => logs.push(c.toString()));
  const base = `http://127.0.0.1:${port}`;
  try {
    const deadline = Date.now() + 15_000;
    for (;;) {
      try {
        const response = await fetch(`${base}/api/healthz`);
        if (response.ok) break;
      } catch { /* not up yet */ }
      if (Date.now() > deadline) throw new Error(`server did not start: ${logs.join('')}`);
      await new Promise(resolve => setTimeout(resolve, 100));
    }

    const { stdout } = await execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', '--url', base, '--token', OPERATOR_TOKEN], { cwd: repoRoot });

    assert.match(stdout, /APP/);
    assert.match(stdout, /QUEUE/);
    assert.match(stdout, /MODEL/);
    assert.match(stdout, /readiness\s+ready/, 'the report must name the readiness phase it observed');
    assert.match(stdout, /abstention/i);
    assert.match(stdout, /failure/i);
    assert.doesNotMatch(stdout, /OPERATOR_TOKEN|report-operator-token/, 'the report must never print the token');
  } finally {
    child.kill('SIGKILL');
    await provider.close();
  }
});

async function startServer(env) {
  const port = await freePort();
  const child = spawn(process.execPath, ['server.mjs'], {
    cwd: repoRoot,
    env: { ...process.env, HOST: '127.0.0.1', PORT: String(port), ...env },
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  const base = `http://127.0.0.1:${port}`;
  const deadline = Date.now() + 15_000;
  for (;;) {
    try {
      const response = await fetch(`${base}/api/healthz`);
      if (response.ok) break;
    } catch { /* not up yet */ }
    if (Date.now() > deadline) throw new Error('server did not start');
    await new Promise(resolve => setTimeout(resolve, 100));
  }
  return { base, stop: () => child.kill('SIGKILL') };
}

test('the report refuses a missing or wrong operator token without echoing it', async () => {
  const tokenless = await startServer({ TIMMY_METRICS_ENABLED: undefined, TIMMY_METRICS_TOKEN: '' });
  try {
    await assert.rejects(
      execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', '--url', tokenless.base, '--token', 'x'], { cwd: repoRoot }),
      error => {
        assert.notEqual(error.code, 0);
        assert.match(error.stderr, /disabled/i, 'a server with no metrics route must say so, not "unauthorized"');
        return true;
      },
    );
  } finally {
    tokenless.stop();
  }

  const guarded = await startServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN });
  try {
    await assert.rejects(
      execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', '--url', guarded.base, '--token', 'not-the-token'], { cwd: repoRoot }),
      error => {
        assert.notEqual(error.code, 0);
        assert.match(error.stderr, /rejected/i);
        assert.doesNotMatch(`${error.stderr}${error.stdout}`, /not-the-token/, 'the rejected token must never be echoed');
        return true;
      },
    );
  } finally {
    guarded.stop();
  }
});