import test from 'node:test'; import assert from 'node:assert/strict'; import http from 'node:http'; import net from 'node:net'; import { spawn } from 'node:child_process'; import { once } from 'node:events'; import { fileURLToPath } from 'node:url'; // Issue #19 at the real boundary: the server that ships, behind a provider stub this test owns, so // "the metric reflects a real request" and "the metric never contains the request" are both // measurable instead of asserted from a mock. const repoRoot = fileURLToPath(new URL('..', import.meta.url)); const OPERATOR_TOKEN = 'test-operator-token-9f2a'; function cleanJpeg({ width = 1200, height = 900 } = {}) { const sof = Buffer.alloc(17); sof.writeUInt16BE(0xffc0, 0); sof.writeUInt16BE(15, 2); sof[4] = 8; sof.writeUInt16BE(height, 5); sof.writeUInt16BE(width, 7); sof[9] = 3; for (let i = 0; i < 3; i += 1) { sof[10 + i * 3] = i + 1; sof[11 + i * 3] = 0x11; } return Buffer.concat([Buffer.from([0xff, 0xd8]), sof, Buffer.from([0xff, 0xd9])]); } const dataUrl = (mime, bytes) => `data:${mime};base64,${bytes.toString('base64')}`; async function freePort() { const server = net.createServer(); server.listen(0, '127.0.0.1'); await once(server, 'listening'); const { port } = server.address(); server.close(); return port; } async function startProvider() { const state = { calls: 0 }; const server = http.createServer(async (req, res) => { if (req.url === '/v1/models') { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ data: [{ id: 'metrics-test-model' }] })); return; } state.calls += 1; for await (const _ of req) { /* drain */ } res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ choices: [{ message: { content: JSON.stringify({ isStool: true, bristolType: 4, color: 'brown', confidence: 0.8, imageQuality: 'good', observations: 'Synthetic test card.', }), }, }], })); }); server.listen(0, '127.0.0.1'); await once(server, 'listening'); return { state, port: server.address().port, close: () => new Promise(resolve => { server.closeAllConnections?.(); server.close(resolve); }), }; } async function withServer(env, run) { const provider = await startProvider(); const port = await freePort(); const child = spawn(process.execPath, ['server.mjs'], { cwd: repoRoot, env: { ...process.env, HOST: '127.0.0.1', PORT: String(port), TIMMY_VISION_ENABLED: 'true', TIMMY_VISION_PROFILE: 'selfhost', TIMMY_VISION_BASE_URL: `http://127.0.0.1:${provider.port}/v1`, TIMMY_VISION_MODEL: 'metrics-test-model', ...env, }, stdio: ['ignore', 'pipe', 'pipe'], }); const logs = []; child.stdout.on('data', chunk => logs.push(chunk.toString())); child.stderr.on('data', chunk => logs.push(chunk.toString())); const base = `http://127.0.0.1:${port}`; try { const deadline = Date.now() + 15_000; for (;;) { try { const response = await fetch(`${base}/api/healthz`); if (response.ok) break; } catch { /* not up yet */ } if (Date.now() > deadline) throw new Error(`server did not start: ${logs.join('')}`); await new Promise(resolve => setTimeout(resolve, 100)); } await run({ base, provider, logs }); } finally { child.kill('SIGKILL'); await provider.close(); } } const postAnalyze = (base, payload) => fetch(`${base}/api/analyze`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(payload), }); const getMetrics = (base, token = OPERATOR_TOKEN) => fetch(`${base}/api/metrics`, { headers: token ? { authorization: `Bearer ${token}` } : {}, }); test('the metrics snapshot reports the request the server actually analysed', async () => { await withServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN }, async ({ base, provider }) => { const analyzed = await postAnalyze(base, { consent: true, imageDataUrl: dataUrl('image/jpeg', cleanJpeg()) }); assert.equal(analyzed.status, 200); assert.equal(provider.state.calls, 1); const response = await getMetrics(base); const snapshot = await response.json(); assert.equal(response.status, 200); assert.equal(snapshot.app.analyzeRequests, 1); assert.equal(snapshot.queue.accepted, 1); assert.equal(snapshot.queue.inFlight, 0, 'a finished request must release its slot'); assert.equal(snapshot.model.outcomes.suggestion, 1); assert.equal(snapshot.model.latency.count, 1); assert.equal(snapshot.model.abstentionRate, 0); assert.equal(snapshot.model.readiness.ready, true, 'readiness is probed from the configured worker'); }); }); test('the metrics route is operator-only, and absent when no token is configured', async () => { await withServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN }, async ({ base }) => { const wrong = await getMetrics(base, 'not-the-operator-token'); assert.equal(wrong.status, 403); assert.doesNotMatch(await wrong.text(), /analyzeRequests/, 'a rejected caller learns nothing about traffic'); const anonymous = await getMetrics(base, ''); assert.equal(anonymous.status, 403); }); await withServer({}, async ({ base }) => { const response = await getMetrics(base, ''); assert.equal(response.status, 404, 'an unconfigured deployment exposes no metrics route at all'); }); }); test('the snapshot proves the privacy boundary from the outside', async () => { await withServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN }, async ({ base }) => { // A real synthetic photo goes through the real path, so any leak would have real bytes to leak. const bytes = cleanJpeg(); const analyzed = await postAnalyze(base, { consent: true, note: 'fictional note about a very private symptom', imageDataUrl: dataUrl('image/jpeg', bytes), }); assert.equal(analyzed.status, 200); const snapshot = await getMetrics(base).then(response => response.json()); const serialized = JSON.stringify(snapshot); assert.doesNotMatch(serialized, /very private symptom/i, 'note text must never be a metric'); const base64Run = bytes.toString('base64').slice(0, 24); assert.ok(base64Run.length === 24, 'sanity: the synthetic image has bytes to leak'); assert.doesNotMatch(serialized, new RegExp(base64Run.replace(/[+/=]/g, '\\$&')), 'image bytes must never be a metric'); assert.doesNotMatch(serialized, /data:image/, 'no data URL may appear in the snapshot'); assert.doesNotMatch(serialized, /127\.0\.0\.1|::1/, 'no caller address may appear in the snapshot'); assert.doesNotMatch(serialized, /image|note|photo/i, 'no request-derived key may appear in the snapshot'); }); });