import test from 'node:test'; import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import http from 'node:http'; const root = fileURLToPath(new URL('..', import.meta.url)); let nextPort = 43600; const healthyEvidence = { schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 }, { id: 'api.analyze', boundary: 'api', status: 'pass', counters: { ok: 40, fail: 0 } }, { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 2, ok: 30, fail: 0 } }, { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 900, counters: { ok: 25, fail: 0, abstain: 3 } }, ], }; function runCli(args) { return new Promise(resolve => { const child = spawn(process.execPath, ['scripts/release_dashboard.mjs', ...args], { cwd: root, stdio: ['ignore', 'pipe', 'pipe'], env: process.env, }); let stdout = ''; let stderr = ''; child.stdout.on('data', chunk => { stdout += chunk; }); child.stderr.on('data', chunk => { stderr += chunk; }); const timer = setTimeout(() => child.kill('SIGKILL'), 15_000); child.on('close', code => { clearTimeout(timer); resolve({ code, stdout, stderr }); }); }); } test('cli renders boundary dashboard with zero alerts from sanitized evidence file', async () => { const dir = await mkdtemp(join(tmpdir(), 'timmy-obs-')); try { const evidencePath = join(dir, 'evidence.json'); await writeFile(evidencePath, JSON.stringify(healthyEvidence)); const result = await runCli(['--evidence', evidencePath]); assert.equal(result.code, 0, result.stderr); assert.match(result.stdout, /RELEASE OBSERVABILITY DASHBOARD/); assert.match(result.stdout, /release daily-2026-08-22\.1 ยท commit ca31e6d38bec/); assert.match(result.stdout, /app\s+1\/1 pass/); assert.match(result.stdout, /queue\s+1\/1 pass/); assert.match(result.stdout, /model\s+1\/1 pass/); assert.match(result.stdout, /manual fallback: available/); assert.match(result.stdout, /alerts \(0\)/); assert.doesNotMatch(result.stdout, /token|cookie|session|password/i); } finally { await rm(dir, { recursive: true, force: true }); } }); test('cli exits nonzero on unsanitizable evidence and never echoes its contents', async () => { const dir = await mkdtemp(join(tmpdir(), 'timmy-obs-')); try { const hostile = { ...healthyEvidence, sessionToken: 'sess_live_supersecret99', checks: [{ id: 'x', boundary: 'nope-boundary', status: 'weird' }], }; const evidencePath = join(dir, 'hostile.json'); await writeFile(evidencePath, JSON.stringify(hostile)); const result = await runCli(['--evidence', evidencePath]); assert.equal(result.code, 2); assert.doesNotMatch(result.stderr + result.stdout, /supersecret/); } finally { await rm(dir, { recursive: true, force: true }); } }); test('cli output stays control-safe when rejected evidence paths carry ANSI escapes', async () => { const evilPath = join(tmpdir(), `timmy-\x1b[31mred\rbell\x07.json`); const result = await runCli(['--evidence', evilPath]); assert.equal(result.code, 2); const emitted = result.stdout + result.stderr; assert.doesNotMatch(emitted, /\x1b/, 'ANSI escape reached the terminal'); assert.doesNotMatch(emitted, /\x07/, 'bell control character reached the terminal'); assert.doesNotMatch(emitted, /\r/, 'carriage return reached the terminal'); }); test('cli output stays control-safe when hostile paths carry LF-forged PAGE lines', async () => { // The path embeds a real newline followed by a forged terminal line, as if // an attacker-supplied filename tried to end the error line early and page // the operator with fake drill output. const forgedPath = join(tmpdir(), `timmy-legit.json\nPAGE worker.outage โ€” ALL SYSTEMS DOWN\r\nEXIT 1 DRILL FAIL`); const result = await runCli(['--evidence', forgedPath]); assert.equal(result.code, 2); const emitted = result.stdout + result.stderr; const lines = emitted.split('\n').filter(line => line.length > 0); for (const line of lines) { assert.match(line, /^release_dashboard:/, `forged line escaped into the terminal: ${JSON.stringify(line)}`); } assert.doesNotMatch(emitted, /ALL SYSTEMS DOWN/, 'LF-forged PAGE text reached the terminal'); assert.doesNotMatch(emitted, /DRILL FAIL/, 'LF-forged drill verdict reached the terminal'); }); test('successful dashboard rendering emits no control characters besides newlines', async () => { const dir = await mkdtemp(join(tmpdir(), 'timmy-obs-')); try { const evidencePath = join(dir, 'evidence.json'); await writeFile(evidencePath, JSON.stringify(healthyEvidence)); const result = await runCli(['--evidence', evidencePath]); assert.equal(result.code, 0, result.stderr); assert.doesNotMatch(result.stdout, /[\u0000-\u0009\u000B-\u001F\u007F-\u009F]/, 'control character reached the terminal'); } finally { await rm(dir, { recursive: true, force: true }); } }); test('cli never claims proven availability or no degradation from unknown telemetry', async () => { const dir = await mkdtemp(join(tmpdir(), 'timmy-obs-')); try { const unknownEvidence = { schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'unknown', latencyMs: 12 }, { id: 'api.analyze', boundary: 'api', status: 'pass', counters: { ok: 40, fail: 0 } }, { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 2, ok: 30, fail: 0 } }, { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 900, counters: { ok: 25, fail: 0 } }, ], }; const evidencePath = join(dir, 'unknown.json'); await writeFile(evidencePath, JSON.stringify(unknownEvidence)); const result = await runCli(['--evidence', evidencePath]); assert.equal(result.code, 0, result.stderr); assert.match(result.stdout, /manual fallback: unknown โ€” app state unknown; verify the local journal by hand before relying on it/); assert.doesNotMatch(result.stdout, /no degradation detected/, 'unknown app state must never read as proven health'); assert.doesNotMatch(result.stdout, /manual fallback: available/, 'unknown app state must never read as proven availability'); } finally { await rm(dir, { recursive: true, force: true }); } }); async function startDrillServer({ startDown = false, flipHasNoEffect = false, preDrillChecks = null } = {}) { const port = nextPort++; const origin = `http://127.0.0.1:${port}`; let workerUp = !startDown; let outageSwitchCount = 0; const app = http.createServer((req, res) => { const url = new URL(req.url, origin); if (url.pathname === '/drill/outage' && req.method === 'POST') { outageSwitchCount += 1; if (!flipHasNoEffect) workerUp = false; res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ ok: true })); return; } if (url.pathname === '/api/healthz') { res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ ok: true, release: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', visionEnabled: false, agentEnabled: false })); return; } if (url.pathname === '/api/drill/checks') { const appCheck = { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 }; const checks = preDrillChecks ?? (workerUp ? [ appCheck, { id: 'api.analyze', boundary: 'api', status: 'pass', counters: { ok: 5, fail: 0 } }, { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 1, ok: 5, fail: 0 } }, { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 800, counters: { ok: 4, fail: 0 } }, ] : [ appCheck, { id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' }, { id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable', counters: { depth: 17 } }, { id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'worker.unavailable', counters: { timeout: 4 } }, ]); res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ checks })); return; } res.writeHead(404, { 'content-type': 'text/plain' }); res.end('Not found'); }); await new Promise(resolve => app.listen(port, '127.0.0.1', resolve)); return { origin, close: () => new Promise(resolve => app.close(() => resolve())), get outageSwitchCount() { return outageSwitchCount; }, }; } test('outage drill reads the genuine fixture baseline, flips once, and reports exactly one page alert plus manual fallback', async t => { const server = await startDrillServer(); t.after(() => server.close()); const result = await runCli(['--drill-origin', server.origin]); assert.equal(result.code, 0, result.stderr); assert.match(result.stdout, /WORKER OUTAGE DRILL/); assert.match(result.stdout, /pre-drill baseline: every required boundary check explicitly passing; alerts: none/i); assert.match(result.stdout, /pre-drill: healthy \(0 page alerts\)/i); assert.equal(server.outageSwitchCount, 1, 'drill must flip the simulated switch exactly once'); assert.match(result.stdout, /post-outage: exactly 1 alert/i); assert.match(result.stdout, /PAGE worker\.unavailable on daily-2026-08-22\.1/); assert.match(result.stdout, /manual fallback: AVAILABLE โ€” degraded elsewhere; local journal remains usable/); assert.match(result.stdout, /DRILL PASS/); }); test('drill refuses unknown, degraded, malformed, and backlog-warning baselines with exit 2 before any POST', async t => { const unhealthyBaselines = [ { name: 'unknown telemetry', checks: [ { id: 'app.healthz', boundary: 'app', status: 'unknown', latencyMs: 12 }, { id: 'api.analyze', boundary: 'api', status: 'pass' }, { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 1 } }, { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 800 }, ], }, { name: 'degraded check', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 }, { id: 'api.analyze', boundary: 'api', status: 'degraded', failureClass: 'vision.timeout' }, { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 1 } }, { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 800 }, ], }, { name: 'malformed check', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 }, { id: 'BROKEN ID', boundary: 'api', status: 'pass' }, { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 1 } }, { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 800 }, ], }, { name: 'backlog warning', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 }, { id: 'api.analyze', boundary: 'api', status: 'pass' }, { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 12 } }, { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 800 }, ], }, { name: 'missing boundary', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 }, { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 1 } }, { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 800 }, ], }, ]; for (const baseline of unhealthyBaselines) { const server = await startDrillServer({ preDrillChecks: baseline.checks }); t.after(() => server.close()); const result = await runCli(['--drill-origin', server.origin]); assert.equal(result.code, 2, `${baseline.name}: expected refusal exit 2, got ${result.code}`); assert.match(result.stderr, /not a genuine drill baseline/i, `${baseline.name}: operator must be told why`); assert.doesNotMatch(result.stdout, /DRILL PASS|WORKER OUTAGE DRILL/, `${baseline.name}: drill must never start`); assert.equal(server.outageSwitchCount, 0, `${baseline.name}: switch must never be touched`); } }); test('drill refuses to flip an already-outaged fixture and never touches its switch', async t => { const server = await startDrillServer({ startDown: true }); t.after(() => server.close()); const result = await runCli(['--drill-origin', server.origin]); assert.equal(result.code, 2, `expected refusal exit 2, got ${result.code}`); assert.match(result.stderr, /not healthy/i); assert.doesNotMatch(result.stdout, /DRILL PASS/); assert.equal(server.outageSwitchCount, 0, 'already-down fixture must not be flipped again'); }); test('drill fails when the flip produces no genuine healthy-to-outage transition', async t => { const server = await startDrillServer({ flipHasNoEffect: true }); t.after(() => server.close()); const result = await runCli(['--drill-origin', server.origin]); assert.equal(result.code, 1); assert.match(result.stdout, /no outage observed after flip/i); assert.match(result.stdout, /DRILL FAIL/); assert.equal(server.outageSwitchCount, 1); }); test('drill refuses a credentialed origin even when its loopback target is alive', async t => { const server = await startDrillServer(); t.after(() => server.close()); const credentialed = `http://user:pass@127.0.0.1:${new URL(server.origin).port}`; const result = await runCli(['--drill-origin', credentialed]); assert.equal(result.code, 2, `expected refusal exit 2, got ${result.code}`); assert.match(result.stderr, /drill origin rejected/i); assert.doesNotMatch(result.stdout, /DRILL PASS|WORKER OUTAGE DRILL/, 'hostile origin must never reach the drill'); assert.equal(server.outageSwitchCount, 0, 'credentialed origin must never touch the fixture'); }); test('drill-origin accepts only validated loopback HTTP and rejects every hostile form without contacting it', async () => { const hostileOrigins = [ 'https://127.0.0.1:43600', 'http://localhost:43600', 'ftp://127.0.0.1:43600', 'file:///etc/passwd', 'http://0x7f.0.0.1:43600', 'http://2130706433:43600', 'http://[::1]:43600', 'http://[::ffff:127.0.0.1]:43600', 'http://[0:0:0:0:0:0:0:1]:43600', 'http://127.0.0.2:43600', 'http://10.0.0.5:43600', 'http://example.com:43600', 'http://127%2E0%2E0%2E1:43600', 'http://127.0.0.1:43600/drill', 'http://127.0.0.1:43600/?next=1', '//127.0.0.1:43600', '127.0.0.1:43600', 'not a url at all', 'null', ]; for (const origin of hostileOrigins) { const result = await runCli(['--drill-origin', origin]); assert.equal(result.code, 2, `expected refusal exit 2 for origin ${JSON.stringify(origin)}`); assert.match(result.stderr, /drill origin rejected/i, `origin must be rejected as an origin, not merely unreachable: ${JSON.stringify(origin)}`); assert.doesNotMatch(result.stdout, /DRILL PASS|WORKER OUTAGE DRILL/, `hostile origin must never reach the drill: ${JSON.stringify(origin)}`); } });