#!/usr/bin/env node // RC attack suite (#40): attack the release candidate before users do. // // Every case drives the shipping `server.mjs` with synthetic hostile bytes while a stub worker counts // the calls that reach it. The suite records what it observed and derives a verdict from the // observation; it never asserts a result it did not see. No real medical image, no live user data, // no third-party provider is involved, and the provider is a counting stub, so "reached the worker" // is provable rather than inferred. // // Usage: node scripts/security_attack_suite.mjs [--json ] [--base-url ] // Exit: 0 = GO, 1 = BLOCKED (a P0 finding, or a required category with no executable evidence). import { randomBytes } from 'node:crypto'; import { spawn } from 'node:child_process'; import { access, mkdir, readFile, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import http from 'node:http'; import { fileURLToPath } from 'node:url'; import { buildReport, containsImageBytes, latencyPercentiles } from '../src/release-security-gate.js'; const repoRoot = fileURLToPath(new URL('..', import.meta.url)); const argument = (name, fallback) => { const index = process.argv.indexOf(name); return index === -1 ? fallback : process.argv[index + 1]; }; const reportPath = argument('--json', join(repoRoot, 'artifacts/security/rc-attack-report.json')); const explicitBaseUrl = argument('--base-url', ''); function freePort() { return new Promise(resolve => { const probe = http.createServer(); probe.listen(0, '127.0.0.1', () => { const { port } = probe.address(); probe.close(() => resolve(port)); }); }); } // A counting OpenAI-compatible worker. `state.content` is the answer the next request receives, and // `state.requests` is the evidence that a request crossed the provider boundary at all. function startStubWorker() { const state = { content: '', requests: 0 }; const server = http.createServer((req, res) => { if (req.url.endsWith('/models')) { res.writeHead(200, { 'content-type': 'application/json' }); return res.end(JSON.stringify({ data: [{ id: 'stub-vision-worker' }] })); } req.resume(); req.on('end', () => { state.requests += 1; res.writeHead(200, { 'content-type': 'application/json' }); res.end(JSON.stringify({ choices: [{ message: { content: state.content } }] })); }); }); return new Promise(resolve => server.listen(0, '127.0.0.1', () => resolve({ state, port: server.address().port, close: () => new Promise(done => server.close(done)), }))); } // The real server, with its own stdout/stderr captured from the first byte: the log scan is only // worth something if it read a log that actually exists. async function startTarget({ providerPort, port, env = {} }) { const child = spawn(process.execPath, ['server.mjs'], { cwd: repoRoot, env: { ...process.env, HOST: '127.0.0.1', PORT: String(port), TIMMY_VISION_ENABLED: 'true', TIMMY_VISION_PROFILE: 'selfhost', TIMMY_VISION_BASE_URL: `http://127.0.0.1:${providerPort}/v1`, TIMMY_VISION_MODEL: 'stub-vision-worker', TIMMY_VISION_TIMEOUT_MS: '5000', ...env, }, stdio: ['ignore', 'pipe', 'pipe'], }); const log = []; child.stdout.on('data', chunk => log.push(chunk.toString('utf8'))); child.stderr.on('data', chunk => log.push(chunk.toString('utf8'))); for (let attempt = 0; attempt < 100; attempt += 1) { if (log.join('').includes('Timmy is listening')) return { child, log }; try { const response = await fetch(`http://127.0.0.1:${port}/api/healthz`); if (response.ok) return { child, log }; } catch {} await new Promise(resolve => setTimeout(resolve, 100)); } child.kill(); throw new Error('attack suite target did not become ready'); } const stop = child => new Promise(resolve => { if (!child || child.exitCode !== null) return resolve(); child.kill(); child.once('exit', resolve); }); async function main() { const worker = await startStubWorker(); const port = explicitBaseUrl ? null : await freePort(); const baseUrl = explicitBaseUrl || `http://127.0.0.1:${port}`; const target = explicitBaseUrl ? null : await startTarget({ providerPort: worker.port, port }); const photo = await readFile(join(repoRoot, 'tests/fixtures/synthetic-type4.jpg')); const imageDataUrl = `data:image/jpeg;base64,${photo.toString('base64')}`; const post = async (body, targetBase = baseUrl) => { const before = worker.state.requests; const response = await fetch(`${targetBase}/api/analyze`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: typeof body === 'string' ? body : JSON.stringify(body), }); const text = await response.text(); return { status: response.status, text, providerCalls: worker.state.requests - before }; }; const cases = []; const record = (entry) => { cases.push(entry); return entry; }; try { const health = await fetch(`${baseUrl}/api/healthz`).then(response => response.json()); if (!health.visionEnabled) throw new Error('the attack suite needs vision analysis enabled on the target'); // 1. An oversized body must die at the request cap, before the photo is parsed or forwarded. const oversized = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${'A'.repeat(6.6 * 1024 * 1024)}` }); record({ id: 'ingress-oversized-body', category: 'oversized', severity: 'P0', requirement: 'A body over the request cap is refused before any provider work.', expected: { status: 413, providerCalls: 0 }, observed: { status: oversized.status, providerCalls: oversized.providerCalls }, snippet: oversized.text.slice(0, 120), }); // 2. Bytes decide the format, not the data-URL label: HTML dressed as a JPEG is a polyglot. const html = Buffer.from(''); const polyglot = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${html.toString('base64')}` }); record({ id: 'ingress-polyglot-html', category: 'polyglot', severity: 'P0', requirement: 'A payload whose bytes are HTML under an image/jpeg label is refused before the provider sees it.', expected: { status: 400, providerCalls: 0 }, observed: { status: polyglot.status, providerCalls: polyglot.providerCalls }, snippet: polyglot.text.slice(0, 120), }); // 3. Random bytes under a JPEG label are not an image either. const garbage = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${randomBytes(64).toString('base64')}` }); record({ id: 'ingress-garbage-bytes', category: 'truncated', severity: 'P0', requirement: 'Undecodable bytes are refused rather than forwarded and interpreted.', expected: { status: 400, providerCalls: 0 }, observed: { status: garbage.status, providerCalls: garbage.providerCalls }, snippet: garbage.text.slice(0, 120), }); // 4. Consent is a precondition, and it is cheap to prove: no consent, no provider call. const noConsent = await post({ consent: false, imageDataUrl }); record({ id: 'consent-required', category: 'consent', severity: 'P0', requirement: 'Analysis without explicit consent is refused and never leaves the boundary.', expected: { status: 400, providerCalls: 0 }, observed: { status: noConsent.status, providerCalls: noConsent.providerCalls }, snippet: noConsent.text.slice(0, 120), }); // 5. A worker answer outside the schema must fail closed, and the provider's words must not // reach the user or the evidence file. worker.state.content = 'garbage-completion-token: isStool yes maybe, bristolType ten'; const malformed = await post({ consent: true, imageDataUrl }); record({ id: 'model-output-malformed', category: 'malformed', severity: 'P0', requirement: 'A non-conforming provider answer is replaced by neutral copy and never echoed.', expected: { status: 503, providerTextLeaked: false }, observed: { status: malformed.status, providerTextLeaked: /garbage-completion-token/.test(malformed.text) }, snippet: malformed.text.slice(0, 120), }); // 6. Replay: this boundary holds no session state, so a replay is a second request. What must // hold is that it is answered identically and that nothing accumulates server-side. worker.state.content = JSON.stringify({ isStool: true, bristolType: 4, color: 'brown', confidence: 0.82, imageQuality: 'good', observations: 'Smooth, formed appearance.' }); const first = await post({ consent: true, imageDataUrl }); const second = await post({ consent: true, imageDataUrl }); record({ id: 'replay-duplicate-request', category: 'replay', severity: 'P2', requirement: 'A byte-identical replay is answered identically and stores nothing on the server.', expected: { status: 200, providerCalls: 1, repeatedAnswerIdentical: true }, observed: { status: second.status, providerCalls: second.providerCalls, repeatedAnswerIdentical: first.text === second.text }, snippet: second.text.slice(0, 120), }); // 7. Flood: concurrent requests must meet a bound instead of all reaching the worker at once. const beforeFlood = worker.state.requests; const flood = await Promise.all(Array.from({ length: 12 }, () => post({ consent: true, imageDataUrl }))); record({ id: 'flood-concurrency', category: 'flood', severity: 'P1', requirement: 'A burst of concurrent analyses meets a bounded queue or a rate refusal.', expected: { refused: true }, observed: { refused: flood.some(item => item.status === 429 || item.status === 503), providerCalls: worker.state.requests - beforeFlood }, snippet: `statuses ${[...new Set(flood.map(item => item.status))].sort().join(',')}`, }); // 8. Log inspection: the operator log is committed evidence, so it must not contain the photo. const capturedLog = (target?.log || []).join(''); if (!capturedLog.trim()) throw new Error('no server log was captured, so the log scan would prove nothing'); record({ id: 'log-leak-scan', category: 'log', severity: 'P0', requirement: 'Server logs contain no image bytes, base64, or note text.', expected: { imageBytesInLog: false }, observed: { imageBytesInLog: containsImageBytes(capturedLog) }, snippet: `${capturedLog.trim().split('\n').length} log line(s) inspected`, }); // 9. Deletion is governed by the contribution ledger, not by this HTTP boundary. Covering it // here means naming the executable test that does it, and only counting it once verified. let evidenceVerified = false; try { await access(join(repoRoot, 'tests/training-contribution.test.js')); evidenceVerified = true; } catch {} record({ id: 'deletion-withdrawal', category: 'deletion', severity: 'P0', surface: 'out-of-surface', requirement: 'Withdrawal deletes the contribution and the server holds no copy of the photo.', evidence: 'tests/training-contribution.test.js', evidenceVerified, snippet: evidenceVerified ? 'covered by the contribution ledger suite' : 'named evidence file is missing', }); // Load: sequential valid analyses, abstaining answers, latency measured with the same boundary. // The abuse cases above spend the ingress request budget, so latency is measured against a second // target that starts with the budget wide open; otherwise these samples would be 429 refusals. const loadPort = explicitBaseUrl ? null : await freePort(); const loadTarget = explicitBaseUrl ? null : await startTarget({ providerPort: worker.port, port: loadPort, env: { TIMMY_ANALYZE_RATE_LIMIT: '100000', TIMMY_ANALYZE_RATE_WINDOW_MS: '60000' }, }); const loadBase = explicitBaseUrl || `http://127.0.0.1:${loadPort}`; worker.state.content = JSON.stringify({ isStool: false, confidence: 0.9 }); const samples = []; let errors = 0; let abstentions = 0; try { for (let index = 0; index < 8; index += 1) { const started = performance.now(); const result = await post({ consent: true, imageDataUrl }, loadBase); samples.push(Math.round(performance.now() - started)); if (result.status !== 200) errors += 1; else { try { if (JSON.parse(result.text).status === 'needs_user_input') abstentions += 1; } catch {} } } } finally { await stop(loadTarget?.child); } const percentiles = latencyPercentiles(samples); const load = { requests: samples.length, errors, abstentions, p50: percentiles.p50, p95: percentiles.p95, max: percentiles.max, note: 'Latency measured against a counting stub on loopback; it measures this boundary, not a model.', }; // Issue #16 adds a per-caller request budget at the ingress, so a suite that spends its budget on // the abuse cases above would measure 429 refusals here and publish them as analysis latency. // A refusal is not a sample: the load phase runs against its own permissive target instead. if (load.errors > 0) { throw new Error(`the load phase measured ${load.errors}/${load.requests} refusals instead of analyses, so it cannot publish latency`); } const report = buildReport({ generatedAt: new Date().toISOString(), target: { baseUrl, provider: 'stub counting worker (synthetic answers, no model loaded)', visionProfile: 'selfhost', release: health.release, commit: health.commit, }, cases, load, }); await mkdir(dirname(reportPath), { recursive: true }); await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8'); const width = Math.max(...cases.map(item => item.id.length)); for (const item of report.cases) { const detail = item.status === 'fail' ? ` — ${item.mismatch}` : ''; console.log(`${item.id.padEnd(width)} ${item.severity} ${item.status}${detail}`); } console.log(`load: ${load.requests} requests · p50 ${load.p50}ms · p95 ${load.p95}ms · errors ${load.errors} · abstentions ${load.abstentions}`); console.log(`report: ${reportPath}`); console.log(`VERDICT: ${report.verdict}`); if (report.verdict === 'BLOCKED') { for (const item of [...report.blocking, ...report.unproven]) console.log(` blocks RC: ${item.id} (${item.severity}) ${item.reason}`); } return report.verdict === 'GO' ? 0 : 1; } finally { await stop(target?.child); await worker.close(); } } main().then(code => { process.exitCode = code; }).catch(error => { console.error(`attack suite could not complete: ${error.message}`); process.exitCode = 2; });