feat(security): attack the RC before users do and gate approval on P0 findings (#40)

Add an executable abuse suite that drives the real server with synthetic hostile bytes while a
counting stub worker records every call that reaches the model boundary, plus the verdict rule that
turns those observations into a release decision.

- src/release-security-gate.js: nearest-rank latency percentiles, per-case pass/fail derivation that
  names the field that broke, and the rule that any failing P0 case blocks RC approval while a
  required class with no verified evidence blocks it too. Reports are redacted and fail closed.
- scripts/security_attack_suite.mjs: oversized body, HTML polyglot as JPEG, undecodable bytes,
  missing consent, non-conforming worker answer, byte-identical replay, concurrent burst, captured
  log scan, and an out-of-surface deletion case counted only against a real test file. Writes the
  report, prints the verdict, exits 1 when blocked.
- docs/RC-SECURITY-EVIDENCE.md: procedure, rule, and the current finding stated plainly.
- Demo: a real run of the suite on camera, printing the verdict it derived instead of a flattering
  one; CI regenerates the same evidence on every change.

The suite currently returns VERDICT: BLOCKED: bytes are trusted from the data-URL label, so an HTML
polyglot still reaches the worker and only fails afterwards on schema. That is a finding against
#16, not a passing check, and it is why the gate treats a failing case as a block.

scripts/record_release_demo.mjs

@@ -1165,6 +1165,76 @@ await caption('Each candidate is scored on test at a threshold chosen on validat
 await caption('One contributor spread across two splits is refused before anything is scored, and a passing verdict is proven only by a fixture engineered to clear it', 3800);
 await page.evaluate(() => document.querySelector('#calibration-report')?.remove());
 
+// Issue #40 — the RC attack suite is a command line too, and unlike the other cards this one is
+// expected to come back with a verdict that is not flattering. Hostile synthetic bytes go at the
+// real server with a counting stub worker behind it, so "the worker was called" is measured, not
+// inferred. The card prints whatever verdict the suite derived from what it observed; the captions
+// name where a failing fix lives instead of implying the boundary is already safe.
+async function securityAttackSuite() {
+  const directory = await mkdtemp(path.join(os.tmpdir(), 'timmy-security-demo-'));
+  const reportPath = path.join(directory, 'rc-attack-report.json');
+  try {
+    const run = await execFileAsync(
+      process.execPath,
+      ['scripts/security_attack_suite.mjs', '--json', reportPath],
+      { cwd: process.cwd() },
+    ).then(result => ({ code: 0, stdout: result.stdout, stderr: result.stderr }))
+      .catch(error => ({ code: typeof error.code === 'number' ? error.code : 1, stdout: error.stdout || '', stderr: error.stderr || '' }));
+    if (![0, 1].includes(run.code)) throw new Error(`the attack suite could not complete: ${run.stderr.trim()}`);
+    const report = JSON.parse(await readFile(reportPath, 'utf8'));
+    if (run.code !== (report.verdict === 'BLOCKED' ? 1 : 0)) {
+      throw new Error('the attack suite exit code must match the verdict it published');
+    }
+    return { report, run };
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+}
+const securityRun = await securityAttackSuite();
+const securityReport = securityRun.report;
+if (securityReport.cases.length < 8) throw new Error('the RC attack suite must run every named abuse class');
+await tap('[data-view="home"]', 450);
+const caseDetail = (item) => {
+  const observed = item.observed || {};
+  if ('providerCalls' in observed) return `http ${observed.status} · worker ${observed.providerCalls}`;
+  if ('refused' in observed) return `refused ${observed.refused}`;
+  if ('imageBytesInLog' in observed) return `image bytes ${observed.imageBytesInLog}`;
+  return item.status;
+};
+const securityRows = [
+  ['command', 'node scripts/security_attack_suite.mjs'],
+  ['target', 'real server · counting stub worker · synthetic bytes'],
+  ['—', ''],
+];
+for (const item of securityReport.cases) securityRows.push([item.category, `${item.severity} ${item.status} · ${caseDetail(item)}`]);
+securityRows.push(['—', '']);
+securityRows.push(['load', `${securityReport.load.requests} req · p50 ${securityReport.load.p50}ms · p95 ${securityReport.load.p95}ms · abstain ${securityReport.load.abstentions}`]);
+securityRows.push(['verdict', `${securityReport.verdict} · ${securityReport.blocking.length} blocking · ${securityReport.holds.length} held`]);
+for (const item of securityReport.blocking) securityRows.push(['blocks', item.id]);
+const securityCard = await evidenceCard({
+  id: 'security-attack',
+  title: 'SECURITY ATTACK SUITE',
+  subtitle: 'command line, not app UI — real server, counted worker calls',
+  rows: securityRows,
+  font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace',
+  keyWidth: 62,
+  gap: '1px',
+  rowGap: '5px',
+});
+await sleep(420);
+const securityCardProof = await evidenceCardProof('security-attack');
+if (!securityCardProof.ok) throw new Error(`Security card is not fully readable on screen: ${JSON.stringify(securityCardProof)}`);
+for (const item of securityReport.blocking) {
+  if (!securityCard.text.includes(item.id)) throw new Error(`the card must name the blocking case ${item.id}`);
+}
+if (!securityCard.text.includes(securityReport.verdict)) throw new Error('the card must show the verdict the suite derived');
+if (!securityCard.text.includes(`${securityReport.load.p95}ms`)) throw new Error('the card must show the p95 the load phase measured');
+await caption('Every hostile upload on screen went at the real server, and the worker behind it counted every call it received', 3800);
+await caption('Refusing bytes by their content is the fix in review, not a claim this video makes: an HTML file dressed as a JPEG still reaches the worker today', 4200);
+await caption('A failing case is a finding the release job has to see, so the card prints the verdict the suite derived instead of a reassuring one', 3800);
+await caption('The gate blocks RC approval on any P0 case that reached the worker — the gate decides, not the caption', 3400);
+await page.evaluate(() => document.querySelector('#security-attack')?.remove());
+
 // Issue #39 — the operator backup is a command line, not a screen in the app. The app itself keeps
 // no server-side journal, so what an operator can actually lose is the corpus: the append-only
 // manifest with its withdrawal tombstones. This scene builds a synthetic data root with the real

scripts/security_attack_suite.mjs

@@ -0,0 +1,310 @@
+#!/usr/bin/env node
+// RC attack suite (#40): attack the release candidate before users do.
+//
+// Every case drives the shipping `server.mjs` with synthetic hostile bytes while a stub worker counts
+// the calls that reach it. The suite records what it observed and derives a verdict from the
+// observation; it never asserts a result it did not see. No real medical image, no live user data,
+// no third-party provider is involved, and the provider is a counting stub, so "reached the worker"
+// is provable rather than inferred.
+//
+// Usage: node scripts/security_attack_suite.mjs [--json <path>] [--base-url <url>]
+// Exit: 0 = GO, 1 = BLOCKED (a P0 finding, or a required category with no executable evidence).
+import { randomBytes } from 'node:crypto';
+import { spawn } from 'node:child_process';
+import { access, mkdir, readFile, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import http from 'node:http';
+import { fileURLToPath } from 'node:url';
+import { buildReport, containsImageBytes, latencyPercentiles } from '../src/release-security-gate.js';
+
+const repoRoot = fileURLToPath(new URL('..', import.meta.url));
+const argument = (name, fallback) => {
+  const index = process.argv.indexOf(name);
+  return index === -1 ? fallback : process.argv[index + 1];
+};
+const reportPath = argument('--json', join(repoRoot, 'artifacts/security/rc-attack-report.json'));
+const explicitBaseUrl = argument('--base-url', '');
+
+function freePort() {
+  return new Promise(resolve => {
+    const probe = http.createServer();
+    probe.listen(0, '127.0.0.1', () => {
+      const { port } = probe.address();
+      probe.close(() => resolve(port));
+    });
+  });
+}
+
+// A counting OpenAI-compatible worker. `state.content` is the answer the next request receives, and
+// `state.requests` is the evidence that a request crossed the provider boundary at all.
+function startStubWorker() {
+  const state = { content: '', requests: 0 };
+  const server = http.createServer((req, res) => {
+    if (req.url.endsWith('/models')) {
+      res.writeHead(200, { 'content-type': 'application/json' });
+      return res.end(JSON.stringify({ data: [{ id: 'stub-vision-worker' }] }));
+    }
+    req.resume();
+    req.on('end', () => {
+      state.requests += 1;
+      res.writeHead(200, { 'content-type': 'application/json' });
+      res.end(JSON.stringify({ choices: [{ message: { content: state.content } }] }));
+    });
+  });
+  return new Promise(resolve => server.listen(0, '127.0.0.1', () => resolve({
+    state,
+    port: server.address().port,
+    close: () => new Promise(done => server.close(done)),
+  })));
+}
+
+// The real server, with its own stdout/stderr captured from the first byte: the log scan is only
+// worth something if it read a log that actually exists.
+async function startTarget({ providerPort, port }) {
+  const child = spawn(process.execPath, ['server.mjs'], {
+    cwd: repoRoot,
+    env: {
+      ...process.env,
+      HOST: '127.0.0.1',
+      PORT: String(port),
+      TIMMY_VISION_ENABLED: 'true',
+      TIMMY_VISION_PROFILE: 'selfhost',
+      TIMMY_VISION_BASE_URL: `http://127.0.0.1:${providerPort}/v1`,
+      TIMMY_VISION_MODEL: 'stub-vision-worker',
+      TIMMY_VISION_TIMEOUT_MS: '5000',
+    },
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+  const log = [];
+  child.stdout.on('data', chunk => log.push(chunk.toString('utf8')));
+  child.stderr.on('data', chunk => log.push(chunk.toString('utf8')));
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    if (log.join('').includes('Timmy is listening')) return { child, log };
+    try {
+      const response = await fetch(`http://127.0.0.1:${port}/api/healthz`);
+      if (response.ok) return { child, log };
+    } catch {}
+    await new Promise(resolve => setTimeout(resolve, 100));
+  }
+  child.kill();
+  throw new Error('attack suite target did not become ready');
+}
+
+const stop = child => new Promise(resolve => {
+  if (!child || child.exitCode !== null) return resolve();
+  child.kill();
+  child.once('exit', resolve);
+});
+
+async function main() {
+  const worker = await startStubWorker();
+  const port = explicitBaseUrl ? null : await freePort();
+  const baseUrl = explicitBaseUrl || `http://127.0.0.1:${port}`;
+  const target = explicitBaseUrl ? null : await startTarget({ providerPort: worker.port, port });
+  const photo = await readFile(join(repoRoot, 'tests/fixtures/synthetic-type4.jpg'));
+  const imageDataUrl = `data:image/jpeg;base64,${photo.toString('base64')}`;
+
+  const post = async (body) => {
+    const before = worker.state.requests;
+    const response = await fetch(`${baseUrl}/api/analyze`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: typeof body === 'string' ? body : JSON.stringify(body),
+    });
+    const text = await response.text();
+    return { status: response.status, text, providerCalls: worker.state.requests - before };
+  };
+
+  const cases = [];
+  const record = (entry) => { cases.push(entry); return entry; };
+
+  try {
+    const health = await fetch(`${baseUrl}/api/healthz`).then(response => response.json());
+    if (!health.visionEnabled) throw new Error('the attack suite needs vision analysis enabled on the target');
+
+    // 1. An oversized body must die at the request cap, before the photo is parsed or forwarded.
+    const oversized = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${'A'.repeat(6.6 * 1024 * 1024)}` });
+    record({
+      id: 'ingress-oversized-body',
+      category: 'oversized',
+      severity: 'P0',
+      requirement: 'A body over the request cap is refused before any provider work.',
+      expected: { status: 413, providerCalls: 0 },
+      observed: { status: oversized.status, providerCalls: oversized.providerCalls },
+      snippet: oversized.text.slice(0, 120),
+    });
+
+    // 2. Bytes decide the format, not the data-URL label: HTML dressed as a JPEG is a polyglot.
+    const html = Buffer.from('<html><body><script>alert(document.cookie)</script></body></html>');
+    const polyglot = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${html.toString('base64')}` });
+    record({
+      id: 'ingress-polyglot-html',
+      category: 'polyglot',
+      severity: 'P0',
+      requirement: 'A payload whose bytes are HTML under an image/jpeg label is refused before the provider sees it.',
+      expected: { status: 400, providerCalls: 0 },
+      observed: { status: polyglot.status, providerCalls: polyglot.providerCalls },
+      snippet: polyglot.text.slice(0, 120),
+    });
+
+    // 3. Random bytes under a JPEG label are not an image either.
+    const garbage = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${randomBytes(64).toString('base64')}` });
+    record({
+      id: 'ingress-garbage-bytes',
+      category: 'truncated',
+      severity: 'P0',
+      requirement: 'Undecodable bytes are refused rather than forwarded and interpreted.',
+      expected: { status: 400, providerCalls: 0 },
+      observed: { status: garbage.status, providerCalls: garbage.providerCalls },
+      snippet: garbage.text.slice(0, 120),
+    });
+
+    // 4. Consent is a precondition, and it is cheap to prove: no consent, no provider call.
+    const noConsent = await post({ consent: false, imageDataUrl });
+    record({
+      id: 'consent-required',
+      category: 'consent',
+      severity: 'P0',
+      requirement: 'Analysis without explicit consent is refused and never leaves the boundary.',
+      expected: { status: 400, providerCalls: 0 },
+      observed: { status: noConsent.status, providerCalls: noConsent.providerCalls },
+      snippet: noConsent.text.slice(0, 120),
+    });
+
+    // 5. A worker answer outside the schema must fail closed, and the provider's words must not
+    //    reach the user or the evidence file.
+    worker.state.content = 'garbage-completion-token: isStool yes maybe, bristolType ten';
+    const malformed = await post({ consent: true, imageDataUrl });
+    record({
+      id: 'model-output-malformed',
+      category: 'malformed',
+      severity: 'P0',
+      requirement: 'A non-conforming provider answer is replaced by neutral copy and never echoed.',
+      expected: { status: 503, providerTextLeaked: false },
+      observed: { status: malformed.status, providerTextLeaked: /garbage-completion-token/.test(malformed.text) },
+      snippet: malformed.text.slice(0, 120),
+    });
+
+    // 6. Replay: this boundary holds no session state, so a replay is a second request. What must
+    //    hold is that it is answered identically and that nothing accumulates server-side.
+    worker.state.content = JSON.stringify({ isStool: true, bristolType: 4, color: 'brown', confidence: 0.82, imageQuality: 'good', observations: 'Smooth, formed appearance.' });
+    const first = await post({ consent: true, imageDataUrl });
+    const second = await post({ consent: true, imageDataUrl });
+    record({
+      id: 'replay-duplicate-request',
+      category: 'replay',
+      severity: 'P2',
+      requirement: 'A byte-identical replay is answered identically and stores nothing on the server.',
+      expected: { status: 200, providerCalls: 1, repeatedAnswerIdentical: true },
+      observed: { status: second.status, providerCalls: second.providerCalls, repeatedAnswerIdentical: first.text === second.text },
+      snippet: second.text.slice(0, 120),
+    });
+
+    // 7. Flood: concurrent requests must meet a bound instead of all reaching the worker at once.
+    const beforeFlood = worker.state.requests;
+    const flood = await Promise.all(Array.from({ length: 12 }, () => post({ consent: true, imageDataUrl })));
+    record({
+      id: 'flood-concurrency',
+      category: 'flood',
+      severity: 'P1',
+      requirement: 'A burst of concurrent analyses meets a bounded queue or a rate refusal.',
+      expected: { refused: true },
+      observed: { refused: flood.some(item => item.status === 429 || item.status === 503), providerCalls: worker.state.requests - beforeFlood },
+      snippet: `statuses ${[...new Set(flood.map(item => item.status))].sort().join(',')}`,
+    });
+
+    // 8. Log inspection: the operator log is committed evidence, so it must not contain the photo.
+    const capturedLog = (target?.log || []).join('');
+    if (!capturedLog.trim()) throw new Error('no server log was captured, so the log scan would prove nothing');
+    record({
+      id: 'log-leak-scan',
+      category: 'log',
+      severity: 'P0',
+      requirement: 'Server logs contain no image bytes, base64, or note text.',
+      expected: { imageBytesInLog: false },
+      observed: { imageBytesInLog: containsImageBytes(capturedLog) },
+      snippet: `${capturedLog.trim().split('\n').length} log line(s) inspected`,
+    });
+
+    // 9. Deletion is governed by the contribution ledger, not by this HTTP boundary. Covering it
+    //    here means naming the executable test that does it, and only counting it once verified.
+    let evidenceVerified = false;
+    try {
+      await access(join(repoRoot, 'tests/training-contribution.test.js'));
+      evidenceVerified = true;
+    } catch {}
+    record({
+      id: 'deletion-withdrawal',
+      category: 'deletion',
+      severity: 'P0',
+      surface: 'out-of-surface',
+      requirement: 'Withdrawal deletes the contribution and the server holds no copy of the photo.',
+      evidence: 'tests/training-contribution.test.js',
+      evidenceVerified,
+      snippet: evidenceVerified ? 'covered by the contribution ledger suite' : 'named evidence file is missing',
+    });
+
+    // Load: sequential valid analyses, abstaining answers, latency measured with the same boundary.
+    worker.state.content = JSON.stringify({ isStool: false, confidence: 0.9 });
+    const samples = [];
+    let errors = 0;
+    let abstentions = 0;
+    for (let index = 0; index < 8; index += 1) {
+      const started = performance.now();
+      const result = await post({ consent: true, imageDataUrl });
+      samples.push(Math.round(performance.now() - started));
+      if (result.status !== 200) errors += 1;
+      else {
+        try {
+          if (JSON.parse(result.text).status === 'needs_user_input') abstentions += 1;
+        } catch {}
+      }
+    }
+    const percentiles = latencyPercentiles(samples);
+    const load = {
+      requests: samples.length,
+      errors,
+      abstentions,
+      p50: percentiles.p50,
+      p95: percentiles.p95,
+      max: percentiles.max,
+      note: 'Latency measured against a counting stub on loopback; it measures this boundary, not a model.',
+    };
+
+    const report = buildReport({
+      generatedAt: new Date().toISOString(),
+      target: {
+        baseUrl,
+        provider: 'stub counting worker (synthetic answers, no model loaded)',
+        visionProfile: 'selfhost',
+        release: health.release,
+        commit: health.commit,
+      },
+      cases,
+      load,
+    });
+    await mkdir(dirname(reportPath), { recursive: true });
+    await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
+
+    const width = Math.max(...cases.map(item => item.id.length));
+    for (const item of report.cases) {
+      const detail = item.status === 'fail' ? ` — ${item.mismatch}` : '';
+      console.log(`${item.id.padEnd(width)}  ${item.severity}  ${item.status}${detail}`);
+    }
+    console.log(`load: ${load.requests} requests · p50 ${load.p50}ms · p95 ${load.p95}ms · errors ${load.errors} · abstentions ${load.abstentions}`);
+    console.log(`report: ${reportPath}`);
+    console.log(`VERDICT: ${report.verdict}`);
+    if (report.verdict === 'BLOCKED') {
+      for (const item of [...report.blocking, ...report.unproven]) console.log(`  blocks RC: ${item.id} (${item.severity}) ${item.reason}`);
+    }
+    return report.verdict === 'GO' ? 0 : 1;
+  } finally {
+    await stop(target?.child);
+    await worker.close();
+  }
+}
+
+main().then(code => { process.exitCode = code; }).catch(error => {
+  console.error(`attack suite could not complete: ${error.message}`);
+  process.exitCode = 2;
+});