timmy-talking-turd/scripts/security_attack_suite.mjs

#!/usr/bin/env node
// RC attack suite (#40): attack the release candidate before users do.
//
// Every case drives the shipping `server.mjs` with synthetic hostile bytes while a stub worker counts
// the calls that reach it. The suite records what it observed and derives a verdict from the
// observation; it never asserts a result it did not see. No real medical image, no live user data,
// no third-party provider is involved, and the provider is a counting stub, so "reached the worker"
// is provable rather than inferred.
//
// Usage: node scripts/security_attack_suite.mjs [--json <path>] [--base-url <url>]
// Exit: 0 = GO, 1 = BLOCKED (a P0 finding, or a required category with no executable evidence).
import { randomBytes } from 'node:crypto';
import { spawn } from 'node:child_process';
import { access, mkdir, readFile, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import http from 'node:http';
import { fileURLToPath } from 'node:url';
import { buildReport, containsImageBytes, latencyPercentiles } from '../src/release-security-gate.js';

const repoRoot = fileURLToPath(new URL('..', import.meta.url));
const argument = (name, fallback) => {
  const index = process.argv.indexOf(name);
  return index === -1 ? fallback : process.argv[index + 1];
};
const reportPath = argument('--json', join(repoRoot, 'artifacts/security/rc-attack-report.json'));
const explicitBaseUrl = argument('--base-url', '');

function freePort() {
  return new Promise(resolve => {
    const probe = http.createServer();
    probe.listen(0, '127.0.0.1', () => {
      const { port } = probe.address();
      probe.close(() => resolve(port));
    });
  });
}

// A counting OpenAI-compatible worker. `state.content` is the answer the next request receives, and
// `state.requests` is the evidence that a request crossed the provider boundary at all.
function startStubWorker() {
  const state = { content: '', requests: 0 };
  const server = http.createServer((req, res) => {
    if (req.url.endsWith('/models')) {
      res.writeHead(200, { 'content-type': 'application/json' });
      return res.end(JSON.stringify({ data: [{ id: 'stub-vision-worker' }] }));
    }
    req.resume();
    req.on('end', () => {
      state.requests += 1;
      res.writeHead(200, { 'content-type': 'application/json' });
      res.end(JSON.stringify({ choices: [{ message: { content: state.content } }] }));
    });
  });
  return new Promise(resolve => server.listen(0, '127.0.0.1', () => resolve({
    state,
    port: server.address().port,
    close: () => new Promise(done => server.close(done)),
  })));
}

// The real server, with its own stdout/stderr captured from the first byte: the log scan is only
// worth something if it read a log that actually exists.
async function startTarget({ providerPort, port, env = {} }) {
  const child = spawn(process.execPath, ['server.mjs'], {
    cwd: repoRoot,
    env: {
      ...process.env,
      HOST: '127.0.0.1',
      PORT: String(port),
      TIMMY_VISION_ENABLED: 'true',
      TIMMY_VISION_PROFILE: 'selfhost',
      TIMMY_VISION_BASE_URL: `http://127.0.0.1:${providerPort}/v1`,
      TIMMY_VISION_MODEL: 'stub-vision-worker',
      TIMMY_VISION_TIMEOUT_MS: '5000',
      ...env,
    },
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  const log = [];
  child.stdout.on('data', chunk => log.push(chunk.toString('utf8')));
  child.stderr.on('data', chunk => log.push(chunk.toString('utf8')));
  for (let attempt = 0; attempt < 100; attempt += 1) {
    if (log.join('').includes('Timmy is listening')) return { child, log };
    try {
      const response = await fetch(`http://127.0.0.1:${port}/api/healthz`);
      if (response.ok) return { child, log };
    } catch {}
    await new Promise(resolve => setTimeout(resolve, 100));
  }
  child.kill();
  throw new Error('attack suite target did not become ready');
}

const stop = child => new Promise(resolve => {
  if (!child || child.exitCode !== null) return resolve();
  child.kill();
  child.once('exit', resolve);
});

async function main() {
  const worker = await startStubWorker();
  const port = explicitBaseUrl ? null : await freePort();
  const baseUrl = explicitBaseUrl || `http://127.0.0.1:${port}`;
  const target = explicitBaseUrl ? null : await startTarget({ providerPort: worker.port, port });
  const photo = await readFile(join(repoRoot, 'tests/fixtures/synthetic-type4.jpg'));
  const imageDataUrl = `data:image/jpeg;base64,${photo.toString('base64')}`;

  const post = async (body, targetBase = baseUrl) => {
    const before = worker.state.requests;
    const response = await fetch(`${targetBase}/api/analyze`, {
      method: 'POST',
      headers: { 'content-type': 'application/json' },
      body: typeof body === 'string' ? body : JSON.stringify(body),
    });
    const text = await response.text();
    return { status: response.status, text, providerCalls: worker.state.requests - before };
  };

  const cases = [];
  const record = (entry) => { cases.push(entry); return entry; };

  try {
    const health = await fetch(`${baseUrl}/api/healthz`).then(response => response.json());
    if (!health.visionEnabled) throw new Error('the attack suite needs vision analysis enabled on the target');

    // 1. An oversized body must die at the request cap, before the photo is parsed or forwarded.
    const oversized = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${'A'.repeat(6.6 * 1024 * 1024)}` });
    record({
      id: 'ingress-oversized-body',
      category: 'oversized',
      severity: 'P0',
      requirement: 'A body over the request cap is refused before any provider work.',
      expected: { status: 413, providerCalls: 0 },
      observed: { status: oversized.status, providerCalls: oversized.providerCalls },
      snippet: oversized.text.slice(0, 120),
    });

    // 2. Bytes decide the format, not the data-URL label: HTML dressed as a JPEG is a polyglot.
    const html = Buffer.from('<html><body><script>alert(document.cookie)</script></body></html>');
    const polyglot = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${html.toString('base64')}` });
    record({
      id: 'ingress-polyglot-html',
      category: 'polyglot',
      severity: 'P0',
      requirement: 'A payload whose bytes are HTML under an image/jpeg label is refused before the provider sees it.',
      expected: { status: 400, providerCalls: 0 },
      observed: { status: polyglot.status, providerCalls: polyglot.providerCalls },
      snippet: polyglot.text.slice(0, 120),
    });

    // 3. Random bytes under a JPEG label are not an image either.
    const garbage = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${randomBytes(64).toString('base64')}` });
    record({
      id: 'ingress-garbage-bytes',
      category: 'truncated',
      severity: 'P0',
      requirement: 'Undecodable bytes are refused rather than forwarded and interpreted.',
      expected: { status: 400, providerCalls: 0 },
      observed: { status: garbage.status, providerCalls: garbage.providerCalls },
      snippet: garbage.text.slice(0, 120),
    });

    // 4. Consent is a precondition, and it is cheap to prove: no consent, no provider call.
    const noConsent = await post({ consent: false, imageDataUrl });
    record({
      id: 'consent-required',
      category: 'consent',
      severity: 'P0',
      requirement: 'Analysis without explicit consent is refused and never leaves the boundary.',
      expected: { status: 400, providerCalls: 0 },
      observed: { status: noConsent.status, providerCalls: noConsent.providerCalls },
      snippet: noConsent.text.slice(0, 120),
    });

    // 5. A worker answer outside the schema must fail closed, and the provider's words must not
    //    reach the user or the evidence file.
    worker.state.content = 'garbage-completion-token: isStool yes maybe, bristolType ten';
    const malformed = await post({ consent: true, imageDataUrl });
    record({
      id: 'model-output-malformed',
      category: 'malformed',
      severity: 'P0',
      requirement: 'A non-conforming provider answer is replaced by neutral copy and never echoed.',
      expected: { status: 503, providerTextLeaked: false },
      observed: { status: malformed.status, providerTextLeaked: /garbage-completion-token/.test(malformed.text) },
      snippet: malformed.text.slice(0, 120),
    });

    // 6. Replay: this boundary holds no session state, so a replay is a second request. What must
    //    hold is that it is answered identically and that nothing accumulates server-side.
    worker.state.content = JSON.stringify({ isStool: true, bristolType: 4, color: 'brown', confidence: 0.82, imageQuality: 'good', observations: 'Smooth, formed appearance.' });
    const first = await post({ consent: true, imageDataUrl });
    const second = await post({ consent: true, imageDataUrl });
    record({
      id: 'replay-duplicate-request',
      category: 'replay',
      severity: 'P2',
      requirement: 'A byte-identical replay is answered identically and stores nothing on the server.',
      expected: { status: 200, providerCalls: 1, repeatedAnswerIdentical: true },
      observed: { status: second.status, providerCalls: second.providerCalls, repeatedAnswerIdentical: first.text === second.text },
      snippet: second.text.slice(0, 120),
    });

    // 7. Flood: concurrent requests must meet a bound instead of all reaching the worker at once.
    const beforeFlood = worker.state.requests;
    const flood = await Promise.all(Array.from({ length: 12 }, () => post({ consent: true, imageDataUrl })));
    record({
      id: 'flood-concurrency',
      category: 'flood',
      severity: 'P1',
      requirement: 'A burst of concurrent analyses meets a bounded queue or a rate refusal.',
      expected: { refused: true },
      observed: { refused: flood.some(item => item.status === 429 || item.status === 503), providerCalls: worker.state.requests - beforeFlood },
      snippet: `statuses ${[...new Set(flood.map(item => item.status))].sort().join(',')}`,
    });

    // 8. Log inspection: the operator log is committed evidence, so it must not contain the photo.
    const capturedLog = (target?.log || []).join('');
    if (!capturedLog.trim()) throw new Error('no server log was captured, so the log scan would prove nothing');
    record({
      id: 'log-leak-scan',
      category: 'log',
      severity: 'P0',
      requirement: 'Server logs contain no image bytes, base64, or note text.',
      expected: { imageBytesInLog: false },
      observed: { imageBytesInLog: containsImageBytes(capturedLog) },
      snippet: `${capturedLog.trim().split('\n').length} log line(s) inspected`,
    });

    // 9. Deletion is governed by the contribution ledger, not by this HTTP boundary. Covering it
    //    here means naming the executable test that does it, and only counting it once verified.
    let evidenceVerified = false;
    try {
      await access(join(repoRoot, 'tests/training-contribution.test.js'));
      evidenceVerified = true;
    } catch {}
    record({
      id: 'deletion-withdrawal',
      category: 'deletion',
      severity: 'P0',
      surface: 'out-of-surface',
      requirement: 'Withdrawal deletes the contribution and the server holds no copy of the photo.',
      evidence: 'tests/training-contribution.test.js',
      evidenceVerified,
      snippet: evidenceVerified ? 'covered by the contribution ledger suite' : 'named evidence file is missing',
    });

    // Load: sequential valid analyses, abstaining answers, latency measured with the same boundary.
    // The abuse cases above spend the ingress request budget, so latency is measured against a second
    // target that starts with the budget wide open; otherwise these samples would be 429 refusals.
    const loadPort = explicitBaseUrl ? null : await freePort();
    const loadTarget = explicitBaseUrl
      ? null
      : await startTarget({
        providerPort: worker.port,
        port: loadPort,
        env: { TIMMY_ANALYZE_RATE_LIMIT: '100000', TIMMY_ANALYZE_RATE_WINDOW_MS: '60000' },
      });
    const loadBase = explicitBaseUrl || `http://127.0.0.1:${loadPort}`;
    worker.state.content = JSON.stringify({ isStool: false, confidence: 0.9 });
    const samples = [];
    let errors = 0;
    let abstentions = 0;
    try {
      for (let index = 0; index < 8; index += 1) {
        const started = performance.now();
        const result = await post({ consent: true, imageDataUrl }, loadBase);
        samples.push(Math.round(performance.now() - started));
        if (result.status !== 200) errors += 1;
        else {
          try {
            if (JSON.parse(result.text).status === 'needs_user_input') abstentions += 1;
          } catch {}
        }
      }
    } finally {
      await stop(loadTarget?.child);
    }
    const percentiles = latencyPercentiles(samples);
    const load = {
      requests: samples.length,
      errors,
      abstentions,
      p50: percentiles.p50,
      p95: percentiles.p95,
      max: percentiles.max,
      note: 'Latency measured against a counting stub on loopback; it measures this boundary, not a model.',
    };
    // Issue #16 adds a per-caller request budget at the ingress, so a suite that spends its budget on
    // the abuse cases above would measure 429 refusals here and publish them as analysis latency.
    // A refusal is not a sample: the load phase runs against its own permissive target instead.
    if (load.errors > 0) {
      throw new Error(`the load phase measured ${load.errors}/${load.requests} refusals instead of analyses, so it cannot publish latency`);
    }

    const report = buildReport({
      generatedAt: new Date().toISOString(),
      target: {
        baseUrl,
        provider: 'stub counting worker (synthetic answers, no model loaded)',
        visionProfile: 'selfhost',
        release: health.release,
        commit: health.commit,
      },
      cases,
      load,
    });
    await mkdir(dirname(reportPath), { recursive: true });
    await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');

    const width = Math.max(...cases.map(item => item.id.length));
    for (const item of report.cases) {
      const detail = item.status === 'fail' ? ` — ${item.mismatch}` : '';
      console.log(`${item.id.padEnd(width)}  ${item.severity}  ${item.status}${detail}`);
    }
    console.log(`load: ${load.requests} requests · p50 ${load.p50}ms · p95 ${load.p95}ms · errors ${load.errors} · abstentions ${load.abstentions}`);
    console.log(`report: ${reportPath}`);
    console.log(`VERDICT: ${report.verdict}`);
    if (report.verdict === 'BLOCKED') {
      for (const item of [...report.blocking, ...report.unproven]) console.log(`  blocks RC: ${item.id} (${item.severity}) ${item.reason}`);
    }
    return report.verdict === 'GO' ? 0 : 1;
  } finally {
    await stop(target?.child);
    await worker.close();
  }
}

main().then(code => { process.exitCode = code; }).catch(error => {
  console.error(`attack suite could not complete: ${error.message}`);
  process.exitCode = 2;
});