timmy-talking-turd/tests/threat-model.test.js

import test from 'node:test';
import assert from 'node:assert/strict';
import { readFile } from 'node:fs/promises';

const threatModelPath = new URL('../docs/PHOTO-DATAFLOW-THREAT-MODEL.md', import.meta.url);
const productDecisionPath = new URL('../docs/PRODUCT-DECISIONS.md', import.meta.url);
const roadmapPath = new URL('../ROADMAP.md', import.meta.url);

// Every hop that can hold, move, or leak a user's photo bytes has to be named
// in the diagram, not only in prose.
const REQUIRED_DIAGRAM_NODES = [
  ['browser', /\bbrowser\b/i],
  ['API ingress', /\bapi[- ]ingress\b/i],
  ['model worker', /\bmodel worker\b/i],
  ['transient buffers', /\btransient buffer/i],
  ['training opt-in store', /\btraining opt-in\b/i],
  ['logs', /\blogs?\b/i],
  ['exports', /\bexports?\b/i],
  ['deletion', /\bdeletion\b/i],
];

// The abuse cases the ticket names explicitly, in the order they appear there.
const REQUIRED_THREATS = [
  ['replay', /\breplay\b/i],
  ['oversized input', /\boversized input\b/i],
  ['metadata leakage', /\bmetadata leakage\b/i],
  ['poisoning', /\bpoisoning\b/i],
  ['unauthorized training', /\bunauthorized training\b/i],
  ['operator access', /\boperator access\b/i],
];

function diagramBody(doc) {
  const match = doc.match(/```mermaid\r?\n([\s\S]*?)```/);
  assert.ok(match, 'the threat model must contain a fenced mermaid data-flow diagram');
  return match[1];
}

function threatBlocks(doc) {
  const blocks = new Map();
  const pattern = /^###\s+(T\d+)\s+—\s+(.+)$/gm;
  const headings = [...doc.matchAll(pattern)];
  headings.forEach((heading, index) => {
    const start = heading.index + heading[0].length;
    const end = index + 1 < headings.length ? headings[index + 1].index : doc.length;
    blocks.set(heading[1], { name: heading[2].trim(), body: doc.slice(start, end) });
  });
  return blocks;
}

function knownIssueNumbers(roadmap) {
  return new Set([...roadmap.matchAll(/issues\/(\d+)/g)].map((match) => Number(match[1])));
}

test('photo data-flow threat model diagrams every component that can hold or leak image bytes', async () => {
  const doc = await readFile(threatModelPath, 'utf8');
  const diagram = diagramBody(doc);

  for (const [label, pattern] of REQUIRED_DIAGRAM_NODES) {
    assert.match(diagram, pattern, `the data-flow diagram must show the ${label} component`);
  }

  // The diagram has to end somewhere: the model worker's bytes must not flow on.
  assert.match(diagram, /no retention|dropped|discarded|never persist/i,
    'the diagram must show that worker input is not retained');
});

test('every required abuse case has a mitigation, an owner, and a real follow-up ticket', async () => {
  const [doc, roadmap] = await Promise.all([
    readFile(threatModelPath, 'utf8'),
    readFile(roadmapPath, 'utf8'),
  ]);
  const known = knownIssueNumbers(roadmap);
  const blocks = threatBlocks(doc);

  assert.ok(blocks.size > 0, 'threats must be numbered headings so their coverage is checkable');
  const named = [...blocks.values()].map((block) => block.name).join('\n');

  for (const [label, pattern] of REQUIRED_THREATS) {
    assert.match(named, pattern, `the ${label} abuse case must be its own numbered threat`);

    const block = [...blocks.values()].find((entry) => pattern.test(entry.name));
    assert.match(block.body, /^\s*-\s*\*\*Owner:\*\*\s*\S+/m,
      `${label} needs a named owner`);
    assert.match(block.body, /^\s*-\s*\*\*Testable follow-up:\*\*\s*#\d+/m,
      `${label} needs a testable follow-up ticket`);
    assert.match(block.body, /^\s*-\s*\*\*Mitigation:\*\*\s*\S+/m,
      `${label} needs a concrete mitigation, not a restatement of the risk`);
    assert.match(block.body, /^\s*-\s*\*\*Evidence:\*\*\s*\S+/m,
      `${label} needs the artifact that will prove the mitigation works`);

    const followUp = Number(block.body.match(/^\s*-\s*\*\*Testable follow-up:\*\*\s*#(\d+)/m)[1]);
    assert.ok(known.has(followUp),
      `${label} points at #${followUp}, which is not a tracked ticket in ROADMAP.md`);
    assert.notEqual(followUp, 20, `${label} must delegate to work other than this threat model itself`);
  }
});

test('threat model keeps the photo path non-diagnostic and free of real medical images', async () => {
  const doc = await readFile(threatModelPath, 'utf8');

  assert.match(doc, /non-diagnostic/i);
  assert.match(doc, /never diagnos/i);
  assert.match(doc, /provisional/i);
  assert.match(doc, /synthetic/i, 'fixtures must be synthetic, not real medical images');
  assert.match(doc, /no real (?:medical |stool )?images? (?:are |may )?(?:ever )?(?:added|used|shipped|included)/i,
    'the model must state plainly that no real medical images enter the repo');
  assert.match(doc, /deterministic (?:red-flag )?(?:escalation|urgent-symptom)/i,
    'the model must note that escalation is deterministic and never model-gated');
  assert.doesNotMatch(doc, /\b(?:bathroom photo|patient photo|clinical image|real stool image)\b/i,
    'the model must not promise or reference real medical imagery');
  assert.doesNotMatch(doc, /\b(?:clinically validated|clinically proven|medical device|FDA-cleared|CE-marked)\b/i,
    'the model must not overstate the clinical status of the vision path');
});

test('the accepted product decision record links the photo data-flow threat model', async () => {
  const decision = await readFile(productDecisionPath, 'utf8');
  assert.match(decision, /\.\/PHOTO-DATAFLOW-THREAT-MODEL\.md/,
    'the governing decision record must point reviewers at the threat model');
});