From f0a9a734db801feab5fb8fc6b5beb2cf13bb1c4f Mon Sep 17 00:00:00 2001 From: Timmy Date: Fri, 30 Oct 2026 11:12:52 +0000 Subject: [PATCH] feat(security): attack the RC before users do and gate approval on P0 findings (#40) Add an executable abuse suite that drives the real server with synthetic hostile bytes while a counting stub worker records every call that reaches the model boundary, plus the verdict rule that turns those observations into a release decision. - src/release-security-gate.js: nearest-rank latency percentiles, per-case pass/fail derivation that names the field that broke, and the rule that any failing P0 case blocks RC approval while a required class with no verified evidence blocks it too. Reports are redacted and fail closed. - scripts/security_attack_suite.mjs: oversized body, HTML polyglot as JPEG, undecodable bytes, missing consent, non-conforming worker answer, byte-identical replay, concurrent burst, captured log scan, and an out-of-surface deletion case counted only against a real test file. Writes the report, prints the verdict, exits 1 when blocked. - docs/RC-SECURITY-EVIDENCE.md: procedure, rule, and the current finding stated plainly. - Demo: a real run of the suite on camera, printing the verdict it derived instead of a flattering one; CI regenerates the same evidence on every change. The suite currently returns VERDICT: BLOCKED: bytes are trusted from the data-URL label, so an HTML polyglot still reaches the worker and only fails afterwards on schema. That is a finding against #16, not a passing check, and it is why the gate treats a failing case as a block. --- .gitea/workflows/quality.yml | 1 + .gitignore | 1 + README.md | 3 + docs/RC-SECURITY-EVIDENCE.md | 78 +++++++ package.json | 7 +- scripts/record_release_demo.mjs | 70 ++++++ scripts/security_attack_suite.mjs | 310 +++++++++++++++++++++++++++ src/release-security-gate.js | 95 ++++++++ tests/ci-workflow.test.js | 11 + tests/release-demo.test.js | 16 ++ tests/release-security-gate.test.js | 104 +++++++++ tests/security-attack.acceptance.mjs | 89 ++++++++ 12 files changed, 782 insertions(+), 3 deletions(-) create mode 100644 docs/RC-SECURITY-EVIDENCE.md create mode 100644 scripts/security_attack_suite.mjs create mode 100644 src/release-security-gate.js create mode 100644 tests/release-security-gate.test.js create mode 100644 tests/security-attack.acceptance.mjs diff --git a/.gitea/workflows/quality.yml b/.gitea/workflows/quality.yml index 8eb3ad0..4229066 100644 --- a/.gitea/workflows/quality.yml +++ b/.gitea/workflows/quality.yml @@ -32,6 +32,7 @@ jobs: - name: Unit and security tests run: | npm test + npm run test:security-attack python3 tests/staging-deploy.test.py -v python3 tests/corpus-manifest.test.py -v python3 tests/corpus-review.test.py -v diff --git a/.gitignore b/.gitignore index a8758b0..f64aec2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ *.pyc .env .env.* +artifacts/security/ diff --git a/README.md b/README.md index 1abae5f..527dcbb 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ npm run test:grounded-chat # chat answers cite the ledger, or refuse to answe npm run test:photo-lifecycle # photo presence, explicit reveal, remove-photo-keeps-log, delete-all npm run test:calibration # the release-gate harness: gates, threshold protocol, refused corpora npm run test:review # blind review queue, held-back model answer, an append-only ledger, withdrawals +npm run test:security-attack # hostile synthetic uploads at the real boundary, with a counting worker (#40) npm audit --audit-level=high ``` @@ -160,6 +161,8 @@ The reviewer tool ([scripts/review_corpus.py](scripts/review_corpus.py), [contra What the corpus is, and what a label may assert, is published rather than implied: [docs/DATASET_CARD.md](docs/DATASET_CARD.md) is generated from the manifest by [scripts/dataset_card.py](scripts/dataset_card.py), so every count on it is read back from real records and every fact the pipeline does not collect (devices, contexts, licenses) is printed as `pending` instead of estimated — a card for a corpus where one contributor straddles two splits is refused, not published. [docs/LABELING_GUIDE.md](docs/LABELING_GUIDE.md) is the reviewer's rulebook: visible form, broad visible color, and photo quality only, visible color is not blood, nothing is diagnosed, and a model suggestion is provisional input to a human decision rather than the label. +The release candidate is attacked before anyone is asked to trust it. [scripts/security_attack_suite.mjs](scripts/security_attack_suite.mjs) drives the real server with synthetic hostile bytes — an oversized body, an HTML polyglot dressed as a JPEG, undecodable bytes, a request without consent, a worker answer that breaks the schema, a byte-identical replay, and a concurrent burst — while a counting stub worker records every call that reaches the model boundary. [docs/RC-SECURITY-EVIDENCE.md](docs/RC-SECURITY-EVIDENCE.md) owns the rule: any failing P0 case blocks RC approval, and a required class with no executable evidence blocks it too. **It currently returns `VERDICT: BLOCKED`** — bytes are still trusted from the data-URL label, so a hostile file reaches the worker before it fails on schema. That is a published finding against [#16](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/16), not a passing check. + ## Architecture ```text diff --git a/docs/RC-SECURITY-EVIDENCE.md b/docs/RC-SECURITY-EVIDENCE.md new file mode 100644 index 0000000..e1efb79 --- /dev/null +++ b/docs/RC-SECURITY-EVIDENCE.md @@ -0,0 +1,78 @@ +# RC Security & Abuse Evidence (#40) + +The release candidate is attacked before users get to it. This page is the procedure and the rule, +not a stored result: a pasted-in "all clear" is exactly the kind of evidence that stops being true +after the next commit. + +## Run it + +```bash +node scripts/security_attack_suite.mjs --json artifacts/security/rc-attack-report.json +``` + +Exit codes are part of the contract: + +| Exit | Meaning | +|---:|---| +| `0` | `VERDICT: GO` — every case matched its expectation. | +| `1` | `VERDICT: BLOCKED` — at least one P0 case failed, or a required category has no executable evidence. | +| `2` | The suite could not complete (target never became ready, vision disabled, no log captured). | + +The suite is a child process of nothing: it starts its own copy of `server.mjs` on a free loopback +port, points it at a **counting stub worker**, and drives it with synthetic hostile bytes. It never +contacts a real model provider, never uses a real medical image, and never touches a live user's data. + +## What it does and does not prove + +- It proves what this HTTP boundary did with the bytes it was handed, and whether the worker behind + it was called. The worker is a stub, so "the provider saw it" is a counted fact, not an inference. +- It does **not** measure model quality, clinical accuracy, or the safety of the medical/non-diagnostic + boundary. Those live in the vision, escalation, and clinical-review issues. +- Latency numbers describe the loopback boundary against a stub. They are not a model benchmark. + +## Cases + +| Case | Class | Severity | Expectation | +|---|---|---:|---| +| `ingress-oversized-body` | oversized | P0 | `413`, worker never called | +| `ingress-polyglot-html` | polyglot | P0 | `400`, worker never called | +| `ingress-garbage-bytes` | truncated | P0 | `400`, worker never called | +| `consent-required` | consent | P0 | `400`, worker never called | +| `model-output-malformed` | malformed | P0 | `503` neutral copy, provider text never echoed | +| `replay-duplicate-request` | replay | P2 | identical answer, nothing stored server-side | +| `flood-concurrency` | flood | P1 | burst meets a bound (`429`/overload), not an open pipe | +| `log-leak-scan` | log | P0 | captured server log carries no image bytes | +| `deletion-withdrawal` | deletion | P0 | out of surface — counted only when `tests/training-contribution.test.js` exists | + +## The rule + +`src/release-security-gate.js` owns the verdict, so it is testable rather than asserted: + +- A case passes only when **every** expected field matches the observation. A partial match fails and + names the field that broke (`status 400 expected, observed 200`). +- **Any** failing P0 case blocks RC approval. P1/P2 failures are recorded as holds. +- A required class with no executable evidence is `unproven` and blocks the release. Coverage may be + borrowed from another suite (`out-of-surface` + an evidence path) only after the runner verified + that file exists — otherwise it blocks. +- The report is redacted before it is written, and assembly fails closed if any image bytes survive + redaction. Evidence that carries a photo is worse than no evidence. + +## Current state (read the run, not this line) + +The suite is expected to come back **BLOCKED** until the ingress hardening in +[#16](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/16) lands: bytes +are still trusted from the data-URL label, so an HTML file dressed as a JPEG reaches the worker and +only fails afterwards on schema. That is the finding, and it is why the release gate treats a failing +case as a block instead of a warning. Queue bounding is +[#17](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/17) and is held +here as P1. + +Deletion is governed by the contribution ledger, not by this HTTP boundary; the case stays +`covered-elsewhere` against `tests/training-contribution.test.js` and blocks if that file disappears. + +## Evidence hygiene + +- The report is written to `artifacts/security/` and must never be committed with a real photo in it. +- The captured server log is scanned by the same detector (`containsImageBytes`) before it is quoted. +- Error bodies are recorded as a 120-character snippet; the suite refuses to write a report that + still contains a data URI or a long base64 run. diff --git a/package.json b/package.json index 644461b..a5c31ad 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "private": true, "type": "module", "scripts": { - "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js tests/training-contribution.test.js tests/portability.test.js tests/abstention-policy.test.js", + "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js tests/training-contribution.test.js tests/portability.test.js tests/abstention-policy.test.js tests/release-security-gate.test.js", "test:ui": "node tests/ui.acceptance.mjs", "test:photo": "node tests/photo-first.acceptance.mjs", "test:mobile-capture": "node tests/mobile-capture.acceptance.mjs", @@ -18,7 +18,7 @@ "test:grounded-chat": "node tests/grounded-chat.acceptance.mjs", "test:photo-lifecycle": "node tests/photo-lifecycle.acceptance.mjs", "test:training-contribution": "node tests/training-contribution.acceptance.mjs", - "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check src/training-contribution.js && node --check src/abstention-policy.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/corpus_manifest.py scripts/review_corpus.py scripts/dataset_card.py scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py scripts/calibration_report.py scripts/operator_backup.py && node --check scripts/record_worker_benchmark.mjs && node --check scripts/abstention_policy_receipt.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/training-contribution.acceptance.mjs && node --check tests/portability.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh", + "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check src/training-contribution.js && node --check src/abstention-policy.js && node --check src/release-security-gate.js && node --check scripts/security_attack_suite.mjs && node --check tests/security-attack.acceptance.mjs && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/corpus_manifest.py scripts/review_corpus.py scripts/dataset_card.py scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py scripts/calibration_report.py scripts/operator_backup.py && node --check scripts/record_worker_benchmark.mjs && node --check scripts/abstention_policy_receipt.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/training-contribution.acceptance.mjs && node --check tests/portability.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh", "check:diff": "bash scripts/check_diff.sh", "start": "node server.mjs", "test:worker-readiness": "node tests/worker-readiness.acceptance.mjs", @@ -29,7 +29,8 @@ "test:review": "python3 tests/corpus-review.test.py", "test:dataset-card": "python3 tests/dataset-card.test.py", "test:backup": "python3 tests/backup-restore.test.py", - "test:runbook": "python3 tests/production-runbook.test.py" + "test:runbook": "python3 tests/production-runbook.test.py", + "test:security-attack": "node tests/security-attack.acceptance.mjs" }, "devDependencies": { "playwright": "^1.62.1" diff --git a/scripts/record_release_demo.mjs b/scripts/record_release_demo.mjs index b965f8e..63b46ae 100644 --- a/scripts/record_release_demo.mjs +++ b/scripts/record_release_demo.mjs @@ -1165,6 +1165,76 @@ await caption('Each candidate is scored on test at a threshold chosen on validat await caption('One contributor spread across two splits is refused before anything is scored, and a passing verdict is proven only by a fixture engineered to clear it', 3800); await page.evaluate(() => document.querySelector('#calibration-report')?.remove()); +// Issue #40 — the RC attack suite is a command line too, and unlike the other cards this one is +// expected to come back with a verdict that is not flattering. Hostile synthetic bytes go at the +// real server with a counting stub worker behind it, so "the worker was called" is measured, not +// inferred. The card prints whatever verdict the suite derived from what it observed; the captions +// name where a failing fix lives instead of implying the boundary is already safe. +async function securityAttackSuite() { + const directory = await mkdtemp(path.join(os.tmpdir(), 'timmy-security-demo-')); + const reportPath = path.join(directory, 'rc-attack-report.json'); + try { + const run = await execFileAsync( + process.execPath, + ['scripts/security_attack_suite.mjs', '--json', reportPath], + { cwd: process.cwd() }, + ).then(result => ({ code: 0, stdout: result.stdout, stderr: result.stderr })) + .catch(error => ({ code: typeof error.code === 'number' ? error.code : 1, stdout: error.stdout || '', stderr: error.stderr || '' })); + if (![0, 1].includes(run.code)) throw new Error(`the attack suite could not complete: ${run.stderr.trim()}`); + const report = JSON.parse(await readFile(reportPath, 'utf8')); + if (run.code !== (report.verdict === 'BLOCKED' ? 1 : 0)) { + throw new Error('the attack suite exit code must match the verdict it published'); + } + return { report, run }; + } finally { + await rm(directory, { recursive: true, force: true }); + } +} +const securityRun = await securityAttackSuite(); +const securityReport = securityRun.report; +if (securityReport.cases.length < 8) throw new Error('the RC attack suite must run every named abuse class'); +await tap('[data-view="home"]', 450); +const caseDetail = (item) => { + const observed = item.observed || {}; + if ('providerCalls' in observed) return `http ${observed.status} · worker ${observed.providerCalls}`; + if ('refused' in observed) return `refused ${observed.refused}`; + if ('imageBytesInLog' in observed) return `image bytes ${observed.imageBytesInLog}`; + return item.status; +}; +const securityRows = [ + ['command', 'node scripts/security_attack_suite.mjs'], + ['target', 'real server · counting stub worker · synthetic bytes'], + ['—', ''], +]; +for (const item of securityReport.cases) securityRows.push([item.category, `${item.severity} ${item.status} · ${caseDetail(item)}`]); +securityRows.push(['—', '']); +securityRows.push(['load', `${securityReport.load.requests} req · p50 ${securityReport.load.p50}ms · p95 ${securityReport.load.p95}ms · abstain ${securityReport.load.abstentions}`]); +securityRows.push(['verdict', `${securityReport.verdict} · ${securityReport.blocking.length} blocking · ${securityReport.holds.length} held`]); +for (const item of securityReport.blocking) securityRows.push(['blocks', item.id]); +const securityCard = await evidenceCard({ + id: 'security-attack', + title: 'SECURITY ATTACK SUITE', + subtitle: 'command line, not app UI — real server, counted worker calls', + rows: securityRows, + font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace', + keyWidth: 62, + gap: '1px', + rowGap: '5px', +}); +await sleep(420); +const securityCardProof = await evidenceCardProof('security-attack'); +if (!securityCardProof.ok) throw new Error(`Security card is not fully readable on screen: ${JSON.stringify(securityCardProof)}`); +for (const item of securityReport.blocking) { + if (!securityCard.text.includes(item.id)) throw new Error(`the card must name the blocking case ${item.id}`); +} +if (!securityCard.text.includes(securityReport.verdict)) throw new Error('the card must show the verdict the suite derived'); +if (!securityCard.text.includes(`${securityReport.load.p95}ms`)) throw new Error('the card must show the p95 the load phase measured'); +await caption('Every hostile upload on screen went at the real server, and the worker behind it counted every call it received', 3800); +await caption('Refusing bytes by their content is the fix in review, not a claim this video makes: an HTML file dressed as a JPEG still reaches the worker today', 4200); +await caption('A failing case is a finding the release job has to see, so the card prints the verdict the suite derived instead of a reassuring one', 3800); +await caption('The gate blocks RC approval on any P0 case that reached the worker — the gate decides, not the caption', 3400); +await page.evaluate(() => document.querySelector('#security-attack')?.remove()); + // Issue #39 — the operator backup is a command line, not a screen in the app. The app itself keeps // no server-side journal, so what an operator can actually lose is the corpus: the append-only // manifest with its withdrawal tombstones. This scene builds a synthetic data root with the real diff --git a/scripts/security_attack_suite.mjs b/scripts/security_attack_suite.mjs new file mode 100644 index 0000000..8378c84 --- /dev/null +++ b/scripts/security_attack_suite.mjs @@ -0,0 +1,310 @@ +#!/usr/bin/env node +// RC attack suite (#40): attack the release candidate before users do. +// +// Every case drives the shipping `server.mjs` with synthetic hostile bytes while a stub worker counts +// the calls that reach it. The suite records what it observed and derives a verdict from the +// observation; it never asserts a result it did not see. No real medical image, no live user data, +// no third-party provider is involved, and the provider is a counting stub, so "reached the worker" +// is provable rather than inferred. +// +// Usage: node scripts/security_attack_suite.mjs [--json ] [--base-url ] +// Exit: 0 = GO, 1 = BLOCKED (a P0 finding, or a required category with no executable evidence). +import { randomBytes } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { access, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import http from 'node:http'; +import { fileURLToPath } from 'node:url'; +import { buildReport, containsImageBytes, latencyPercentiles } from '../src/release-security-gate.js'; + +const repoRoot = fileURLToPath(new URL('..', import.meta.url)); +const argument = (name, fallback) => { + const index = process.argv.indexOf(name); + return index === -1 ? fallback : process.argv[index + 1]; +}; +const reportPath = argument('--json', join(repoRoot, 'artifacts/security/rc-attack-report.json')); +const explicitBaseUrl = argument('--base-url', ''); + +function freePort() { + return new Promise(resolve => { + const probe = http.createServer(); + probe.listen(0, '127.0.0.1', () => { + const { port } = probe.address(); + probe.close(() => resolve(port)); + }); + }); +} + +// A counting OpenAI-compatible worker. `state.content` is the answer the next request receives, and +// `state.requests` is the evidence that a request crossed the provider boundary at all. +function startStubWorker() { + const state = { content: '', requests: 0 }; + const server = http.createServer((req, res) => { + if (req.url.endsWith('/models')) { + res.writeHead(200, { 'content-type': 'application/json' }); + return res.end(JSON.stringify({ data: [{ id: 'stub-vision-worker' }] })); + } + req.resume(); + req.on('end', () => { + state.requests += 1; + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ choices: [{ message: { content: state.content } }] })); + }); + }); + return new Promise(resolve => server.listen(0, '127.0.0.1', () => resolve({ + state, + port: server.address().port, + close: () => new Promise(done => server.close(done)), + }))); +} + +// The real server, with its own stdout/stderr captured from the first byte: the log scan is only +// worth something if it read a log that actually exists. +async function startTarget({ providerPort, port }) { + const child = spawn(process.execPath, ['server.mjs'], { + cwd: repoRoot, + env: { + ...process.env, + HOST: '127.0.0.1', + PORT: String(port), + TIMMY_VISION_ENABLED: 'true', + TIMMY_VISION_PROFILE: 'selfhost', + TIMMY_VISION_BASE_URL: `http://127.0.0.1:${providerPort}/v1`, + TIMMY_VISION_MODEL: 'stub-vision-worker', + TIMMY_VISION_TIMEOUT_MS: '5000', + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const log = []; + child.stdout.on('data', chunk => log.push(chunk.toString('utf8'))); + child.stderr.on('data', chunk => log.push(chunk.toString('utf8'))); + for (let attempt = 0; attempt < 100; attempt += 1) { + if (log.join('').includes('Timmy is listening')) return { child, log }; + try { + const response = await fetch(`http://127.0.0.1:${port}/api/healthz`); + if (response.ok) return { child, log }; + } catch {} + await new Promise(resolve => setTimeout(resolve, 100)); + } + child.kill(); + throw new Error('attack suite target did not become ready'); +} + +const stop = child => new Promise(resolve => { + if (!child || child.exitCode !== null) return resolve(); + child.kill(); + child.once('exit', resolve); +}); + +async function main() { + const worker = await startStubWorker(); + const port = explicitBaseUrl ? null : await freePort(); + const baseUrl = explicitBaseUrl || `http://127.0.0.1:${port}`; + const target = explicitBaseUrl ? null : await startTarget({ providerPort: worker.port, port }); + const photo = await readFile(join(repoRoot, 'tests/fixtures/synthetic-type4.jpg')); + const imageDataUrl = `data:image/jpeg;base64,${photo.toString('base64')}`; + + const post = async (body) => { + const before = worker.state.requests; + const response = await fetch(`${baseUrl}/api/analyze`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }); + const text = await response.text(); + return { status: response.status, text, providerCalls: worker.state.requests - before }; + }; + + const cases = []; + const record = (entry) => { cases.push(entry); return entry; }; + + try { + const health = await fetch(`${baseUrl}/api/healthz`).then(response => response.json()); + if (!health.visionEnabled) throw new Error('the attack suite needs vision analysis enabled on the target'); + + // 1. An oversized body must die at the request cap, before the photo is parsed or forwarded. + const oversized = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${'A'.repeat(6.6 * 1024 * 1024)}` }); + record({ + id: 'ingress-oversized-body', + category: 'oversized', + severity: 'P0', + requirement: 'A body over the request cap is refused before any provider work.', + expected: { status: 413, providerCalls: 0 }, + observed: { status: oversized.status, providerCalls: oversized.providerCalls }, + snippet: oversized.text.slice(0, 120), + }); + + // 2. Bytes decide the format, not the data-URL label: HTML dressed as a JPEG is a polyglot. + const html = Buffer.from(''); + const polyglot = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${html.toString('base64')}` }); + record({ + id: 'ingress-polyglot-html', + category: 'polyglot', + severity: 'P0', + requirement: 'A payload whose bytes are HTML under an image/jpeg label is refused before the provider sees it.', + expected: { status: 400, providerCalls: 0 }, + observed: { status: polyglot.status, providerCalls: polyglot.providerCalls }, + snippet: polyglot.text.slice(0, 120), + }); + + // 3. Random bytes under a JPEG label are not an image either. + const garbage = await post({ consent: true, imageDataUrl: `data:image/jpeg;base64,${randomBytes(64).toString('base64')}` }); + record({ + id: 'ingress-garbage-bytes', + category: 'truncated', + severity: 'P0', + requirement: 'Undecodable bytes are refused rather than forwarded and interpreted.', + expected: { status: 400, providerCalls: 0 }, + observed: { status: garbage.status, providerCalls: garbage.providerCalls }, + snippet: garbage.text.slice(0, 120), + }); + + // 4. Consent is a precondition, and it is cheap to prove: no consent, no provider call. + const noConsent = await post({ consent: false, imageDataUrl }); + record({ + id: 'consent-required', + category: 'consent', + severity: 'P0', + requirement: 'Analysis without explicit consent is refused and never leaves the boundary.', + expected: { status: 400, providerCalls: 0 }, + observed: { status: noConsent.status, providerCalls: noConsent.providerCalls }, + snippet: noConsent.text.slice(0, 120), + }); + + // 5. A worker answer outside the schema must fail closed, and the provider's words must not + // reach the user or the evidence file. + worker.state.content = 'garbage-completion-token: isStool yes maybe, bristolType ten'; + const malformed = await post({ consent: true, imageDataUrl }); + record({ + id: 'model-output-malformed', + category: 'malformed', + severity: 'P0', + requirement: 'A non-conforming provider answer is replaced by neutral copy and never echoed.', + expected: { status: 503, providerTextLeaked: false }, + observed: { status: malformed.status, providerTextLeaked: /garbage-completion-token/.test(malformed.text) }, + snippet: malformed.text.slice(0, 120), + }); + + // 6. Replay: this boundary holds no session state, so a replay is a second request. What must + // hold is that it is answered identically and that nothing accumulates server-side. + worker.state.content = JSON.stringify({ isStool: true, bristolType: 4, color: 'brown', confidence: 0.82, imageQuality: 'good', observations: 'Smooth, formed appearance.' }); + const first = await post({ consent: true, imageDataUrl }); + const second = await post({ consent: true, imageDataUrl }); + record({ + id: 'replay-duplicate-request', + category: 'replay', + severity: 'P2', + requirement: 'A byte-identical replay is answered identically and stores nothing on the server.', + expected: { status: 200, providerCalls: 1, repeatedAnswerIdentical: true }, + observed: { status: second.status, providerCalls: second.providerCalls, repeatedAnswerIdentical: first.text === second.text }, + snippet: second.text.slice(0, 120), + }); + + // 7. Flood: concurrent requests must meet a bound instead of all reaching the worker at once. + const beforeFlood = worker.state.requests; + const flood = await Promise.all(Array.from({ length: 12 }, () => post({ consent: true, imageDataUrl }))); + record({ + id: 'flood-concurrency', + category: 'flood', + severity: 'P1', + requirement: 'A burst of concurrent analyses meets a bounded queue or a rate refusal.', + expected: { refused: true }, + observed: { refused: flood.some(item => item.status === 429 || item.status === 503), providerCalls: worker.state.requests - beforeFlood }, + snippet: `statuses ${[...new Set(flood.map(item => item.status))].sort().join(',')}`, + }); + + // 8. Log inspection: the operator log is committed evidence, so it must not contain the photo. + const capturedLog = (target?.log || []).join(''); + if (!capturedLog.trim()) throw new Error('no server log was captured, so the log scan would prove nothing'); + record({ + id: 'log-leak-scan', + category: 'log', + severity: 'P0', + requirement: 'Server logs contain no image bytes, base64, or note text.', + expected: { imageBytesInLog: false }, + observed: { imageBytesInLog: containsImageBytes(capturedLog) }, + snippet: `${capturedLog.trim().split('\n').length} log line(s) inspected`, + }); + + // 9. Deletion is governed by the contribution ledger, not by this HTTP boundary. Covering it + // here means naming the executable test that does it, and only counting it once verified. + let evidenceVerified = false; + try { + await access(join(repoRoot, 'tests/training-contribution.test.js')); + evidenceVerified = true; + } catch {} + record({ + id: 'deletion-withdrawal', + category: 'deletion', + severity: 'P0', + surface: 'out-of-surface', + requirement: 'Withdrawal deletes the contribution and the server holds no copy of the photo.', + evidence: 'tests/training-contribution.test.js', + evidenceVerified, + snippet: evidenceVerified ? 'covered by the contribution ledger suite' : 'named evidence file is missing', + }); + + // Load: sequential valid analyses, abstaining answers, latency measured with the same boundary. + worker.state.content = JSON.stringify({ isStool: false, confidence: 0.9 }); + const samples = []; + let errors = 0; + let abstentions = 0; + for (let index = 0; index < 8; index += 1) { + const started = performance.now(); + const result = await post({ consent: true, imageDataUrl }); + samples.push(Math.round(performance.now() - started)); + if (result.status !== 200) errors += 1; + else { + try { + if (JSON.parse(result.text).status === 'needs_user_input') abstentions += 1; + } catch {} + } + } + const percentiles = latencyPercentiles(samples); + const load = { + requests: samples.length, + errors, + abstentions, + p50: percentiles.p50, + p95: percentiles.p95, + max: percentiles.max, + note: 'Latency measured against a counting stub on loopback; it measures this boundary, not a model.', + }; + + const report = buildReport({ + generatedAt: new Date().toISOString(), + target: { + baseUrl, + provider: 'stub counting worker (synthetic answers, no model loaded)', + visionProfile: 'selfhost', + release: health.release, + commit: health.commit, + }, + cases, + load, + }); + await mkdir(dirname(reportPath), { recursive: true }); + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8'); + + const width = Math.max(...cases.map(item => item.id.length)); + for (const item of report.cases) { + const detail = item.status === 'fail' ? ` — ${item.mismatch}` : ''; + console.log(`${item.id.padEnd(width)} ${item.severity} ${item.status}${detail}`); + } + console.log(`load: ${load.requests} requests · p50 ${load.p50}ms · p95 ${load.p95}ms · errors ${load.errors} · abstentions ${load.abstentions}`); + console.log(`report: ${reportPath}`); + console.log(`VERDICT: ${report.verdict}`); + if (report.verdict === 'BLOCKED') { + for (const item of [...report.blocking, ...report.unproven]) console.log(` blocks RC: ${item.id} (${item.severity}) ${item.reason}`); + } + return report.verdict === 'GO' ? 0 : 1; + } finally { + await stop(target?.child); + await worker.close(); + } +} + +main().then(code => { process.exitCode = code; }).catch(error => { + console.error(`attack suite could not complete: ${error.message}`); + process.exitCode = 2; +}); diff --git a/src/release-security-gate.js b/src/release-security-gate.js new file mode 100644 index 0000000..6e17379 --- /dev/null +++ b/src/release-security-gate.js @@ -0,0 +1,95 @@ +// Release security gate: the arithmetic and the verdict rule for the RC attack suite (#40). +// +// The suite observes the real server; this module only turns observations into a verdict. Keeping +// the rule here (not in the runner) is what makes "a P0 finding blocks RC approval" testable. + +/** Nearest-rank percentile: the returned value is always a sample that was actually measured. */ +export function latencyPercentiles(samples) { + const sorted = [...samples].sort((a, b) => a - b); + if (!sorted.length) return { count: 0, p50: null, p95: null, max: null }; + const rank = (p) => sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)]; + return { count: sorted.length, p50: rank(50), p95: rank(95), max: sorted[sorted.length - 1] }; +} + +/** One case against one expectation. A partial match is a failure, and it says which field broke. */ +export function evaluateCase(record) { + const mismatches = Object.keys(record.expected) + .filter(field => record.observed?.[field] !== record.expected[field]) + .map(field => `${field} ${record.expected[field]} expected, observed ${record.observed?.[field]}`); + return { ...record, status: mismatches.length ? 'fail' : 'pass', mismatch: mismatches.join('; ') }; +} + +const SEVERITY_BLOCKS = (severity) => severity === 'P0'; + +/** The RC rule: every P0 failure blocks approval, unproven categories block it too, P1/P2 only hold. */ +export function releaseVerdict({ cases, load = null }) { + const evaluated = cases.map(record => (record.surface === 'out-of-surface' + ? { ...record, status: record.evidenceVerified ? 'covered-elsewhere' : 'unproven' } + : evaluateCase(record))); + const blocking = evaluated.filter(item => item.status === 'fail' && SEVERITY_BLOCKS(item.severity)) + .map(item => ({ id: item.id, severity: item.severity, reason: item.mismatch })); + const holds = evaluated.filter(item => item.status === 'fail' && !SEVERITY_BLOCKS(item.severity)) + .map(item => ({ id: item.id, severity: item.severity, reason: item.mismatch })); + const unproven = evaluated.filter(item => item.status === 'unproven') + .map(item => ({ id: item.id, severity: item.severity, reason: 'no executable evidence' })); + return { + verdict: blocking.length || unproven.length ? 'BLOCKED' : 'GO', + blocking, + holds, + unproven, + coveredElsewhere: evaluated.filter(item => item.status === 'covered-elsewhere').map(item => item.id), + proven: evaluated.filter(item => item.status === 'pass').length, + cases: evaluated, + load, + }; +} + +// Evidence hygiene: the report and the captured server log are committed and quoted. A photo that +// survived into either one is a leak of the exact data this suite exists to protect. +const IMAGE_DATA_URI = /data:image\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+/gi; +const LONG_BASE64 = /[A-Za-z0-9+/]{200,}={0,2}/; + +export function containsImageBytes(text) { + const value = String(text ?? ''); + return new RegExp(IMAGE_DATA_URI.source, 'i').test(value) || LONG_BASE64.test(value); +} + +export function redactImageBytes(text) { + return String(text ?? '') + .replace(IMAGE_DATA_URI, '[redacted image bytes]') + .replace(new RegExp(LONG_BASE64.source, 'g'), '[redacted image bytes]'); +} + +/** Fails closed rather than emitting evidence that carries a photo. */ +export function assertNoImageBytes(label, text) { + if (containsImageBytes(text)) throw new Error(`${label} contains image bytes; refusing to write release evidence.`); + return true; +} + +/** + * The receipt the release job quotes. It is redacted after assembly, so a case that captured a + * response body still cannot carry a photo in it. + */ +export function buildReport({ generatedAt, target, cases, load, suite = 'scripts/security_attack_suite.mjs' }) { + const verdict = releaseVerdict({ cases, load }); + const report = { + schema: 'timmy-rc-security-attack/1', + issue: 40, + generatedAt, + suite, + target, + verdict: verdict.verdict, + blocking: verdict.blocking, + holds: verdict.holds, + unproven: verdict.unproven, + coveredElsewhere: verdict.coveredElsewhere, + proven: verdict.proven, + cases: verdict.cases.map(({ id, category, severity, requirement, expected, observed, status, mismatch, snippet, surface, evidence }) => + ({ id, category, severity, requirement, expected, observed, status, mismatch, surface, evidence, snippet: snippet ? redactImageBytes(snippet) : undefined })), + load, + disclaimer: 'Synthetic hostile bytes only. Every case drives the real server against a stub worker; no real medical image and no live user data is involved.', + }; + const clean = JSON.parse(redactImageBytes(JSON.stringify(report))); + assertNoImageBytes('security report', JSON.stringify(clean)); + return clean; +} diff --git a/tests/ci-workflow.test.js b/tests/ci-workflow.test.js index 45fe625..1750733 100644 --- a/tests/ci-workflow.test.js +++ b/tests/ci-workflow.test.js @@ -202,3 +202,14 @@ test('the calibration harness is wired into the unit and syntax gates', async () assert.match(builder, /tests\/calibration-harness\.test\.py/); assert.match(builder, /"calibration_harness_gates": "passed"/); }); + +test('CI runs the RC attack suite so its evidence is reproducible on every change', async () => { + const workflow = await readFile(workflowPath, 'utf8'); + const packageJson = JSON.parse(await readFile(packagePath, 'utf8')); + + // Issue #40: the attack report is evidence, not a claim. CI has to be able to regenerate it. + assert.match(workflow, /npm run test:security-attack/); + assert.equal(packageJson.scripts['test:security-attack'], 'node tests/security-attack.acceptance.mjs'); + assert.match(packageJson.scripts.test, /tests\/release-security-gate\.test\.js/); + assert.match(packageJson.scripts['check:syntax'], /node --check src\/release-security-gate\.js/); +}); diff --git a/tests/release-demo.test.js b/tests/release-demo.test.js index e742339..d104856 100644 --- a/tests/release-demo.test.js +++ b/tests/release-demo.test.js @@ -520,3 +520,19 @@ test('release demo runs the operator backup and restore on synthetic records ins assert.match(builder, /The operator backup is a command line, not a screen in the app/); assert.match(builder, /A restore returns the same records, including the withdrawal tombstone a trainer must never lose/); }); + +test('release demo runs the RC attack suite on camera and prints the verdict it derived', async () => { + const demo = await readFile(demoPath, 'utf8'); + + // Issue #40 turns hostile uploads into a release gate. The scene must run the real suite, not + // describe it, and the card must show the verdict the suite derived rather than a flattering one. + assert.match(demo, /scripts\/security_attack_suite\.mjs', '--json'/); + assert.match(demo, /SECURITY ATTACK SUITE/); + assert.match(demo, /Every hostile upload on screen went at the real server, and the worker behind it counted every call it received/); + assert.match(demo, /A failing case is a finding the release job has to see, so the card prints the verdict the suite derived/); + assert.match(demo, /The gate blocks RC approval on any P0 case that reached the worker/); + assert.match(demo, /securityReport\.verdict/); + assert.match(demo, /securityCardProof/); + // The finding must name where the fix lives instead of implying the boundary is already safe. + assert.match(demo, /Refusing bytes by their content is the fix in review, not a claim this video makes/); +}); diff --git a/tests/release-security-gate.test.js b/tests/release-security-gate.test.js new file mode 100644 index 0000000..8f6b5a1 --- /dev/null +++ b/tests/release-security-gate.test.js @@ -0,0 +1,104 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { buildReport, containsImageBytes, evaluateCase, latencyPercentiles, redactImageBytes, releaseVerdict } from '../src/release-security-gate.js'; + +test('latency percentiles use nearest-rank so p95 is an observed sample, not an interpolation', () => { + const samples = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]; + assert.deepEqual(latencyPercentiles(samples), { count: 10, p50: 50, p95: 100, max: 100 }); +}); + +// A case is a promise the suite makes before it runs: this input must land here, and the provider +// must (or must not) see exactly this many calls. The verdict is derived, never typed. +const polyglot = { + id: 'ingress-polyglot-html', + severity: 'P0', + expected: { status: 400, providerCalls: 0 }, + observed: { status: 400, providerCalls: 0 }, +}; + +test('a case whose observation matches its expectation passes', () => { + assert.equal(evaluateCase(polyglot).status, 'pass'); +}); + +test('a case that reached the provider is a failure that names the field that broke', () => { + const result = evaluateCase({ ...polyglot, observed: { status: 200, providerCalls: 1 } }); + assert.equal(result.status, 'fail'); + assert.match(result.mismatch, /status 400 expected, observed 200/); + assert.match(result.mismatch, /providerCalls 0 expected, observed 1/); +}); + +test('a P0 failure blocks the release, and a P1 failure only holds it', () => { + const leaked = { ...polyglot, observed: { status: 200, providerCalls: 1 } }; + const blocked = releaseVerdict({ cases: [leaked, { ...polyglot, id: 'flood', severity: 'P1', observed: { status: 200, providerCalls: 12 } }] }); + assert.equal(blocked.verdict, 'BLOCKED'); + assert.deepEqual(blocked.blocking.map(item => item.id), ['ingress-polyglot-html']); + assert.deepEqual(blocked.holds.map(item => item.id), ['flood']); +}); + +test('a suite with no failures is a GO, and counts say how many cases were proven', () => { + const verdict = releaseVerdict({ cases: [polyglot] }); + assert.equal(verdict.verdict, 'GO'); + assert.deepEqual(verdict.blocking, []); + assert.equal(verdict.proven, 1); +}); + +test('a required category that no case exercised blocks the release instead of passing by omission', () => { + const deletion = { id: 'deletion-withdrawal', severity: 'P0', surface: 'out-of-surface', evidence: 'tests/training-contribution.test.js', evidenceVerified: false }; + const verdict = releaseVerdict({ cases: [polyglot, deletion] }); + assert.equal(verdict.verdict, 'BLOCKED'); + assert.deepEqual(verdict.unproven.map(item => item.id), ['deletion-withdrawal']); +}); + +test('coverage borrowed from another test file only counts once the runner verified that file exists', () => { + const verified = { id: 'deletion-withdrawal', severity: 'P0', surface: 'out-of-surface', evidence: 'tests/training-contribution.test.js', evidenceVerified: true }; + const verdict = releaseVerdict({ cases: [polyglot, verified] }); + assert.equal(verdict.verdict, 'GO'); + assert.deepEqual(verdict.coveredElsewhere, ['deletion-withdrawal']); +}); + +// The report and the server log are shipped as release evidence. Neither may carry a photo. +const JPEG_DATA_URI = 'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/wAALCAABAAEBAREA/8QAFQABAQAAAAAAAAAAAAAAAAAAAAn/xAAUEAEAAAAAAAAAAAAAAAAAAAAJ/9oACAEBAAA/AKp//9k='; + +test('a data URI or a long base64 run is recognised as image bytes', () => { + assert.equal(containsImageBytes(`posted ${JPEG_DATA_URI} to the worker`), true); + assert.equal(containsImageBytes('chunk ' + 'A'.repeat(64) + 'yQ'.repeat(200)), true); +}); + +test('ordinary log lines, hex digests, and short tokens are not image bytes', () => { + assert.equal(containsImageBytes('Timmy is listening on http://127.0.0.1:4173 · vision configured · Hermes locked and ready'), false); + assert.equal(containsImageBytes('sha256 ' + 'a1b2c3d4'.repeat(8)), false); + assert.equal(containsImageBytes('case ingress-polyglot-html observed status 200'), false); +}); + +test('redaction removes the bytes and keeps the sentence around them honest', () => { + const redacted = redactImageBytes(`provider received ${JPEG_DATA_URI} at 12:01`); + assert.equal(containsImageBytes(redacted), false); + assert.match(redacted, /\[redacted image bytes\]/); + assert.match(redacted, /^provider received .* at 12:01$/); +}); + +test('the report carries the verdict, the target it ran against, and the measured load numbers', () => { + const report = buildReport({ + generatedAt: '2026-10-30T10:48:00.000Z', + target: { baseUrl: 'http://127.0.0.1:4173', provider: 'stub counting worker' }, + cases: [polyglot], + load: { requests: 8, errors: 0, abstentions: 8, p50: 12, p95: 40, max: 40 }, + }); + assert.equal(report.schema, 'timmy-rc-security-attack/1'); + assert.equal(report.verdict, 'GO'); + assert.equal(report.generatedAt, '2026-10-30T10:48:00.000Z'); + assert.equal(report.target.provider, 'stub counting worker'); + assert.equal(report.load.p95, 40); + assert.equal(containsImageBytes(JSON.stringify(report)), false); +}); + +test('a case that quotes a response body cannot smuggle the photo into the report', () => { + const report = buildReport({ + generatedAt: '2026-10-30T10:48:00.000Z', + target: { baseUrl: 'http://127.0.0.1:4173', provider: 'stub' }, + cases: [{ ...polyglot, observed: { status: 400, providerCalls: 0 }, snippet: `refused ${JPEG_DATA_URI}` }], + load: null, + }); + assert.equal(containsImageBytes(JSON.stringify(report)), false); + assert.match(JSON.stringify(report), /redacted image bytes/); +}); diff --git a/tests/security-attack.acceptance.mjs b/tests/security-attack.acceptance.mjs new file mode 100644 index 0000000..5371f4a --- /dev/null +++ b/tests/security-attack.acceptance.mjs @@ -0,0 +1,89 @@ +// Acceptance: the RC attack suite must drive the real server and then tell the truth about what it +// saw. Nothing here trusts the runner's own summary line: every case status is re-derived from the +// recorded observation with the same rule the release gate uses, and the report is scanned for photo +// bytes. If the suite ever marks a case pass while observing a mismatch, this test fails. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { containsImageBytes, evaluateCase, releaseVerdict } from '../src/release-security-gate.js'; + +const repoRoot = new URL('..', import.meta.url).pathname; + +async function runSuite() { + const directory = await mkdtemp(join(tmpdir(), 'timmy-rc-attack-')); + const reportPath = join(directory, 'report.json'); + const run = await new Promise(resolve => { + execFile(process.execPath, ['scripts/security_attack_suite.mjs', '--json', reportPath], { + cwd: repoRoot, + timeout: 180_000, + }, (error, stdout, stderr) => resolve({ + code: error?.code ?? 0, + stdout, + stderr, + })); + }); + const report = JSON.parse(await readFile(reportPath, 'utf8')); + await rm(directory, { recursive: true, force: true }); + return { run, report }; +} + +const suite = await runSuite(); +const byId = new Map(suite.report.cases.map(item => [item.id, item])); + +test('the suite exits non-zero exactly when its own verdict blocks the release', () => { + assert.ok([0, 1].includes(suite.run.code), `unexpected exit ${suite.run.code}: ${suite.run.stderr}`); + assert.match(suite.run.stdout, new RegExp(`VERDICT: ${suite.report.verdict}`)); + assert.equal(suite.run.code, suite.report.verdict === 'BLOCKED' ? 1 : 0); +}); + +test('every reported case status is re-derivable from the observation it recorded', () => { + for (const item of suite.report.cases.filter(entry => entry.surface !== 'out-of-surface')) { + const derived = evaluateCase(item); + assert.equal(item.status, derived.status, `${item.id} claims ${item.status} but observed ${JSON.stringify(item.observed)} against ${JSON.stringify(item.expected)}`); + } + assert.equal(suite.report.verdict, releaseVerdict({ cases: suite.report.cases }).verdict); +}); + +test('the suite covers the abuse classes the ticket names, not a convenient subset', () => { + const required = ['oversized', 'polyglot', 'malformed', 'flood', 'log', 'deletion', 'replay']; + const categories = new Set(suite.report.cases.map(item => item.category)); + for (const category of required) assert.ok(categories.has(category), `no case covers ${category}`); + assert.ok(suite.report.cases.length >= 8, `only ${suite.report.cases.length} cases ran`); +}); + +test('a request without consent is refused and the worker is never called', () => { + const consent = byId.get('consent-required'); + assert.equal(consent.observed.status, 400); + assert.equal(consent.observed.providerCalls, 0); + assert.equal(consent.status, 'pass'); +}); + +test('an oversized body is refused before any provider work', () => { + const oversized = byId.get('ingress-oversized-body'); + assert.equal(oversized.observed.status, 413); + assert.equal(oversized.observed.providerCalls, 0); +}); + +test('a provider answer that breaks the schema fails closed without quoting the provider', () => { + const malformed = byId.get('model-output-malformed'); + assert.equal(malformed.observed.status, 503); + assert.equal(malformed.observed.providerTextLeaked, false); + assert.ok(!JSON.stringify(malformed).includes('garbage-completion-token'), 'the report must not quote the provider'); +}); + +test('the captured server log and the report itself carry no photo bytes', () => { + assert.equal(byId.get('log-leak-scan').observed.imageBytesInLog, false); + assert.equal(containsImageBytes(JSON.stringify(suite.report)), false); +}); + +test('the load phase recorded percentiles, errors, and abstentions without any image body', () => { + const load = suite.report.load; + assert.ok(load.requests >= 8, `only ${load.requests} load requests ran`); + assert.equal(typeof load.p50, 'number'); + assert.equal(typeof load.p95, 'number'); + assert.equal(typeof load.abstentions, 'number'); + assert.equal(containsImageBytes(JSON.stringify(load)), false); +}); -- 2.43.0