timmy
f0a9a734db
Quality gates / quality (pull_request) Canceled after 0s
Add an executable abuse suite that drives the real server with synthetic hostile bytes while a counting stub worker records every call that reaches the model boundary, plus the verdict rule that turns those observations into a release decision. - src/release-security-gate.js: nearest-rank latency percentiles, per-case pass/fail derivation that names the field that broke, and the rule that any failing P0 case blocks RC approval while a required class with no verified evidence blocks it too. Reports are redacted and fail closed. - scripts/security_attack_suite.mjs: oversized body, HTML polyglot as JPEG, undecodable bytes, missing consent, non-conforming worker answer, byte-identical replay, concurrent burst, captured log scan, and an out-of-surface deletion case counted only against a real test file. Writes the report, prints the verdict, exits 1 when blocked. - docs/RC-SECURITY-EVIDENCE.md: procedure, rule, and the current finding stated plainly. - Demo: a real run of the suite on camera, printing the verdict it derived instead of a flattering one; CI regenerates the same evidence on every change. The suite currently returns VERDICT: BLOCKED: bytes are trusted from the data-URL label, so an HTML polyglot still reaches the worker and only fails afterwards on schema. That is a finding against #16, not a passing check, and it is why the gate treats a failing case as a block.
90 lines
4.3 KiB
JavaScript
90 lines
4.3 KiB
JavaScript
// Acceptance: the RC attack suite must drive the real server and then tell the truth about what it
|
|
// saw. Nothing here trusts the runner's own summary line: every case status is re-derived from the
|
|
// recorded observation with the same rule the release gate uses, and the report is scanned for photo
|
|
// bytes. If the suite ever marks a case pass while observing a mismatch, this test fails.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { execFile } from 'node:child_process';
|
|
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { containsImageBytes, evaluateCase, releaseVerdict } from '../src/release-security-gate.js';
|
|
|
|
const repoRoot = new URL('..', import.meta.url).pathname;
|
|
|
|
async function runSuite() {
|
|
const directory = await mkdtemp(join(tmpdir(), 'timmy-rc-attack-'));
|
|
const reportPath = join(directory, 'report.json');
|
|
const run = await new Promise(resolve => {
|
|
execFile(process.execPath, ['scripts/security_attack_suite.mjs', '--json', reportPath], {
|
|
cwd: repoRoot,
|
|
timeout: 180_000,
|
|
}, (error, stdout, stderr) => resolve({
|
|
code: error?.code ?? 0,
|
|
stdout,
|
|
stderr,
|
|
}));
|
|
});
|
|
const report = JSON.parse(await readFile(reportPath, 'utf8'));
|
|
await rm(directory, { recursive: true, force: true });
|
|
return { run, report };
|
|
}
|
|
|
|
const suite = await runSuite();
|
|
const byId = new Map(suite.report.cases.map(item => [item.id, item]));
|
|
|
|
test('the suite exits non-zero exactly when its own verdict blocks the release', () => {
|
|
assert.ok([0, 1].includes(suite.run.code), `unexpected exit ${suite.run.code}: ${suite.run.stderr}`);
|
|
assert.match(suite.run.stdout, new RegExp(`VERDICT: ${suite.report.verdict}`));
|
|
assert.equal(suite.run.code, suite.report.verdict === 'BLOCKED' ? 1 : 0);
|
|
});
|
|
|
|
test('every reported case status is re-derivable from the observation it recorded', () => {
|
|
for (const item of suite.report.cases.filter(entry => entry.surface !== 'out-of-surface')) {
|
|
const derived = evaluateCase(item);
|
|
assert.equal(item.status, derived.status, `${item.id} claims ${item.status} but observed ${JSON.stringify(item.observed)} against ${JSON.stringify(item.expected)}`);
|
|
}
|
|
assert.equal(suite.report.verdict, releaseVerdict({ cases: suite.report.cases }).verdict);
|
|
});
|
|
|
|
test('the suite covers the abuse classes the ticket names, not a convenient subset', () => {
|
|
const required = ['oversized', 'polyglot', 'malformed', 'flood', 'log', 'deletion', 'replay'];
|
|
const categories = new Set(suite.report.cases.map(item => item.category));
|
|
for (const category of required) assert.ok(categories.has(category), `no case covers ${category}`);
|
|
assert.ok(suite.report.cases.length >= 8, `only ${suite.report.cases.length} cases ran`);
|
|
});
|
|
|
|
test('a request without consent is refused and the worker is never called', () => {
|
|
const consent = byId.get('consent-required');
|
|
assert.equal(consent.observed.status, 400);
|
|
assert.equal(consent.observed.providerCalls, 0);
|
|
assert.equal(consent.status, 'pass');
|
|
});
|
|
|
|
test('an oversized body is refused before any provider work', () => {
|
|
const oversized = byId.get('ingress-oversized-body');
|
|
assert.equal(oversized.observed.status, 413);
|
|
assert.equal(oversized.observed.providerCalls, 0);
|
|
});
|
|
|
|
test('a provider answer that breaks the schema fails closed without quoting the provider', () => {
|
|
const malformed = byId.get('model-output-malformed');
|
|
assert.equal(malformed.observed.status, 503);
|
|
assert.equal(malformed.observed.providerTextLeaked, false);
|
|
assert.ok(!JSON.stringify(malformed).includes('garbage-completion-token'), 'the report must not quote the provider');
|
|
});
|
|
|
|
test('the captured server log and the report itself carry no photo bytes', () => {
|
|
assert.equal(byId.get('log-leak-scan').observed.imageBytesInLog, false);
|
|
assert.equal(containsImageBytes(JSON.stringify(suite.report)), false);
|
|
});
|
|
|
|
test('the load phase recorded percentiles, errors, and abstentions without any image body', () => {
|
|
const load = suite.report.load;
|
|
assert.ok(load.requests >= 8, `only ${load.requests} load requests ran`);
|
|
assert.equal(typeof load.p50, 'number');
|
|
assert.equal(typeof load.p95, 'number');
|
|
assert.equal(typeof load.abstentions, 'number');
|
|
assert.equal(containsImageBytes(JSON.stringify(load)), false);
|
|
});
|