Quality gates / quality (pull_request) Canceled after 0s
Add an executable abuse suite that drives the real server with synthetic hostile bytes while a counting stub worker records every call that reaches the model boundary, plus the verdict rule that turns those observations into a release decision. - src/release-security-gate.js: nearest-rank latency percentiles, per-case pass/fail derivation that names the field that broke, and the rule that any failing P0 case blocks RC approval while a required class with no verified evidence blocks it too. Reports are redacted and fail closed. - scripts/security_attack_suite.mjs: oversized body, HTML polyglot as JPEG, undecodable bytes, missing consent, non-conforming worker answer, byte-identical replay, concurrent burst, captured log scan, and an out-of-surface deletion case counted only against a real test file. Writes the report, prints the verdict, exits 1 when blocked. - docs/RC-SECURITY-EVIDENCE.md: procedure, rule, and the current finding stated plainly. - Demo: a real run of the suite on camera, printing the verdict it derived instead of a flattering one; CI regenerates the same evidence on every change. The suite currently returns VERDICT: BLOCKED: bytes are trusted from the data-URL label, so an HTML polyglot still reaches the worker and only fails afterwards on schema. That is a finding against #16, not a passing check, and it is why the gate treats a failing case as a block.
96 lines
4.6 KiB
JavaScript
96 lines
4.6 KiB
JavaScript
// Release security gate: the arithmetic and the verdict rule for the RC attack suite (#40).
|
|
//
|
|
// The suite observes the real server; this module only turns observations into a verdict. Keeping
|
|
// the rule here (not in the runner) is what makes "a P0 finding blocks RC approval" testable.
|
|
|
|
/** Nearest-rank percentile: the returned value is always a sample that was actually measured. */
|
|
export function latencyPercentiles(samples) {
|
|
const sorted = [...samples].sort((a, b) => a - b);
|
|
if (!sorted.length) return { count: 0, p50: null, p95: null, max: null };
|
|
const rank = (p) => sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)];
|
|
return { count: sorted.length, p50: rank(50), p95: rank(95), max: sorted[sorted.length - 1] };
|
|
}
|
|
|
|
/** One case against one expectation. A partial match is a failure, and it says which field broke. */
|
|
export function evaluateCase(record) {
|
|
const mismatches = Object.keys(record.expected)
|
|
.filter(field => record.observed?.[field] !== record.expected[field])
|
|
.map(field => `${field} ${record.expected[field]} expected, observed ${record.observed?.[field]}`);
|
|
return { ...record, status: mismatches.length ? 'fail' : 'pass', mismatch: mismatches.join('; ') };
|
|
}
|
|
|
|
const SEVERITY_BLOCKS = (severity) => severity === 'P0';
|
|
|
|
/** The RC rule: every P0 failure blocks approval, unproven categories block it too, P1/P2 only hold. */
|
|
export function releaseVerdict({ cases, load = null }) {
|
|
const evaluated = cases.map(record => (record.surface === 'out-of-surface'
|
|
? { ...record, status: record.evidenceVerified ? 'covered-elsewhere' : 'unproven' }
|
|
: evaluateCase(record)));
|
|
const blocking = evaluated.filter(item => item.status === 'fail' && SEVERITY_BLOCKS(item.severity))
|
|
.map(item => ({ id: item.id, severity: item.severity, reason: item.mismatch }));
|
|
const holds = evaluated.filter(item => item.status === 'fail' && !SEVERITY_BLOCKS(item.severity))
|
|
.map(item => ({ id: item.id, severity: item.severity, reason: item.mismatch }));
|
|
const unproven = evaluated.filter(item => item.status === 'unproven')
|
|
.map(item => ({ id: item.id, severity: item.severity, reason: 'no executable evidence' }));
|
|
return {
|
|
verdict: blocking.length || unproven.length ? 'BLOCKED' : 'GO',
|
|
blocking,
|
|
holds,
|
|
unproven,
|
|
coveredElsewhere: evaluated.filter(item => item.status === 'covered-elsewhere').map(item => item.id),
|
|
proven: evaluated.filter(item => item.status === 'pass').length,
|
|
cases: evaluated,
|
|
load,
|
|
};
|
|
}
|
|
|
|
// Evidence hygiene: the report and the captured server log are committed and quoted. A photo that
|
|
// survived into either one is a leak of the exact data this suite exists to protect.
|
|
const IMAGE_DATA_URI = /data:image\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+/gi;
|
|
const LONG_BASE64 = /[A-Za-z0-9+/]{200,}={0,2}/;
|
|
|
|
export function containsImageBytes(text) {
|
|
const value = String(text ?? '');
|
|
return new RegExp(IMAGE_DATA_URI.source, 'i').test(value) || LONG_BASE64.test(value);
|
|
}
|
|
|
|
export function redactImageBytes(text) {
|
|
return String(text ?? '')
|
|
.replace(IMAGE_DATA_URI, '[redacted image bytes]')
|
|
.replace(new RegExp(LONG_BASE64.source, 'g'), '[redacted image bytes]');
|
|
}
|
|
|
|
/** Fails closed rather than emitting evidence that carries a photo. */
|
|
export function assertNoImageBytes(label, text) {
|
|
if (containsImageBytes(text)) throw new Error(`${label} contains image bytes; refusing to write release evidence.`);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* The receipt the release job quotes. It is redacted after assembly, so a case that captured a
|
|
* response body still cannot carry a photo in it.
|
|
*/
|
|
export function buildReport({ generatedAt, target, cases, load, suite = 'scripts/security_attack_suite.mjs' }) {
|
|
const verdict = releaseVerdict({ cases, load });
|
|
const report = {
|
|
schema: 'timmy-rc-security-attack/1',
|
|
issue: 40,
|
|
generatedAt,
|
|
suite,
|
|
target,
|
|
verdict: verdict.verdict,
|
|
blocking: verdict.blocking,
|
|
holds: verdict.holds,
|
|
unproven: verdict.unproven,
|
|
coveredElsewhere: verdict.coveredElsewhere,
|
|
proven: verdict.proven,
|
|
cases: verdict.cases.map(({ id, category, severity, requirement, expected, observed, status, mismatch, snippet, surface, evidence }) =>
|
|
({ id, category, severity, requirement, expected, observed, status, mismatch, surface, evidence, snippet: snippet ? redactImageBytes(snippet) : undefined })),
|
|
load,
|
|
disclaimer: 'Synthetic hostile bytes only. Every case drives the real server against a stub worker; no real medical image and no live user data is involved.',
|
|
};
|
|
const clean = JSON.parse(redactImageBytes(JSON.stringify(report)));
|
|
assertNoImageBytes('security report', JSON.stringify(clean));
|
|
return clean;
|
|
}
|