timmy
f0a9a734db
Quality gates / quality (pull_request) Canceled after 0s
Add an executable abuse suite that drives the real server with synthetic hostile bytes while a counting stub worker records every call that reaches the model boundary, plus the verdict rule that turns those observations into a release decision. - src/release-security-gate.js: nearest-rank latency percentiles, per-case pass/fail derivation that names the field that broke, and the rule that any failing P0 case blocks RC approval while a required class with no verified evidence blocks it too. Reports are redacted and fail closed. - scripts/security_attack_suite.mjs: oversized body, HTML polyglot as JPEG, undecodable bytes, missing consent, non-conforming worker answer, byte-identical replay, concurrent burst, captured log scan, and an out-of-surface deletion case counted only against a real test file. Writes the report, prints the verdict, exits 1 when blocked. - docs/RC-SECURITY-EVIDENCE.md: procedure, rule, and the current finding stated plainly. - Demo: a real run of the suite on camera, printing the verdict it derived instead of a flattering one; CI regenerates the same evidence on every change. The suite currently returns VERDICT: BLOCKED: bytes are trusted from the data-URL label, so an HTML polyglot still reaches the worker and only fails afterwards on schema. That is a finding against #16, not a passing check, and it is why the gate treats a failing case as a block.
216 lines
11 KiB
JavaScript
216 lines
11 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { readFile } from 'node:fs/promises';
|
|
|
|
const workflowPath = new URL('../.gitea/workflows/quality.yml', import.meta.url);
|
|
const diffCheckPath = new URL('../scripts/check_diff.sh', import.meta.url);
|
|
const pythonRequirementsPath = new URL('../requirements-test.txt', import.meta.url);
|
|
const packagePath = new URL('../package.json', import.meta.url);
|
|
|
|
test('Gitea CI gates pull requests and main with the reproducible quality suite', async () => {
|
|
const workflow = await readFile(workflowPath, 'utf8');
|
|
|
|
assert.match(workflow, /^name: Quality gates$/m);
|
|
assert.match(workflow, /^on:\n push:\n branches: \[main\]\n pull_request:\n branches: \[main\]$/m);
|
|
assert.match(workflow, /^permissions:\n contents: read$/m);
|
|
assert.match(workflow, /^ timeout-minutes: 15$/m);
|
|
assert.match(workflow, /fetch-depth: 0/);
|
|
assert.match(workflow, /npm ci/);
|
|
assert.match(workflow, /python3 -m pip install --break-system-packages -r requirements-test\.txt/);
|
|
assert.match(workflow, /npm test/);
|
|
assert.match(workflow, /npm run test:ui/);
|
|
assert.match(workflow, /npm run test:photo/);
|
|
assert.match(workflow, /npm run test:mobile-capture/);
|
|
assert.match(workflow, /npm run test:sleek/);
|
|
assert.match(workflow, /npm run test:manual-fallback/);
|
|
assert.match(workflow, /npm audit --audit-level=high/);
|
|
assert.match(workflow, /npm run check:syntax/);
|
|
assert.match(workflow, /npm run check:diff/);
|
|
});
|
|
|
|
test('CI pins the Python image dependency required by the full unit suite', async () => {
|
|
const requirements = await readFile(pythonRequirementsPath, 'utf8');
|
|
|
|
assert.match(requirements, /^Pillow==12\.3\.0$/m);
|
|
});
|
|
|
|
test('the dataset card generator and its suite are CI- and release-gated', async () => {
|
|
const workflow = await readFile(workflowPath, 'utf8');
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
// Issue #27 publishes a page a stranger reads about the corpus, so the generator that decides its
|
|
// numbers runs in CI and is invoked by its own suite, not only by whoever remembers.
|
|
assert.match(workflow, /python3 tests\/dataset-card\.test\.py -v/);
|
|
assert.match(packageJson.scripts['test:dataset-card'], /dataset-card\.test\.py/);
|
|
assert.match(packageJson.scripts['check:syntax'], /py_compile[^&]*scripts\/dataset_card\.py/);
|
|
});
|
|
|
|
test('default unit suite includes the self-host bootstrap contract', async () => {
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/selfhost-bootstrap\.test\.js/);
|
|
});
|
|
|
|
test('default unit suite covers the manual fallback and draft contracts', async () => {
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/draft\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/vision-readiness\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:manual-fallback'], 'node tests/manual-fallback.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/draft\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/vision-readiness\.js/);
|
|
});
|
|
|
|
test('provisional AI suggestion state is a unit-and-browser-gated contract', async () => {
|
|
const [packageJsonText, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/suggestion\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:provisional'], 'node tests/provisional-suggestion.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/suggestion\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/provisional-suggestion\.acceptance\.mjs/);
|
|
assert.match(workflow, /npm run test:provisional/);
|
|
});
|
|
|
|
test('hard-stool follow-up is a unit-and-browser-gated safety contract', async () => {
|
|
const [packageJsonText, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/followup\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:followup'], 'node tests/followup.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/followup\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/followup\.acceptance\.mjs/);
|
|
assert.match(workflow, /npm run test:followup/);
|
|
});
|
|
|
|
test('staging smoke is an explicit syntax-gated promotion command, never a PR network call', async () => {
|
|
const [packageJson, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8').then(JSON.parse),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
assert.match(workflow, /python3 tests\/staging-deploy\.test\.py -v/);
|
|
assert.match(packageJson.scripts.test, /tests\/staging-config\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:staging-smoke'], 'node tests/staging.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/staging\.acceptance\.mjs/);
|
|
assert.match(workflow, /node --check tests\/staging\.acceptance\.mjs/);
|
|
assert.doesNotMatch(workflow, /test:staging-smoke|TIMMY_STAGING_URL/);
|
|
});
|
|
|
|
test('diff hygiene checks only the pull request or latest commit range', async () => {
|
|
const script = await readFile(diffCheckPath, 'utf8');
|
|
|
|
assert.match(script, /origin\/\$GITHUB_BASE_REF\.\.\.HEAD/);
|
|
assert.match(script, /HEAD\^\.\.HEAD/);
|
|
assert.doesNotMatch(script, /hash-object -t tree/);
|
|
});
|
|
|
|
test('the vision corpus and its refusal boundary are unit-, browser-, and release-gated', async () => {
|
|
const [packageJsonText, workflow, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/vision-corpus\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/vision-boundary\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:vision-refusal'], 'node tests/vision-refusal.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/vision-corpus\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/vision-refusal\.acceptance\.mjs/);
|
|
assert.match(packageJson.scripts['check:syntax'], /py_compile[^&]*scripts\/build_vision_corpus\.py/);
|
|
assert.match(workflow, /npm run test:vision-refusal/);
|
|
// The release builder must verify the committed corpus hashes and run the refusal suite itself.
|
|
assert.match(builder, /"test:vision-refusal"/);
|
|
assert.match(builder, /"vision_corpus_refusal": "passed"/);
|
|
assert.match(builder, /build_vision_corpus\.py", "--verify"/);
|
|
});
|
|
|
|
test('the pinned worker container, post-load gate, and GPU benchmark are gated end to end', async () => {
|
|
const [packageJsonText, workflow, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/worker-container\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/worker-ready\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/worker-benchmark\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:worker-readiness'], 'node tests/worker-readiness.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check scripts\/record_worker_benchmark\.mjs/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/worker-readiness\.acceptance\.mjs/);
|
|
for (const script of ['scripts/worker_ready.sh', 'deploy/worker/entrypoint.sh', 'deploy/worker/build_image.sh', 'deploy/worker/run_worker_container.sh']) {
|
|
assert.ok(packageJson.scripts['check:syntax'].includes(`bash -n ${script}`), `${script} must be syntax-gated`);
|
|
}
|
|
assert.match(workflow, /npm run test:worker-readiness/);
|
|
assert.match(builder, /"test:worker-readiness"/);
|
|
assert.match(builder, /"worker_container_receipts": "passed"/);
|
|
assert.match(builder, /"worker_post_load_readiness": "passed"/);
|
|
assert.match(builder, /"scripts\/worker_ready\.sh"/);
|
|
});
|
|
|
|
test('explainable pattern summaries are a unit-, browser-, and release-gated contract', async () => {
|
|
const [packageJsonText, workflow, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/summary\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:pattern-summary'], 'node tests/pattern-summary.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/summary\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/pattern-summary\.acceptance\.mjs/);
|
|
assert.match(workflow, /npm run test:pattern-summary/);
|
|
assert.match(builder, /"test:pattern-summary"/);
|
|
assert.match(builder, /"longitudinal_pattern_summary": "passed"/);
|
|
assert.match(builder, /"src\/summary\.js"/);
|
|
});
|
|
|
|
test('the deterministic symptom escalation regression suite runs in the default unit gate', async () => {
|
|
const [packageJsonText, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
// Issue #21: the escalation boundary is only as good as the suite that is actually run.
|
|
assert.match(packageJson.scripts.test, /tests\/escalation-regression\.test\.js/);
|
|
assert.match(builder, /"src\/domain\.js"/);
|
|
assert.match(builder, /"src\/hermes-agent-service\.js"/);
|
|
});
|
|
|
|
test('the calibration harness is wired into the unit and syntax gates', async () => {
|
|
const [packageJsonText, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
// Issue #29: a release gate nothing runs is a claim, not a gate.
|
|
assert.equal(packageJson.scripts['test:calibration'], 'python3 tests/calibration-harness.test.py');
|
|
assert.match(packageJson.scripts['check:syntax'], /py_compile[^&]*scripts\/calibration_report\.py/);
|
|
assert.match(workflow, /python3 tests\/calibration-harness\.test\.py -v/);
|
|
// The release builder has to run it too, or the gate exists only in CI.
|
|
const builder = await readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8');
|
|
assert.match(builder, /tests\/calibration-harness\.test\.py/);
|
|
assert.match(builder, /"calibration_harness_gates": "passed"/);
|
|
});
|
|
|
|
test('CI runs the RC attack suite so its evidence is reproducible on every change', async () => {
|
|
const workflow = await readFile(workflowPath, 'utf8');
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
// Issue #40: the attack report is evidence, not a claim. CI has to be able to regenerate it.
|
|
assert.match(workflow, /npm run test:security-attack/);
|
|
assert.equal(packageJson.scripts['test:security-attack'], 'node tests/security-attack.acceptance.mjs');
|
|
assert.match(packageJson.scripts.test, /tests\/release-security-gate\.test\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/release-security-gate\.js/);
|
|
});
|