timmy
9dabad495b
Quality gates / quality (pull_request) Canceled after 0s
The corpus had an append-only manifest, a reviewer tool, and an evaluation harness, but nothing a stranger could read: what is in the corpus, what a label means, and how leakage is prevented lived in code and in reviewers' heads. scripts/dataset_card.py generates docs/DATASET_CARD.md from manifest.jsonl. Every count is read back from real records — active samples, contributors, per-split contributor and sample counts, class balance over active samples only — and every fact the pipeline does not collect (devices, contexts, licenses) is printed as pending rather than estimated. The generator refuses to render a card for a corpus where one contributor straddles two splits or where a recorded split is not the one the subject key derives, and it never publishes a subject key or a withdrawn sample's labels. --check fails when the committed page is not the generator's own output, so the card cannot drift into a hand-written document. docs/LABELING_GUIDE.md is the reviewer's rulebook: visible form, broad visible color, and photo quality; visible color is not blood; nothing is diagnosed; a model suggestion is provisional input to a human decision. Exclusions are named, and the split policy is stated where it is enforced. The demo's new DATASET CARD scene builds a throwaway corpus with the real ingest command from committed synthetic fixtures, publishes the card, and shows the refusal a planted contributor earns, then re-checks the committed page. Its captions are release gates in build_release.py, which also runs tests/dataset-card.test.py before a release exists. tests/dataset-card.test.py covers the counts, the pending facts, the withdrawal receipt, both refusals, determinism, the empty-corpus case, the committed-page identity, and both documents' prohibitions. Every fixture is synthetic. Closes #27
205 lines
11 KiB
JavaScript
205 lines
11 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { readFile } from 'node:fs/promises';
|
|
|
|
const workflowPath = new URL('../.gitea/workflows/quality.yml', import.meta.url);
|
|
const diffCheckPath = new URL('../scripts/check_diff.sh', import.meta.url);
|
|
const pythonRequirementsPath = new URL('../requirements-test.txt', import.meta.url);
|
|
const packagePath = new URL('../package.json', import.meta.url);
|
|
|
|
test('Gitea CI gates pull requests and main with the reproducible quality suite', async () => {
|
|
const workflow = await readFile(workflowPath, 'utf8');
|
|
|
|
assert.match(workflow, /^name: Quality gates$/m);
|
|
assert.match(workflow, /^on:\n push:\n branches: \[main\]\n pull_request:\n branches: \[main\]$/m);
|
|
assert.match(workflow, /^permissions:\n contents: read$/m);
|
|
assert.match(workflow, /^ timeout-minutes: 15$/m);
|
|
assert.match(workflow, /fetch-depth: 0/);
|
|
assert.match(workflow, /npm ci/);
|
|
assert.match(workflow, /python3 -m pip install --break-system-packages -r requirements-test\.txt/);
|
|
assert.match(workflow, /npm test/);
|
|
assert.match(workflow, /npm run test:ui/);
|
|
assert.match(workflow, /npm run test:photo/);
|
|
assert.match(workflow, /npm run test:mobile-capture/);
|
|
assert.match(workflow, /npm run test:sleek/);
|
|
assert.match(workflow, /npm run test:manual-fallback/);
|
|
assert.match(workflow, /npm audit --audit-level=high/);
|
|
assert.match(workflow, /npm run check:syntax/);
|
|
assert.match(workflow, /npm run check:diff/);
|
|
});
|
|
|
|
test('CI pins the Python image dependency required by the full unit suite', async () => {
|
|
const requirements = await readFile(pythonRequirementsPath, 'utf8');
|
|
|
|
assert.match(requirements, /^Pillow==12\.3\.0$/m);
|
|
});
|
|
|
|
test('the dataset card generator and its suite are CI- and release-gated', async () => {
|
|
const workflow = await readFile(workflowPath, 'utf8');
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
// Issue #27 publishes a page a stranger reads about the corpus, so the generator that decides its
|
|
// numbers runs in CI and is invoked by its own suite, not only by whoever remembers.
|
|
assert.match(workflow, /python3 tests\/dataset-card\.test\.py -v/);
|
|
assert.match(packageJson.scripts['test:dataset-card'], /dataset-card\.test\.py/);
|
|
assert.match(packageJson.scripts['check:syntax'], /py_compile[^&]*scripts\/dataset_card\.py/);
|
|
});
|
|
|
|
test('default unit suite includes the self-host bootstrap contract', async () => {
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/selfhost-bootstrap\.test\.js/);
|
|
});
|
|
|
|
test('default unit suite covers the manual fallback and draft contracts', async () => {
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/draft\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/vision-readiness\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:manual-fallback'], 'node tests/manual-fallback.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/draft\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/vision-readiness\.js/);
|
|
});
|
|
|
|
test('provisional AI suggestion state is a unit-and-browser-gated contract', async () => {
|
|
const [packageJsonText, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/suggestion\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:provisional'], 'node tests/provisional-suggestion.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/suggestion\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/provisional-suggestion\.acceptance\.mjs/);
|
|
assert.match(workflow, /npm run test:provisional/);
|
|
});
|
|
|
|
test('hard-stool follow-up is a unit-and-browser-gated safety contract', async () => {
|
|
const [packageJsonText, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/followup\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:followup'], 'node tests/followup.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/followup\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/followup\.acceptance\.mjs/);
|
|
assert.match(workflow, /npm run test:followup/);
|
|
});
|
|
|
|
test('staging smoke is an explicit syntax-gated promotion command, never a PR network call', async () => {
|
|
const [packageJson, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8').then(JSON.parse),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
assert.match(workflow, /python3 tests\/staging-deploy\.test\.py -v/);
|
|
assert.match(packageJson.scripts.test, /tests\/staging-config\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:staging-smoke'], 'node tests/staging.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/staging\.acceptance\.mjs/);
|
|
assert.match(workflow, /node --check tests\/staging\.acceptance\.mjs/);
|
|
assert.doesNotMatch(workflow, /test:staging-smoke|TIMMY_STAGING_URL/);
|
|
});
|
|
|
|
test('diff hygiene checks only the pull request or latest commit range', async () => {
|
|
const script = await readFile(diffCheckPath, 'utf8');
|
|
|
|
assert.match(script, /origin\/\$GITHUB_BASE_REF\.\.\.HEAD/);
|
|
assert.match(script, /HEAD\^\.\.HEAD/);
|
|
assert.doesNotMatch(script, /hash-object -t tree/);
|
|
});
|
|
|
|
test('the vision corpus and its refusal boundary are unit-, browser-, and release-gated', async () => {
|
|
const [packageJsonText, workflow, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/vision-corpus\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/vision-boundary\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:vision-refusal'], 'node tests/vision-refusal.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/vision-corpus\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/vision-refusal\.acceptance\.mjs/);
|
|
assert.match(packageJson.scripts['check:syntax'], /py_compile[^&]*scripts\/build_vision_corpus\.py/);
|
|
assert.match(workflow, /npm run test:vision-refusal/);
|
|
// The release builder must verify the committed corpus hashes and run the refusal suite itself.
|
|
assert.match(builder, /"test:vision-refusal"/);
|
|
assert.match(builder, /"vision_corpus_refusal": "passed"/);
|
|
assert.match(builder, /build_vision_corpus\.py", "--verify"/);
|
|
});
|
|
|
|
test('the pinned worker container, post-load gate, and GPU benchmark are gated end to end', async () => {
|
|
const [packageJsonText, workflow, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/worker-container\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/worker-ready\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/worker-benchmark\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:worker-readiness'], 'node tests/worker-readiness.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check scripts\/record_worker_benchmark\.mjs/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/worker-readiness\.acceptance\.mjs/);
|
|
for (const script of ['scripts/worker_ready.sh', 'deploy/worker/entrypoint.sh', 'deploy/worker/build_image.sh', 'deploy/worker/run_worker_container.sh']) {
|
|
assert.ok(packageJson.scripts['check:syntax'].includes(`bash -n ${script}`), `${script} must be syntax-gated`);
|
|
}
|
|
assert.match(workflow, /npm run test:worker-readiness/);
|
|
assert.match(builder, /"test:worker-readiness"/);
|
|
assert.match(builder, /"worker_container_receipts": "passed"/);
|
|
assert.match(builder, /"worker_post_load_readiness": "passed"/);
|
|
assert.match(builder, /"scripts\/worker_ready\.sh"/);
|
|
});
|
|
|
|
test('explainable pattern summaries are a unit-, browser-, and release-gated contract', async () => {
|
|
const [packageJsonText, workflow, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/summary\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:pattern-summary'], 'node tests/pattern-summary.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/summary\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/pattern-summary\.acceptance\.mjs/);
|
|
assert.match(workflow, /npm run test:pattern-summary/);
|
|
assert.match(builder, /"test:pattern-summary"/);
|
|
assert.match(builder, /"longitudinal_pattern_summary": "passed"/);
|
|
assert.match(builder, /"src\/summary\.js"/);
|
|
});
|
|
|
|
test('the deterministic symptom escalation regression suite runs in the default unit gate', async () => {
|
|
const [packageJsonText, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
// Issue #21: the escalation boundary is only as good as the suite that is actually run.
|
|
assert.match(packageJson.scripts.test, /tests\/escalation-regression\.test\.js/);
|
|
assert.match(builder, /"src\/domain\.js"/);
|
|
assert.match(builder, /"src\/hermes-agent-service\.js"/);
|
|
});
|
|
|
|
test('the calibration harness is wired into the unit and syntax gates', async () => {
|
|
const [packageJsonText, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
// Issue #29: a release gate nothing runs is a claim, not a gate.
|
|
assert.equal(packageJson.scripts['test:calibration'], 'python3 tests/calibration-harness.test.py');
|
|
assert.match(packageJson.scripts['check:syntax'], /py_compile[^&]*scripts\/calibration_report\.py/);
|
|
assert.match(workflow, /python3 tests\/calibration-harness\.test\.py -v/);
|
|
// The release builder has to run it too, or the gate exists only in CI.
|
|
const builder = await readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8');
|
|
assert.match(builder, /tests\/calibration-harness\.test\.py/);
|
|
assert.match(builder, /"calibration_harness_gates": "passed"/);
|
|
});
|