Timmy
46f961668f
Quality gates / quality (pull_request) Successful in 3m29s
A worker that answers /v1/models has not necessarily loaded its weights, so the app was offering an AI control that could only fail after the upload. Readiness now passes only once the pinned model is loaded, and the not-loaded state reaches manual Bristol entry in one tap. Verified against the real pinned llama.cpp build: it serves 503 while loading and the gate reports the distinct not-loaded state. Adds the reproducible container recipe (digest-pinned bases, pinned llama.cpp commit, hash-verified read-only weights, unprivileged, loopback-only publish), one shared post-load readiness gate used by the host bootstrap and the image healthcheck, and a target-GPU benchmark that refuses to write a receipt it did not measure. Refs #15
146 lines
7.6 KiB
JavaScript
146 lines
7.6 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { readFile } from 'node:fs/promises';
|
|
|
|
const workflowPath = new URL('../.gitea/workflows/quality.yml', import.meta.url);
|
|
const diffCheckPath = new URL('../scripts/check_diff.sh', import.meta.url);
|
|
const pythonRequirementsPath = new URL('../requirements-test.txt', import.meta.url);
|
|
const packagePath = new URL('../package.json', import.meta.url);
|
|
|
|
test('Gitea CI gates pull requests and main with the reproducible quality suite', async () => {
|
|
const workflow = await readFile(workflowPath, 'utf8');
|
|
|
|
assert.match(workflow, /^name: Quality gates$/m);
|
|
assert.match(workflow, /^on:\n push:\n branches: \[main\]\n pull_request:\n branches: \[main\]$/m);
|
|
assert.match(workflow, /^permissions:\n contents: read$/m);
|
|
assert.match(workflow, /^ timeout-minutes: 15$/m);
|
|
assert.match(workflow, /fetch-depth: 0/);
|
|
assert.match(workflow, /npm ci/);
|
|
assert.match(workflow, /python3 -m pip install --break-system-packages -r requirements-test\.txt/);
|
|
assert.match(workflow, /npm test/);
|
|
assert.match(workflow, /npm run test:ui/);
|
|
assert.match(workflow, /npm run test:photo/);
|
|
assert.match(workflow, /npm run test:mobile-capture/);
|
|
assert.match(workflow, /npm run test:sleek/);
|
|
assert.match(workflow, /npm run test:manual-fallback/);
|
|
assert.match(workflow, /npm audit --audit-level=high/);
|
|
assert.match(workflow, /npm run check:syntax/);
|
|
assert.match(workflow, /npm run check:diff/);
|
|
});
|
|
|
|
test('CI pins the Python image dependency required by the full unit suite', async () => {
|
|
const requirements = await readFile(pythonRequirementsPath, 'utf8');
|
|
|
|
assert.match(requirements, /^Pillow==12\.3\.0$/m);
|
|
});
|
|
|
|
test('default unit suite includes the self-host bootstrap contract', async () => {
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/selfhost-bootstrap\.test\.js/);
|
|
});
|
|
|
|
test('default unit suite covers the manual fallback and draft contracts', async () => {
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/draft\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/vision-readiness\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:manual-fallback'], 'node tests/manual-fallback.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/draft\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/vision-readiness\.js/);
|
|
});
|
|
|
|
test('provisional AI suggestion state is a unit-and-browser-gated contract', async () => {
|
|
const [packageJsonText, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/suggestion\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:provisional'], 'node tests/provisional-suggestion.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/suggestion\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/provisional-suggestion\.acceptance\.mjs/);
|
|
assert.match(workflow, /npm run test:provisional/);
|
|
});
|
|
|
|
test('hard-stool follow-up is a unit-and-browser-gated safety contract', async () => {
|
|
const [packageJsonText, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/followup\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:followup'], 'node tests/followup.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/followup\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/followup\.acceptance\.mjs/);
|
|
assert.match(workflow, /npm run test:followup/);
|
|
});
|
|
|
|
test('staging smoke is an explicit syntax-gated promotion command, never a PR network call', async () => {
|
|
const [packageJson, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8').then(JSON.parse),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
assert.match(workflow, /python3 tests\/staging-deploy\.test\.py -v/);
|
|
assert.match(packageJson.scripts.test, /tests\/staging-config\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:staging-smoke'], 'node tests/staging.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/staging\.acceptance\.mjs/);
|
|
assert.match(workflow, /node --check tests\/staging\.acceptance\.mjs/);
|
|
assert.doesNotMatch(workflow, /test:staging-smoke|TIMMY_STAGING_URL/);
|
|
});
|
|
|
|
test('diff hygiene checks only the pull request or latest commit range', async () => {
|
|
const script = await readFile(diffCheckPath, 'utf8');
|
|
|
|
assert.match(script, /origin\/\$GITHUB_BASE_REF\.\.\.HEAD/);
|
|
assert.match(script, /HEAD\^\.\.HEAD/);
|
|
assert.doesNotMatch(script, /hash-object -t tree/);
|
|
});
|
|
|
|
test('the vision corpus and its refusal boundary are unit-, browser-, and release-gated', async () => {
|
|
const [packageJsonText, workflow, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/vision-corpus\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/vision-boundary\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:vision-refusal'], 'node tests/vision-refusal.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/vision-corpus\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/vision-refusal\.acceptance\.mjs/);
|
|
assert.match(packageJson.scripts['check:syntax'], /py_compile[^&]*scripts\/build_vision_corpus\.py/);
|
|
assert.match(workflow, /npm run test:vision-refusal/);
|
|
// The release builder must verify the committed corpus hashes and run the refusal suite itself.
|
|
assert.match(builder, /"test:vision-refusal"/);
|
|
assert.match(builder, /"vision_corpus_refusal": "passed"/);
|
|
assert.match(builder, /build_vision_corpus\.py", "--verify"/);
|
|
});
|
|
|
|
test('the pinned worker container, post-load gate, and GPU benchmark are gated end to end', async () => {
|
|
const [packageJsonText, workflow, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/worker-container\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/worker-ready\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/worker-benchmark\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:worker-readiness'], 'node tests/worker-readiness.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check scripts\/record_worker_benchmark\.mjs/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/worker-readiness\.acceptance\.mjs/);
|
|
for (const script of ['scripts/worker_ready.sh', 'deploy/worker/entrypoint.sh', 'deploy/worker/build_image.sh', 'deploy/worker/run_worker_container.sh']) {
|
|
assert.ok(packageJson.scripts['check:syntax'].includes(`bash -n ${script}`), `${script} must be syntax-gated`);
|
|
}
|
|
assert.match(workflow, /npm run test:worker-readiness/);
|
|
assert.match(builder, /"test:worker-readiness"/);
|
|
assert.match(builder, /"worker_container_receipts": "passed"/);
|
|
assert.match(builder, /"worker_post_load_readiness": "passed"/);
|
|
assert.match(builder, /"scripts\/worker_ready\.sh"/);
|
|
});
|