timmy
d9f82299d0
Quality gates / quality (pull_request) Successful in 2m26s
The worker contract was only proven by one happy-path fixture and a handful of inline assertions, so nothing continuously checked that a reply Timmy cannot use is refused instead of guessed at. Corpus: a CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) generated deterministically by scripts/build_vision_corpus.py. Every fixture records its sha256, provenance, licence, case class, the provider reply it replays, and the outcome that reply must produce. loadVisionCorpus refuses a fixture whose bytes no longer match the manifest, so the corpus cannot be edited into a different test. No real medical imagery, camera, or third-party content is involved. Contract fixes the corpus exposed: - A schema-valid reply with a null form and colour is now an abstention (abstainedFor: unusable_visual_fields) instead of a hard provider failure. The declared response schema permits those nulls, so treating the reply as an error was wrong; it now abstains calmly and still prefill nothing. - Fail-closed replies carry a machine-readable code (invalid_json, invalid_fields) instead of prose matching, and visionFailureResponse() maps them to plain, user-safe copy with no provider text, image bytes, or credentials echoed. A running server now returns that copy for unparsable and out-of-range replies. Evidence: unit corpus contract and hash integrity, corpus-wide conformance through the real analysis path, the same corpus through a running server.mjs with a stub worker (200 abstain vs 503 fail-closed, no leaked bytes), and a real browser acceptance run for the refusal UI. The release demo replays three corpus cases through the real code path so the video shows the status the server would actually send, with the on-camera caption gated by the release builder.
122 lines
6.2 KiB
JavaScript
122 lines
6.2 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { readFile } from 'node:fs/promises';
|
|
|
|
const workflowPath = new URL('../.gitea/workflows/quality.yml', import.meta.url);
|
|
const diffCheckPath = new URL('../scripts/check_diff.sh', import.meta.url);
|
|
const pythonRequirementsPath = new URL('../requirements-test.txt', import.meta.url);
|
|
const packagePath = new URL('../package.json', import.meta.url);
|
|
|
|
test('Gitea CI gates pull requests and main with the reproducible quality suite', async () => {
|
|
const workflow = await readFile(workflowPath, 'utf8');
|
|
|
|
assert.match(workflow, /^name: Quality gates$/m);
|
|
assert.match(workflow, /^on:\n push:\n branches: \[main\]\n pull_request:\n branches: \[main\]$/m);
|
|
assert.match(workflow, /^permissions:\n contents: read$/m);
|
|
assert.match(workflow, /^ timeout-minutes: 15$/m);
|
|
assert.match(workflow, /fetch-depth: 0/);
|
|
assert.match(workflow, /npm ci/);
|
|
assert.match(workflow, /python3 -m pip install --break-system-packages -r requirements-test\.txt/);
|
|
assert.match(workflow, /npm test/);
|
|
assert.match(workflow, /npm run test:ui/);
|
|
assert.match(workflow, /npm run test:photo/);
|
|
assert.match(workflow, /npm run test:mobile-capture/);
|
|
assert.match(workflow, /npm run test:sleek/);
|
|
assert.match(workflow, /npm run test:manual-fallback/);
|
|
assert.match(workflow, /npm audit --audit-level=high/);
|
|
assert.match(workflow, /npm run check:syntax/);
|
|
assert.match(workflow, /npm run check:diff/);
|
|
});
|
|
|
|
test('CI pins the Python image dependency required by the full unit suite', async () => {
|
|
const requirements = await readFile(pythonRequirementsPath, 'utf8');
|
|
|
|
assert.match(requirements, /^Pillow==12\.3\.0$/m);
|
|
});
|
|
|
|
test('default unit suite includes the self-host bootstrap contract', async () => {
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/selfhost-bootstrap\.test\.js/);
|
|
});
|
|
|
|
test('default unit suite covers the manual fallback and draft contracts', async () => {
|
|
const packageJson = JSON.parse(await readFile(packagePath, 'utf8'));
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/draft\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/vision-readiness\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:manual-fallback'], 'node tests/manual-fallback.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/draft\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/vision-readiness\.js/);
|
|
});
|
|
|
|
test('provisional AI suggestion state is a unit-and-browser-gated contract', async () => {
|
|
const [packageJsonText, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/suggestion\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:provisional'], 'node tests/provisional-suggestion.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/suggestion\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/provisional-suggestion\.acceptance\.mjs/);
|
|
assert.match(workflow, /npm run test:provisional/);
|
|
});
|
|
|
|
test('hard-stool follow-up is a unit-and-browser-gated safety contract', async () => {
|
|
const [packageJsonText, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/followup\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:followup'], 'node tests/followup.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/followup\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/followup\.acceptance\.mjs/);
|
|
assert.match(workflow, /npm run test:followup/);
|
|
});
|
|
|
|
test('staging smoke is an explicit syntax-gated promotion command, never a PR network call', async () => {
|
|
const [packageJson, workflow] = await Promise.all([
|
|
readFile(packagePath, 'utf8').then(JSON.parse),
|
|
readFile(workflowPath, 'utf8'),
|
|
]);
|
|
assert.match(workflow, /python3 tests\/staging-deploy\.test\.py -v/);
|
|
assert.match(packageJson.scripts.test, /tests\/staging-config\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:staging-smoke'], 'node tests/staging.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/staging\.acceptance\.mjs/);
|
|
assert.match(workflow, /node --check tests\/staging\.acceptance\.mjs/);
|
|
assert.doesNotMatch(workflow, /test:staging-smoke|TIMMY_STAGING_URL/);
|
|
});
|
|
|
|
test('diff hygiene checks only the pull request or latest commit range', async () => {
|
|
const script = await readFile(diffCheckPath, 'utf8');
|
|
|
|
assert.match(script, /origin\/\$GITHUB_BASE_REF\.\.\.HEAD/);
|
|
assert.match(script, /HEAD\^\.\.HEAD/);
|
|
assert.doesNotMatch(script, /hash-object -t tree/);
|
|
});
|
|
|
|
test('the vision corpus and its refusal boundary are unit-, browser-, and release-gated', async () => {
|
|
const [packageJsonText, workflow, builder] = await Promise.all([
|
|
readFile(packagePath, 'utf8'),
|
|
readFile(workflowPath, 'utf8'),
|
|
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
|
]);
|
|
const packageJson = JSON.parse(packageJsonText);
|
|
|
|
assert.match(packageJson.scripts.test, /tests\/vision-corpus\.test\.js/);
|
|
assert.match(packageJson.scripts.test, /tests\/vision-boundary\.test\.js/);
|
|
assert.equal(packageJson.scripts['test:vision-refusal'], 'node tests/vision-refusal.acceptance.mjs');
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check src\/vision-corpus\.js/);
|
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/vision-refusal\.acceptance\.mjs/);
|
|
assert.match(packageJson.scripts['check:syntax'], /py_compile[^&]*scripts\/build_vision_corpus\.py/);
|
|
assert.match(workflow, /npm run test:vision-refusal/);
|
|
// The release builder must verify the committed corpus hashes and run the refusal suite itself.
|
|
assert.match(builder, /"test:vision-refusal"/);
|
|
assert.match(builder, /"vision_corpus_refusal": "passed"/);
|
|
assert.match(builder, /build_vision_corpus\.py", "--verify"/);
|
|
});
|