timmy
179f8fe6e9
Quality gates / quality (pull_request) Successful in 13m25s
The ROADMAP gates (macro-F1 >= 0.80, precision@coverage >= 0.90, ECE <= 0.05) were prose with no tool behind them. scripts/calibration_report.py now computes them from prediction records and refuses input it cannot honestly score: - a split is re-derived from the pseudonymous subject key with the ingestion rule, so a corpus cannot re-label its own splits; a contributor spanning two splits is refused before scoring - the threshold is selected on validation only, and selecting on test or train is refused - coverage, precision-at-coverage, macro-F1, per-class precision/recall, confusion, and expected calibration error are computed on accepted suggestions, with classes that abstained away named in excludedClasses instead of being scored as a silent zero - exit codes carry the verdict: 0 GO, 3 BLOCKED, 2 refused The release demo records the real harness against committed synthetic fixtures: a blocked three-candidate comparison, a fixture engineered to clear every gate, and a leaked corpus refused. Every number on the card is read back from the report the run wrote, and the captions state that no model was measured, because a fixture is not evidence about accuracy. The harness suite is wired into CI, check:syntax, and the release builder's own gates.
33 lines
3.9 KiB
JSON
33 lines
3.9 KiB
JSON
{
|
|
"name": "timmy-talking-turd",
|
|
"version": "0.1.0",
|
|
"private": true,
|
|
"type": "module",
|
|
"scripts": {
|
|
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js tests/training-contribution.test.js tests/portability.test.js",
|
|
"test:ui": "node tests/ui.acceptance.mjs",
|
|
"test:photo": "node tests/photo-first.acceptance.mjs",
|
|
"test:mobile-capture": "node tests/mobile-capture.acceptance.mjs",
|
|
"test:sleek": "node tests/sleek-chat.acceptance.mjs",
|
|
"test:manual-fallback": "node tests/manual-fallback.acceptance.mjs",
|
|
"test:provisional": "node tests/provisional-suggestion.acceptance.mjs",
|
|
"test:followup": "node tests/followup.acceptance.mjs",
|
|
"test:vision-refusal": "node tests/vision-refusal.acceptance.mjs",
|
|
"test:staging-smoke": "node tests/staging.acceptance.mjs",
|
|
"test:pattern-summary": "node tests/pattern-summary.acceptance.mjs",
|
|
"test:grounded-chat": "node tests/grounded-chat.acceptance.mjs",
|
|
"test:photo-lifecycle": "node tests/photo-lifecycle.acceptance.mjs",
|
|
"test:training-contribution": "node tests/training-contribution.acceptance.mjs",
|
|
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check src/training-contribution.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/corpus_manifest.py scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py scripts/calibration_report.py && node --check scripts/record_worker_benchmark.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/training-contribution.acceptance.mjs && node --check tests/portability.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh",
|
|
"check:diff": "bash scripts/check_diff.sh",
|
|
"start": "node server.mjs",
|
|
"test:worker-readiness": "node tests/worker-readiness.acceptance.mjs",
|
|
"test:portability": "node tests/portability.acceptance.mjs",
|
|
"test:corpus": "python3 tests/corpus-manifest.test.py",
|
|
"test:calibration": "python3 tests/calibration-harness.test.py"
|
|
},
|
|
"devDependencies": {
|
|
"playwright": "^1.62.1"
|
|
}
|
|
}
|