From 6b1e24fec42ead8fac0b6d31e77326a6ca0ba787 Mon Sep 17 00:00:00 2001 From: ledger Date: Mon, 2 Nov 2026 22:58:39 +0000 Subject: [PATCH] volt: root CHECK-* sweep 2026-11-02 -- 32 entries, 20 OK / 12 FAULT at e44d49a sweep.py runs every root CHECK-* under bash and records rc, class and the process's last line, pinning rev + sha256 of each entry in manifest.json. report.md is the table; TRIAGE.md groups the 12 reds into four causes. check-root-sweep.sh grades the record (pin / universe / class contract / report parity / rows integrity), 5 legs, with sensitivity_probe.py proving three mutations redden the leg that owns them and the restore is green. --- check-root-sweep.sh | 14 ++ wizard-work/volt/root-check-sweep/TRIAGE.md | 98 ++++++++ .../volt/root-check-sweep/manifest.json | 209 ++++++++++++++++ wizard-work/volt/root-check-sweep/report.md | 59 +++++ wizard-work/volt/root-check-sweep/results.tsv | 33 +++ .../root-check-sweep/sensitivity_probe.py | 107 ++++++++ wizard-work/volt/root-check-sweep/sweep.py | 232 ++++++++++++++++++ .../volt/root-check-sweep/verify_sweep.py | 204 +++++++++++++++ 8 files changed, 956 insertions(+) create mode 100755 check-root-sweep.sh create mode 100644 wizard-work/volt/root-check-sweep/TRIAGE.md create mode 100644 wizard-work/volt/root-check-sweep/manifest.json create mode 100644 wizard-work/volt/root-check-sweep/report.md create mode 100644 wizard-work/volt/root-check-sweep/results.tsv create mode 100644 wizard-work/volt/root-check-sweep/sensitivity_probe.py create mode 100644 wizard-work/volt/root-check-sweep/sweep.py create mode 100644 wizard-work/volt/root-check-sweep/verify_sweep.py diff --git a/check-root-sweep.sh b/check-root-sweep.sh new file mode 100755 index 0000000..5212592 --- /dev/null +++ b/check-root-sweep.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# check-root-sweep.sh -- grade the root CHECK-* sweep artifact. +# +# The sweep (wizard-work/volt/root-check-sweep/sweep.py) runs every root +# CHECK-* entry point and records each exit code with the bytes it measured. +# Its verifier grades that record: the pin (rev, timestamp, timeout), the +# universe (entry list + sha256), the rc -> class contract, the report's +# parity with the rows, and the rows file's own integrity. +# +# Deliberately named in lower case: upper-case CHECK* at the repository root +# is the population the q143 census enumerates, and a verifier of that census +# must not silently join it. +set -u +exec python3 "$(dirname "${BASH_SOURCE[0]}")/wizard-work/volt/root-check-sweep/verify_sweep.py" diff --git a/wizard-work/volt/root-check-sweep/TRIAGE.md b/wizard-work/volt/root-check-sweep/TRIAGE.md new file mode 100644 index 0000000..06d2b0e --- /dev/null +++ b/wizard-work/volt/root-check-sweep/TRIAGE.md @@ -0,0 +1,98 @@ +# Root CHECK sweep -- 2026-11-02, handover + +Measured at `e44d49aeeca2b61c5a03ab33d0b8cef66f423d73` -- see `manifest.json` for the full +40-hex rev, the UTC stamp and the sha256 of every entry point that was run. The sweep ran **32** +root `CHECK-*` entry points under `bash `: **20 OK, 12 FAULT, 0 UNEXPECTED, +0 TIMEOUT, 0 could-not-grade** (531.3 s). Rows: `results.tsv`. Table: `report.md`. + +The 12 reds are not 12 defects. They are **four causes**, measured below, each with the +fault line the entry point itself printed in this run. + +## A. The Q159 landing is in the worktree, not in the tree (q143, q148, q159) + + CHECK-q143 rc=1 FAIL: MISSING CHECK-q159 -- baseline names it, the tree does not have it + CHECK-q148 rc=1 FAIL: MISSING CHECK-q159 -- baseline names it, the tree does not have it + CHECK-q159 rc=1 census FAULT rc=1 :: paths: 141 entry point(s), 140 baseline path(s), 3 fault(s) + +`wizard-work/volt/q143-check-sweep/check_entry_baseline.tsv` carries the Q159 guard row and +`CHECK-q159` exists on disk, but `check_sweep.py` discovers **tracked** entry points and +`CHECK-q159` is untracked -- so the baseline names a path the tree does not hold. The same +two commands also report the next cause. + +Fix: commit `CHECK-q159` with the baseline row that names it (and the Q159 witnesses), or drop +the row until it is committed. Nothing else in this cluster is a code fault. + +## B. Two tracked entry points have no baseline row (q143, q159) + + FAIL: NEW wizard-work/ledger/CHECK-2026-11-02-jobs-table -- in the tree, not in the baseline + FAIL: NEW wizard-work/sol/CHECK-untracked-file-classify-2026-11-02 -- in the tree, not in the baseline + +Both are committed and therefore discoverable, but neither has a row a human signed. Fix: +run each once, record its rc and first fault line in `check_entry_baseline.tsv` through the +row-level re-take path (the file's own convention), or the census is red for every reader. + +## C. `CHECK-q-checkparen` pins a tally the rubric outgrew (q-checkparen) + + FAULT selftest rc=0 selftest: 51 cells, 51 passed, 0 failed + FAULT pin ['check_lint.py\t51\tcells'] + FAULT falsifier rc=1 selftest: 51 cells, 49 passed, 2 failed + +The subject is green (51/51) and the falsifier leg proves the rubric can red (49/51 on the +mutated copy), so what is stale is the verifier's expected literal, not `check_lint.py`. This +is the Q158 shape a second time: a verifier that asserts a cell tally goes red when the +rubric grows, on a tree where the refactor it grades is untouched. + +## D. Beam lane: coverage and bindings (q149, q153, q85) + + CHECK-q149 rc=1 FAIL beam-gate red (rc=1) + CHECK-q153 rc=1 FAIL lane-check bindings: 185 file(s) graded repo-wide at WORKTREE, 109 test file(s) in wizard-work/beam/ + CHECK-q85 rc=1 coverage: 118 module(s): ... 2 run by CHECK, 6 exempt, 1 fault(s) (all three live passes) + +`CHECK-q153`'s `verify` leg is green (6/6) and `import_resolve_check` grades 185 files with 0 +faults; the red is the `lane-check` binding leg (185 graded vs 109 test files in the lane). +`CHECK-q85`'s coverage authority reports 1 fault over 118 modules in every one of its three +live runs, so it is not a flake. These three are one subject: the beam lane's module set is +larger than the set its collected tests and its binding census can account for. + +## E. `CHECK-q150`: q65's ledger and rc probe (q150) + + FAIL runner-ledger q65: 8 fault(s) + FAIL rc-probe q65_rc_probe: 5 fault(s) + PASS no-verdict q65_no_verdict: 0 fault(s) + +Two of three legs red, and the two `.tsv` files they read +(`wizard-work/volt/q65_runner_baseline.tsv`, `q65_rc_probe.tsv`) are **modified in this +worktree** (uncommitted), so the first question is whether this red is the modification or +the tree. Not settled here. + +## F. Frozen untracked snapshots (sol-untracked-classify, untracked-triad, untracked-triage) + + CHECK-sol-untracked-classify rc=1 frozen path missing on disk: 'verify_wordcount_test.py' + CHECK-sol-untracked-classify rc=1 frozen path not present under repo: 'wizard-work/null/.hermes-tmp.RqZp3z' + CHECK-untracked-triad-2026-11-01 rc=1 id=1 path=CHECK-q146 is not currently untracked + CHECK-untracked-triage-2026-11-01 rc=1 frozen path missing on disk: ['wizard-work/null/.hermes-tmp.RqZp3z'] + +Three snapshots of "the untracked set at rev X" graded against the live set. Two of the paths +they froze no longer exist (`wizard-work/null/.hermes-tmp.RqZp3z` is a temp directory; +`verify_wordcount_test.py` left the root) and `CHECK-q146` was committed, so it stopped being +untracked. The classification was never wrong; the universe moved. This is Q157's shape +exactly, in three more files. + +## G. `CHECK-sol-profile-triage-2026-11-02`: artifact vs live profile (sol-profile) + + CHECK FAIL categories mismatch: + artifact state_db 11 / config_data 1 + live state_db 10 / config_data 2 + +One live profile path moved between the two categories after the artifact was taken. The +baseline already signs this row `KNOWN RED: artifact categories != live disk (sol profile +changed after the artifact was taken)`, so this sweep confirms the signed reading rather than +finding a new one. + +## What this sweep does not claim + +- It grades the **worktree**, not the commit: 106 paths differ from `e44d49a` (modified + + untracked). A green row is green for those bytes, which `manifest.json` pins by sha256. +- It does not re-run the checks that were green; it records the process and its last line. +- It does not say a red row is a defect. A, C, F and G are stale instruments on a moved + universe; B is unregistered work; D and E are not attributed here. diff --git a/wizard-work/volt/root-check-sweep/manifest.json b/wizard-work/volt/root-check-sweep/manifest.json new file mode 100644 index 0000000..9e08f8f --- /dev/null +++ b/wizard-work/volt/root-check-sweep/manifest.json @@ -0,0 +1,209 @@ +{ + "counts": { + "FAULT": 12, + "OK": 20 + }, + "entries": [ + { + "bytes": 761, + "name": "CHECK", + "sha256": "5d039e42302a6d7a2618cf271e6f4a03450ed6df6ee9a0144c2295b7a6fa6975", + "tracked_at_rev": true + }, + { + "bytes": 365, + "name": "CHECK-2026-11-01-ledger-dir-census", + "sha256": "0a524960e582563431bba85d15f3955ef1f9c501c1e921068c070060d1edf62d", + "tracked_at_rev": true + }, + { + "bytes": 428, + "name": "CHECK-2026-11-02-openmeteo-summary", + "sha256": "c50bfff355a56d5fa427fef8ac8c270e71519d84261eb438598f6631fa6cf884", + "tracked_at_rev": false + }, + { + "bytes": 1276, + "name": "CHECK-2026-11-02-untracked-classify", + "sha256": "fdce9d0887932498611e6fd72c1fb9f97516fb9647738c6bf74dd79c0f8083e4", + "tracked_at_rev": true + }, + { + "bytes": 524, + "name": "CHECK-ledger-dir-census", + "sha256": "0db9b52e919cc17e7b5a2b063137edd04b1e3767d284912ce665a2fbb9e1f47d", + "tracked_at_rev": true + }, + { + "bytes": 466, + "name": "CHECK-q-checkparen", + "sha256": "63035f0c2df959ad5b54021fa4d86abb3fb79868510a7abeb8c6a09f3b9f4b54", + "tracked_at_rev": true + }, + { + "bytes": 3743, + "name": "CHECK-q143", + "sha256": "5d8b3f6a5647959c1d04fefedac35f6fd5fe325168ff9dd578a4486c2571c43e", + "tracked_at_rev": true + }, + { + "bytes": 3556, + "name": "CHECK-q146", + "sha256": "b64ff06524745e8fd6d2845ab69303d7472f579b1b18605efb272d42c5594d7c", + "tracked_at_rev": true + }, + { + "bytes": 2947, + "name": "CHECK-q147", + "sha256": "5aa35a79a4da211a6beb758598375b9c7fbcc653ea197d68dac58dafe0389ab0", + "tracked_at_rev": true + }, + { + "bytes": 5783, + "name": "CHECK-q148", + "sha256": "231d04cff956786092b4ea3fd4972d186e6b382463a9c10051ec2603b4f97b24", + "tracked_at_rev": true + }, + { + "bytes": 6178, + "name": "CHECK-q149", + "sha256": "2cf788980c1626e6e34fe7e9d99e729b3030653cd515f314270f95addef37785", + "tracked_at_rev": true + }, + { + "bytes": 4626, + "name": "CHECK-q150", + "sha256": "0a7975b85e8130f35a3c0d1e90d12a867effc4420b59d4f6806f8f7b6b134ff7", + "tracked_at_rev": true + }, + { + "bytes": 2873, + "name": "CHECK-q151", + "sha256": "59ef476b23d885b3caa93e3e843eff6a97210814f7a6f587bbe20b2caebf1cfe", + "tracked_at_rev": true + }, + { + "bytes": 3315, + "name": "CHECK-q152", + "sha256": "b9de094511787b806115581041523b6188ea0be6ad8fefdc75328caa6f8560ef", + "tracked_at_rev": true + }, + { + "bytes": 2600, + "name": "CHECK-q153", + "sha256": "062dced3a83f657e65c5f9fe224e4e8904c3905b707c685620a2348b1ef8f20e", + "tracked_at_rev": true + }, + { + "bytes": 5909, + "name": "CHECK-q154", + "sha256": "0bbf23fb865312b8648b22b5a77a88131beddae23f9fd80d0b18a13a86b828cf", + "tracked_at_rev": false + }, + { + "bytes": 2575, + "name": "CHECK-q155", + "sha256": "54617458249263e3f10b22d2833c285434f0f4a92345abb22768f9bc1aaab7b7", + "tracked_at_rev": true + }, + { + "bytes": 3447, + "name": "CHECK-q156", + "sha256": "383ee3d94361ce01297e1079e6ea8ca0755f20376d29404bf45a21f52a63cf79", + "tracked_at_rev": true + }, + { + "bytes": 3312, + "name": "CHECK-q157", + "sha256": "4744b5b5327abbab7a6be5662ea87473b306a3fef01196f8583da497023f9047", + "tracked_at_rev": true + }, + { + "bytes": 6925, + "name": "CHECK-q158", + "sha256": "f80552e1f098208bab404743142c19b2ab058b23e3ac5c358de4c7a6b7451a08", + "tracked_at_rev": true + }, + { + "bytes": 4956, + "name": "CHECK-q159", + "sha256": "acb287d3fe74dd8ec8f555dfb9b952fad8fe4fc0e43d8720f5811dbc318e45d0", + "tracked_at_rev": false + }, + { + "bytes": 5528, + "name": "CHECK-q85", + "sha256": "4980bbd76f171ccf7af4584d84175f1f144872f28a58b73cb33e5892be0916c7", + "tracked_at_rev": true + }, + { + "bytes": 6438, + "name": "CHECK-quill-draft-compliance", + "sha256": "a1beb354a32e70f759f4e1375a2444b055fa437a085fa288456d2a50b76bad21", + "tracked_at_rev": true + }, + { + "bytes": 689, + "name": "CHECK-sol-lane-triage-2026-10-29", + "sha256": "24800b074f4be5471e1d75e94ca8200dd96228b71fc862deb9027994883f66a0", + "tracked_at_rev": true + }, + { + "bytes": 673, + "name": "CHECK-sol-lane-triage-2026-11-01", + "sha256": "198a8ca344c37a4883abc584f409bc4dce54f57b4d273b8ee30fa874e4852007", + "tracked_at_rev": true + }, + { + "bytes": 602, + "name": "CHECK-sol-lane-triage-2026-11-02", + "sha256": "1031f3c20fb31b5b46f36e63f9b9e240901090fb75c07f0cc2dca9122f1a3bc5", + "tracked_at_rev": true + }, + { + "bytes": 527, + "name": "CHECK-sol-profile-triage-2026-11-02", + "sha256": "e6bb09c6a98708c84abafe4c4cf7de2fbe457c3c0a54ff8ac886e31aeecb6c5e", + "tracked_at_rev": true + }, + { + "bytes": 457, + "name": "CHECK-sol-untracked-classify", + "sha256": "4e383e4b40f99115830cfaab37267786a8b8639a22590a308368f25e31e156ba", + "tracked_at_rev": false + }, + { + "bytes": 358, + "name": "CHECK-triage-2026-10-29", + "sha256": "aa3e442abdf491038888fa3e1319c13b218ae1013ec60fb4b62c0c300cd83c15", + "tracked_at_rev": false + }, + { + "bytes": 386, + "name": "CHECK-triage-2026-11-01", + "sha256": "f6dba366ba5c5a1de2be1efb7e3d413acaca73e0f98aec7a1fbcc7e4bedd0384", + "tracked_at_rev": true + }, + { + "bytes": 503, + "name": "CHECK-untracked-triad-2026-11-01", + "sha256": "001cb873fe11c58ece2a0785e51a30b1ba7d7f6a6595fca59af94e9ec84f8ad2", + "tracked_at_rev": true + }, + { + "bytes": 533, + "name": "CHECK-untracked-triage-2026-11-01", + "sha256": "f1a013f31a3a5aa05302eefe977b5865424c942b78eeff01824fbd563503214c", + "tracked_at_rev": true + } + ], + "entry_count": 32, + "generated_utc": "2026-11-02T22:38:46Z", + "repo": "troupe-standup", + "results_sha256": "540fea8d1bea8bcfd3f5b7572c11804c11cd36ceabecc61eedce23dcc2658331", + "rev": "e44d49aeeca2b61c5a03ab33d0b8cef66f423d73", + "rev_subject": "beam: content-based duplicate file detector with tests", + "timeout_s": 300, + "tool": "wizard-work/volt/root-check-sweep/sweep.py", + "worktree_dirty_paths": 106 +} diff --git a/wizard-work/volt/root-check-sweep/report.md b/wizard-work/volt/root-check-sweep/report.md new file mode 100644 index 0000000..9240f2a --- /dev/null +++ b/wizard-work/volt/root-check-sweep/report.md @@ -0,0 +1,59 @@ +# Root CHECK-* entry-point sweep + +- measured at: `2026-11-02T22:38:46Z` +- rev: `e44d49aeeca2b61c5a03ab33d0b8cef66f423d73` -- beam: content-based duplicate file detector with tests +- worktree: 106 path(s) differ from that rev (modified + untracked); the worktree is what was graded +- entry points run: **32** +- FAULT: **12** +- OK: **20** +- total wall time: **531.3 s** + +| entry | rc | s | class | last line | +|---|---|---|---|---| +| `CHECK` | 0 | 2.3 | OK | verify_selftest_sweep_q93_refactor: ALL GREEN (0 failed, all checks) | +| `CHECK-2026-11-01-ledger-dir-census` | 0 | 1.67 | OK | VERDICT: rc 0 -- the committed CSV is the derivation at the pinned rev | +| `CHECK-2026-11-02-openmeteo-summary` | 0 | 0.19 | OK | PASS: 6 rows match source-derived table; header exact | +| `CHECK-2026-11-02-untracked-classify` | 0 | 1.24 | OK | CHECK OK: 306 pinned paths; manifest==snapshot in order; categories re-derive by rule; rule agrees with the generator; no pinned path was tracked at 4a640e1 | +| `CHECK-ledger-dir-census` | 0 | 2.28 | OK | VERDICT: rc 0 -- the committed CSV is the derivation at the pinned rev | +| `CHECK-q-checkparen` | 1 | 2.0 | FAULT | verify_fix: 3 leg(s) faulted | +| `CHECK-q143` | 1 | 31.17 | FAULT | q143: 2 leg(s), 1 fault(s) | +| `CHECK-q146` | 0 | 2.79 | OK | q146: 4 leg(s), no fault | +| `CHECK-q147` | 0 | 5.33 | OK | q147: 4 of 4 leg(s) passed | +| `CHECK-q148` | 1 | 24.26 | FAULT | q148: 4 of 5 leg(s) passed | +| `CHECK-q149` | 1 | 107.43 | FAULT | q149: 5 of 6 leg(s) passed | +| `CHECK-q150` | 1 | 47.26 | FAULT | q150: 6 leg(s), 2 fault(s) | +| `CHECK-q151` | 0 | 12.33 | OK | q151: 3 leg(s), all passed | +| `CHECK-q152` | 0 | 31.81 | OK | q152: 4 leg(s), all passed | +| `CHECK-q153` | 1 | 68.72 | FAULT | q153: 2 leg(s), 1 fault(s) | +| `CHECK-q154` | 0 | 14.47 | OK | CHECK-q154: 5 leg(s) passed, 0 fault(s), 0 ungradable | +| `CHECK-q155` | 0 | 7.16 | OK | q155: 3 leg(s), all passed | +| `CHECK-q156` | 0 | 63.19 | OK | q156: 6 leg(s), all passed | +| `CHECK-q157` | 0 | 2.74 | OK | q157: 6 leg(s), all passed | +| `CHECK-q158` | 0 | 4.17 | OK | CHECK-q158: 6 leg(s) passed, 0 fault(s), 0 ungradable | +| `CHECK-q159` | 1 | 4.05 | FAULT | q159: 2 passed, 3 fault(s) | +| `CHECK-q85` | 1 | 69.59 | FAULT | q85: 3 leg(s) passed, 2 fault(s), 0 ungradable | +| `CHECK-quill-draft-compliance` | 0 | 2.83 | OK | CHECK-quill-draft-compliance: 5 leg(s) passed, 0 fault(s), 0 ungradable | +| `CHECK-sol-lane-triage-2026-10-29` | 0 | 2.84 | OK | VERIFY OK (216 files, categories re-derived, all paths present) | +| `CHECK-sol-lane-triage-2026-11-01` | 0 | 4.62 | OK | VERIFY OK (339 files, categories re-derived, all paths present) | +| `CHECK-sol-lane-triage-2026-11-02` | 0 | 6.67 | OK | VERIFY OK (382 files, categories re-derived, all paths present) | +| `CHECK-sol-profile-triage-2026-11-02` | 1 | 0.4 | FAULT | - categories mismatch: artifact={'identity': 2, 'config': 3, 'state_db': 11, 'soul': 1, 'script': 20, 'cache': 3, 'backups': 1, 'extension_dir': 18, 'config_dat | +| `CHECK-sol-untracked-classify` | 1 | 0.2 | FAULT | - frozen path not present under repo: 'wizard-work/null/.hermes-tmp.RqZp3z' | +| `CHECK-triage-2026-10-29` | 0 | 0.17 | OK | PASS: Triage-2026-10-29.json re-derives 8/8 items from evidence; decision_counts={'approve': 6, 'table': 2, 'reject': 0}, categories={'archive_fix': 5, 'app_pr' | +| `CHECK-triage-2026-11-01` | 0 | 0.24 | OK | PASS: Triage-2026-11-01.json re-derives 8/8 items from seed evidence; decision_counts={'approve': 6, 'table': 2}, categories={'archive_fix': 5, 'app_pr': 1, 'ge | +| `CHECK-untracked-triad-2026-11-01` | 1 | 7.0 | FAULT | - id=20 path=wizard-work/null/.hermes-tmp.RqZp3z is not currently untracked | +| `CHECK-untracked-triage-2026-11-01` | 1 | 0.18 | FAULT | - frozen path missing on disk: ['wizard-work/null/.hermes-tmp.RqZp3z'] | + +## Not OK + +- `CHECK-q-checkparen` rc=1 (FAULT): verify_fix: 3 leg(s) faulted +- `CHECK-q143` rc=1 (FAULT): q143: 2 leg(s), 1 fault(s) +- `CHECK-q148` rc=1 (FAULT): q148: 4 of 5 leg(s) passed +- `CHECK-q149` rc=1 (FAULT): q149: 5 of 6 leg(s) passed +- `CHECK-q150` rc=1 (FAULT): q150: 6 leg(s), 2 fault(s) +- `CHECK-q153` rc=1 (FAULT): q153: 2 leg(s), 1 fault(s) +- `CHECK-q159` rc=1 (FAULT): q159: 2 passed, 3 fault(s) +- `CHECK-q85` rc=1 (FAULT): q85: 3 leg(s) passed, 2 fault(s), 0 ungradable +- `CHECK-sol-profile-triage-2026-11-02` rc=1 (FAULT): - categories mismatch: artifact={'identity': 2, 'config': 3, 'state_db': 11, 'soul': 1, 'script': 20, 'cache': 3, 'backups': 1, 'extension_dir': 18, 'config_dat +- `CHECK-sol-untracked-classify` rc=1 (FAULT): - frozen path not present under repo: 'wizard-work/null/.hermes-tmp.RqZp3z' +- `CHECK-untracked-triad-2026-11-01` rc=1 (FAULT): - id=20 path=wizard-work/null/.hermes-tmp.RqZp3z is not currently untracked +- `CHECK-untracked-triage-2026-11-01` rc=1 (FAULT): - frozen path missing on disk: ['wizard-work/null/.hermes-tmp.RqZp3z'] diff --git a/wizard-work/volt/root-check-sweep/results.tsv b/wizard-work/volt/root-check-sweep/results.tsv new file mode 100644 index 0000000..4629f16 --- /dev/null +++ b/wizard-work/volt/root-check-sweep/results.tsv @@ -0,0 +1,33 @@ +entry rc secs class traceback last_stdout last_stderr +CHECK 0 2.3 OK 0 verify_selftest_sweep_q93_refactor: ALL GREEN (0 failed, all checks) - +CHECK-2026-11-01-ledger-dir-census 0 1.67 OK 0 VERDICT: rc 0 -- the committed CSV is the derivation at the pinned rev - +CHECK-2026-11-02-openmeteo-summary 0 0.19 OK 0 PASS: 6 rows match source-derived table; header exact - +CHECK-2026-11-02-untracked-classify 0 1.24 OK 0 CHECK OK: 306 pinned paths; manifest==snapshot in order; categories re-derive by rule; rule agrees with the generator; no pinned path was tracked at 4a640e1 - +CHECK-ledger-dir-census 0 2.28 OK 0 VERDICT: rc 0 -- the committed CSV is the derivation at the pinned rev - +CHECK-q-checkparen 1 2.0 FAULT 0 verify_fix: 3 leg(s) faulted - +CHECK-q143 1 31.17 FAULT 0 q143: 2 leg(s), 1 fault(s) - +CHECK-q146 0 2.79 OK 0 q146: 4 leg(s), no fault - +CHECK-q147 0 5.33 OK 0 q147: 4 of 4 leg(s) passed - +CHECK-q148 1 24.26 FAULT 0 q148: 4 of 5 leg(s) passed - +CHECK-q149 1 107.43 FAULT 0 q149: 5 of 6 leg(s) passed - +CHECK-q150 1 47.26 FAULT 0 q150: 6 leg(s), 2 fault(s) - +CHECK-q151 0 12.33 OK 0 q151: 3 leg(s), all passed - +CHECK-q152 0 31.81 OK 0 q152: 4 leg(s), all passed - +CHECK-q153 1 68.72 FAULT 0 q153: 2 leg(s), 1 fault(s) - +CHECK-q154 0 14.47 OK 0 CHECK-q154: 5 leg(s) passed, 0 fault(s), 0 ungradable - +CHECK-q155 0 7.16 OK 0 q155: 3 leg(s), all passed - +CHECK-q156 0 63.19 OK 0 q156: 6 leg(s), all passed - +CHECK-q157 0 2.74 OK 0 q157: 6 leg(s), all passed - +CHECK-q158 0 4.17 OK 0 CHECK-q158: 6 leg(s) passed, 0 fault(s), 0 ungradable - +CHECK-q159 1 4.05 FAULT 0 q159: 2 passed, 3 fault(s) - +CHECK-q85 1 69.59 FAULT 0 q85: 3 leg(s) passed, 2 fault(s), 0 ungradable - +CHECK-quill-draft-compliance 0 2.83 OK 0 CHECK-quill-draft-compliance: 5 leg(s) passed, 0 fault(s), 0 ungradable - +CHECK-sol-lane-triage-2026-10-29 0 2.84 OK 0 VERIFY OK (216 files, categories re-derived, all paths present) - +CHECK-sol-lane-triage-2026-11-01 0 4.62 OK 0 VERIFY OK (339 files, categories re-derived, all paths present) - +CHECK-sol-lane-triage-2026-11-02 0 6.67 OK 0 VERIFY OK (382 files, categories re-derived, all paths present) - +CHECK-sol-profile-triage-2026-11-02 1 0.4 FAULT 0 - categories mismatch: artifact={'identity': 2, 'config': 3, 'state_db': 11, 'soul': 1, 'script': 20, 'cache': 3, 'backups': 1, 'extension_dir': 18, 'config_dat - +CHECK-sol-untracked-classify 1 0.2 FAULT 0 - frozen path not present under repo: 'wizard-work/null/.hermes-tmp.RqZp3z' - +CHECK-triage-2026-10-29 0 0.17 OK 0 PASS: Triage-2026-10-29.json re-derives 8/8 items from evidence; decision_counts={'approve': 6, 'table': 2, 'reject': 0}, categories={'archive_fix': 5, 'app_pr' - +CHECK-triage-2026-11-01 0 0.24 OK 0 PASS: Triage-2026-11-01.json re-derives 8/8 items from seed evidence; decision_counts={'approve': 6, 'table': 2}, categories={'archive_fix': 5, 'app_pr': 1, 'ge - +CHECK-untracked-triad-2026-11-01 1 7.0 FAULT 0 - id=20 path=wizard-work/null/.hermes-tmp.RqZp3z is not currently untracked - +CHECK-untracked-triage-2026-11-01 1 0.18 FAULT 0 - frozen path missing on disk: ['wizard-work/null/.hermes-tmp.RqZp3z'] - diff --git a/wizard-work/volt/root-check-sweep/sensitivity_probe.py b/wizard-work/volt/root-check-sweep/sensitivity_probe.py new file mode 100644 index 0000000..f43f7bb --- /dev/null +++ b/wizard-work/volt/root-check-sweep/sensitivity_probe.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""sensitivity_probe.py -- prove the sweep verifier can print a red. + +A verifier that has only ever printed PASS is not evidence: it may be a print +statement with a five-leg shape. This probe mutates a copy of the artifact in +three independent places -- a class label, a pinned sha256, a stated count -- +and requires `check-root-sweep.sh` to exit 1 and name the leg that owns the +mutated field. Every mutation is restored from a pristine copy, and the last +assertion is that the restored artifact is green again. + +Run: python3 sensitivity_probe.py (exit 0 = all three mutations reddened the + leg that owns them and the restore is green) +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile + +ART = os.path.dirname(os.path.abspath(__file__)) +ROOT = subprocess.run(["git", "-C", ART, "rev-parse", "--show-toplevel"], + capture_output=True, text=True).stdout.strip() +CHECK = os.path.join(ROOT, "check-root-sweep.sh") +FILES = ["results.tsv", "manifest.json", "report.md"] + + +def run_check() -> tuple[int, str]: + p = subprocess.run(["bash", CHECK], capture_output=True, text=True, + errors="replace") + return p.returncode, p.stdout + p.stderr + + +def leg_line(out: str, leg: str) -> str: + for line in out.splitlines(): + if line.strip().startswith(leg + " "): + return line.strip() + return "" + + +def mutate(name: str, path: str, old: str, new: str) -> None: + text = open(path, encoding="utf-8").read() + if text.count(old) != 1: + raise SystemExit(f"{name}: mutation site occurs {text.count(old)}x, want 1") + open(path, "w", encoding="utf-8").write(text.replace(old, new)) + + +def main() -> int: + pristine = tempfile.mkdtemp(prefix="sweep-pristine-") + for f in FILES: + shutil.copy2(os.path.join(ART, f), os.path.join(pristine, f)) + + rows = [] + try: + # 1 -- a class label that contradicts its rc. + name = "class-label" + path = os.path.join(ART, "results.tsv") + first_ok = open(path, encoding="utf-8").read().splitlines()[1] + cols = first_ok.split("\t") + mutate(name, path, "\t".join(cols), "\t".join( + cols[:3] + ["FAULT"] + cols[4:])) + rc, out = run_check() + rows.append((name, "classes", rc, leg_line(out, "classes"))) + shutil.copy2(os.path.join(pristine, "results.tsv"), path) + + # 2 -- a pinned sha256 that no longer describes the file. + name = "pinned-sha" + mpath = os.path.join(ART, "manifest.json") + manifest = json.load(open(mpath, encoding="utf-8")) + good = manifest["entries"][0]["sha256"] + mutate(name, mpath, good, "0" * 64) + rc, out = run_check() + rows.append((name, "universe", rc, leg_line(out, "universe"))) + shutil.copy2(os.path.join(pristine, "manifest.json"), mpath) + + # 3 -- a stated count that disagrees with the rows file. + name = "report-count" + rpath = os.path.join(ART, "report.md") + mutate(name, rpath, f"- entry points run: **{len(manifest['entries'])}**", + f"- entry points run: **{len(manifest['entries']) - 1}**") + rc, out = run_check() + rows.append((name, "report", rc, leg_line(out, "report"))) + shutil.copy2(os.path.join(pristine, "report.md"), rpath) + + # 4 -- the restore: the artifact as shipped must be green again. + rc, out = run_check() + rows.append(("restored", "-", rc, out.strip().splitlines()[-1])) + finally: + shutil.rmtree(pristine, ignore_errors=True) + + ok = True + for name, leg, rc, line in rows: + if name == "restored": + good = rc == 0 + else: + good = rc == 1 and leg in line and "FAULT" in line + ok = ok and good + print(f" {name:<13} {'PASS' if good else 'FAULT'} rc={rc} {line}") + print(f"\nsensitivity: {sum(1 for r in rows if (r[0] == 'restored' and r[2] == 0) or (r[0] != 'restored' and r[2] == 1))}/4 cell(s) as expected") + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wizard-work/volt/root-check-sweep/sweep.py b/wizard-work/volt/root-check-sweep/sweep.py new file mode 100644 index 0000000..15182b2 --- /dev/null +++ b/wizard-work/volt/root-check-sweep/sweep.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""sweep.py -- run every root CHECK-* entry point and classify its exit code. + +WHY + The repo gate (`tools/verify.py --fast`) runs a fixed set of legs; none of + them runs the committed `CHECK-*` entry points sitting at the repository + root. Each of those was written as a deliverable's own verification command, + so the question "does the tree still satisfy the checks it shipped with?" + has no answer from the gate. This sweep answers it by running each one under + the interpreter every caller in this tree uses (`bash `) and + classifying the exit code against this repo's published contract: + + rc 0 OK the check graded clean + rc 1 FAULT the check graded a fault (a real, reportable red) + rc 2 COULD_NOT_GRADE the check could not grade its subject + other UNEXPECTED a shell/driver error (126, 127, signal, ...) + timeout TIMEOUT killed after --timeout seconds + + Nothing here is a stub: every row is a real process with a real exit code, + and stdout/stderr tails are the process's own bytes. + +WHAT IS PINNED + A verdict is only worth the tree it was measured on, so every run writes + `manifest.json` beside its rows: the full HEAD sha and subject, the UTC + timestamp, the timeout, and the sha256 of every swept entry-point file, plus + a flag for whether that path exists at HEAD. The report prints the same rev. + The bytes that were run are therefore recoverable, not implied. + + An interrupted run is partial evidence, not no evidence: rows are flushed as + they finish, so a sweep killed at the wall clock leaves every completed row. + +USAGE + python3 sweep.py --out [--repo ] [--timeout 300] [--json] + + Writes /results.tsv, /manifest.json and /report.md, and + prints one summary line. Exit code of the sweep itself: 0 if every row + graded 0, 1 if any row graded 1 or worse, 2 if the sweep could not run (no + entry points found, or a manifest could not be captured). +""" + +from __future__ import annotations + +import argparse +import datetime as _dt +import glob +import hashlib +import json +import os +import subprocess +import sys +import time + +CLASS = {0: "OK", 1: "FAULT", 2: "COULD_NOT_GRADE"} + + +def discover(repo: str) -> list[str]: + rows = [] + for path in sorted(glob.glob(os.path.join(repo, "CHECK*"))): + if os.path.isfile(path): + rows.append(os.path.basename(path)) + return rows + + +def tail(text: str, limit: int = 160) -> str: + text = (text or "").strip().replace("\t", " ").replace("\r", " ") + lines = [ln for ln in text.splitlines() if ln.strip()] + if not lines: + return "-" + return lines[-1].strip()[:limit] + + +def run_one(repo: str, name: str, timeout: int) -> dict: + started = time.time() + try: + proc = subprocess.run( + ["bash", name], + cwd=repo, + capture_output=True, + text=True, + timeout=timeout, + errors="replace", + ) + rc, out, err = proc.returncode, proc.stdout, proc.stderr + except subprocess.TimeoutExpired as exc: + rc = "timeout" + out = (exc.stdout or "") if isinstance(exc.stdout, str) else "" + err = (exc.stderr or "") if isinstance(exc.stderr, str) else "" + secs = round(time.time() - started, 2) + klass = "TIMEOUT" if rc == "timeout" else CLASS.get(rc, "UNEXPECTED") + return { + "name": name, + "rc": rc, + "secs": secs, + "class": klass, + "last_out": tail(out), + "last_err": tail(err), + "traceback": "Traceback (most recent call last)" in (err or ""), + } + + +def git_out(repo: str, *args: str) -> str: + try: + p = subprocess.run(["git", "-C", repo, *args], capture_output=True, + text=True, timeout=30, errors="replace") + except (OSError, subprocess.SubprocessError): + return "" + return p.stdout.strip() if p.returncode == 0 else "" + + +def sha256_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def capture_manifest(repo: str, names: list[str], timeout: int) -> dict: + rev = git_out(repo, "rev-parse", "HEAD") + if len(rev) != 40: + raise RuntimeError(f"cannot resolve HEAD in {repo!r}") + subject = git_out(repo, "log", "-1", "--format=%s", "HEAD") + tracked = set(git_out(repo, "ls-tree", "-r", "--name-only", "HEAD") + .splitlines()) + entries = [] + for name in names: + path = os.path.join(repo, name) + entries.append({ + "name": name, + "sha256": sha256_file(path), + "bytes": os.path.getsize(path), + "tracked_at_rev": name in tracked, + }) + dirty = [ln for ln in git_out(repo, "status", "--porcelain").splitlines() + if ln.strip()] + return { + "tool": "wizard-work/volt/root-check-sweep/sweep.py", + "generated_utc": _dt.datetime.now(_dt.timezone.utc) + .strftime("%Y-%m-%dT%H:%M:%SZ"), + "repo": os.path.basename(repo.rstrip("/")), + "rev": rev, + "rev_subject": subject, + "worktree_dirty_paths": len(dirty), + "timeout_s": timeout, + "entry_count": len(entries), + "entries": entries, + } + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--repo", default=os.path.dirname(os.path.abspath(__file__))) + ap.add_argument("--out", required=True) + ap.add_argument("--timeout", type=int, default=300) + ap.add_argument("--json", action="store_true") + args = ap.parse_args(argv) + + repo = os.path.abspath(args.repo) + names = discover(repo) + if not names: + print(f"sweep: no CHECK* entry points under {repo}") + return 2 + try: + manifest = capture_manifest(repo, names, args.timeout) + except (OSError, RuntimeError) as exc: + print(f"sweep: cannot capture manifest: {exc}", file=sys.stderr) + return 2 + + rows: list[dict] = [] + os.makedirs(args.out, exist_ok=True) + tsv = os.path.join(args.out, "results.tsv") + with open(tsv, "w", encoding="utf-8") as fh: + fh.write("entry\trc\tsecs\tclass\ttraceback\tlast_stdout\tlast_stderr\n") + fh.flush() + for name in names: + r = run_one(repo, name, args.timeout) + rows.append(r) + fh.write( + f"{r['name']}\t{r['rc']}\t{r['secs']}\t{r['class']}\t" + f"{int(r['traceback'])}\t{r['last_out']}\t{r['last_err']}\n" + ) + fh.flush() + print(f" {r['class']:<16} rc={r['rc']:<8} {r['secs']:>7.2f}s {r['name']}", + flush=True) + + counts: dict[str, int] = {} + for r in rows: + counts[r["class"]] = counts.get(r["class"], 0) + 1 + bad = [r for r in rows if r["class"] != "OK"] + + manifest["results_sha256"] = sha256_file(tsv) + manifest["counts"] = dict(sorted(counts.items())) + with open(os.path.join(args.out, "manifest.json"), "w", encoding="utf-8") as fh: + json.dump(manifest, fh, indent=2, sort_keys=True) + fh.write("\n") + + report = os.path.join(args.out, "report.md") + with open(report, "w", encoding="utf-8") as fh: + fh.write("# Root CHECK-* entry-point sweep\n\n") + fh.write(f"- measured at: `{manifest['generated_utc']}`\n") + fh.write(f"- rev: `{manifest['rev']}` -- {manifest['rev_subject']}\n") + fh.write(f"- worktree: {manifest['worktree_dirty_paths']} path(s) differ " + f"from that rev (modified + untracked); the worktree is what " + f"was graded\n") + fh.write(f"- entry points run: **{len(rows)}**\n") + for k in sorted(counts): + fh.write(f"- {k}: **{counts[k]}**\n") + fh.write(f"- total wall time: **{round(sum(r['secs'] for r in rows), 1)} s**\n\n") + fh.write("| entry | rc | s | class | last line |\n|---|---|---|---|---|\n") + for r in rows: + last = r["last_err"] if r["last_err"] != "-" else r["last_out"] + fh.write( + f"| `{r['name']}` | {r['rc']} | {r['secs']} | {r['class']} | " + f"{last.replace('|', '/')} |\n" + ) + if bad: + fh.write("\n## Not OK\n\n") + for r in bad: + fh.write(f"- `{r['name']}` rc={r['rc']} ({r['class']}): " + f"{r['last_err'] if r['last_err'] != '-' else r['last_out']}\n") + + if args.json: + print(json.dumps({"manifest": manifest, "rows": rows}, indent=2)) + else: + detail = ", ".join(f"{k} {counts[k]}" for k in sorted(counts)) + print(f"sweep: {len(rows)} entry point(s) -- {detail} " + f"({round(sum(r['secs'] for r in rows), 1)} s) at {manifest['rev'][:12]}") + return 0 if not bad else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wizard-work/volt/root-check-sweep/verify_sweep.py b/wizard-work/volt/root-check-sweep/verify_sweep.py new file mode 100644 index 0000000..c07ae3f --- /dev/null +++ b/wizard-work/volt/root-check-sweep/verify_sweep.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""verify_sweep.py -- grade the root CHECK-* sweep artifact against itself. + +WHAT THIS IS + `sweep.py` runs every root `CHECK-*` entry point and classifies its exit + code. Its verdict is only worth the bytes it measured, so this verifier + grades the artifact, not the repository: the pin (rev, timestamp, timeout), + the universe (which entry points, and their sha256), the class mapping + (rc -> class, the repo's published contract), the report (one table row per + entry, counts and rev stated), and the integrity of the row file itself. + + It does NOT re-run the entry points -- that is the sweep's job and it costs + minutes. It answers "is this artifact a faithful record of a run over the + universe it names", which is the question a reader of a table actually has. + +EXIT + 0 every leg passed; 1 a fault; 2 the artifact is not there to grade. +""" + +from __future__ import annotations + +import datetime as _dt +import hashlib +import json +import os +import re +import subprocess +import sys + +ART = os.path.dirname(os.path.abspath(__file__)) +MANIFEST = os.path.join(ART, "manifest.json") +RESULTS = os.path.join(ART, "results.tsv") +REPORT = os.path.join(ART, "report.md") + +CLASS = {0: "OK", 1: "FAULT", 2: "COULD_NOT_GRADE"} +HEX40 = re.compile(r"^[0-9a-f]{40}$") +UTC = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$") + +passed = 0 +faults: list[str] = [] + + +def leg(name: str, problems: list[str], detail: str) -> None: + global passed + if problems: + print(f" {name:<10} FAULT {'; '.join(problems[:4])}") + faults.extend(problems) + else: + print(f" {name:<10} PASS {detail}") + passed += 1 + + +def sha256_file(path: str) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def git(*args: str) -> str: + p = subprocess.run(["git", "-C", ART, *args], capture_output=True, + text=True, errors="replace") + return p.stdout.strip() if p.returncode == 0 else "" + + +def git_ok(*args: str) -> bool: + p = subprocess.run(["git", "-C", ART, *args], capture_output=True, + text=True, errors="replace") + return p.returncode == 0 + + +def read_rows() -> list[dict]: + rows = [] + with open(RESULTS, encoding="utf-8") as fh: + header = fh.readline().rstrip("\n").split("\t") + want = ["entry", "rc", "secs", "class", "traceback", + "last_stdout", "last_stderr"] + if header != want: + raise SystemExit(f"results.tsv header {header!r} != {want!r}") + for line in fh: + parts = line.rstrip("\n").split("\t") + if len(parts) != 7: + raise SystemExit(f"results.tsv row has {len(parts)} fields: {line!r}") + rows.append(dict(zip(want, parts))) + return rows + + +def main() -> int: + for path in (MANIFEST, RESULTS, REPORT): + if not os.path.isfile(path): + print(f"verify_sweep: no {path} -- cannot grade", file=sys.stderr) + return 2 + manifest = json.load(open(MANIFEST, encoding="utf-8")) + rows = read_rows() + report = open(REPORT, encoding="utf-8").read() + + # 1 -- the pin: a full rev that resolves, a real UTC stamp, a coherent count. + problems = [] + rev = str(manifest.get("rev", "")) + if not HEX40.match(rev): + problems.append(f"rev is not a full 40-hex sha: {rev!r}") + elif not git_ok("cat-file", "-e", f"{rev}^{{commit}}"): + problems.append(f"rev {rev} does not resolve in this repo") + stamp = str(manifest.get("generated_utc", "")) + if not UTC.match(stamp): + problems.append(f"generated_utc is not YYYY-MM-DDTHH:MM:SSZ: {stamp!r}") + else: + try: + _dt.datetime.fromisoformat(stamp.replace("Z", "+00:00")) + except ValueError: + problems.append(f"generated_utc does not parse: {stamp!r}") + entries = manifest.get("entries") or [] + if manifest.get("entry_count") != len(entries): + problems.append(f"entry_count {manifest.get('entry_count')} != " + f"{len(entries)} manifest entries") + if len(entries) != len(rows): + problems.append(f"manifest names {len(entries)} entries, results has {len(rows)} rows") + if not isinstance(manifest.get("timeout_s"), int) or manifest["timeout_s"] <= 0: + problems.append(f"timeout_s is not a positive int: {manifest.get('timeout_s')!r}") + leg("pin", problems, f"{len(rows)} rows at {rev[:12]} {stamp}") + + # 2 -- the universe: same names, same order, same bytes, same trackedness. + problems = [] + names = [r["entry"] for r in rows] + if names != [e.get("name") for e in entries]: + problems.append("results row order/names differ from manifest entries") + root = git("rev-parse", "--show-toplevel") or os.path.dirname(ART) + tracked = set(git("ls-tree", "-r", "--full-tree", "--name-only", rev).splitlines()) if HEX40.match(rev) else set() + for e in entries: + path = os.path.join(root, e.get("name", "")) + if not os.path.isfile(path): + problems.append(f"{e.get('name')}: file absent") + continue + if sha256_file(path) != e.get("sha256"): + problems.append(f"{e.get('name')}: sha256 moved since the sweep") + if bool(e.get("tracked_at_rev")) != (e.get("name") in tracked): + problems.append(f"{e.get('name')}: tracked_at_rev disagrees with ls-tree {rev[:8]}") + leg("universe", problems, f"{len(entries)} entry files byte-identical to the manifest pin") + + # 3 -- the class mapping: rc -> class is the contract, not a label. + problems = [] + for r in rows: + try: + rc = int(r["rc"]) + except ValueError: + problems.append(f"{r['entry']}: rc {r['rc']!r} is not an int") + continue + want = CLASS.get(rc, "UNEXPECTED") + if r["class"] != want: + problems.append(f"{r['entry']}: rc {rc} labelled {r['class']}, contract says {want}") + if r["traceback"] not in ("0", "1"): + problems.append(f"{r['entry']}: traceback flag {r['traceback']!r}") + try: + if float(r["secs"]) < 0: + problems.append(f"{r['entry']}: negative seconds") + except ValueError: + problems.append(f"{r['entry']}: secs {r['secs']!r}") + leg("classes", problems, "every rc maps to its class under the published contract") + + # 4 -- the report: one table row per entry, counts and rev as measured. + problems = [] + counts: dict[str, int] = {} + for r in rows: + counts[r["class"]] = counts.get(r["class"], 0) + 1 + table = re.findall(r"^\| `([^`]+)` \| ([^|]+?) \| ([^|]+?) \| ([A-Z_]+) \|", report, re.M) + if [t[0] for t in table] != names: + problems.append(f"report table names {len(table)} rows, results has {len(names)}") + if f"- entry points run: **{len(rows)}**" not in report: + problems.append("report does not state entry points run") + for klass, count in counts.items(): + if f"- {klass}: **{count}**" not in report: + problems.append(f"report does not state {klass} count {count}") + if rev and rev not in report: + problems.append("report does not name the pinned rev") + leg("report", problems, f"{len(table)} table rows, counts and rev agree with the rows file") + + # 5 -- substance: the rows file is the file the manifest hashed and no row + # is silent, so a green row is a process that said something. + problems = [] + if sha256_file(RESULTS) != manifest.get("results_sha256"): + problems.append("results.tsv sha256 differs from the manifest") + for r in rows: + if r["class"] in ("OK", "FAULT") and r["last_stdout"].strip() in ("", "-") \ + and r["last_stderr"].strip() in ("", "-"): + problems.append(f"{r['entry']}: {r['class']} row carries no output at all") + bad = sorted(r["entry"] for r in rows if r["class"] != "OK") + bullets = re.findall(r"^- `([^`]+)` rc=", report, re.M) + if sorted(bullets) != bad: + problems.append(f"Not-OK bullets {sorted(bullets)} != rows {bad}") + if not any(r["class"] == "OK" for r in rows): + problems.append("no OK row -- a sweep that cannot pass anything is not evidence") + leg("substance", problems, f"{len(bad)} non-OK row(s) enumerated, every row carries output") + + print() + if faults: + print(f"sweep-check: {passed} passed, {len(faults)} fault(s)") + return 1 + print(f"sweep-check: 5 leg(s), all passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main())