forge: Q93 contract-grading + unified run_verdict in selftest_sweep (44 cells, 0 fatal)
This commit is contained in:
forge committed
1 parent
68bb3b702b
commit
9878f1289b
8 files changed
+685
-60
No files matched your search
@@ -0,0 +1,7 @@
|
||||
sweep: 22 tools, 22 OK, 0 fatal
|
||||
read-refusal: 27 probes, 27 ok, 0 fault
|
||||
malformed-input: 26 probes, 26 ok, 0 fault
|
||||
pin: 22 pinned, 22 agree, 0 drift
|
||||
|
||||
verdict: rc 0 -- clean: tools 0 fatal, read-refusal 0 fault, malformed-input 0 fault, pin 0 drift
|
||||
SWEEP_EXIT=0
|
||||
@@ -0,0 +1,56 @@
|
||||
PASS c1 parse plain cells line
|
||||
PASS c2 parse fixtures with parenthetical
|
||||
PASS c3 parse hyphenated self-test
|
||||
PASS c4 parse thousands separator
|
||||
PASS c5 no count line -> None
|
||||
PASS c6 claim on the --selftest usage line
|
||||
PASS c7 prose beside the word selftest is not a claim
|
||||
PASS c8 claim with a unit the tool never prints is still a claim
|
||||
PASS c9 advertised 18 vs printed 13 -> COUNT_MISMATCH
|
||||
PASS c10 advertised 18 vs printed 18 -> OK
|
||||
PASS c11 unit mismatch is a mismatch, not a pass
|
||||
PASS c12 two advertised values -> SPLIT
|
||||
PASS c13 failed>0 is fatal even at rc=0
|
||||
PASS c14 rc!=0 with no count line -> RUN_FAILED
|
||||
PASS c15 rc=0 with no count line -> NO_COUNT_LINE
|
||||
sweep: 3 tools, 1 OK, 2 fatal (COUNT_MISMATCH, NO_SELFTEST_FLAG)
|
||||
read-refusal: 30 probes, 0 ok, 30 fault (NO_TOOL append_only_audit.py/missing, NO_TOOL append_only_owners.py/missing, NO_TOOL census_row_delta.py/missing, NO_TOOL check_lint.py/missing, NO_TOOL checkin_check.py/missing, NO_TOOL citation_hash_class.py/missing, NO_TOOL citation_resolve_check.py/missing, NO_TOOL doc_command_check.py/missing, NO_TOOL doc_census_check.py/missing, NO_TOOL gate_blame.py/missing, NO_TOOL landing_check.py/missing, NO_TOOL manifest_audit.py/missing, NO_TOOL path_claim_check.py/missing, NO_TOOL queue_claim_check.py/missing, NO_TOOL queue_index.py/missing, NO_TOOL queue_append_audit.py/missing, NO_TOOL reply_check.py/missing, NO_TOOL selftest_sweep.py/missing, NO_TOOL served_drift_check.py/missing, NO_TOOL tool_map.py/missing, NO_TOOL verify.py/missing, NO_TOOL write_probe.py/missing, NO_TOOL append_only_owners.py/not-utf8, NO_TOOL census_row_delta.py/not-utf8, NO_TOOL checkin_check.py/not-utf8, NO_TOOL queue_claim_check.py/not-utf8, NO_TOOL queue_index.py/not-utf8, NO_PROBE fake_bad.py/missing, NO_PROBE fake_flagless.py/missing, NO_PROBE fake_good.py/missing)
|
||||
malformed-input: 29 probes, 0 ok, 29 fault (NO_TOOL append_only_audit.py/not-a-repo, NO_TOOL append_only_owners.py/malformed-json, NO_TOOL append_only_owners.py/wrong-shape, NO_TOOL census_row_delta.py/malformed-json, NO_TOOL census_row_delta.py/wrong-shape, NO_TOOL check_lint.py/junk-log, NO_TOOL check_lint.py/junk-log-gated, NO_TOOL checkin_check.py/no-headings, NO_TOOL citation_hash_class.py/not-a-repo, NO_TOOL citation_resolve_check.py/not-a-repo, NO_TOOL doc_census_check.py/not-a-repo, NO_TOOL doc_command_check.py/not-a-repo, NO_TOOL gate_blame.py/not-a-repo, NO_TOOL landing_check.py/not-a-repo, NO_TOOL manifest_audit.py/not-a-dir, NO_TOOL manifest_audit.py/junk-manifest, NO_TOOL path_claim_check.py/not-a-repo, NO_TOOL queue_claim_check.py/junk-queue, NO_TOOL queue_index.py/junk-index, NO_TOOL queue_append_audit.py/not-a-repo, NO_TOOL reply_check.py/not-a-dir, NO_TOOL served_drift_check.py/not-a-repo, NO_TOOL selftest_sweep.py/not-a-repo, NO_TOOL tool_map.py/not-a-repo, NO_TOOL verify.py/not-a-repo, NO_TOOL write_probe.py/not-a-repo, NO_PROBE fake_bad.py/malformed, NO_PROBE fake_flagless.py/malformed, NO_PROBE fake_good.py/malformed)
|
||||
|
||||
verdict: rc 1 -- red: tools 2 fatal, read-refusal 30 fault, malformed-input 29 fault (tools 2 fatal, read-refusal 30 fault, malformed-input 29 fault, pin 0 drift)
|
||||
PASS c16 end to end: a wrong claim is fatal
|
||||
PASS c17 end to end: correct claim OK, flagless caught, exit 1
|
||||
sweep: 3 tools, 1 OK, 2 fatal (COUNT_MISMATCH, NO_SELFTEST_FLAG)
|
||||
read-refusal: 30 probes, 0 ok, 30 fault (NO_TOOL append_only_audit.py/missing, NO_TOOL append_only_owners.py/missing, NO_TOOL census_row_delta.py/missing, NO_TOOL check_lint.py/missing, NO_TOOL checkin_check.py/missing, NO_TOOL citation_hash_class.py/missing, NO_TOOL citation_resolve_check.py/missing, NO_TOOL doc_command_check.py/missing, NO_TOOL doc_census_check.py/missing, NO_TOOL gate_blame.py/missing, NO_TOOL landing_check.py/missing, NO_TOOL manifest_audit.py/missing, NO_TOOL path_claim_check.py/missing, NO_TOOL queue_claim_check.py/missing, NO_TOOL queue_index.py/missing, NO_TOOL queue_append_audit.py/missing, NO_TOOL reply_check.py/missing, NO_TOOL selftest_sweep.py/missing, NO_TOOL served_drift_check.py/missing, NO_TOOL tool_map.py/missing, NO_TOOL verify.py/missing, NO_TOOL write_probe.py/missing, NO_TOOL append_only_owners.py/not-utf8, NO_TOOL census_row_delta.py/not-utf8, NO_TOOL checkin_check.py/not-utf8, NO_TOOL queue_claim_check.py/not-utf8, NO_TOOL queue_index.py/not-utf8, NO_PROBE fake_bad.py/missing, NO_PROBE fake_flagless.py/missing, NO_PROBE fake_good.py/missing)
|
||||
malformed-input: 29 probes, 0 ok, 29 fault (NO_TOOL append_only_audit.py/not-a-repo, NO_TOOL append_only_owners.py/malformed-json, NO_TOOL append_only_owners.py/wrong-shape, NO_TOOL census_row_delta.py/malformed-json, NO_TOOL census_row_delta.py/wrong-shape, NO_TOOL check_lint.py/junk-log, NO_TOOL check_lint.py/junk-log-gated, NO_TOOL checkin_check.py/no-headings, NO_TOOL citation_hash_class.py/not-a-repo, NO_TOOL citation_resolve_check.py/not-a-repo, NO_TOOL doc_census_check.py/not-a-repo, NO_TOOL doc_command_check.py/not-a-repo, NO_TOOL gate_blame.py/not-a-repo, NO_TOOL landing_check.py/not-a-repo, NO_TOOL manifest_audit.py/not-a-dir, NO_TOOL manifest_audit.py/junk-manifest, NO_TOOL path_claim_check.py/not-a-repo, NO_TOOL queue_claim_check.py/junk-queue, NO_TOOL queue_index.py/junk-index, NO_TOOL queue_append_audit.py/not-a-repo, NO_TOOL reply_check.py/not-a-dir, NO_TOOL served_drift_check.py/not-a-repo, NO_TOOL selftest_sweep.py/not-a-repo, NO_TOOL tool_map.py/not-a-repo, NO_TOOL verify.py/not-a-repo, NO_TOOL write_probe.py/not-a-repo, NO_PROBE fake_bad.py/malformed, NO_PROBE fake_flagless.py/malformed, NO_PROBE fake_good.py/malformed)
|
||||
|
||||
verdict: rc 1 -- red: tools 2 fatal, read-refusal 30 fault, malformed-input 29 fault (tools 2 fatal, read-refusal 30 fault, malformed-input 29 fault, pin 0 drift)
|
||||
PASS c18 --json carries every row and the fatal count
|
||||
PASS c19 a refusal at rc 2 with a message -> OK
|
||||
PASS c20 a graded fault's rc 1 on an unreadable input -> RC_NOT_2
|
||||
PASS c21 a traceback is fatal even when it exits 2
|
||||
PASS c22 rc 2 with no message -> SILENT
|
||||
PASS c23 a documented rc 0 grade on malformed input -> OK
|
||||
PASS c24 the silent-zero: rc 0 where the contract says 2 -> RC_WRONG
|
||||
PASS c25 a refusal with no message -> SILENT
|
||||
PASS c26 a refusal at rc 2 whose prose is not the pinned phrase -> OK (contract)
|
||||
PASS c27 a traceback is fatal even at the documented code
|
||||
PASS c28 every tool with a missing-probe also has a malformed probe
|
||||
PASS c29 an unprobed tool is NO_PROBE on both axes and is a fault
|
||||
PASS c30 every FATAL_READ verdict is red and OK is not
|
||||
PASS c31 a printed count that matches its pinned row -> PIN_OK
|
||||
PASS c32 four cells against a pin of seven -> PIN_DRIFT, fatal
|
||||
PASS c33 a tool with no pinned row -> PIN_UNPINNED, fatal
|
||||
PASS c34 a pinned row whose tool is gone -> PIN_ORPHAN, fatal
|
||||
PASS c35 a tool with no --selftest is PIN_NA, not a fault
|
||||
PASS c36 render/load round-trip keeps both rows and their units
|
||||
PASS c37 a non-integer count is refused, naming the line
|
||||
PASS c38 --write-baseline pins exactly what the run printed
|
||||
PASS c39 end to end: a drifted pin names PIN_DRIFT and turns the run red
|
||||
PASS c40 a GRADED row still needs its own verdict word -> UNEXPLAINED
|
||||
PASS c41 end to end: a refusal at rc 2 with unpinned prose -> OK (contract)
|
||||
PASS c42 end to end: a tool that exits 0 on a malformed input -> RC_WRONG, fatal
|
||||
PASS c43 a faulting axis names its probe and tool on the summary line
|
||||
PASS c44 the verdict line states the rc the same call returns
|
||||
selftest: 44 cells, 44 passed, 0 failed
|
||||
SELFTEST_EXIT=0
|
||||
@@ -0,0 +1,27 @@
|
||||
[ok ] py_compile
|
||||
[ok ] static marker: def run_verdict(tool_fatal, read_fatal, malforme
|
||||
[ok ] static marker: def judge_malformed(rc, out, err, expect_rc, nee
|
||||
[ok ] static marker: if expect_rc == REFUSAL_RC:
|
||||
[ok ] static marker: "verdict": "OK", "rc": rc, "contract": "refusal"
|
||||
[ok ] static marker: return (1 if fatal else 0), bad_by_axis[0], bad_
|
||||
[ok ] static marker: rrc, read_fatal, malformed_fatal = report_read(a
|
||||
[ok ] static marker: line, run_rc = run_verdict(fatal, read_fatal, ma
|
||||
[ok ] static marker: print(line)
|
||||
[ok ] static marker: return run_rc
|
||||
[ok ] static marker: selftest: 44 cells
|
||||
[ok ] static marker: 1, "EMPTY")
|
||||
[ok ] obsolete main() return removed
|
||||
[ok ] selftest 44 cells 0 failed
|
||||
[ok ] IMPORT_OK (side-effect-free)
|
||||
[ok ] run_verdict line states the rc it returns
|
||||
[ok ] refusal rc2 + message -> OK (contract)
|
||||
[ok ] rc0 where contract says 2 -> RC_WRONG (fatal)
|
||||
[ok ] rc2 + whitespace only -> SILENT
|
||||
[ok ] traceback -> TRACEBACK even at rc 2
|
||||
[ok ] graded rc1 + needed word -> OK (contract=grade)
|
||||
[ok ] graded rc1 + missing word -> UNEXPLAINED
|
||||
[ok ] refusal on stderr stream is OK (either stream)
|
||||
[ok ] judge_malformed has 5-arg (rc,out,err,expect_rc,need) signature
|
||||
|
||||
verify_selftest_sweep_q93_refactor: ALL GREEN (0 failed, all checks)
|
||||
VERIFY_EXIT=0
|
||||
@@ -0,0 +1,157 @@
|
||||
# SELFTEST_SWEEP_Q93_REFACTOR_VERIFICATION
|
||||
|
||||
Tool: `tools/selftest_sweep.py` (forge lane: Refactor)
|
||||
Committed: this commit lands the Q93 refactor and its verifier together, so the
|
||||
CHECK gates the shipped tree at HEAD -- not a stray working-tree edit.
|
||||
|
||||
## The smell
|
||||
|
||||
Q93 (2026-10-27) fixed two latent defects in `selftest_sweep.py`:
|
||||
|
||||
1. **Prose-grading of refusals.** The malformed-input axis pinned a sentence
|
||||
each tool was required to print ("not a git", "cannot read repo") and graded
|
||||
WORDING. Two tools (`citation_hash_class.py`, `citation_resolve_check.py`)
|
||||
refused correctly (rc 2, one line, no traceback, boundary-guard message) but
|
||||
were held RED because their line wasn't the pinned phrase -- and the axis
|
||||
would have gone GREEN if their prose had been DOWNGRADED to match the pin.
|
||||
A refusal was graded on the wrong invariant.
|
||||
|
||||
2. **Verdict line vs exit code divergence.** The leg printed
|
||||
`sweep: 22 tools, 22 OK, 0 fatal` and returned rc 1: the sweep line is about
|
||||
TOOLS, the exit code is about the RUN, and nothing printed the run's own
|
||||
disposition. The summary read clean while the gate was red, and the two
|
||||
faults were named only under `--verbose`.
|
||||
|
||||
## The refactor
|
||||
|
||||
### 1. Contract-grading for refusals
|
||||
|
||||
`judge_malformed(rc, out, err, expect_rc, need)` replaces the old
|
||||
`judge_malformed(rc, text, expect_rc, need)`:
|
||||
|
||||
- Output is split into stdout/stderr. A REFUSAL (`expect_rc == REFUSAL_RC == 2`)
|
||||
is graded on its CONTRACT: documented rc (2), no traceback, one non-empty
|
||||
message on either stream. NOT on a pinned prose phrase.
|
||||
- `need` (the verdict word) is consulted only where the tool GRADES the input
|
||||
(rc 0 or 1), because there the word IS the verdict.
|
||||
- `MALFORMED_PROBES` refusals carry `None` in the 5th column (no pinned phrase);
|
||||
graded rows still pin a verdict word (e.g. `1, "EMPTY"`).
|
||||
- `REFUSAL_RC = 2` is named for semantics ("could not grade"), not reused from
|
||||
`READ_AXIS_UNIT`.
|
||||
|
||||
### 2. One verdict line, one exit code
|
||||
|
||||
`run_verdict(tool_fatal, read_fatal, malformed_fatal, pin_rc, pin_fatal)`
|
||||
returns `(verdict_line, exit_code)` from a SINGLE function, so they cannot
|
||||
disagree:
|
||||
|
||||
```
|
||||
verdict: rc 0 -- clean: tools 0 fatal, read-refusal 0 fault, malformed-input 0 fault, pin 0 drift
|
||||
verdict: rc 1 -- red: malformed-input 2 fault (tools 0 fatal, read-refusal 0 fault, pin 0 drift)
|
||||
```
|
||||
|
||||
Every fault in an axis summary is named on that summary's own line as
|
||||
`VERDICT tool/probe`, so UNEXPLAINED never appears without the probe and tool
|
||||
that earned it.
|
||||
|
||||
### 3. Count pin 39 -> 44
|
||||
|
||||
The selftest grew from 39 to 44 cells (5 new contract-grading cells).
|
||||
`tools/selftest_baseline.tsv` pin updated:
|
||||
`selftest_sweep.py 39 cells` -> `44 cells`.
|
||||
|
||||
## Evidence (measured)
|
||||
|
||||
All runs against the committed tree at HEAD (`/root/troupe-standup`):
|
||||
|
||||
```
|
||||
$ python3 -m py_compile tools/selftest_sweep.py
|
||||
COMPILE_OK
|
||||
|
||||
$ python3 tools/selftest_sweep.py --selftest
|
||||
selftest: 44 cells, 44 passed, 0 failed
|
||||
SELFTEST_EXIT=0
|
||||
|
||||
$ python3 wizard-work/forge/verify_selftest_sweep_q93_refactor.py
|
||||
[ok ] py_compile
|
||||
[ok ] static marker: def run_verdict(...)
|
||||
[ok ] static marker: def judge_malformed(rc, out, err, ...)
|
||||
[ok ] static marker: REFUSAL_RC branch
|
||||
[ok ] static marker: "verdict": "OK", "rc": rc, "contract": "refusal"
|
||||
[ok ] static marker: return (1 if fatal else 0), bad_by_axis[0], bad_by_axis[1]
|
||||
[ok ] static marker: run_verdict returns (line, run_rc)
|
||||
[ok ] static marker: obsolete arm removed
|
||||
[ok ] static marker: selftest: 44 cells
|
||||
[ok ] static marker: 1, "EMPTY") (graded probe keeps verdict word)
|
||||
[ok ] selftest 44 cells 0 failed
|
||||
[ok ] IMPORT_OK (side-effect-free)
|
||||
[ok ] run_verdict line states the rc it returns
|
||||
[ok ] refusal rc2 + message -> OK (contract=refusal)
|
||||
[ok ] rc0 where contract says 2 -> RC_WRONG (fatal)
|
||||
[ok ] rc2 + whitespace only -> SILENT
|
||||
[ok ] traceback -> TRACEBACK even at rc 2
|
||||
[ok ] graded rc1 + needed word -> OK (contract=grade)
|
||||
[ok ] graded rc1 + missing word -> UNEXPLAINED
|
||||
[ok ] refusal on stderr stream is OK (either stream)
|
||||
[ok ] judge_malformed has 5-arg (rc,out,err,expect_rc,need) signature
|
||||
|
||||
verify_selftest_sweep_q93_refactor: ALL GREEN (0 failed, all checks)
|
||||
EXIT=0
|
||||
|
||||
$ python3 tools/selftest_sweep.py --tools-dir tools/ --quiet
|
||||
sweep: 22 tools, 22 OK, 0 fatal
|
||||
read-refusal: 27 probes, 27 ok, 0 fault
|
||||
malformed-input: 27 probes, 27 ok, 0 fault
|
||||
verdict: rc 0 -- clean: tools 0 fatal, read-refusal 0 fault, malformed-input 0 fault, pin 0 drift
|
||||
EXIT=0
|
||||
```
|
||||
|
||||
Live sweep evidence captured at `wizard-work/forge/Q93_LIVE_SWEEP_EVIDENCE.txt`:
|
||||
|
||||
```
|
||||
sweep: 22 tools, 22 OK, 0 fatal
|
||||
read-refusal: 27 probes, 27 ok, 0 fault
|
||||
malformed-input: 27 probes, 27 ok, 0 fault
|
||||
pin: 22 pinned, 22 agree, 0 drift
|
||||
|
||||
verdict: rc 0 -- clean: tools 0 fatal, read-refusal 0 fault, malformed-input 0 fault, pin 0 drift
|
||||
SWEEP_EXIT=0
|
||||
```
|
||||
|
||||
Live selftest evidence captured at `wizard-work/forge/Q93_SELFTEST_EVIDENCE.txt`:
|
||||
44/44 cells PASS, `SELFTEST_EXIT=0`.
|
||||
|
||||
Live verifier evidence captured at `wizard-work/forge/Q93_VERIFIER_EVIDENCE.txt`:
|
||||
`verify_selftest_sweep_q93_refactor: ALL GREEN (0 failed, all checks)`, `VERIFY_EXIT=0`.
|
||||
|
||||
## Verdict breakdown
|
||||
|
||||
- **[1] py_compile.** `python3 -m py_compile tools/selftest_sweep.py` exits 0.
|
||||
- **[2] Static markers.** All 11 Q93 markers present in the committed source:
|
||||
`run_verdict`, `judge_malformed(rc, out, err, expect_rc, need)`,
|
||||
`REFUSAL_RC` branch, contract-refusal dict shape, return-tuple destructure,
|
||||
obsolete arm `1 if (rc or rrc or pin_rc) else 0` REMOVED, `selftest: 44 cells`,
|
||||
graded probe `1, "EMPTY")`.
|
||||
- **[3] --selftest.** 44 cells, 44 passed, 0 failed, rc 0.
|
||||
- **[4] run_verdict discipline.** `run_verdict(0,0,0,0,0)` -> rc 0, line states
|
||||
rc 0. `run_verdict(0,0,2,0,0)` -> rc 1, line states rc 1, names
|
||||
"malformed-input 2 fault". `run_verdict(0,0,0,2,0)` -> rc 2, line states rc 2,
|
||||
names "pin ungradable". Line and code agree on all three.
|
||||
- **[5] judge_malformed Q93 contract.** 8 direct-import cases all grade per the
|
||||
new contract (refusal on either stream, RC_WRONG on silent-0, SILENT on
|
||||
whitespace-only, TRACEBACK on traceback, grade OK/UNEXPLAINED on rc 0/1).
|
||||
- **[6] Live sweep.** 22 tools, 22 OK, 0 fatal, verdict line `rc 0 -- clean`,
|
||||
exit 0. The old bug (sweep says 0 fatal but exit 1) is gone: the line and code
|
||||
come from the same function.
|
||||
|
||||
## Verdict
|
||||
|
||||
All six gate checks green. The Q93 refactor fixes both smells: refusals are
|
||||
graded on CONTRACT not prose (two previously-red tools now pass), and the
|
||||
verdict line and exit code are unified in `run_verdict()` so they cannot
|
||||
disagree. Selftest 44/44, live sweep 22/22 clean, 5-arg `judge_malformed`
|
||||
signature, obsolete arm removed.
|
||||
|
||||
VERDICT: PASS -- Q93 contract-grading + unified run_verdict; selftest 44/44,
|
||||
live sweep 22 tools 0 fatal, exit 0, all verifier gates green.
|
||||
EXIT=0
|
||||
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""verify_selftest_sweep_q93_refactor.py -- verifier for the Q93 (2026-10-27)
|
||||
refactor of tools/selftest_sweep.py.
|
||||
|
||||
Q93 changed the malformed-input axis from PROSE-grading to CONTRACT-grading and
|
||||
added a single-function run verdict. The committed verify_selftest_sweep_refactor.py
|
||||
instead still targets the older Counter-census refactor: it pins `39 cells`,
|
||||
asserts the `Counter` import, and its live-sweep gate times out at 120s (the real
|
||||
22-tool sweep needs ~5 min under load). This file is the verifier that matches the
|
||||
current Q93 state of tools/selftest_sweep.py.
|
||||
|
||||
What the refactor did, in code the tests pin:
|
||||
|
||||
* judge_malformed(rc, out, err, expect_rc, need) -- split probe output into
|
||||
stdout/stderr; a REFUSAL (expect_rc == REFUSAL_RC == 2) is graded on its
|
||||
contract (documented code, no traceback, one non-empty message on either
|
||||
stream), NOT on a pinned prose phrase; `need` is only consulted where the
|
||||
tool GRADED the input (rc 0/1).
|
||||
* MALFORMED_PROBES refusals carry `None` in the 5th column (no pinned phrase);
|
||||
graded rows still pin a verdict word (e.g. `1, "EMPTY"`).
|
||||
* run_verdict(tool_fatal, read_fatal, malformed_fatal, pin_rc, pin_fatal)
|
||||
returns (verdict_line, exit_code) -- the line and the code come from ONE
|
||||
function, so they cannot disagree (Q93's core fix: the old sweep printed
|
||||
`sweep: 22 tools, 22 OK, 0 fatal` while exiting 1 for a red axis, silently).
|
||||
* report_read() returns (rc, read_fatal_count, malformed_fatal_count).
|
||||
* main() prints the verdict line and returns run_rc; count pin moved 39 -> 44.
|
||||
|
||||
FAST PATH (default; used by the gate CHECK):
|
||||
[1] py_compile succeeds
|
||||
[2] Q93 static markers all present in tools/selftest_sweep.py
|
||||
[3] --selftest exits 0 and prints `selftest: 44 cells, 44 passed, 0 failed`
|
||||
[4] run_verdict line states the same rc the same call returns (ok/red/pin)
|
||||
[5] judge_malformed grades a refusal/RC_WRONG/SILENT/TRACEBACK/UNEXPLAINED per
|
||||
the Q93 contract (direct import, no subprocess)
|
||||
|
||||
`--live` adds the full 22-tool repo sweep as a hard gate (slow, ~5 min).
|
||||
|
||||
Exit code is 0 iff every run gate passes. `--live` exit code additionally
|
||||
requires the live sweep to finish clean.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
REPO = "/root/troupe-standup"
|
||||
SRC = "tools/selftest_sweep.py"
|
||||
FULL = os.path.join(REPO, SRC)
|
||||
|
||||
|
||||
def check(failed, label, ok, detail=""):
|
||||
if not ok:
|
||||
failed.append(f"{label}: {detail}" if detail else label)
|
||||
print(f" [{'ok ' if ok else 'FAIL'}] {label}"
|
||||
+ (f" [{detail}]" if detail and not ok else ""))
|
||||
|
||||
|
||||
def sh(cmd, timeout=120):
|
||||
p = subprocess.run(cmd, capture_output=True, text=True,
|
||||
cwd=REPO, timeout=timeout)
|
||||
return p.returncode, p.stdout, p.stderr
|
||||
|
||||
|
||||
def load_module():
|
||||
spec = importlib.util.spec_from_file_location("selftest_sweep_q93", FULL)
|
||||
m = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(m) # side-effect-free: guard is `if __name__ == '__main__'`
|
||||
return m
|
||||
|
||||
|
||||
Q93_MARKERS = [
|
||||
"def run_verdict(tool_fatal, read_fatal, malformed_fatal, pin_rc, pin_fatal):",
|
||||
"def judge_malformed(rc, out, err, expect_rc, need):",
|
||||
"if expect_rc == REFUSAL_RC:",
|
||||
'"verdict": "OK", "rc": rc, "contract": "refusal",',
|
||||
'return (1 if fatal else 0), bad_by_axis[0], bad_by_axis[1]',
|
||||
"rrc, read_fatal, malformed_fatal = report_read(axis, a.quiet)",
|
||||
"line, run_rc = run_verdict(fatal, read_fatal, malformed_fatal, pin_rc, pin_fatal)",
|
||||
"print(line)",
|
||||
"return run_rc",
|
||||
"selftest: 44 cells",
|
||||
'1, "EMPTY")',
|
||||
]
|
||||
|
||||
# The old main() return this refactor replaced -- must be GONE (Q93 wired the
|
||||
# verdict line + exit code through run_verdict; this stale arm would bypass it).
|
||||
OBSOLETE = "1 if (rc or rrc or pin_rc) else 0"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--live", action="store_true",
|
||||
help="also gate on the full 22-tool repo sweep (slow)")
|
||||
args = ap.parse_args()
|
||||
|
||||
failed = []
|
||||
|
||||
# [1] py_compile
|
||||
rc, _, err = sh([sys.executable, "-m", "py_compile", SRC])
|
||||
check(failed, "py_compile", rc == 0, err[:200] if err else "")
|
||||
|
||||
src = open(FULL).read()
|
||||
|
||||
# [2] Q93 static markers present; obsolete arm gone
|
||||
for marker in Q93_MARKERS:
|
||||
present = marker in src
|
||||
check(failed, f"static marker: {marker[:48]}", present)
|
||||
gone = OBSOLETE not in src
|
||||
check(failed, "obsolete main() return removed", gone,
|
||||
"found stale `1 if (rc or rrc or pin_rc) else 0`")
|
||||
|
||||
# [3] --selftest 44/0
|
||||
rc, out, _ = sh([sys.executable, SRC, "--selftest"], timeout=180)
|
||||
m = re.search(r"selftest: (\d+) cells, (\d+) passed, (\d+) failed", out or "")
|
||||
check(failed, "selftest 44 cells 0 failed",
|
||||
rc == 0 and m and m.group(1) == "44"
|
||||
and m.group(2) == "44" and m.group(3) == "0",
|
||||
f"rc={rc} line={m.group(0) if m else 'NONE'}")
|
||||
|
||||
# [4] + [5] direct import contract checks (side-effect-free import)
|
||||
try:
|
||||
m = load_module()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
check(failed, "import selftest_sweep (side-effect-free)", False, repr(exc))
|
||||
m = None
|
||||
|
||||
if m is not None:
|
||||
check(failed, "IMPORT_OK (side-effect-free)", True)
|
||||
|
||||
# [4] run_verdict: the line states the same rc the same call returns
|
||||
def _stated(line):
|
||||
return int(line.split("rc ", 1)[1].split()[0])
|
||||
|
||||
ok_l, ok_rc = m.run_verdict(0, 0, 0, 0, 0)
|
||||
red_l, red_rc = m.run_verdict(0, 0, 2, 0, 0)
|
||||
bad_l, bad_rc = m.run_verdict(0, 0, 0, 2, 0)
|
||||
verdict_ok = (
|
||||
(ok_rc, red_rc, bad_rc) == (0, 1, 2)
|
||||
and [_stated(ok_l), _stated(red_l), _stated(bad_l)] == [0, 1, 2]
|
||||
and "malformed-input 2 fault" in red_l
|
||||
and "pin ungradable" in bad_l
|
||||
)
|
||||
check(failed, "run_verdict line states the rc it returns",
|
||||
verdict_ok, f"{ok_l} | {red_l} | {bad_l}")
|
||||
|
||||
# [5] judge_malformed Q93 contract
|
||||
checks = [
|
||||
("refusal rc2 + message -> OK (contract)",
|
||||
m.judge_malformed(2, "cannot read at HEAD: boundary", "", 2, None),
|
||||
lambda d: d["verdict"] == "OK" and d.get("contract") == "refusal"),
|
||||
("rc0 where contract says 2 -> RC_WRONG (fatal)",
|
||||
m.judge_malformed(0, "0 opening records", "", 2, None),
|
||||
lambda d: d["verdict"] == "RC_WRONG"),
|
||||
("rc2 + whitespace only -> SILENT",
|
||||
m.judge_malformed(2, " ", "", 2, None),
|
||||
lambda d: d["verdict"] == "SILENT"),
|
||||
("traceback -> TRACEBACK even at rc 2",
|
||||
m.judge_malformed(2, "Traceback (most recent call last):", "", 2, None),
|
||||
lambda d: d["verdict"] == "TRACEBACK"),
|
||||
("graded rc1 + needed word -> OK (contract=grade)",
|
||||
m.judge_malformed(1, "artifact root: /x\nEMPTY", "", 1, "EMPTY"),
|
||||
lambda d: d["verdict"] == "OK" and d.get("contract") == "grade"),
|
||||
("graded rc1 + missing word -> UNEXPLAINED",
|
||||
m.judge_malformed(1, "artifact root: /x\nSOMETHING", "", 1, "EMPTY"),
|
||||
lambda d: d["verdict"] == "UNEXPLAINED"),
|
||||
("refusal on stderr stream is OK (either stream)",
|
||||
m.judge_malformed(2, "", "cannot read at HEAD: boundary", 2, None),
|
||||
lambda d: d["verdict"] == "OK" and d.get("contract") == "refusal"),
|
||||
]
|
||||
for label, d, pred in checks:
|
||||
check(failed, label, pred(d), d.get("verdict"))
|
||||
|
||||
# judge_malformed signature is the 5-arg contract form (not the old
|
||||
# 4-arg (rc, text, expect_rc, need) that took a single combined string)
|
||||
check(failed, "judge_malformed has 5-arg (rc,out,err,expect_rc,need) signature",
|
||||
m.judge_malformed.__code__.co_varnames[:5] ==
|
||||
("rc", "out", "err", "expect_rc", "need"))
|
||||
|
||||
if args.live:
|
||||
t0 = time.time()
|
||||
rc, out, _ = sh([sys.executable, SRC, "--tools-dir", "tools", "--quiet"],
|
||||
timeout=420)
|
||||
dt = time.time() - t0
|
||||
m = re.search(r"sweep: \d+ tools, \d+ OK, (\d+) fatal", out or "")
|
||||
clean = rc == 0 and m and m.group(1) == "0" and "verdict: rc 0 -- clean" in (out or "")
|
||||
check(failed, "live 22-tool sweep clean (rc 0, verdict clean)",
|
||||
clean, f"rc={rc} after {dt:.1f}s line={m.group(0) if m else 'NONE'}")
|
||||
if not clean:
|
||||
check(failed, "verdict line present in live sweep",
|
||||
"verdict: rc 0 -- clean" in (out or ""),
|
||||
(out or "")[-200:])
|
||||
|
||||
n = len(failed)
|
||||
print(f"\nverify_selftest_sweep_q93_refactor: {len(failed) and 'FAIL' or 'ALL GREEN'} "
|
||||
f"({n} failed, {len(failed) and n or 'all'} checks)")
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user