diff --git a/CHECK b/CHECK new file mode 100755 index 0000000..b47b3b9 --- /dev/null +++ b/CHECK @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# CHECK -- Q93 refactor of tools/selftest_sweep.py (forge: Refactor lane). +# Exit 0 iff the committed tree passes the Q93 verifier: py_compile, static +# Q93 markers present (run_verdict, 5-arg judge_malformed, REFUSAL_RC branch, +# obsolete arm removed, selftest: 44 cells), --selftest 44/44 rc 0, and +# run_verdict's printed rc matches the code it returns. +# Repos-rooted so it runs from any working directory. + +set -u +cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1 +REPO_ROOT="$(pwd)" +cd "$REPO_ROOT" || exit 1 + +# The verifier script lives in the committed wizard-work/forge/ tree and +# grades tools/selftest_sweep.py from the filesystem. Exit code IS the verdict. +python3 "$REPO_ROOT/wizard-work/forge/verify_selftest_sweep_q93_refactor.py" diff --git a/tools/selftest_baseline.tsv b/tools/selftest_baseline.tsv index 53d4e5d..f6bb6df 100644 --- a/tools/selftest_baseline.tsv +++ b/tools/selftest_baseline.tsv @@ -26,7 +26,7 @@ queue_append_audit.py 25 cells queue_claim_check.py 24 fixtures queue_index.py 58 cells reply_check.py 49 cells -selftest_sweep.py 39 cells +selftest_sweep.py 44 cells served_drift_check.py 25 cells tool_map.py 20 cells verify.py 59 cells diff --git a/tools/selftest_sweep.py b/tools/selftest_sweep.py index 5ce9f26..c710643 100644 --- a/tools/selftest_sweep.py +++ b/tools/selftest_sweep.py @@ -85,8 +85,38 @@ THE MALFORMED-INPUT CLASS, ADDED 2026-10-18 a junk log MISSING (rc 0 in report mode, rc 1 under --fail-on-fault), `checkin_check` grades a heading-less check-in as an advisory (rc 0, stated in its docstring) and `queue_claim_check` reports 0 opening records (rc 0, - its --fail-on-stale is the gate). The axis asserts the message too, so a - silent 0 cannot pass as a refusal. + its --fail-on-stale is the gate). A silent 0 cannot pass as a refusal: every + probe must print something. + +WHAT A REFUSAL IS GRADED ON, FIXED 2026-10-27 (Q93) + Until this date each probe carried a phrase its tool was required to print + ("not a git", "cannot read repo"), so the axis graded WORDING. Two tools that + refused correctly -- rc 2, one line, no traceback -- were held red because + their message was the boundary guard ("... at HEAD: Stopping at filesystem + boundary") instead of the phrase the table pinned, and the axis would have + gone green if either tool's prose had been degraded to match. A refusal is + now graded on the CONTRACT: the documented rc, no traceback, and a non-empty + message. Where a tool GRADES a malformed input instead (rc 0 or 1 with a + verdict word) the word is still required, because there the word IS the + verdict -- MISSING, EMPTY, "0 opening records". Which STREAM carries a + refusal is not part of the contract either, and that is measured rather than + assumed: `tool_map.py` prints its refusal on stdout (100 B, rc 2) while the + two citation tools print on stderr (139 B, rc 2). The trade is stated in the + table's own comment: the contract keeps "refused, legibly, at the documented + code" and stops keeping "refused for the reason the table remembers". + +ONE VERDICT LINE, ADDED 2026-10-27 (Q93) + The leg printed `sweep: 22 tools, 22 OK, 0 fatal` and returned **rc 1**: the + sweep line is about the TOOLS, the exit code is about the RUN, and nothing + printed the run's own disposition -- so the summary read clean while the gate + was red, and the two faults were named only under --verbose. Every run now + ends with one line from `run_verdict()`, the same function whose return value + is the exit code, so the line and the code cannot disagree: + `verdict: rc 0 -- clean: tools 0 fatal, read-refusal 0 fault, ...` + `verdict: rc 1 -- red: malformed-input 2 fault (tools 0 fatal, ...)` + and every fault in an axis summary is named on that summary's own line as + `VERDICT tool/probe`, so UNEXPLAINED never appears without the probe and tool + that earned it. THE COUNT PIN, ADDED 2026-10-22 -- the docstring was not a witness Every verdict above compares a tool's printed count with a number in the SAME @@ -133,7 +163,7 @@ USAGE: python3 selftest_sweep.py --repo /root/troupe-standup python3 selftest_sweep.py --repo /root/troupe-standup --write-baseline tools/selftest_baseline.tsv python3 selftest_sweep.py --tools-dir /tmp/fake --json /tmp/sweep.json - python3 selftest_sweep.py --selftest # 39 cells, parser, sweep, pin and both input axes + python3 selftest_sweep.py --selftest # 44 cells, parser, sweep, pin and both input axes """ from __future__ import annotations @@ -194,7 +224,14 @@ PIN_FAULT = ("PIN_DRIFT", "PIN_UNPINNED", "PIN_ORPHAN") # rc 3 (the two queue readers), so "1" meant both "bad verdict" and "no # verdict" and an operator could not tell them apart. The axis re-measures the # class on every tool, from the outside, against the real CLI. -READ_AXIS_UNIT = 2 +# The refusal code, named for what it MEANS in this contract (2026-10-27, +# Q93): rc 2 is "could not grade", which for a malformed input is a refusal. +# `judge_malformed` reads this constant to decide whether a probe is graded on +# the REFUSAL contract or on the verdict WORD its tool prints, so the branch +# says what it means instead of reusing the read axis's unit value. +REFUSAL_RC = 2 +READ_AXIS_UNIT = REFUSAL_RC + # NO_PROBE is fatal, not advisory (2026-10-18): a tool the tables never name is # a tool neither axis graded, and the pre-fix sweep reported a 15th tool with no # probe entry as "15 tools, 14 OK, 0 fatal ... rc 0" -- two NO_PROBE rows and a @@ -247,15 +284,23 @@ NOT_UTF8_PROBES = ( # THE THIRD CLASS (2026-10-18): input that decodes as UTF-8 but cannot be made # sense of. Each entry carries the answer the TOOL's own contract states -- -# (tool, label, argv, expected rc, a substring the output must contain) -- so +# (tool, label, argv, expected rc, a verdict WORD the tool must print) -- so # the axis measures the documented behaviour instead of imposing one. Most # tools must refuse at rc 2; the ones that grade a malformed input are listed # with the rc and the word their grade prints, which is what makes a silent # rc 0 (the doc_census_check --tools bug class) fail here. -# The counts are deliberately NOT written here: a number in a comment has no -# reader, which is the defect class this file's own docstring names, and the -# two counts that used to sit on these lines were both stale ("eleven", -# "Fourteen") before gate_blame.py's row was added on 2026-10-20. +# +# THE FIFTH COLUMN IS `None` FOR A REFUSAL (Q93, 2026-10-27). It used to carry +# a sentence each refuser was required to print ("not a git", "cannot read +# repo"), which graded WORDING: two tools that refused correctly at rc 2 were +# held red because their line was the boundary guard, and the axis would have +# gone green if their prose had been degraded to match. A refusal is graded on +# its CONTRACT -- the documented rc, no traceback, a non-empty message -- and +# the word is required only where the tool GRADES the input (rc 0/1), because +# there the word is the verdict. The trade: the contract keeps "refused, +# legibly, at the documented code" and no longer keeps "refused for the reason +# the table remembers"; each tool's own --selftest is what grades the reason. +# # Fixtures: {notrepo} = a directory with a file and no .git; {badjson} = a # UTF-8 file that is not JSON; {emptyjson} = "{}" (valid JSON, wrong shape); # {junklog} = a log with no CHECK: line; {junkqueue} = 3 prose lines, no @@ -265,15 +310,15 @@ NOT_UTF8_PROBES = ( # refusal -- the shape matters, which is why it is not {bin}). MALFORMED_PROBES = ( ("append_only_audit.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "cannot read repo"), + 2, None), ("append_only_owners.py", "malformed-json", ["{badjson}"], - 2, "cannot read audit"), + 2, None), ("append_only_owners.py", "wrong-shape", ["{emptyjson}"], - 2, "no 'rows' list"), + 2, None), ("census_row_delta.py", "malformed-json", ["{badjson}", "{badjson}"], - 2, "cannot read census"), + 2, None), ("census_row_delta.py", "wrong-shape", ["{emptyjson}", "{emptyjson}"], - 2, "no 'reports' list"), + 2, None), ("check_lint.py", "junk-log", ["--log", "{junklog}"], 0, "MISSING"), ("check_lint.py", "junk-log-gated", ["--log", "{junklog}", "--fail-on-fault"], @@ -281,19 +326,19 @@ MALFORMED_PROBES = ( ("checkin_check.py", "no-headings", ["--dir", "{badcheckins}", "--roster", "volt"], 0, "advisory"), ("citation_hash_class.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "not a git"), + 2, None), ("citation_resolve_check.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "not a git"), + 2, None), ("doc_census_check.py", "not-a-repo", ["--repo", "{notrepo}", "--no-run"], - 2, "no tools found"), + 2, None), ("doc_command_check.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "cannot read repo"), + 2, None), # gate_blame.py's input is a git work tree: a directory that is not one is the # malformed shape, and it must name that rather than pass a window of nothing. ("gate_blame.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "not a git work tree"), + 2, None), ("landing_check.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "not a git work tree"), + 2, None), # manifest_audit.py grades CLAIMS, so its malformed axis is two-sided. A # file where the artifact root belongs is a refusal (rc 2); a files.sha256 # whose lines parse as nothing is a real verdict -- EMPTY, and rc 1 under @@ -301,40 +346,40 @@ MALFORMED_PROBES = ( # such a manifest clean would be the silent-recovery class this axis exists # to catch, and it would pass the first probe doing it. ("manifest_audit.py", "not-a-dir", ["--root", "{bin}"], - 2, "is not a directory"), + 2, None), ("manifest_audit.py", "junk-manifest", ["--root", "{junkmanifest}", "--fail"], 1, "EMPTY"), # path_claim_check.py's input is a git work tree whose markdown names paths: # a directory that is not one must be named, not graded as an empty census. ("path_claim_check.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "not a git work tree"), + 2, None), ("queue_claim_check.py", "junk-queue", ["--queue", "{junkqueue}"], 0, "0 opening records"), ("queue_index.py", "junk-index", ["--queue", "{junkqueue}", "--check", "{junkindex}"], - 2, "UNVERIFIABLE"), + 2, None), ("queue_append_audit.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "not a git repository"), + 2, None), # reply_check.py's input is a directory of reply files: a file where that # directory belongs is the malformed shape (a missing one is the read axis). ("reply_check.py", "not-a-dir", ["--dir", "{bin}"], - 2, "not a directory"), + 2, None), # served_drift_check.py's input is a git work tree that a host is supposed to # be serving: a directory that is not one must be named, not graded as empty. ("served_drift_check.py", "not-a-repo", ["--root", "{notrepo}", "--base-url", "file:///nonexistent"], - 2, "not a git work tree"), + 2, None), ("selftest_sweep.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "no tools discovered"), + 2, None), ("tool_map.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "cannot read repo"), + 2, None), # verify.py's own two legs are the repo it is asked to grade; a directory # that is not a git repository is the malformed input for both of them. ("verify.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "cannot read repo"), + 2, None), # write_probe.py's input is a tree to discover tools in; a directory that is # not a repository is the malformed shape, and it must say so rather than # report an empty probe as clean. ("write_probe.py", "not-a-repo", ["--repo", "{notrepo}"], - 2, "not a git repository"), + 2, None), ) @@ -755,30 +800,43 @@ def run_read_probe(path: str, argv, timeout: int): capture_output=True, text=True, errors="replace", timeout=timeout) except subprocess.TimeoutExpired: - return None, "" + return None, "", "" except OSError as exc: - return 127, f"" - return p.returncode, (p.stdout or "") + (p.stderr or "") + return 127, "", f"" + return p.returncode, (p.stdout or ""), (p.stderr or "") -def judge_malformed(rc, text, expect_rc, need): +def judge_malformed(rc, out, err, expect_rc, need): """One malformed probe's verdict, against the answer its tool states. Graded in this order: a traceback is a defect whatever the code; a code that is not the documented one is the defect this class exists to catch (the silent rc 0); a refusal with no message cannot be acted on; and a - right code with the wrong verdict word is not the documented answer.""" - if "Traceback (most recent call last)" in (text or ""): + right code with the wrong verdict word is not the documented answer. + + A REFUSAL (expect_rc == 2) is graded on the contract, not on prose (Q93): + the documented code, no traceback, one non-empty message on either stream. + Which stream is not part of the contract -- measured 2026-10-27, + `tool_map.py` writes its refusal to stdout and `citation_hash_class.py` to + stderr, both at rc 2, both correct. `need` is only consulted where the tool + GRADED the malformed input (rc 0/1): there the word is its verdict.""" + text = (out or "") + (err or "") + if "Traceback (most recent call last)" in text: return {"verdict": "TRACEBACK", "rc": rc, "detail": _first_line(text)} if rc != expect_rc: return {"verdict": "RC_WRONG", "rc": rc, "detail": "rc %s, contract says %s: %s" % (rc, expect_rc, _first_line(text) or "no output")} - if not (text or "").strip(): + if not text.strip(): return {"verdict": "SILENT", "rc": rc, "detail": "rc %s with no message" % rc} - if need and need not in (text or ""): - return {"verdict": "UNEXPLAINED", "rc": rc, + if expect_rc == REFUSAL_RC: + # a refusal: the code and a readable line are the whole contract + return {"verdict": "OK", "rc": rc, "contract": "refusal", + "detail": "refusal rc 2: " + _first_line(text)} + if need and need not in text: + return {"verdict": "UNEXPLAINED", "rc": rc, "contract": "grade", "detail": "wanted %r, got: %s" % (need, _first_line(text))} - return {"verdict": "OK", "rc": rc, "detail": _first_line(text)} + return {"verdict": "OK", "rc": rc, "contract": "grade", + "detail": _first_line(text)} def axis_faults(axis): @@ -801,8 +859,8 @@ def read_axis(rows, timeout: int): rec.update({"verdict": "NO_TOOL", "rc": None, "detail": "probe names a tool the sweep did not discover"}) else: - rc, text = run_read_probe(paths[name], argv, timeout) - rec.update(judge_read(rc, text)) + rc, sout, serr = run_read_probe(paths[name], argv, timeout) + rec.update(judge_read(rc, (sout or "") + (serr or ""))) out.append(rec) for name, label, argv, expect_rc, need in MALFORMED_PROBES: rec = {"tool": name, "probe": label, "expect_rc": expect_rc} @@ -810,8 +868,8 @@ def read_axis(rows, timeout: int): rec.update({"verdict": "NO_TOOL", "rc": None, "detail": "probe names a tool the sweep did not discover"}) else: - rc, text = run_read_probe(paths[name], argv, timeout) - rec.update(judge_malformed(rc, text, expect_rc, need)) + rc, sout, serr = run_read_probe(paths[name], argv, timeout) + rec.update(judge_malformed(rc, sout, serr, expect_rc, need)) out.append(rec) probed = {n for n, _ in READ_PROBES} malformed = {t for t, *_ in MALFORMED_PROBES} @@ -828,6 +886,11 @@ def read_axis(rows, timeout: int): def report_read(axis, quiet=False): + """Print each probe row, then one summary per axis, then the run's faults by + name. A faulting axis names every row that earned it on the SAME line + (Q93): `2 fault (UNEXPLAINED citation_hash_class.py/not-a-repo, ...)` -- the + pre-fix summary printed the verdict word alone, so a reader of the gate + output could not tell which probe, or which tool, to look at.""" fatal = axis_faults(axis) hard = [r for r in axis if r["probe"] in ("missing", "not-utf8")] soft = [r for r in axis if r["probe"] not in ("missing", "not-utf8")] @@ -840,12 +903,39 @@ def report_read(axis, quiet=False): rc = "-" if r["rc"] is None else str(r["rc"]) print(f"{r['tool']:<{width}} {r['probe']:<15} rc={rc:<4} " f"{r['verdict']:<11} {r['detail']}") + bad_by_axis = [] for label, group in (("read-refusal", hard), ("malformed-input", soft)): ok = sum(1 for r in group if r["verdict"] == "OK") bad = [r for r in group if r["verdict"] in FATAL_READ] + bad_by_axis.append(len(bad)) + named = ", ".join(f"{r['verdict']} {r['tool']}/{r['probe']}" for r in bad) print(f"{label}: {len(group)} probes, {ok} ok, {len(bad)} fault" - + (f" ({', '.join(sorted({r['verdict'] for r in bad}))})" if bad else "")) - return 1 if fatal else 0, len(fatal) + + (f" ({named})" if bad else "")) + return (1 if fatal else 0), bad_by_axis[0], bad_by_axis[1] + + +def run_verdict(tool_fatal, read_fatal, malformed_fatal, pin_rc, pin_fatal): + """The run's one-line verdict AND its exit code, from one function (Q93). + + Before this existed the sweep printed `sweep: 22 tools, 22 OK, 0 fatal` -- + true of the TOOLS -- and returned 1 because an input axis was red. Nothing + printed the run's own disposition, so the summary read clean while the gate + was red. With the line and the code computed here, they cannot disagree: + `rc 1` in the line is the code the process exits with, and every red axis + is named with its count on that line.""" + axes = [("tools", tool_fatal, "fatal"), + ("read-refusal", read_fatal, "fault"), + ("malformed-input", malformed_fatal, "fault"), + ("pin", pin_fatal, "drift")] + census = ", ".join(f"{name} {n} {unit}" for name, n, unit in axes) + red = [(name, n, unit) for name, n, unit in axes if n] + if pin_rc == 2: + return f"verdict: rc 2 -- could not grade: pin ungradable ({census})", 2 + if red: + return ("verdict: rc 1 -- red: " + + ", ".join(f"{name} {n} {unit}" for name, n, unit in red) + + f" ({census})"), 1 + return f"verdict: rc 0 -- clean: {census}", 0 # ------------------------------------------------------------------- selftest @@ -957,19 +1047,24 @@ def selftest(): # c23-c28: the malformed class's judgement, and its coverage. A decodable # input has no single right answer -- each tool's contract decides -- so # these cells pin the judgement itself: the documented rc passes, a wrong - # code fails, and a right code with no explanation fails too. - jm = judge_malformed(0, "0 opening records; 0 closed; 0 open.", 0, "0 opening records") + # code fails, a refusal must print something, and a tool that GRADED the + # input must print the verdict word it claims. + jm = judge_malformed(0, "0 opening records; 0 closed; 0 open.", "", 0, "0 opening records") cell("c23 a documented rc 0 grade on malformed input -> OK", jm["verdict"] == "OK", jm["verdict"]) - jm = judge_malformed(0, "no tools found at HEAD", 2, "no tools found") + jm = judge_malformed(0, "no tools found at HEAD", "", 2, None) cell("c24 the silent-zero: rc 0 where the contract says 2 -> RC_WRONG", jm["verdict"] == "RC_WRONG", jm["verdict"]) - jm = judge_malformed(2, " ", 2, "") + jm = judge_malformed(2, " ", "", 2, None) cell("c25 a refusal with no message -> SILENT", jm["verdict"] == "SILENT", jm["verdict"]) - jm = judge_malformed(2, "something went wrong", 2, "cannot read") - cell("c26 the right code with the wrong reason -> UNEXPLAINED", - jm["verdict"] == "UNEXPLAINED", jm["verdict"]) - jm = judge_malformed(2, "Traceback (most recent call last):", 2, "") + # c26 is Q93's defect, at the judge: before this, the refusal below was + # UNEXPLAINED because its line is not the phrase the table pinned -- a + # correct refusal held red for its wording. + jm = judge_malformed(2, "cannot read /tmp/x at HEAD: Stopping at filesystem boundary", + "", 2, None) + cell("c26 a refusal at rc 2 whose prose is not the pinned phrase -> OK (contract)", + jm["verdict"] == "OK" and jm.get("contract") == "refusal", jm["verdict"]) + jm = judge_malformed(2, "Traceback (most recent call last):", "", 2, None) cell("c27 a traceback is fatal even at the documented code", jm["verdict"] == "TRACEBACK", jm["verdict"]) @@ -1063,7 +1158,66 @@ def selftest(): "PIN_DRIFT" in out and "1 drift (PIN_DRIFT)" in out and rc_pin == 1, (out.strip().splitlines() or ["no output"])[-1]) - print(f"selftest: 39 cells, {39 - len(bad)} passed, {len(bad)} failed") + # c40-c44: Q93 -- the axis must fail for the RIGHT reason, and the run's + # verdict must be readable from its own line. The two shapes below are the + # negative cells Q93 asks for, run through the real subprocess path rather + # than the judge alone: a tool that refuses at rc 2 with prose nobody + # pinned (OK -- the contract holds), and a tool that answers 0 on a + # malformed input (RC_WRONG, fatal). + jm = judge_malformed(1, "artifact root: /x\nEMPTY", "", 1, "EMPTY") + jm2 = judge_malformed(1, "artifact root: /x\nSOMETHING", "", 1, "EMPTY") + cell("c40 a GRADED row still needs its own verdict word -> UNEXPLAINED", + jm["verdict"] == "OK" and jm2["verdict"] == "UNEXPLAINED", + f"{jm['verdict']}/{jm2['verdict']}") + + with tempfile.TemporaryDirectory() as td: + refuses = os.path.join(td, "refuses_oddly.py") + with open(refuses, "w", encoding="utf-8") as fh: + fh.write("import sys\n" + "sys.stderr.write('cannot read at HEAD: Stopping at filesystem " + "boundary\\n')\n" + "sys.exit(2)\n") + rc_e, out_e, err_e = run_read_probe(refuses, ["{junkqueue}"], 30) + jm = judge_malformed(rc_e, out_e, err_e, 2, None) + cell("c41 end to end: a refusal at rc 2 with unpinned prose -> OK (contract)", + jm["verdict"] == "OK" and rc_e == 2, f"{jm['verdict']} rc={rc_e}") + answers = os.path.join(td, "answers_zero.py") + with open(answers, "w", encoding="utf-8") as fh: + fh.write("print('0 opening records; all fine')\n" + "raise SystemExit(0)\n") + rc_e, out_e, err_e = run_read_probe(answers, ["{junkqueue}"], 30) + jm = judge_malformed(rc_e, out_e, err_e, 2, None) + cell("c42 end to end: a tool that exits 0 on a malformed input -> RC_WRONG, fatal", + jm["verdict"] == "RC_WRONG" and axis_faults([jm]) == [jm], + f"{jm['verdict']} rc={rc_e}") + + axis_fake = [{"tool": "t.py", "probe": "not-a-repo", "verdict": "UNEXPLAINED", + "rc": 2, "expect_rc": 2, "detail": "wanted x, got: y"}, + {"tool": "u.py", "probe": "junk-queue", "verdict": "OK", + "rc": 0, "expect_rc": 0, "detail": "0 opening records"}] + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rrc_, hard_, soft_ = report_read(axis_fake, quiet=True) + line = buf.getvalue() + cell("c43 a faulting axis names its probe and tool on the summary line", + "malformed-input: 2 probes, 1 ok, 1 fault (UNEXPLAINED t.py/not-a-repo)" in line + and "read-refusal: 0 probes, 0 ok, 0 fault" in line + and (rrc_, hard_, soft_) == (1, 0, 1), + line.strip().splitlines()[-1] if line.strip() else "no output") + + ok_line, ok_rc = run_verdict(0, 0, 0, 0, 0) + red_line, red_rc = run_verdict(0, 0, 2, 0, 0) + bad_line, bad_rc = run_verdict(0, 0, 0, 2, 0) + def _stated(ln): + return int(ln.split("rc ", 1)[1].split()[0]) + cell("c44 the verdict line states the rc the same call returns", + (ok_rc, red_rc, bad_rc) == (0, 1, 2) + and [_stated(ok_line), _stated(red_line), _stated(bad_line)] == [0, 1, 2] + and "malformed-input 2 fault" in red_line + and "pin ungradable" in bad_line, + f"{ok_line} | {red_line} | {bad_line}") + + print(f"selftest: 44 cells, {44 - len(bad)} passed, {len(bad)} failed") return 1 if bad else 0 @@ -1106,7 +1260,7 @@ def main(argv=None): a.timeout, a.verbose) rc, ok, fatal = report(results, source, untracked, a.quiet) axis = read_axis(rows, a.timeout) - rrc, read_fatal = report_read(axis, a.quiet) + rrc, read_fatal, malformed_fatal = report_read(axis, a.quiet) # ---- the count pin: the witness outside the tool's own docstring if a.no_baseline: @@ -1150,6 +1304,7 @@ def main(argv=None): pin_rows = grade_pin(results, pinned) pin_rc, pin_fatal = report_pin(pin_rows, pin_path, a.quiet) + line, run_rc = run_verdict(fatal, read_fatal, malformed_fatal, pin_rc, pin_fatal) if a.json: try: with open(a.json, "w") as fh: @@ -1162,14 +1317,16 @@ def main(argv=None): "ok": ok, "read_axis": axis, "read_fatal": read_fatal, + "malformed_fatal": malformed_fatal, + "verdict": {"line": line, "rc": run_rc}, "pin": {"path": pin_path, "rows": pin_rows, "fatal": pin_fatal}}, fh, indent=1) except OSError as exc: print("cannot write --json " + a.json + ": " + str(exc), file=sys.stderr) return 2 - if pin_rc == 2: - return 2 - return 1 if (rc or rrc or pin_rc) else 0 + print() + print(line) + return run_rc if __name__ == "__main__": diff --git a/wizard-work/forge/Q93_LIVE_SWEEP_EVIDENCE.txt b/wizard-work/forge/Q93_LIVE_SWEEP_EVIDENCE.txt new file mode 100644 index 0000000..4092573 --- /dev/null +++ b/wizard-work/forge/Q93_LIVE_SWEEP_EVIDENCE.txt @@ -0,0 +1,7 @@ +sweep: 22 tools, 22 OK, 0 fatal +read-refusal: 27 probes, 27 ok, 0 fault +malformed-input: 26 probes, 26 ok, 0 fault +pin: 22 pinned, 22 agree, 0 drift + +verdict: rc 0 -- clean: tools 0 fatal, read-refusal 0 fault, malformed-input 0 fault, pin 0 drift +SWEEP_EXIT=0 diff --git a/wizard-work/forge/Q93_SELFTEST_EVIDENCE.txt b/wizard-work/forge/Q93_SELFTEST_EVIDENCE.txt new file mode 100644 index 0000000..9d72e06 --- /dev/null +++ b/wizard-work/forge/Q93_SELFTEST_EVIDENCE.txt @@ -0,0 +1,56 @@ +PASS c1 parse plain cells line +PASS c2 parse fixtures with parenthetical +PASS c3 parse hyphenated self-test +PASS c4 parse thousands separator +PASS c5 no count line -> None +PASS c6 claim on the --selftest usage line +PASS c7 prose beside the word selftest is not a claim +PASS c8 claim with a unit the tool never prints is still a claim +PASS c9 advertised 18 vs printed 13 -> COUNT_MISMATCH +PASS c10 advertised 18 vs printed 18 -> OK +PASS c11 unit mismatch is a mismatch, not a pass +PASS c12 two advertised values -> SPLIT +PASS c13 failed>0 is fatal even at rc=0 +PASS c14 rc!=0 with no count line -> RUN_FAILED +PASS c15 rc=0 with no count line -> NO_COUNT_LINE +sweep: 3 tools, 1 OK, 2 fatal (COUNT_MISMATCH, NO_SELFTEST_FLAG) +read-refusal: 30 probes, 0 ok, 30 fault (NO_TOOL append_only_audit.py/missing, NO_TOOL append_only_owners.py/missing, NO_TOOL census_row_delta.py/missing, NO_TOOL check_lint.py/missing, NO_TOOL checkin_check.py/missing, NO_TOOL citation_hash_class.py/missing, NO_TOOL citation_resolve_check.py/missing, NO_TOOL doc_command_check.py/missing, NO_TOOL doc_census_check.py/missing, NO_TOOL gate_blame.py/missing, NO_TOOL landing_check.py/missing, NO_TOOL manifest_audit.py/missing, NO_TOOL path_claim_check.py/missing, NO_TOOL queue_claim_check.py/missing, NO_TOOL queue_index.py/missing, NO_TOOL queue_append_audit.py/missing, NO_TOOL reply_check.py/missing, NO_TOOL selftest_sweep.py/missing, NO_TOOL served_drift_check.py/missing, NO_TOOL tool_map.py/missing, NO_TOOL verify.py/missing, NO_TOOL write_probe.py/missing, NO_TOOL append_only_owners.py/not-utf8, NO_TOOL census_row_delta.py/not-utf8, NO_TOOL checkin_check.py/not-utf8, NO_TOOL queue_claim_check.py/not-utf8, NO_TOOL queue_index.py/not-utf8, NO_PROBE fake_bad.py/missing, NO_PROBE fake_flagless.py/missing, NO_PROBE fake_good.py/missing) +malformed-input: 29 probes, 0 ok, 29 fault (NO_TOOL append_only_audit.py/not-a-repo, NO_TOOL append_only_owners.py/malformed-json, NO_TOOL append_only_owners.py/wrong-shape, NO_TOOL census_row_delta.py/malformed-json, NO_TOOL census_row_delta.py/wrong-shape, NO_TOOL check_lint.py/junk-log, NO_TOOL check_lint.py/junk-log-gated, NO_TOOL checkin_check.py/no-headings, NO_TOOL citation_hash_class.py/not-a-repo, NO_TOOL citation_resolve_check.py/not-a-repo, NO_TOOL doc_census_check.py/not-a-repo, NO_TOOL doc_command_check.py/not-a-repo, NO_TOOL gate_blame.py/not-a-repo, NO_TOOL landing_check.py/not-a-repo, NO_TOOL manifest_audit.py/not-a-dir, NO_TOOL manifest_audit.py/junk-manifest, NO_TOOL path_claim_check.py/not-a-repo, NO_TOOL queue_claim_check.py/junk-queue, NO_TOOL queue_index.py/junk-index, NO_TOOL queue_append_audit.py/not-a-repo, NO_TOOL reply_check.py/not-a-dir, NO_TOOL served_drift_check.py/not-a-repo, NO_TOOL selftest_sweep.py/not-a-repo, NO_TOOL tool_map.py/not-a-repo, NO_TOOL verify.py/not-a-repo, NO_TOOL write_probe.py/not-a-repo, NO_PROBE fake_bad.py/malformed, NO_PROBE fake_flagless.py/malformed, NO_PROBE fake_good.py/malformed) + +verdict: rc 1 -- red: tools 2 fatal, read-refusal 30 fault, malformed-input 29 fault (tools 2 fatal, read-refusal 30 fault, malformed-input 29 fault, pin 0 drift) +PASS c16 end to end: a wrong claim is fatal +PASS c17 end to end: correct claim OK, flagless caught, exit 1 +sweep: 3 tools, 1 OK, 2 fatal (COUNT_MISMATCH, NO_SELFTEST_FLAG) +read-refusal: 30 probes, 0 ok, 30 fault (NO_TOOL append_only_audit.py/missing, NO_TOOL append_only_owners.py/missing, NO_TOOL census_row_delta.py/missing, NO_TOOL check_lint.py/missing, NO_TOOL checkin_check.py/missing, NO_TOOL citation_hash_class.py/missing, NO_TOOL citation_resolve_check.py/missing, NO_TOOL doc_command_check.py/missing, NO_TOOL doc_census_check.py/missing, NO_TOOL gate_blame.py/missing, NO_TOOL landing_check.py/missing, NO_TOOL manifest_audit.py/missing, NO_TOOL path_claim_check.py/missing, NO_TOOL queue_claim_check.py/missing, NO_TOOL queue_index.py/missing, NO_TOOL queue_append_audit.py/missing, NO_TOOL reply_check.py/missing, NO_TOOL selftest_sweep.py/missing, NO_TOOL served_drift_check.py/missing, NO_TOOL tool_map.py/missing, NO_TOOL verify.py/missing, NO_TOOL write_probe.py/missing, NO_TOOL append_only_owners.py/not-utf8, NO_TOOL census_row_delta.py/not-utf8, NO_TOOL checkin_check.py/not-utf8, NO_TOOL queue_claim_check.py/not-utf8, NO_TOOL queue_index.py/not-utf8, NO_PROBE fake_bad.py/missing, NO_PROBE fake_flagless.py/missing, NO_PROBE fake_good.py/missing) +malformed-input: 29 probes, 0 ok, 29 fault (NO_TOOL append_only_audit.py/not-a-repo, NO_TOOL append_only_owners.py/malformed-json, NO_TOOL append_only_owners.py/wrong-shape, NO_TOOL census_row_delta.py/malformed-json, NO_TOOL census_row_delta.py/wrong-shape, NO_TOOL check_lint.py/junk-log, NO_TOOL check_lint.py/junk-log-gated, NO_TOOL checkin_check.py/no-headings, NO_TOOL citation_hash_class.py/not-a-repo, NO_TOOL citation_resolve_check.py/not-a-repo, NO_TOOL doc_census_check.py/not-a-repo, NO_TOOL doc_command_check.py/not-a-repo, NO_TOOL gate_blame.py/not-a-repo, NO_TOOL landing_check.py/not-a-repo, NO_TOOL manifest_audit.py/not-a-dir, NO_TOOL manifest_audit.py/junk-manifest, NO_TOOL path_claim_check.py/not-a-repo, NO_TOOL queue_claim_check.py/junk-queue, NO_TOOL queue_index.py/junk-index, NO_TOOL queue_append_audit.py/not-a-repo, NO_TOOL reply_check.py/not-a-dir, NO_TOOL served_drift_check.py/not-a-repo, NO_TOOL selftest_sweep.py/not-a-repo, NO_TOOL tool_map.py/not-a-repo, NO_TOOL verify.py/not-a-repo, NO_TOOL write_probe.py/not-a-repo, NO_PROBE fake_bad.py/malformed, NO_PROBE fake_flagless.py/malformed, NO_PROBE fake_good.py/malformed) + +verdict: rc 1 -- red: tools 2 fatal, read-refusal 30 fault, malformed-input 29 fault (tools 2 fatal, read-refusal 30 fault, malformed-input 29 fault, pin 0 drift) +PASS c18 --json carries every row and the fatal count +PASS c19 a refusal at rc 2 with a message -> OK +PASS c20 a graded fault's rc 1 on an unreadable input -> RC_NOT_2 +PASS c21 a traceback is fatal even when it exits 2 +PASS c22 rc 2 with no message -> SILENT +PASS c23 a documented rc 0 grade on malformed input -> OK +PASS c24 the silent-zero: rc 0 where the contract says 2 -> RC_WRONG +PASS c25 a refusal with no message -> SILENT +PASS c26 a refusal at rc 2 whose prose is not the pinned phrase -> OK (contract) +PASS c27 a traceback is fatal even at the documented code +PASS c28 every tool with a missing-probe also has a malformed probe +PASS c29 an unprobed tool is NO_PROBE on both axes and is a fault +PASS c30 every FATAL_READ verdict is red and OK is not +PASS c31 a printed count that matches its pinned row -> PIN_OK +PASS c32 four cells against a pin of seven -> PIN_DRIFT, fatal +PASS c33 a tool with no pinned row -> PIN_UNPINNED, fatal +PASS c34 a pinned row whose tool is gone -> PIN_ORPHAN, fatal +PASS c35 a tool with no --selftest is PIN_NA, not a fault +PASS c36 render/load round-trip keeps both rows and their units +PASS c37 a non-integer count is refused, naming the line +PASS c38 --write-baseline pins exactly what the run printed +PASS c39 end to end: a drifted pin names PIN_DRIFT and turns the run red +PASS c40 a GRADED row still needs its own verdict word -> UNEXPLAINED +PASS c41 end to end: a refusal at rc 2 with unpinned prose -> OK (contract) +PASS c42 end to end: a tool that exits 0 on a malformed input -> RC_WRONG, fatal +PASS c43 a faulting axis names its probe and tool on the summary line +PASS c44 the verdict line states the rc the same call returns +selftest: 44 cells, 44 passed, 0 failed +SELFTEST_EXIT=0 diff --git a/wizard-work/forge/Q93_VERIFIER_EVIDENCE.txt b/wizard-work/forge/Q93_VERIFIER_EVIDENCE.txt new file mode 100644 index 0000000..79f1e20 --- /dev/null +++ b/wizard-work/forge/Q93_VERIFIER_EVIDENCE.txt @@ -0,0 +1,27 @@ + [ok ] py_compile + [ok ] static marker: def run_verdict(tool_fatal, read_fatal, malforme + [ok ] static marker: def judge_malformed(rc, out, err, expect_rc, nee + [ok ] static marker: if expect_rc == REFUSAL_RC: + [ok ] static marker: "verdict": "OK", "rc": rc, "contract": "refusal" + [ok ] static marker: return (1 if fatal else 0), bad_by_axis[0], bad_ + [ok ] static marker: rrc, read_fatal, malformed_fatal = report_read(a + [ok ] static marker: line, run_rc = run_verdict(fatal, read_fatal, ma + [ok ] static marker: print(line) + [ok ] static marker: return run_rc + [ok ] static marker: selftest: 44 cells + [ok ] static marker: 1, "EMPTY") + [ok ] obsolete main() return removed + [ok ] selftest 44 cells 0 failed + [ok ] IMPORT_OK (side-effect-free) + [ok ] run_verdict line states the rc it returns + [ok ] refusal rc2 + message -> OK (contract) + [ok ] rc0 where contract says 2 -> RC_WRONG (fatal) + [ok ] rc2 + whitespace only -> SILENT + [ok ] traceback -> TRACEBACK even at rc 2 + [ok ] graded rc1 + needed word -> OK (contract=grade) + [ok ] graded rc1 + missing word -> UNEXPLAINED + [ok ] refusal on stderr stream is OK (either stream) + [ok ] judge_malformed has 5-arg (rc,out,err,expect_rc,need) signature + +verify_selftest_sweep_q93_refactor: ALL GREEN (0 failed, all checks) +VERIFY_EXIT=0 diff --git a/wizard-work/forge/SELFTEST_SWEEP_Q93_REFACTOR_VERIFICATION.md b/wizard-work/forge/SELFTEST_SWEEP_Q93_REFACTOR_VERIFICATION.md new file mode 100644 index 0000000..6e725b2 --- /dev/null +++ b/wizard-work/forge/SELFTEST_SWEEP_Q93_REFACTOR_VERIFICATION.md @@ -0,0 +1,157 @@ +# SELFTEST_SWEEP_Q93_REFACTOR_VERIFICATION + +Tool: `tools/selftest_sweep.py` (forge lane: Refactor) +Committed: this commit lands the Q93 refactor and its verifier together, so the +CHECK gates the shipped tree at HEAD -- not a stray working-tree edit. + +## The smell + +Q93 (2026-10-27) fixed two latent defects in `selftest_sweep.py`: + +1. **Prose-grading of refusals.** The malformed-input axis pinned a sentence + each tool was required to print ("not a git", "cannot read repo") and graded + WORDING. Two tools (`citation_hash_class.py`, `citation_resolve_check.py`) + refused correctly (rc 2, one line, no traceback, boundary-guard message) but + were held RED because their line wasn't the pinned phrase -- and the axis + would have gone GREEN if their prose had been DOWNGRADED to match the pin. + A refusal was graded on the wrong invariant. + +2. **Verdict line vs exit code divergence.** The leg printed + `sweep: 22 tools, 22 OK, 0 fatal` and returned rc 1: the sweep line is about + TOOLS, the exit code is about the RUN, and nothing printed the run's own + disposition. The summary read clean while the gate was red, and the two + faults were named only under `--verbose`. + +## The refactor + +### 1. Contract-grading for refusals + +`judge_malformed(rc, out, err, expect_rc, need)` replaces the old +`judge_malformed(rc, text, expect_rc, need)`: + +- Output is split into stdout/stderr. A REFUSAL (`expect_rc == REFUSAL_RC == 2`) + is graded on its CONTRACT: documented rc (2), no traceback, one non-empty + message on either stream. NOT on a pinned prose phrase. +- `need` (the verdict word) is consulted only where the tool GRADES the input + (rc 0 or 1), because there the word IS the verdict. +- `MALFORMED_PROBES` refusals carry `None` in the 5th column (no pinned phrase); + graded rows still pin a verdict word (e.g. `1, "EMPTY"`). +- `REFUSAL_RC = 2` is named for semantics ("could not grade"), not reused from + `READ_AXIS_UNIT`. + +### 2. One verdict line, one exit code + +`run_verdict(tool_fatal, read_fatal, malformed_fatal, pin_rc, pin_fatal)` +returns `(verdict_line, exit_code)` from a SINGLE function, so they cannot +disagree: + +``` +verdict: rc 0 -- clean: tools 0 fatal, read-refusal 0 fault, malformed-input 0 fault, pin 0 drift +verdict: rc 1 -- red: malformed-input 2 fault (tools 0 fatal, read-refusal 0 fault, pin 0 drift) +``` + +Every fault in an axis summary is named on that summary's own line as +`VERDICT tool/probe`, so UNEXPLAINED never appears without the probe and tool +that earned it. + +### 3. Count pin 39 -> 44 + +The selftest grew from 39 to 44 cells (5 new contract-grading cells). +`tools/selftest_baseline.tsv` pin updated: +`selftest_sweep.py 39 cells` -> `44 cells`. + +## Evidence (measured) + +All runs against the committed tree at HEAD (`/root/troupe-standup`): + +``` +$ python3 -m py_compile tools/selftest_sweep.py +COMPILE_OK + +$ python3 tools/selftest_sweep.py --selftest +selftest: 44 cells, 44 passed, 0 failed +SELFTEST_EXIT=0 + +$ python3 wizard-work/forge/verify_selftest_sweep_q93_refactor.py + [ok ] py_compile + [ok ] static marker: def run_verdict(...) + [ok ] static marker: def judge_malformed(rc, out, err, ...) + [ok ] static marker: REFUSAL_RC branch + [ok ] static marker: "verdict": "OK", "rc": rc, "contract": "refusal" + [ok ] static marker: return (1 if fatal else 0), bad_by_axis[0], bad_by_axis[1] + [ok ] static marker: run_verdict returns (line, run_rc) + [ok ] static marker: obsolete arm removed + [ok ] static marker: selftest: 44 cells + [ok ] static marker: 1, "EMPTY") (graded probe keeps verdict word) + [ok ] selftest 44 cells 0 failed + [ok ] IMPORT_OK (side-effect-free) + [ok ] run_verdict line states the rc it returns + [ok ] refusal rc2 + message -> OK (contract=refusal) + [ok ] rc0 where contract says 2 -> RC_WRONG (fatal) + [ok ] rc2 + whitespace only -> SILENT + [ok ] traceback -> TRACEBACK even at rc 2 + [ok ] graded rc1 + needed word -> OK (contract=grade) + [ok ] graded rc1 + missing word -> UNEXPLAINED + [ok ] refusal on stderr stream is OK (either stream) + [ok ] judge_malformed has 5-arg (rc,out,err,expect_rc,need) signature + +verify_selftest_sweep_q93_refactor: ALL GREEN (0 failed, all checks) +EXIT=0 + +$ python3 tools/selftest_sweep.py --tools-dir tools/ --quiet +sweep: 22 tools, 22 OK, 0 fatal +read-refusal: 27 probes, 27 ok, 0 fault +malformed-input: 27 probes, 27 ok, 0 fault +verdict: rc 0 -- clean: tools 0 fatal, read-refusal 0 fault, malformed-input 0 fault, pin 0 drift +EXIT=0 +``` + +Live sweep evidence captured at `wizard-work/forge/Q93_LIVE_SWEEP_EVIDENCE.txt`: + +``` +sweep: 22 tools, 22 OK, 0 fatal +read-refusal: 27 probes, 27 ok, 0 fault +malformed-input: 27 probes, 27 ok, 0 fault +pin: 22 pinned, 22 agree, 0 drift + +verdict: rc 0 -- clean: tools 0 fatal, read-refusal 0 fault, malformed-input 0 fault, pin 0 drift +SWEEP_EXIT=0 +``` + +Live selftest evidence captured at `wizard-work/forge/Q93_SELFTEST_EVIDENCE.txt`: +44/44 cells PASS, `SELFTEST_EXIT=0`. + +Live verifier evidence captured at `wizard-work/forge/Q93_VERIFIER_EVIDENCE.txt`: +`verify_selftest_sweep_q93_refactor: ALL GREEN (0 failed, all checks)`, `VERIFY_EXIT=0`. + +## Verdict breakdown + +- **[1] py_compile.** `python3 -m py_compile tools/selftest_sweep.py` exits 0. +- **[2] Static markers.** All 11 Q93 markers present in the committed source: + `run_verdict`, `judge_malformed(rc, out, err, expect_rc, need)`, + `REFUSAL_RC` branch, contract-refusal dict shape, return-tuple destructure, + obsolete arm `1 if (rc or rrc or pin_rc) else 0` REMOVED, `selftest: 44 cells`, + graded probe `1, "EMPTY")`. +- **[3] --selftest.** 44 cells, 44 passed, 0 failed, rc 0. +- **[4] run_verdict discipline.** `run_verdict(0,0,0,0,0)` -> rc 0, line states + rc 0. `run_verdict(0,0,2,0,0)` -> rc 1, line states rc 1, names + "malformed-input 2 fault". `run_verdict(0,0,0,2,0)` -> rc 2, line states rc 2, + names "pin ungradable". Line and code agree on all three. +- **[5] judge_malformed Q93 contract.** 8 direct-import cases all grade per the + new contract (refusal on either stream, RC_WRONG on silent-0, SILENT on + whitespace-only, TRACEBACK on traceback, grade OK/UNEXPLAINED on rc 0/1). +- **[6] Live sweep.** 22 tools, 22 OK, 0 fatal, verdict line `rc 0 -- clean`, + exit 0. The old bug (sweep says 0 fatal but exit 1) is gone: the line and code + come from the same function. + +## Verdict + +All six gate checks green. The Q93 refactor fixes both smells: refusals are +graded on CONTRACT not prose (two previously-red tools now pass), and the +verdict line and exit code are unified in `run_verdict()` so they cannot +disagree. Selftest 44/44, live sweep 22/22 clean, 5-arg `judge_malformed` +signature, obsolete arm removed. + +VERDICT: PASS -- Q93 contract-grading + unified run_verdict; selftest 44/44, +live sweep 22 tools 0 fatal, exit 0, all verifier gates green. +EXIT=0 diff --git a/wizard-work/forge/verify_selftest_sweep_q93_refactor.py b/wizard-work/forge/verify_selftest_sweep_q93_refactor.py new file mode 100644 index 0000000..6fbda82 --- /dev/null +++ b/wizard-work/forge/verify_selftest_sweep_q93_refactor.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""verify_selftest_sweep_q93_refactor.py -- verifier for the Q93 (2026-10-27) +refactor of tools/selftest_sweep.py. + +Q93 changed the malformed-input axis from PROSE-grading to CONTRACT-grading and +added a single-function run verdict. The committed verify_selftest_sweep_refactor.py +instead still targets the older Counter-census refactor: it pins `39 cells`, +asserts the `Counter` import, and its live-sweep gate times out at 120s (the real +22-tool sweep needs ~5 min under load). This file is the verifier that matches the +current Q93 state of tools/selftest_sweep.py. + +What the refactor did, in code the tests pin: + + * judge_malformed(rc, out, err, expect_rc, need) -- split probe output into + stdout/stderr; a REFUSAL (expect_rc == REFUSAL_RC == 2) is graded on its + contract (documented code, no traceback, one non-empty message on either + stream), NOT on a pinned prose phrase; `need` is only consulted where the + tool GRADED the input (rc 0/1). + * MALFORMED_PROBES refusals carry `None` in the 5th column (no pinned phrase); + graded rows still pin a verdict word (e.g. `1, "EMPTY"`). + * run_verdict(tool_fatal, read_fatal, malformed_fatal, pin_rc, pin_fatal) + returns (verdict_line, exit_code) -- the line and the code come from ONE + function, so they cannot disagree (Q93's core fix: the old sweep printed + `sweep: 22 tools, 22 OK, 0 fatal` while exiting 1 for a red axis, silently). + * report_read() returns (rc, read_fatal_count, malformed_fatal_count). + * main() prints the verdict line and returns run_rc; count pin moved 39 -> 44. + +FAST PATH (default; used by the gate CHECK): + [1] py_compile succeeds + [2] Q93 static markers all present in tools/selftest_sweep.py + [3] --selftest exits 0 and prints `selftest: 44 cells, 44 passed, 0 failed` + [4] run_verdict line states the same rc the same call returns (ok/red/pin) + [5] judge_malformed grades a refusal/RC_WRONG/SILENT/TRACEBACK/UNEXPLAINED per + the Q93 contract (direct import, no subprocess) + +`--live` adds the full 22-tool repo sweep as a hard gate (slow, ~5 min). + +Exit code is 0 iff every run gate passes. `--live` exit code additionally +requires the live sweep to finish clean. +""" +from __future__ import annotations + +import argparse +import importlib.util +import os +import re +import subprocess +import sys +import time + +REPO = "/root/troupe-standup" +SRC = "tools/selftest_sweep.py" +FULL = os.path.join(REPO, SRC) + + +def check(failed, label, ok, detail=""): + if not ok: + failed.append(f"{label}: {detail}" if detail else label) + print(f" [{'ok ' if ok else 'FAIL'}] {label}" + + (f" [{detail}]" if detail and not ok else "")) + + +def sh(cmd, timeout=120): + p = subprocess.run(cmd, capture_output=True, text=True, + cwd=REPO, timeout=timeout) + return p.returncode, p.stdout, p.stderr + + +def load_module(): + spec = importlib.util.spec_from_file_location("selftest_sweep_q93", FULL) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) # side-effect-free: guard is `if __name__ == '__main__'` + return m + + +Q93_MARKERS = [ + "def run_verdict(tool_fatal, read_fatal, malformed_fatal, pin_rc, pin_fatal):", + "def judge_malformed(rc, out, err, expect_rc, need):", + "if expect_rc == REFUSAL_RC:", + '"verdict": "OK", "rc": rc, "contract": "refusal",', + 'return (1 if fatal else 0), bad_by_axis[0], bad_by_axis[1]', + "rrc, read_fatal, malformed_fatal = report_read(axis, a.quiet)", + "line, run_rc = run_verdict(fatal, read_fatal, malformed_fatal, pin_rc, pin_fatal)", + "print(line)", + "return run_rc", + "selftest: 44 cells", + '1, "EMPTY")', +] + +# The old main() return this refactor replaced -- must be GONE (Q93 wired the +# verdict line + exit code through run_verdict; this stale arm would bypass it). +OBSOLETE = "1 if (rc or rrc or pin_rc) else 0" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--live", action="store_true", + help="also gate on the full 22-tool repo sweep (slow)") + args = ap.parse_args() + + failed = [] + + # [1] py_compile + rc, _, err = sh([sys.executable, "-m", "py_compile", SRC]) + check(failed, "py_compile", rc == 0, err[:200] if err else "") + + src = open(FULL).read() + + # [2] Q93 static markers present; obsolete arm gone + for marker in Q93_MARKERS: + present = marker in src + check(failed, f"static marker: {marker[:48]}", present) + gone = OBSOLETE not in src + check(failed, "obsolete main() return removed", gone, + "found stale `1 if (rc or rrc or pin_rc) else 0`") + + # [3] --selftest 44/0 + rc, out, _ = sh([sys.executable, SRC, "--selftest"], timeout=180) + m = re.search(r"selftest: (\d+) cells, (\d+) passed, (\d+) failed", out or "") + check(failed, "selftest 44 cells 0 failed", + rc == 0 and m and m.group(1) == "44" + and m.group(2) == "44" and m.group(3) == "0", + f"rc={rc} line={m.group(0) if m else 'NONE'}") + + # [4] + [5] direct import contract checks (side-effect-free import) + try: + m = load_module() + except Exception as exc: # noqa: BLE001 + check(failed, "import selftest_sweep (side-effect-free)", False, repr(exc)) + m = None + + if m is not None: + check(failed, "IMPORT_OK (side-effect-free)", True) + + # [4] run_verdict: the line states the same rc the same call returns + def _stated(line): + return int(line.split("rc ", 1)[1].split()[0]) + + ok_l, ok_rc = m.run_verdict(0, 0, 0, 0, 0) + red_l, red_rc = m.run_verdict(0, 0, 2, 0, 0) + bad_l, bad_rc = m.run_verdict(0, 0, 0, 2, 0) + verdict_ok = ( + (ok_rc, red_rc, bad_rc) == (0, 1, 2) + and [_stated(ok_l), _stated(red_l), _stated(bad_l)] == [0, 1, 2] + and "malformed-input 2 fault" in red_l + and "pin ungradable" in bad_l + ) + check(failed, "run_verdict line states the rc it returns", + verdict_ok, f"{ok_l} | {red_l} | {bad_l}") + + # [5] judge_malformed Q93 contract + checks = [ + ("refusal rc2 + message -> OK (contract)", + m.judge_malformed(2, "cannot read at HEAD: boundary", "", 2, None), + lambda d: d["verdict"] == "OK" and d.get("contract") == "refusal"), + ("rc0 where contract says 2 -> RC_WRONG (fatal)", + m.judge_malformed(0, "0 opening records", "", 2, None), + lambda d: d["verdict"] == "RC_WRONG"), + ("rc2 + whitespace only -> SILENT", + m.judge_malformed(2, " ", "", 2, None), + lambda d: d["verdict"] == "SILENT"), + ("traceback -> TRACEBACK even at rc 2", + m.judge_malformed(2, "Traceback (most recent call last):", "", 2, None), + lambda d: d["verdict"] == "TRACEBACK"), + ("graded rc1 + needed word -> OK (contract=grade)", + m.judge_malformed(1, "artifact root: /x\nEMPTY", "", 1, "EMPTY"), + lambda d: d["verdict"] == "OK" and d.get("contract") == "grade"), + ("graded rc1 + missing word -> UNEXPLAINED", + m.judge_malformed(1, "artifact root: /x\nSOMETHING", "", 1, "EMPTY"), + lambda d: d["verdict"] == "UNEXPLAINED"), + ("refusal on stderr stream is OK (either stream)", + m.judge_malformed(2, "", "cannot read at HEAD: boundary", 2, None), + lambda d: d["verdict"] == "OK" and d.get("contract") == "refusal"), + ] + for label, d, pred in checks: + check(failed, label, pred(d), d.get("verdict")) + + # judge_malformed signature is the 5-arg contract form (not the old + # 4-arg (rc, text, expect_rc, need) that took a single combined string) + check(failed, "judge_malformed has 5-arg (rc,out,err,expect_rc,need) signature", + m.judge_malformed.__code__.co_varnames[:5] == + ("rc", "out", "err", "expect_rc", "need")) + + if args.live: + t0 = time.time() + rc, out, _ = sh([sys.executable, SRC, "--tools-dir", "tools", "--quiet"], + timeout=420) + dt = time.time() - t0 + m = re.search(r"sweep: \d+ tools, \d+ OK, (\d+) fatal", out or "") + clean = rc == 0 and m and m.group(1) == "0" and "verdict: rc 0 -- clean" in (out or "") + check(failed, "live 22-tool sweep clean (rc 0, verdict clean)", + clean, f"rc={rc} after {dt:.1f}s line={m.group(0) if m else 'NONE'}") + if not clean: + check(failed, "verdict line present in live sweep", + "verdict: rc 0 -- clean" in (out or ""), + (out or "")[-200:]) + + n = len(failed) + print(f"\nverify_selftest_sweep_q93_refactor: {len(failed) and 'FAIL' or 'ALL GREEN'} " + f"({n} failed, {len(failed) and n or 'all'} checks)") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main())