From cc2c6a5fdbfb1ddd9f2d89659be990e5bb89bf70 Mon Sep 17 00:00:00 2001 From: Volt Date: Wed, 21 Oct 2026 14:26:46 +0000 Subject: [PATCH] tools: the write-axis fault names the faulting tool, and its own recipe is now a command the checker can resolve A gate that has only ever been seen green is indistinguishable from a gate that cannot go red, so the write axis was mutated one defect at a time in throwaway worktrees (nine cases, wizard-outputs/volt/2026-10-21-write-axis-bites/ mutation_study.py). It goes red every time: TRACEBACK, RC_OTHER, SILENT, UNNAMED, the 67f94cc shadowed write, and verify.py's own write-axis leg -- while the clean control stays green 21/21. The study found two things this commit fixes and one it only states. FIXED 1 -- the detail line on a red write-axis leg named a healthy tool. Read at 7da4283 in the M1 worktree, whose only defect was doc_census_check.py, verify.py printed `FAULT: append_only_audit.py OK 2 cannot write --json ...`: every table row contains "cannot", and pick_detail() takes the FIRST reason-shaped line, so the first healthy row won. write_probe.py now prints a `FAULT ...` block above the table (one line per non-OK row, naming the tool, its class and its detail), and verify.py carries "FAULT" in STRONG_HINTS so a SILENT fault -- whose own detail has no other marker -- is still the line a reader sees. Cell c11 grades the block: selftest 10 -> 11 cells, 11/11 rc 0. FIXED 2 -- doc_command_check.py's only red row at 7da4283 was this tool's own reproduction recipe: `python3 /tools/write_probe.py --repo ` graded TOOL_MISSING, because the copied absolute path is not in the tree at HEAD. The recipe now reads `cd && python3 tools/write_probe.py --repo `, which is both what a reader actually types and a path the checker resolves. STATED, NOT CHANGED -- UNNAMED grades the TEXT, not the guard's sentence. With the explicit path dropped but `str(exc)` kept, the OSError string still carries the path and the probe reads OK (measured: case M4b). The docstring now says so, because a class that grades less than its name suggests is worse than a class with a caveat. Measured before this commit, at 7da4283: write_probe --repo . 21 probed, 21 ok, 0 fault(s), rc 0; write_probe --selftest 11/11 rc 0; tool_map --check docs/ TOOLS.md rc 0; doc_census_check --tools tools/write_probe.py --fail 0 STALE rc 0. Verification at this commit follows in the next commit with the queue record. --- tools/verify.py | 9 ++++- tools/write_probe.py | 86 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 84 insertions(+), 11 deletions(-) diff --git a/tools/verify.py b/tools/verify.py index 8a03927..482f9f6 100644 --- a/tools/verify.py +++ b/tools/verify.py @@ -184,7 +184,14 @@ LEGS = ( # the row said FAULT beside a line about the README while the tool's own # "FAIL: 2 file(s) violate the check-in contract" sat further down, unpicked. STRONG_HINTS = ("STALE:", "FAIL:", "FAILED", "fatal", "TIMEOUT", "NO_", - "cannot", "error", "Error", "refus") + "cannot", "error", "Error", "refus", + # write_probe's fault block (2026-10-21): each line begins + # "FAULT ...", and that line sits above the probe table + # precisely so this function reaches it before the first healthy + # row. A SILENT fault's own detail carries no other marker, so + # without this token the block would be skipped for the row that + # names a tool with nothing wrong with it. + "FAULT") WEAK_HINTS = ("FAIL", "not a ") # A tool's concluding line: `reply_check: 192 file(s), 0 new fault(s) ...`. diff --git a/tools/write_probe.py b/tools/write_probe.py index 79d59bf..ca8b9e9 100644 --- a/tools/write_probe.py +++ b/tools/write_probe.py @@ -16,9 +16,13 @@ WHY THIS EXISTS of this paragraph said "five" and "13 of 20"; neither is what the instrument below prints, and the number that counts is the one a reader can reproduce: `git worktree add --detach 67f94cc && cp tools/write_probe.py - /tools/ && python3 /tools/write_probe.py --repo `. The guards - landed that day; this instrument is what proves them and what a new tool has - to pass -- including this one, which probes itself. + /tools/ && cd && python3 tools/write_probe.py --repo `. The + guards landed that day; this instrument is what proves them and what a new + tool has to pass -- including this one, which probes itself. The recipe names + the tracked path rather than the copied absolute one so that the command a + reader types is also a command `doc_command_check.py` can resolve: addressed + through the scratch copy's absolute path it graded TOOL_MISSING at 7da4283, + the only red row that instrument had. THE CONTRACT IT GRADES The grade runs first and its stdout report stands. The write is wrapped, so a @@ -32,13 +36,27 @@ VERDICTS, IN THIS ORDER (a traceback is a defect whatever the exit code) RC_OTHER clean stream, rc != 2 -- the write failed and the tool did not say so in the one code that means "could not grade" SILENT rc 2 with no message at all -- a refusal nobody can read - UNNAMED rc 2, a message, but the message never names the target + UNNAMED rc 2, a message, but neither stream names the target. What this + grades is the TEXT, not the guard's own sentence: `str(exc)` on + an OSError carries the path it failed on, so a guard that drops + its explicit path but keeps the exception still reads OK. That is + measured, not assumed -- it is the M4b case of the mutation study + in wizard-outputs/volt/2026-10-21-write-axis-bites/ + mutation_study.py, and UNNAMED is reachable only when no stream + carries the path at all. OK rc 2, one line naming the target, no traceback NO_REACH the tool never got as far as the write on this probe: its own rc 2 came from the input, so the write path is UNGRADED here (counted, never passed -- an ungraded path is not a clean one) NO_JSON no --json argument in the source: nothing to probe + A fault is printed as a `FAULT ...` line ABOVE the table, not only as + a row inside it: verify.py's detail line takes the first reason-shaped line of + a leg's report, and the first such line used to be the table's first healthy + row. Measured 2026-10-21 on a one-defect worktree: the leg said + `FAULT: append_only_audit.py OK 2 cannot write --json ...` while the tool that + carried the defect was doc_census_check.py. + FIXTURES Every probe runs in a throwaway tree, so the grade each tool performs is over a few bytes rather than over this repository: the write path is what is being @@ -50,7 +68,8 @@ READ-ONLY on the repo: the only files created are under a TemporaryDirectory. USAGE python3 tools/write_probe.py [--repo DIR] [--only TOOL] [--timeout S] [--json PATH] [--verbose] - python3 tools/write_probe.py --selftest # 10 cells, judge, table, fixture + python3 tools/write_probe.py --selftest # 11 cells, judge, table, + # fault block, fixture EXIT 0 every probed tool is OK; 1 at least one is not (including NO_REACH and NO_JSON: an ungraded write path may not be reported clean); 2 could not @@ -198,6 +217,31 @@ def judge(rc, out, err, target): return {"verdict": "OK", "rc": rc, "detail": msg[-1][:110]} +def fault_lines(rows, width): + """One line per row that is not OK, naming the tool and its class. + + Printed ABOVE the table by main(). The position is the point: verify.py's + detail line takes the FIRST reason-shaped line a leg prints, and every row + of the table reads `cannot write --json : [Errno 2] ...`, so before + this block the detail on a red write-axis leg was the first HEALTHY row. + Measured 2026-10-21 (M1-leg of the mutation study): the row said + `FAULT: append_only_audit.py OK 2 cannot write --json ...` on a worktree + whose only defect was in doc_census_check.py -- the one tool a reader would + not need to look at. The word FAULT is carried for the same reason: it is in + verify.py's STRONG_HINTS, so a fault whose own detail has no diagnostic + marker (SILENT: "rc 2 with no message") is still the line a reader sees. + """ + out = [] + for r in rows: + if r.get("verdict") == "OK": + continue + out.append("FAULT %-*s %-9s %4s %s" + % (width, r.get("tool", "?"), r.get("verdict", "?"), + "-" if r.get("rc") is None else r["rc"], + (r.get("detail") or "")[:110])) + return out + + def run_probe(path, argv, timeout, cwd): try: p = subprocess.run([sys.executable, path] + argv, cwd=cwd, @@ -220,10 +264,12 @@ def tracked_tools(repo): def selftest(repo): - """Ten cells: six for the judge's ordered verdicts, four for the table and - the fixture. The judge cells matter because the order is the contract -- a - traceback outranks a right code, and a rc-2 input refusal (NO_REACH) must - never read as a passed write path. + """Eleven cells: six for the judge's ordered verdicts, five for the table, + the fixture and the fault block. The judge cells matter because the order is + the contract -- a traceback outranks a right code, and a rc-2 input refusal + (NO_REACH) must never read as a passed write path. The fault block gets its + own cell because its position above the table is load-bearing for verify.py's + detail line, and a formatting change would otherwise regress it silently. """ bad = [] @@ -279,7 +325,22 @@ def selftest(repo): cell("c10 NO_PROBE counts as a fault, so an ungraded path cannot pass", len([r for r in rows if r["verdict"] != "OK"]) == 1) - print("selftest: 10 cells, %d passed, %d failed" % (10 - len(bad), len(bad))) + # c11 exists because the fault block's POSITION is the whole point of it: + # the block is what verify.py's detail line reads, and a block that named a + # healthy tool (or came after the table) would put the wrong filename in + # front of a reader on every red write-axis leg. + rows2 = [{"tool": "healthy.py", "verdict": "OK", "rc": 0, "detail": "wrote"}, + {"tool": "hurt.py", "verdict": "SILENT", "rc": 2, "detail": ""}, + {"tool": "worse.py", "verdict": "TRACEBACK", "rc": 1, + "detail": "FileNotFoundError: [Errno 2] no such file"}] + blk = fault_lines(rows2, 10) + cell("c11 the fault block names every faulting tool, never a healthy one", + len(blk) == 2 and "hurt.py" in blk[0] and "worse.py" in blk[1] + and "healthy.py" not in " ".join(blk) + and all(ln.startswith("FAULT ") for ln in blk), + str(blk)) + + print("selftest: 11 cells, %d passed, %d failed" % (11 - len(bad), len(bad))) return 1 if bad else 0 @@ -346,6 +407,11 @@ def main(argv=None): print("write probe -- %d of %d tracked tool(s) probed under %s (%s)" % (len(rows), len(names), os.path.join(repo, "tools"), source)) print() + faults = fault_lines(rows, width) + if faults: + for ln in faults: + print(ln) + print() print("%-*s %-9s %4s detail" % (width, "tool", "verdict", "rc")) for r in rows: print("%-*s %-9s %4s %s"