#!/usr/bin/env python3 """Measure the WRITE end of every tool in tools/: `--json PATH` at a target the host cannot write. WHY THIS EXISTS `selftest_sweep.py`'s read axis grades what a tool does with an input it cannot read. Nothing graded the other end. Measured at `67f94cc`, the last revision before the guards landed, over the 20 tools then tracked: **18 answered a `--json` target inside a nonexistent directory with an uncaught FileNotFoundError at rc 1**; `gate_blame.py` returned rc 2 from its own input and never reached the write (the UNNAMED class below, ungraded rather than passed); `queue_append_audit.py` already refused (rc 2, "cannot write json"). A traceback is what this repo's contract calls fatal, and rc 1 is the code that means a graded fault, so those 18 made "the grade is bad" indistinguishable from "the artifact was never written". Two earlier drafts of this paragraph said "five" and "13 of 20"; neither is what the instrument below prints, and the number that counts is the one a reader can reproduce: `git worktree add --detach 67f94cc && cp tools/write_probe.py /tools/ && cd && python3 tools/write_probe.py --repo `. The guards landed that day; this instrument is what proves them and what a new tool has to pass -- including this one, which probes itself. The recipe names the tracked path rather than the copied absolute one so that the command a reader types is also a command `doc_command_check.py` can resolve: addressed through the scratch copy's absolute path it graded TOOL_MISSING at 7da4283, the only red row that instrument had. THE CONTRACT IT GRADES The grade runs first and its stdout report stands. The write is wrapped, so a target this host cannot write prints one line naming the path on stderr and returns 2 -- "could not grade": the artifact was asked for and not delivered. rc 2 outranks the 0/1 the report may have earned, because a caller who asked for a file and got a stack trace cannot know a grade happened at all. VERDICTS, IN THIS ORDER (a traceback is a defect whatever the exit code) TRACEBACK uncaught exception text in either stream RC_OTHER clean stream, rc != 2 -- the write failed and the tool did not say so in the one code that means "could not grade" SILENT rc 2 with no message at all -- a refusal nobody can read UNNAMED rc 2, a message, but neither stream names the target. What this grades is the TEXT, not the guard's own sentence: `str(exc)` on an OSError carries the path it failed on, so a guard that drops its explicit path but keeps the exception still reads OK. That is measured, not assumed -- it is the M4b case of the mutation study in wizard-outputs/volt/2026-10-21-write-axis-bites/ mutation_study.py, and UNNAMED is reachable only when no stream carries the path at all. OK rc 2, one line naming the target, no traceback NO_REACH the tool never got as far as the write on this probe: its own rc 2 came from the input, so the write path is UNGRADED here (counted, never passed -- an ungraded path is not a clean one) NO_JSON no --json argument in the source: nothing to probe A fault is printed as a `FAULT ...` line ABOVE the table, not only as a row inside it: verify.py's detail line takes the first reason-shaped line of a leg's report, and the first such line used to be the table's first healthy row. Measured 2026-10-21 on a one-defect worktree: the leg said `FAULT: append_only_audit.py OK 2 cannot write --json ...` while the tool that carried the defect was doc_census_check.py. FIXTURES Every probe runs in a throwaway tree, so the grade each tool performs is over a few bytes rather than over this repository: the write path is what is being graded, not the tool's subject matter. `{target}` is the only path that does not exist -- it must not. READ-ONLY on the repo: the only files created are under a TemporaryDirectory. USAGE python3 tools/write_probe.py [--repo DIR] [--only TOOL] [--timeout S] [--json PATH] [--verbose] python3 tools/write_probe.py --selftest # 11 cells, judge, table, # fault block, fixture EXIT 0 every probed tool is OK; 1 at least one is not (including NO_REACH and NO_JSON: an ungraded write path may not be reported clean); 2 could not grade -- no tracked tools discovered, or --json itself unwritable. """ import argparse import json import os import re import subprocess import sys import tempfile JSON_FLAG = re.compile(r"add_argument\(\s*[\"']--json[\"']") TRACEBACK = "Traceback (most recent call last)" # (tool, argv) -- `{...}` placeholders are filled by fixture_paths(). One entry # per tracked tool: a tool the table does not name is NO_PROBE (fatal), on the # same rule as the read axis -- an ungraded path may not pass. WRITE_PROBES = ( ("append_only_audit.py", ["--repo", "{tinyrepo}", "--json", "{target}"]), ("append_only_owners.py", ["{audit}", "--json", "{target}"]), ("census_row_delta.py", ["{census_a}", "{census_b}", "--json", "{target}"]), ("check_lint.py", ["--check", "echo ok", "--json", "{target}"]), ("checkin_check.py", ["--dir", "{checkins}", "--roster", "volt", "--json", "{target}"]), ("citation_hash_class.py", ["--repo", "{tinyrepo}", "--json", "{target}"]), ("citation_resolve_check.py", ["--repo", "{tinyrepo}", "--json", "{target}"]), ("doc_census_check.py", ["--repo", "{tinyrepo}", "--no-run", "--json", "{target}"]), ("doc_command_check.py", ["--repo", "{tinyrepo}", "--json", "{target}"]), ("gate_blame.py", ["--repo", "{tinyrepo}", "--leg", "selftest-sweep", "--range", "1", "--json", "{target}"]), ("landing_check.py", ["--repo", "{tinyrepo}", "--json", "{target}"]), ("path_claim_check.py", ["--repo", "{tinyrepo}", "--json", "{target}"]), ("queue_append_audit.py", ["--repo", "{tinyrepo}", "--json", "{target}"]), ("queue_claim_check.py", ["--queue", "{queue}", "--json", "{target}"]), ("queue_index.py", ["--queue", "{queue}", "--json", "{target}"]), ("reply_check.py", ["--dir", "{replies}", "--json", "{target}"]), ("selftest_sweep.py", ["--repo", "{tinyrepo}", "--json", "{target}"]), ("served_drift_check.py", ["--root", "{tinyrepo}", "--base-url", "{baseurl}", "--json", "{target}"]), ("tool_map.py", ["--repo", "{tinyrepo}", "--json", "{target}"]), ("verify.py", ["--repo", "{tinyrepo}", "--fast", "--json", "{target}"]), # It grades itself: run over the one-tool fixture repo, this probe reports # that fixture tool as NO_PROBE (rc 1), then turns to its own --json write, # which is the path being graded. Committing the instrument without this # row would make the table's own NO_PROBE rule fail on its author. ("write_probe.py", ["--repo", "{tinyrepo}", "--json", "{target}"]), ) def _write(path, text): d = os.path.dirname(path) if d and not os.path.isdir(d): os.makedirs(d, exist_ok=True) with open(path, "w", encoding="utf-8") as fh: fh.write(text) def build_fixtures(td, real_repo): """A one-commit git repo, one fake instrument, one reply, one check-in, one log, one queue and two census files: enough for every tool in the table to earn a 0 or a 1 and then turn to its write. Sizes are deliberately tiny.""" p = {} tiny = os.path.join(td, "tiny") os.makedirs(os.path.join(tiny, "docs"), exist_ok=True) os.makedirs(os.path.join(tiny, "checkins"), exist_ok=True) os.makedirs(os.path.join(tiny, "replies"), exist_ok=True) os.makedirs(os.path.join(tiny, "tools"), exist_ok=True) _write(os.path.join(tiny, "README.md"), "# tiny\n\nA fixture repo.\n") _write(os.path.join(tiny, "WORK-QUEUE.md"), "# Work queue\n\n**Q1 -- OPEN (pipe, 2026-01-01). One item.**\n\n" "Body.\n") _write(os.path.join(tiny, "docs", "queue-index.md"), "# Index\n\n| id | state |\n| --- | --- |\n| Q1 | OPEN |\n") _write(os.path.join(tiny, "tools", "fake_tool.py"), '#!/usr/bin/env python3\n"""fake_tool.py -- a fake instrument.\n\n' 'USAGE:\n python3 fake_tool.py --selftest # 1 cells\n"""\n' 'import sys\nif "--selftest" in sys.argv:\n' ' print("selftest: 1 cells, 1 passed, 0 failed")\n' ' sys.exit(0)\nprint("fake")\n') _write(os.path.join(tiny, "checkins", "2026-01-01-volt.md"), "# volt check-in 2026-01-01\n\n" "- Built: the fixture.\n- Next: nothing.\n") _write(os.path.join(tiny, "replies", "2026-01-01-to-merlin-01.md"), "# reply 01 to merlin\n\n## 1. Subject\n\nBody text, one finding.\n\n" "-- volt\n") for cmd in (["init", "-q", "-b", "main"], ["add", "-A"], ["-c", "user.email=v@example.invalid", "-c", "user.name=volt", "commit", "-q", "-m", "fixture"]): subprocess.run(["git", "-C", tiny] + cmd, capture_output=True, text=True) p["tinyrepo"] = tiny p["queue"] = os.path.join(tiny, "WORK-QUEUE.md") p["index"] = os.path.join(tiny, "docs", "queue-index.md") p["replies"] = os.path.join(tiny, "replies") p["checkins"] = os.path.join(tiny, "checkins") p["toolsdir"] = os.path.join(tiny, "tools") p["audit"] = os.path.join(td, "audit.json") _write(p["audit"], json.dumps({"rows": []}) + "\n") p["log"] = os.path.join(td, "log.txt") _write(p["log"], "checked 2026-01-01 Q1 clean\n") for key in ("census_a", "census_b"): p[key] = os.path.join(td, key + ".json") _write(p[key], json.dumps({"rev": "abc1234", "repo": tiny, "reports": [], "rows": []}) + "\n") p["baseurl"] = "file://" + tiny p["target"] = os.path.join(td, "no-such-dir", "out.json") # The probe is meaningless if the target's parent exists: an existing # directory would let the write succeed and grade the wrong thing. assert not os.path.isdir(os.path.dirname(p["target"])) p["repo"] = real_repo return p def judge(rc, out, err, target): text = (out or "") + (err or "") if TRACEBACK in text: tail = [l for l in text.splitlines() if l.strip()][-1:] return {"verdict": "TRACEBACK", "rc": rc, "detail": (tail[0] if tail else "")[:110]} if rc != 2: first = next((l.strip() for l in (err or out or "").splitlines() if l.strip()), "") return {"verdict": "RC_OTHER", "rc": rc, "detail": (first or "rc %s, no message" % rc)[:110]} msg = [l for l in (err or "").splitlines() if l.strip()] or \ [l for l in (out or "").splitlines() if l.strip()] if not msg: return {"verdict": "SILENT", "rc": rc, "detail": "rc 2 with no message"} if target not in text: return {"verdict": "UNNAMED", "rc": rc, "detail": "rc 2 but the message never names the target: " + msg[-1][:64]} if "cannot write" not in text and "cannot be written" not in text: # rc 2 and the path is named, but not as a write refusal: the tool # refused its INPUT. The write path is therefore ungraded here. return {"verdict": "NO_REACH", "rc": rc, "detail": "rc 2 on the input, never reached the write: " + msg[-1][:64]} return {"verdict": "OK", "rc": rc, "detail": msg[-1][:110]} def fault_lines(rows, width): """One line per row that is not OK, naming the tool and its class. Printed ABOVE the table by main(). The position is the point: verify.py's detail line takes the FIRST reason-shaped line a leg prints, and every row of the table reads `cannot write --json : [Errno 2] ...`, so before this block the detail on a red write-axis leg was the first HEALTHY row. Measured 2026-10-21 (M1-leg of the mutation study): the row said `FAULT: append_only_audit.py OK 2 cannot write --json ...` on a worktree whose only defect was in doc_census_check.py -- the one tool a reader would not need to look at. The word FAULT is carried for the same reason: it is in verify.py's STRONG_HINTS, so a fault whose own detail has no diagnostic marker (SILENT: "rc 2 with no message") is still the line a reader sees. """ out = [] for r in rows: if r.get("verdict") == "OK": continue out.append("FAULT %-*s %-9s %4s %s" % (width, r.get("tool", "?"), r.get("verdict", "?"), "-" if r.get("rc") is None else r["rc"], (r.get("detail") or "")[:110])) return out def run_probe(path, argv, timeout, cwd): try: p = subprocess.run([sys.executable, path] + argv, cwd=cwd, capture_output=True, text=True, errors="replace", timeout=timeout) return p.returncode, p.stdout or "", p.stderr or "" except subprocess.TimeoutExpired: return None, "", "TIMEOUT after %ss" % timeout except OSError as exc: return None, "", "" % exc def tracked_tools(repo): p = subprocess.run(["git", "-C", repo, "ls-tree", "-r", "--name-only", "HEAD", "tools/"], capture_output=True, text=True) if p.returncode != 0: return None, (p.stderr or "git ls-tree failed").strip() return [os.path.basename(x) for x in p.stdout.splitlines() if x.endswith(".py")], "git ls-tree HEAD tools/" def selftest(repo): """Eleven cells: six for the judge's ordered verdicts, five for the table, the fixture and the fault block. The judge cells matter because the order is the contract -- a traceback outranks a right code, and a rc-2 input refusal (NO_REACH) must never read as a passed write path. The fault block gets its own cell because its position above the table is load-bearing for verify.py's detail line, and a formatting change would otherwise regress it silently. """ bad = [] def cell(name, ok, why=""): print((" ok " if ok else " FAIL ") + name + ("" if ok else " [%s]" % why)) if not ok: bad.append(name) t = os.path.join("no-such-dir", "out.json") tb = TRACEBACK + "\n File \"x.py\", line 1\nFileNotFoundError: [Errno 2]" cell("c1 a traceback is TRACEBACK whatever the exit code", judge(2, "", tb, t)["verdict"] == "TRACEBACK") cell("c2 a clean rc 1 is RC_OTHER, not a pass", judge(1, "", "cannot write --json " + t + "\n", t)["verdict"] == "RC_OTHER") cell("c3 rc 2 with no message is SILENT", judge(2, "", "", t)["verdict"] == "SILENT") cell("c4 rc 2 that never names the target is UNNAMED", judge(2, "", "gate_blame: cannot grade: all 1 commit(s) UNGRADABLE\n", t)["verdict"] == "UNNAMED") cell("c5 rc 2 naming the target but refusing INPUT is NO_REACH (ungraded)", judge(2, "", "cannot read " + t + ": no such file\n", t)["verdict"] == "NO_REACH") cell("c6 rc 2 naming the target as a WRITE refusal is OK", judge(2, "", "cannot write --json " + t + ": [Errno 2]\n", t)["verdict"] == "OK") names, source = tracked_tools(repo) table = dict(WRITE_PROBES) dupes = len([n for n, _ in WRITE_PROBES]) - len(table) cell("c7 the table names every tracked tool once, and no tool it cannot see", names is not None and not dupes and set(table) == set(names), "%s: table-only=%s tool-only=%s dupes=%s" % (source, sorted(set(table) - set(names or [])), sorted(set(names or []) - set(table)), dupes)) with tempfile.TemporaryDirectory(prefix="write-selftest-") as td: fx = build_fixtures(td, repo) cell("c8 the probe target's parent does not exist (or the write would succeed)", not os.path.isdir(os.path.dirname(fx["target"])), fx["target"]) if names and "check_lint.py" in names: rc, out, err = run_probe(os.path.join(repo, "tools", "check_lint.py"), ["--check", "echo ok", "--json", fx["target"]], 120, fx["tinyrepo"]) v = judge(rc, out, err, fx["target"]) cell("c9 a real tool, probed: rc 2, one line naming the target, no traceback", v["verdict"] == "OK", str(v)) else: cell("c9 a real tool, probed: rc 2, one line naming the target", False, "check_lint.py not discovered under %s" % repo) rows = [{"tool": "x", "verdict": "OK"}, {"tool": "y", "verdict": "NO_PROBE"}] cell("c10 NO_PROBE counts as a fault, so an ungraded path cannot pass", len([r for r in rows if r["verdict"] != "OK"]) == 1) # c11 exists because the fault block's POSITION is the whole point of it: # the block is what verify.py's detail line reads, and a block that named a # healthy tool (or came after the table) would put the wrong filename in # front of a reader on every red write-axis leg. rows2 = [{"tool": "healthy.py", "verdict": "OK", "rc": 0, "detail": "wrote"}, {"tool": "hurt.py", "verdict": "SILENT", "rc": 2, "detail": ""}, {"tool": "worse.py", "verdict": "TRACEBACK", "rc": 1, "detail": "FileNotFoundError: [Errno 2] no such file"}] blk = fault_lines(rows2, 10) cell("c11 the fault block names every faulting tool, never a healthy one", len(blk) == 2 and "hurt.py" in blk[0] and "worse.py" in blk[1] and "healthy.py" not in " ".join(blk) and all(ln.startswith("FAULT ") for ln in blk), str(blk)) print("selftest: 11 cells, %d passed, %d failed" % (11 - len(bad), len(bad))) return 1 if bad else 0 def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) ap.add_argument("--repo", default="/root/troupe-standup") ap.add_argument("--only", default=None, metavar="TOOL") ap.add_argument("--timeout", type=int, default=180, metavar="S") ap.add_argument("--json", default=None, metavar="PATH") ap.add_argument("--selftest", action="store_true", help="prove the judge and the table, then exit") ap.add_argument("--verbose", action="store_true") a = ap.parse_args(argv) repo = os.path.abspath(a.repo) if a.selftest: return selftest(repo) names, source = tracked_tools(repo) if names is None: print("cannot read: " + source, file=sys.stderr) return 2 if not names: print("cannot read: no tracked tools under " + repo, file=sys.stderr) return 2 table = dict(WRITE_PROBES) rows = [] with tempfile.TemporaryDirectory(prefix="write-probe-") as td: fx = build_fixtures(td, repo) for name in names: if a.only and name != a.only: continue path = os.path.join(repo, "tools", name) try: with open(path, encoding="utf-8") as fh: src = fh.read() except OSError as exc: rows.append({"tool": name, "verdict": "UNREADABLE", "rc": None, "detail": str(exc)[:110]}) continue if name not in table: rows.append({"tool": name, "verdict": "NO_PROBE", "rc": None, "detail": "no write probe in the table -- ungraded"}) continue if not JSON_FLAG.search(src): rows.append({"tool": name, "verdict": "NO_JSON", "rc": None, "detail": "no --json argument in source"}) continue argv = [re.sub(r"\{(\w+)\}", lambda m: fx[m.group(1)], x) for x in table[name]] rc, out, err = run_probe(path, argv, a.timeout, fx["tinyrepo"]) rec = {"tool": name, "probe": table[name]} rec.update(judge(rc, out, err, fx["target"])) rows.append(rec) if a.verbose: print(" . %-24s %-9s rc=%s" % (name, rec["verdict"], rc)) if not rows: print("cannot read: no tool matched --only " + str(a.only), file=sys.stderr) return 2 bad = [r for r in rows if r["verdict"] != "OK"] width = max([len(r["tool"]) for r in rows] + [4]) print("write probe -- %d of %d tracked tool(s) probed under %s (%s)" % (len(rows), len(names), os.path.join(repo, "tools"), source)) print() faults = fault_lines(rows, width) if faults: for ln in faults: print(ln) print() print("%-*s %-9s %4s detail" % (width, "tool", "verdict", "rc")) for r in rows: print("%-*s %-9s %4s %s" % (width, r["tool"], r["verdict"], "-" if r.get("rc") is None else r["rc"], r["detail"])) print() print("write axis: %d probed, %d ok, %d fault(s)" % (len(rows), len(rows) - len(bad), len(bad)) + ("" if not bad else " (" + ", ".join(sorted({r["verdict"] for r in bad})) + ")")) if a.json: payload = {"repo": repo, "discovery": source, "probed": len(rows), "ok": len(rows) - len(bad), "faults": len(bad), "tools": rows} try: with open(a.json, "w", encoding="utf-8") as fh: json.dump(payload, fh, indent=1, sort_keys=True) fh.write("\n") except OSError as exc: print("cannot write --json " + a.json + ": " + str(exc), file=sys.stderr) return 2 return 1 if bad else 0 if __name__ == "__main__": sys.exit(main())