diff --git a/tools/selftest_sweep.py b/tools/selftest_sweep.py index a282e7b..7bbf4a3 100644 --- a/tools/selftest_sweep.py +++ b/tools/selftest_sweep.py @@ -45,7 +45,9 @@ THE READ-REFUSAL AXIS, ADDED 2026-10-17 a fault, rc 2 could not grade. The axis runs each tool against a path that does not exist (and, for the four file readers, against bytes that are not UTF-8) and grades the answer OK / RC_NOT_2 / TRACEBACK / SILENT; the last - three are fatal. rc 3 is retired: the two queue readers used it for "cannot + three are fatal. So is NO_PROBE (below): a tool the tables never name is a + tool this axis never graded, and an ungraded tool may not be reported clean. + rc 3 is retired: the two queue readers used it for "cannot read the queue", so one meaning carried two codes and a stale index (rc 1) could be confused with a queue that was never read. @@ -58,7 +60,7 @@ THE MALFORMED-INPUT CLASS, ADDED 2026-10-18 return 0, which reads exactly like a clean pass (`doc_census_check`'s `--tools` bug, `76043b7`, was this class). The axis now runs 17 such probes and grades each against ITS OWN stated answer, because "must exit 2" is not - a universal truth: ten tools must refuse at rc 2, while `check_lint` grades + a universal truth: eleven tools must refuse at rc 2, while `check_lint` grades a junk log MISSING (rc 0 in report mode, rc 1 under --fail-on-fault), `checkin_check` grades a heading-less check-in as an advisory (rc 0, stated in its docstring) and `queue_claim_check` reports 0 opening records (rc 0, @@ -84,11 +86,12 @@ DISCOVERY, AND WHY UNTRACKED FILES ARE PRINTED READ-ONLY: the only write path is `--json PATH`. It does not edit a tool, and it does not decide that a claim SHOULD exist -- NO_CLAIM is advisory by design. +NO_PROBE is not: an ungraded tool cannot be reported clean (see the read axis). USAGE: python3 selftest_sweep.py --repo /root/troupe-standup python3 selftest_sweep.py --tools-dir /tmp/fake --json /tmp/sweep.json - python3 selftest_sweep.py --selftest # 28 cells, parser, sweep and both input axes + python3 selftest_sweep.py --selftest # 30 cells, parser, sweep and both input axes """ from __future__ import annotations @@ -137,8 +140,14 @@ FATAL = ( # verdict" and an operator could not tell them apart. The axis re-measures the # class on every tool, from the outside, against the real CLI. READ_AXIS_UNIT = 2 -FATAL_READ = ("TRACEBACK", "RC_NOT_2", "SILENT", "NO_TOOL", "RC_WRONG", "UNEXPLAINED") -NO_PROBE = "NO_PROBE" # advisory: a tool with no probe defined +# NO_PROBE is fatal, not advisory (2026-10-18): a tool the tables never name is +# a tool neither axis graded, and the pre-fix sweep reported a 15th tool with no +# probe entry as "15 tools, 14 OK, 0 fatal ... rc 0" -- two NO_PROBE rows and a +# green gate. An ungraded tool may not be reported clean; the tables are the +# gate's only cover, so a gap in them has to fail like a fault in a tool. +FATAL_READ = ("TRACEBACK", "RC_NOT_2", "SILENT", "NO_TOOL", "RC_WRONG", + "UNEXPLAINED", "NO_PROBE") +NO_PROBE = "NO_PROBE" # fatal: a tool with no probe defined # {nope} = a path that does not exist; {bin} = a file that is not UTF-8; # {bindir} = a directory holding one non-UTF-8 *.md (added 2026-10-18 with @@ -171,7 +180,7 @@ NOT_UTF8_PROBES = ( # THE THIRD CLASS (2026-10-18): input that decodes as UTF-8 but cannot be made # sense of. Each entry carries the answer the TOOL's own contract states -- # (tool, label, argv, expected rc, a substring the output must contain) -- so -# the axis measures the documented behaviour instead of imposing one. Nine +# the axis measures the documented behaviour instead of imposing one. Eleven # tools must refuse at rc 2; the three that grade a malformed input are listed # with the rc and the word their grade prints, which is what makes a silent # rc 0 (the doc_census_check --tools bug class) fail here. @@ -520,10 +529,17 @@ def judge_malformed(rc, text, expect_rc, need): return {"verdict": "OK", "rc": rc, "detail": _first_line(text)} +def axis_faults(axis): + """The rows that make the axis red. NO_PROBE is one of them: a tool with no + probe entry has not been graded, and 'not graded' is not 'clean'.""" + return [r for r in axis if r["verdict"] in FATAL_READ] + + def read_axis(rows, timeout: int): """One row per (tool, probe). Tools the sweep discovered are probed by path; a probe naming a tool that was not discovered is NO_TOOL (fatal -- the table - and the probes have drifted apart).""" + and the probes have drifted apart), and a discovered tool the tables never + name is NO_PROBE on each axis (also fatal: the tool is uncovered).""" paths = dict(rows) out = [] for probe, specs in (("missing", READ_PROBES), ("not-utf8", NOT_UTF8_PROBES)): @@ -550,15 +566,17 @@ def read_axis(rows, timeout: int): for name, _p in rows: if name not in probed: out.append({"tool": name, "probe": "missing", "verdict": NO_PROBE, - "rc": None, "detail": "no read probe defined (advisory)"}) + "rc": None, + "detail": "no read probe defined -- ungraded, fatal"}) if name not in malformed: out.append({"tool": name, "probe": "malformed", "verdict": NO_PROBE, - "rc": None, "detail": "no malformed probe defined (advisory)"}) + "rc": None, + "detail": "no malformed probe defined -- ungraded, fatal"}) return out def report_read(axis, quiet=False): - fatal = [r for r in axis if r["verdict"] in FATAL_READ] + fatal = axis_faults(axis) hard = [r for r in axis if r["probe"] in ("missing", "not-utf8")] soft = [r for r in axis if r["probe"] not in ("missing", "not-utf8")] if not quiet: @@ -709,7 +727,23 @@ def selftest(): missing_probed == malformed_probed, str(sorted(missing_probed ^ malformed_probed))) - print(f"selftest: 28 cells, {28 - len(bad)} passed, {len(bad)} failed") + # c29/c30: the NO_PROBE hole, measured from the other side -- a tool the + # tables never name must come out uncovered on BOTH axes and must make the + # axis red. Before this cell existed, read_axis emitted two advisory rows and + # a 15-tool directory graded "14 OK, 0 fatal, rc 0". + alone = read_axis([("zzz_unprobed.py", "/nonexistent/zzz_unprobed.py")], 5) + mine = [r for r in alone if r["tool"] == "zzz_unprobed.py"] + cell("c29 an unprobed tool is NO_PROBE on both axes and is a fault", + sorted(r["probe"] for r in mine) == ["malformed", "missing"] + and len(axis_faults(mine)) == 2, + str([(r["probe"], r["verdict"]) for r in mine])) + cell("c30 every FATAL_READ verdict is red and OK is not", + all(axis_faults([{"verdict": v}]) for v in FATAL_READ) + and not axis_faults([{"verdict": "OK"}]) + and NO_PROBE in FATAL_READ, + str(FATAL_READ)) + + print(f"selftest: 30 cells, {30 - len(bad)} passed, {len(bad)} failed") return 1 if bad else 0