diff --git a/tools/selftest_sweep.py b/tools/selftest_sweep.py index 67412a2..0efb318 100644 --- a/tools/selftest_sweep.py +++ b/tools/selftest_sweep.py @@ -49,6 +49,22 @@ THE READ-REFUSAL AXIS, ADDED 2026-10-17 read the queue", so one meaning carried two codes and a stale index (rc 1) could be confused with a queue that was never read. +THE MALFORMED-INPUT CLASS, ADDED 2026-10-18 + A missing path and non-UTF-8 bytes are the easy two. The class the axis left + unmeasured was the one in between: an input that DECODES but cannot be made + sense of -- a `.json` that is not JSON, a valid `{}` where an audit file + should be, a directory that is not a git repository, a queue file with no + records. Those are the inputs where a tool is most tempted to answer "0" and + return 0, which reads exactly like a clean pass (`doc_census_check`'s + `--tools` bug, `76043b7`, was this class). The axis now runs 15 such probes + and grades each against ITS OWN stated answer, because "must exit 2" is not + a universal truth: nine tools must refuse at rc 2, while `check_lint` grades + a junk log MISSING (rc 0 in report mode, rc 1 under --fail-on-fault), + `checkin_check` grades a heading-less check-in as an advisory (rc 0, stated + in its docstring) and `queue_claim_check` reports 0 opening records (rc 0, + its --fail-on-stale is the gate). The axis asserts the message too, so a + silent 0 cannot pass as a refusal. + WHAT COUNTS AS A CLAIM, PRECISELY Only the module docstring (read through `ast.get_docstring`, so a fixture string in the BODY is not a claim -- `doc_census_check.py` contains the literal @@ -72,7 +88,7 @@ does not decide that a claim SHOULD exist -- NO_CLAIM is advisory by design. USAGE: python3 selftest_sweep.py --repo /root/troupe-standup python3 selftest_sweep.py --tools-dir /tmp/fake --json /tmp/sweep.json - python3 selftest_sweep.py --selftest # 22 cells, parser, sweep and read axis + python3 selftest_sweep.py --selftest # 28 cells, parser, sweep and both input axes """ from __future__ import annotations @@ -121,7 +137,7 @@ FATAL = ( # verdict" and an operator could not tell them apart. The axis re-measures the # class on every tool, from the outside, against the real CLI. READ_AXIS_UNIT = 2 -FATAL_READ = ("TRACEBACK", "RC_NOT_2", "SILENT", "NO_TOOL") +FATAL_READ = ("TRACEBACK", "RC_NOT_2", "SILENT", "NO_TOOL", "RC_WRONG", "UNEXPLAINED") NO_PROBE = "NO_PROBE" # advisory: a tool with no probe defined # {nope} = a path that does not exist; {bin} = a file that is not UTF-8; @@ -150,6 +166,51 @@ NOT_UTF8_PROBES = ( ("queue_index.py", ["--queue", "{bin}", "--check", "{bin}"]), ) +# THE THIRD CLASS (2026-10-18): input that decodes as UTF-8 but cannot be made +# sense of. Each entry carries the answer the TOOL's own contract states -- +# (tool, label, argv, expected rc, a substring the output must contain) -- so +# the axis measures the documented behaviour instead of imposing one. Nine +# tools must refuse at rc 2; the three that grade a malformed input are listed +# with the rc and the word their grade prints, which is what makes a silent +# rc 0 (the doc_census_check --tools bug class) fail here. +# Fixtures: {notrepo} = a directory with a file and no .git; {badjson} = a +# UTF-8 file that is not JSON; {emptyjson} = "{}" (valid JSON, wrong shape); +# {junklog} = a log with no CHECK: line; {junkqueue} = 3 prose lines, no +# records; {junkindex} = a file with no pinned parse; {badcheckins} = a +# directory holding one heading-less check-in. +MALFORMED_PROBES = ( + ("append_only_audit.py", "not-a-repo", ["--repo", "{notrepo}"], + 2, "cannot read repo"), + ("append_only_owners.py", "malformed-json", ["{badjson}"], + 2, "cannot read audit"), + ("append_only_owners.py", "wrong-shape", ["{emptyjson}"], + 2, "no 'rows' list"), + ("census_row_delta.py", "malformed-json", ["{badjson}", "{badjson}"], + 2, "cannot read census"), + ("census_row_delta.py", "wrong-shape", ["{emptyjson}", "{emptyjson}"], + 2, "no 'reports' list"), + ("check_lint.py", "junk-log", ["--log", "{junklog}"], + 0, "MISSING"), + ("check_lint.py", "junk-log-gated", ["--log", "{junklog}", "--fail-on-fault"], + 1, "MISSING"), + ("checkin_check.py", "no-headings", ["--dir", "{badcheckins}", "--roster", "volt"], + 0, "advisory"), + ("citation_hash_class.py", "not-a-repo", ["--repo", "{notrepo}"], + 2, "not a git"), + ("citation_resolve_check.py", "not-a-repo", ["--repo", "{notrepo}"], + 2, "not a git"), + ("doc_census_check.py", "not-a-repo", ["--repo", "{notrepo}", "--no-run"], + 2, "no tools found"), + ("landing_check.py", "not-a-repo", ["--repo", "{notrepo}"], + 2, "not a git work tree"), + ("queue_claim_check.py", "junk-queue", ["--queue", "{junkqueue}"], + 0, "0 opening records"), + ("queue_index.py", "junk-index", ["--queue", "{junkqueue}", "--check", "{junkindex}"], + 2, "UNVERIFIABLE"), + ("selftest_sweep.py", "not-a-repo", ["--repo", "{notrepo}"], + 2, "no tools discovered"), +) + def _int(text: str) -> int: return int(text.replace(",", "")) @@ -387,8 +448,39 @@ def run_read_probe(path: str, argv, timeout: int): with open(os.path.join(bindir, "2026-01-01-volt.md"), "wb") as fh: fh.write(b"\xff\xfe\x00binary\x00\n") nope = os.path.join(td, "there-is-no-such-path") - real = [a.replace("{nope}", nope).replace("{bin}", binpath) - .replace("{bindir}", bindir) for a in argv] + + # the malformed class: each of these DECODES, none of it parses + notrepo = os.path.join(td, "not-a-repo") + os.makedirs(notrepo, exist_ok=True) + with open(os.path.join(notrepo, "README.txt"), "w") as fh: + fh.write("files, but no .git here\n") + paths = {"badjson": os.path.join(td, "bad.json"), + "emptyjson": os.path.join(td, "empty.json"), + "junklog": os.path.join(td, "log.txt"), + "junkqueue": os.path.join(td, "q.md"), + "junkindex": os.path.join(td, "i.md")} + bodies = {"badjson": "not json at all\n", + "emptyjson": "{}\n", + "junklog": "garbage line\nnot a lint entry\n", + "junkqueue": "this is not a queue\nno headings here at all\nQ\n", + "junkindex": "not an index either\n"} + for key, body in bodies.items(): + with open(paths[key], "w", encoding="utf-8") as fh: + fh.write(body) + badcheckins = os.path.join(td, "checkins") + os.makedirs(badcheckins, exist_ok=True) + with open(os.path.join(badcheckins, "2026-01-01-volt.md"), "w", + encoding="utf-8") as fh: + fh.write("no heading, no template\n") + + repl = {"{nope}": nope, "{bin}": binpath, "{bindir}": bindir, + "{notrepo}": notrepo, "{badcheckins}": badcheckins} + repl.update({"{" + k + "}": v for k, v in paths.items()}) + real = [] + for a in argv: + for key, val in repl.items(): + a = a.replace(key, val) + real.append(a) try: p = subprocess.run([sys.executable, path] + real, cwd=td, capture_output=True, text=True, errors="replace", @@ -400,6 +492,26 @@ def run_read_probe(path: str, argv, timeout: int): return p.returncode, (p.stdout or "") + (p.stderr or "") +def judge_malformed(rc, text, expect_rc, need): + """One malformed probe's verdict, against the answer its tool states. + Graded in this order: a traceback is a defect whatever the code; a code + that is not the documented one is the defect this class exists to catch + (the silent rc 0); a refusal with no message cannot be acted on; and a + right code with the wrong verdict word is not the documented answer.""" + if "Traceback (most recent call last)" in (text or ""): + return {"verdict": "TRACEBACK", "rc": rc, "detail": _first_line(text)} + if rc != expect_rc: + return {"verdict": "RC_WRONG", "rc": rc, + "detail": "rc %s, contract says %s: %s" + % (rc, expect_rc, _first_line(text) or "no output")} + if not (text or "").strip(): + return {"verdict": "SILENT", "rc": rc, "detail": "rc %s with no message" % rc} + if need and need not in (text or ""): + return {"verdict": "UNEXPLAINED", "rc": rc, + "detail": "wanted %r, got: %s" % (need, _first_line(text))} + return {"verdict": "OK", "rc": rc, "detail": _first_line(text)} + + def read_axis(rows, timeout: int): """One row per (tool, probe). Tools the sweep discovered are probed by path; a probe naming a tool that was not discovered is NO_TOOL (fatal -- the table @@ -416,16 +528,31 @@ def read_axis(rows, timeout: int): rc, text = run_read_probe(paths[name], argv, timeout) rec.update(judge_read(rc, text)) out.append(rec) + for name, label, argv, expect_rc, need in MALFORMED_PROBES: + rec = {"tool": name, "probe": label, "expect_rc": expect_rc} + if name not in paths: + rec.update({"verdict": "NO_TOOL", "rc": None, + "detail": "probe names a tool the sweep did not discover"}) + else: + rc, text = run_read_probe(paths[name], argv, timeout) + rec.update(judge_malformed(rc, text, expect_rc, need)) + out.append(rec) probed = {n for n, _ in READ_PROBES} + malformed = {t for t, *_ in MALFORMED_PROBES} for name, _p in rows: if name not in probed: out.append({"tool": name, "probe": "missing", "verdict": NO_PROBE, "rc": None, "detail": "no read probe defined (advisory)"}) + if name not in malformed: + out.append({"tool": name, "probe": "malformed", "verdict": NO_PROBE, + "rc": None, "detail": "no malformed probe defined (advisory)"}) return out def report_read(axis, quiet=False): fatal = [r for r in axis if r["verdict"] in FATAL_READ] + hard = [r for r in axis if r["probe"] in ("missing", "not-utf8")] + soft = [r for r in axis if r["probe"] not in ("missing", "not-utf8")] if not quiet: print() print(f"read-refusal axis: an unreadable input exits {READ_AXIS_UNIT}, " @@ -433,11 +560,13 @@ def report_read(axis, quiet=False): width = max([len(r["tool"]) for r in axis] + [4]) for r in axis: rc = "-" if r["rc"] is None else str(r["rc"]) - print(f"{r['tool']:<{width}} {r['probe']:<9} rc={rc:<4} " - f"{r['verdict']:<10} {r['detail']}") - ok = sum(1 for r in axis if r["verdict"] == "OK") - print(f"read-refusal: {len(axis)} probes, {ok} ok, {len(fatal)} fault" - + (f" ({', '.join(sorted({r['verdict'] for r in fatal}))})" if fatal else "")) + print(f"{r['tool']:<{width}} {r['probe']:<15} rc={rc:<4} " + f"{r['verdict']:<11} {r['detail']}") + for label, group in (("read-refusal", hard), ("malformed-input", soft)): + ok = sum(1 for r in group if r["verdict"] == "OK") + bad = [r for r in group if r["verdict"] in FATAL_READ] + print(f"{label}: {len(group)} probes, {ok} ok, {len(bad)} fault" + + (f" ({', '.join(sorted({r['verdict'] for r in bad}))})" if bad else "")) return 1 if fatal else 0 @@ -547,7 +676,32 @@ def selftest(): jr = judge_read(2, "") cell("c22 rc 2 with no message -> SILENT", jr["verdict"] == "SILENT", jr["verdict"]) - print(f"selftest: 22 cells, {22 - len(bad)} passed, {len(bad)} failed") + # c23-c28: the malformed class's judgement, and its coverage. A decodable + # input has no single right answer -- each tool's contract decides -- so + # these cells pin the judgement itself: the documented rc passes, a wrong + # code fails, and a right code with no explanation fails too. + jm = judge_malformed(0, "0 opening records; 0 closed; 0 open.", 0, "0 opening records") + cell("c23 a documented rc 0 grade on malformed input -> OK", + jm["verdict"] == "OK", jm["verdict"]) + jm = judge_malformed(0, "no tools found at HEAD", 2, "no tools found") + cell("c24 the silent-zero: rc 0 where the contract says 2 -> RC_WRONG", + jm["verdict"] == "RC_WRONG", jm["verdict"]) + jm = judge_malformed(2, " ", 2, "") + cell("c25 a refusal with no message -> SILENT", jm["verdict"] == "SILENT", jm["verdict"]) + jm = judge_malformed(2, "something went wrong", 2, "cannot read") + cell("c26 the right code with the wrong reason -> UNEXPLAINED", + jm["verdict"] == "UNEXPLAINED", jm["verdict"]) + jm = judge_malformed(2, "Traceback (most recent call last):", 2, "") + cell("c27 a traceback is fatal even at the documented code", + jm["verdict"] == "TRACEBACK", jm["verdict"]) + + missing_probed = {t for t, _ in READ_PROBES} + malformed_probed = {t for t, *_ in MALFORMED_PROBES} + cell("c28 every tool with a missing-probe also has a malformed probe", + missing_probed == malformed_probed, + str(sorted(missing_probed ^ malformed_probed))) + + print(f"selftest: 28 cells, {28 - len(bad)} passed, {len(bad)} failed") return 1 if bad else 0