#!/usr/bin/env python3 """doc_command_check.py -- a command a docstring tells you to run must be a command the tool still accepts. WHY THIS EXISTS Every tool in `tools/` documents itself with commands a reader is meant to paste: `python3 checkin_check.py --repo . --fail --baseline ...`. Those commands are the tool's interface as its readers actually meet it, and they are PROSE: nothing executes them. `doc_census_check.py` grades ` ` claims, `selftest_sweep.py` grades the advertised cell count, and neither looks at a flag name. So a renamed or removed option leaves every quoted command in the docstring pointing at an argument argparse rejects -- and the tool still exits 0 on everything it is asked to grade, because no run in this archive ever passes `--flag` from a docstring to argparse. This is the same class the archive already grades twice under other notations: a number one notation away from the net that exists to catch it (the doc pin at `queue_claim_check.py`), and a count written as a word with no reader at all (the 2026-10-19 reply_check defect). A command written as prose is the third notation of that same defect. WHAT IT MEASURES -- one verdict per documented command The oracle is the tool's own `--help`, which is the only statement of its interface that cannot drift from the interface: argparse prints it from the parser it will use. Every `--long` / `-s` flag on a documented command line is looked up in that set. OK every documented flag is an option the tool prints FLAG_NOT_ACCEPTED a documented flag is not in the tool's `--help`: the command as written is rejected by argparse. A FAULT. TOOL_MISSING the command names a .py file that is not in the tree at the revision read: a reader cannot run it. A FAULT. NO_ORACLE the tool prints no usable `--help` (it parses argv by hand, or refuses to start), so its documented flags cannot be adjudicated. ADVISORY: an absent oracle is not a false command. A flag is only CHECKED against a tool that HAS an oracle, so a tool without argparse can never be accused of rejecting its own documentation. READ-ONLY, and it runs nothing but `--help`. `--help` is the one invocation of a tool that does no work by construction; this checker never runs a documented command, so it cannot write, post, or deploy anything a docstring asks for. The only write path is `--json PATH`. REVISIONS. `--rev` selects the tree the docstrings are read from AND the tree the named tools are taken from, so the claim and its oracle come from the same revision: a command documented at `R` is checked against the tool at `R`. The default is `HEAD`. A revision that does not materialise is rc 2, never an empty clean run. `--rev WORKTREE` is not a revision: it reads the files on disk and lists them from the INDEX, so it grades the tree you are about to commit. `HEAD` cannot see an uncommitted docstring, which is the window a lane writes in -- the same blind spot `tool_map.py --rev WORKTREE` exists for. The two are measured apart in the selftest: one document, edited on disk but not committed, is rc 0 at `HEAD` and rc 1 at `WORKTREE`. EXIT CODES (the tools/ contract) 0 every documented command was adjudicated and none was rejected 1 at least one documented command is a fault (FLAG_NOT_ACCEPTED, TOOL_MISSING) 2 could not grade: not a git work tree, a revision that will not materialise, or a `--help` run that outlived its timeout USAGE python3 doc_command_check.py --repo . [--rev REV] [--tools a.py,b.py] [--json OUT] [--fail] python3 doc_command_check.py --selftest # 18 cells, oracle and verdicts """ import argparse import ast import contextlib import io import json import os import re import subprocess import sys import tempfile from collections import Counter # A documented command: `python3 .py `. The path may be written # as `tools/x.py` from the repo root or as `x.py` from inside `tools/`, which # is how both forms appear in this archive; `resolve_target` accepts either. CMD_RE = re.compile(r"python3?\s+(\S*?[\w.-]+\.py)((?:[ \t]+[^\s#|;&<>]+)*)") # A flag: `--long` or `-s`. A bare `-` (stdin) and a negative number are not # flags, and `--` is the separator, not an option. FLAG_RE = re.compile(r"(? the tool paths in the tree at `rev`.""" if rev == WORKTREE: # The index, not the disk: an untracked `tools/x.py` is not part of the # commit this mode grades, and including it would grade a file that is # about to be left behind. `git ls-files` fails outside a work tree, # which is the rc 2 this mode owes a caller who ran it anywhere else. proc = subprocess.run(["git", "-C", repo, "ls-files", "--", "tools"], capture_output=True, text=True) else: proc = subprocess.run( ["git", "-C", repo, "ls-tree", "-r", "--name-only", rev, "--", "tools"], capture_output=True, text=True) if proc.returncode != 0: raise CannotGrade("cannot read repo: %s at %s: %s" % (repo, rev, (proc.stderr or "").strip()[:200])) return [p for p in proc.stdout.splitlines() if p.startswith("tools/") and p.endswith(".py")] def blob(repo, rev, path): """The bytes of `path` at `rev`, or None when the tree does not hold it.""" if rev == WORKTREE: try: with open(os.path.join(repo, *path.split("/")), "r", encoding="utf-8") as fh: return fh.read() except FileNotFoundError: return None except OSError as exc: raise CannotGrade("cannot read %s at WORKTREE: %s" % (path, exc)) except UnicodeDecodeError as exc: raise CannotGrade("cannot read %s at WORKTREE: not UTF-8: %s" % (path, exc)) proc = subprocess.run(["git", "-C", repo, "show", "%s:%s" % (rev, path)], capture_output=True) if proc.returncode != 0: return None try: return proc.stdout.decode("utf-8") except UnicodeDecodeError as exc: raise CannotGrade("cannot read %s at %s: not UTF-8: %s" % (path, rev, exc)) def docstring_of(source): """The module docstring, or "" -- a file that will not parse has none.""" try: return ast.get_docstring(ast.parse(source)) or "" except SyntaxError: return "" def documented_commands(doc): """[(path, [flags])] for every command line in a docstring. The argument run stops at a shell separator by construction (CMD_RE's character class), so `--emit-baseline > tools/x.tsv` documents `--emit-baseline` and not the redirect target. """ out = [] for line in (doc or "").splitlines(): for m in CMD_RE.finditer(line): target, rest = m.group(1), m.group(2) or "" flags = FLAG_RE.findall(rest) out.append((target, flags)) return out def resolve_target(repo, rev, target, tools): """The tree path a documented `x.py` names, or None. `tools/x.py` resolves directly. A bare `x.py` resolves to `tools/x.py`, because docstrings in this archive are written from inside `tools/` -- and only if that path is actually in the tree at `rev`, so a command naming a file that does not exist stays TOOL_MISSING instead of resolving to something plausible. """ cand = [target.lstrip("./")] if "/" not in cand[0]: cand.append("tools/" + cand[0]) for c in cand: if c in tools: return c return None def help_options(repo, rev, path): """(set of option strings, None) or (None, why-not) for one tool at `rev`. The tool is run from a TEMPORARY COPY of the blob at `rev`, never from the worktree: a docstring read at `R` must be adjudicated against the parser at `R`, or the check silently grades the past against the present. """ source = blob(repo, rev, path) if source is None: return None, "not in the tree at %s" % rev with tempfile.TemporaryDirectory() as td: copy = os.path.join(td, os.path.basename(path)) with open(copy, "w", encoding="utf-8") as fh: fh.write(source) try: proc = subprocess.run([sys.executable, copy, "--help"], capture_output=True, text=True, timeout=HELP_TIMEOUT, cwd=repo) except subprocess.TimeoutExpired: raise CannotGrade("cannot grade: %s --help at %s outlived %ds" % (path, rev, HELP_TIMEOUT)) if proc.returncode != 0 and not proc.stdout.strip(): return None, "no usable --help (rc %d)" % proc.returncode opts = set(OPT_RE.findall(proc.stdout)) if not opts: return None, "no --help output" return opts, None def census(repo, rev, only=None): """The report: one row per documented command, plus the per-tool oracles.""" tools = tracked_tools(repo, rev) if only: tools = [t for t in tools if os.path.basename(t) in only] rows, oracles = [], {} for path in tools: doc = docstring_of(blob(repo, rev, path) or "") for target, flags in documented_commands(doc): real = resolve_target(repo, rev, target, tools) if real is None: rows.append({"doc": path, "target": target, "verdict": "TOOL_MISSING", "missing": [], "detail": "no such file at %s" % rev}) continue if real not in oracles: oracles[real] = help_options(repo, rev, real) opts, why = oracles[real] if opts is None: rows.append({"doc": path, "target": real, "verdict": "NO_ORACLE", "missing": [], "detail": why}) continue missing = sorted(f for f in flags if f not in opts) rows.append({"doc": path, "target": real, "verdict": "FLAG_NOT_ACCEPTED" if missing else "OK", "missing": missing, "detail": ""}) return {"rev": rev, "rows": rows, "oracles": len(oracles)} def fatal_rows(rep): return [r for r in rep["rows"] if r["verdict"] in ("FLAG_NOT_ACCEPTED", "TOOL_MISSING")] def report(rep, out=sys.stdout): for r in rep["rows"]: if r["verdict"] == "OK": continue miss = (" missing=%s" % (r["missing"],)) if r["missing"] else "" print("%-18s %-30s -> %-26s %s%s" % (r["verdict"], r["doc"], r["target"], r["detail"], miss), file=out) # `collections.Counter` replaces the hand-rolled tally loop: every entry's # verdict is counted exactly as `counts[k] = counts.get(k, 0) + 1` did, and # `sorted(counts)` still orders the keys, so the rendered summary is # byte-identical -- this is a deletion of a duplicate loop, not a rewrite. counts = Counter(r["verdict"] for r in rep["rows"]) print("doc commands at %s: %d command(s) over %d oracle(s) -- %s" % (rep["rev"], len(rep["rows"]), rep["oracles"], ", ".join("%s %d" % (k, counts[k]) for k in sorted(counts)) or "none")) # ------------------------------------------------------------------ selftest def selftest(): """Fixtures for the reader, the oracle and both verdicts, on one document.""" cells = [] def cell(label, ok, detail=None): cells.append((label, bool(ok), detail)) doc = ( "USAGE\n" " python3 selfcheck_tool.py --repo . --fail --baseline b.tsv\n" " python3 tools/selfcheck_tool.py --emit-baseline > tools/b.tsv\n" " python3 gone.py --whatever\n" " python3 selfcheck_tool.py --selftest # 12 cells, and a \n" " prose line that merely mentions --not-a-command in passing\n") cmds = documented_commands(doc) cell("every python3 command line is read", len(cmds) == 4, len(cmds)) cell("flags are read in order, values are not flags", cmds[0] == ("selfcheck_tool.py", ["--repo", "--fail", "--baseline"]), cmds[0] if cmds else None) cell("a redirect target is not read as an argument", cmds[1] == ("tools/selfcheck_tool.py", ["--emit-baseline"]), cmds[1]) cell("a bare filename is read as written", cmds[2][0] == "gone.py", cmds[2]) cell("prose mentions are not commands", all(c[0].endswith(".py") for c in cmds)) opts = {"-h", "--repo", "--fail", "--baseline", "--emit-baseline", "--selftest"} cell("an accepted flag is not missing", not [f for f in cmds[0][1] if f not in opts]) cell("a documented flag outside the oracle is named", [f for f in ["--renamed"] if f not in opts] == ["--renamed"]) # resolve_target, on a tree that holds only tools/selfcheck_tool.py tree = ["tools/selfcheck_tool.py"] cell("tools/x.py resolves in the tree", resolve_target(".", "HEAD", "tools/selfcheck_tool.py", tree) == "tools/selfcheck_tool.py") cell("a bare x.py resolves to tools/x.py", resolve_target(".", "HEAD", "selfcheck_tool.py", tree) == "tools/selfcheck_tool.py") cell("a bare x.py that is NOT in the tree stays unresolved", resolve_target(".", "HEAD", "gone.py", tree) is None) # argparse's own text is the oracle: check the reader against a real parser src = ( "import argparse\n" "ap = argparse.ArgumentParser()\n" "ap.add_argument('--repo', default='.')\n" "ap.add_argument('--long-only-name')\n" "ap.print_help()\n") cell("--help text is harvested into option strings", set(OPT_RE.findall(src)) >= {"--repo", "--long-only-name"}) harvested = set(OPT_RE.findall("usage: x [-h] [--fail] [--json JSON]\n")) cell("bracketed usage options are harvested", harvested >= {"-h", "--fail", "--json"}) # WORKTREE: the tree you are about to commit, not the last commit. One # document, edited on disk and NOT committed: HEAD reads the blob (clean), # WORKTREE reads the disk (the documented flag its parser never learned). with tempfile.TemporaryDirectory(prefix="doc-cmd-wt-") as td: repo = os.path.join(td, "repo") os.makedirs(os.path.join(repo, "tools")) tool = os.path.join(repo, "tools", "oracle_tool.py") def write_doc(extra=""): with open(tool, "w", encoding="utf-8") as fh: fh.write("#!/usr/bin/env python3\n" '"""USAGE\n' " python3 oracle_tool.py --present\n" + extra + '"""\n' "import argparse\n" "ap = argparse.ArgumentParser()\n" "ap.add_argument('--present')\n" "ap.parse_args()\n") def git(*args): subprocess.run(["git", "-C", repo] + list(args), capture_output=True, text=True) write_doc() git("init", "-q") git("add", "-A") git("-c", "user.name=selftest", "-c", "user.email=selftest@invalid", "-c", "commit.gpgsign=false", "commit", "-q", "-m", "fixture") write_doc(" python3 oracle_tool.py --gone-after-edit\n") head_rep = census(repo, "HEAD") wt_rep = census(repo, WORKTREE) cell("WORKTREE reads the document on disk, HEAD reads the blob at HEAD", [r["verdict"] for r in wt_rep["rows"]] == ["OK", "FLAG_NOT_ACCEPTED"] and [r["verdict"] for r in head_rep["rows"]] == ["OK"], ([r["verdict"] for r in head_rep["rows"]], [r["verdict"] for r in wt_rep["rows"]])) cell("the uncommitted command's flag is the one named", [r["missing"] for r in wt_rep["rows"] if r["missing"]] == [["--gone-after-edit"]], [r["missing"] for r in wt_rep["rows"] if r["missing"]]) outsider = os.path.join(td, "not-a-repo") os.makedirs(outsider) buf = io.StringIO() with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(buf): rc_nogit = run(["--repo", outsider, "--rev", WORKTREE, "--fail"]) cell("WORKTREE outside a git work tree is rc 2, not a clean run", rc_nogit == 2 and "cannot read repo" in buf.getvalue(), (rc_nogit, buf.getvalue().strip()[:80])) # verdicts rep = {"rev": "HEAD", "oracles": 1, "rows": [ {"doc": "tools/a.py", "target": "tools/a.py", "verdict": "OK", "missing": [], "detail": ""}, {"doc": "tools/a.py", "target": "tools/a.py", "verdict": "FLAG_NOT_ACCEPTED", "missing": ["--gone"], "detail": ""}, {"doc": "tools/a.py", "target": "gone.py", "verdict": "TOOL_MISSING", "missing": [], "detail": ""}, {"doc": "tools/a.py", "target": "tools/a.py", "verdict": "NO_ORACLE", "missing": [], "detail": "no --help output"}, ]} cell("both faults are graded, NO_ORACLE is not", [r["verdict"] for r in fatal_rows(rep)] == ["FLAG_NOT_ACCEPTED", "TOOL_MISSING"], [r["verdict"] for r in fatal_rows(rep)]) # the reader applied to this file's own docstring: every command that names # THIS tool must use a flag this tool accepts. A command quoting a sibling # (`checkin_check.py --baseline ...`) is prose about that sibling and is # checked against its own `--help`, not against this one's vocabulary. own = [c for c in documented_commands(__doc__) if os.path.basename(c[0]) == os.path.basename(__file__)] cell("this docstring documents commands of its own", len(own) >= 2, len(own)) known_here = {"--repo", "--rev", "--tools", "--json", "--fail", "--selftest", "-h", "--help"} bad = sorted({f for _, fs in own for f in fs if f not in known_here}) cell("every flag this docstring documents is one this tool accepts", not bad, bad) failed = [(lab, det) for lab, ok, det in cells if not ok] print("selftest: %d cells, %d passed, %d failed" % (len(cells), len(cells) - len(failed), len(failed))) for lab, det in failed: print(" FAILED: %s" % lab) if det is not None: print(" got: %r" % (det,)) return 1 if failed else 0 # ---------------------------------------------------------------------- main def run(argv): ap = argparse.ArgumentParser( description="grade the commands a tool's docstring tells a reader to run") ap.add_argument("--repo", default=".", help="the work tree (default: .)") ap.add_argument("--rev", default="HEAD", help="revision the docstrings AND the oracles are read from") ap.add_argument("--tools", default=None, help="comma-separated basenames to limit the scan to") ap.add_argument("--json", default=None, help="write the full report here") ap.add_argument("--fail", action="store_true", help="exit 1 when a documented command is rejected") ap.add_argument("--selftest", action="store_true", help="grade this tool's own cells and exit") args = ap.parse_args(argv) if args.selftest: return selftest() only = [t.strip() for t in args.tools.split(",") if t.strip()] if args.tools else None try: if not os.path.isdir(args.repo): raise CannotGrade("cannot read repo: %s: no such directory" % args.repo) rep = census(args.repo, args.rev, only) except CannotGrade as exc: print(str(exc), file=sys.stderr) return 2 report(rep) if args.json: try: with open(args.json, "w", encoding="utf-8") as fh: json.dump(rep, fh, indent=1, sort_keys=True) fh.write("\n") except OSError as exc: print("cannot write --json " + args.json + ": " + str(exc), file=sys.stderr) return 2 bad = fatal_rows(rep) if bad and args.fail: print("FAIL: %d documented command(s) this tool would reject" % len(bad)) return 1 return 0 if __name__ == "__main__": sys.exit(run(sys.argv[1:]))