diff --git a/tools/doc_census_check.py b/tools/doc_census_check.py index 645d07a..9ccfcdf 100644 --- a/tools/doc_census_check.py +++ b/tools/doc_census_check.py @@ -19,11 +19,17 @@ WHY THIS EXISTS WHAT IT MEASURES (read-only: this tool only writes --json) Two independent verdicts per tool, and they do not share evidence. - 1. INTERNAL SPLIT -- no oracle, no run. Within one docstring, every numeric - claim of the form ` ` is collected; two claims of ONE unit - that a single revision scope covers, carrying two different values, is a - split. This is the verdict that catches the 441/471 instance, and it is - why the tool is useful with no subprocess at all. + 1. INTERNAL SPLIT -- one unit, one scope, two values. Within one docstring, + every numeric claim of the form ` ` is collected; two claims + of ONE unit that a single revision scope covers, carrying two different + values, is a split. This catches the 441/471 instance. A split is GRADED + only when a leg verified the unit (`CHECKABLE`: the run printed it with + exactly one value); every other split -- no oracle (`LABEL_ONLY`, + `--no-run`), a unit no leg prints (`NOT_A_CENSUS_UNIT`), a unit the tool + prints per section (`SECTION_SPLIT_BY_TOOL`) -- is printed with its values + as ADVISORY and never reaches the gate. 2026-10-18: the old build failed + `--fail --no-run` on the 471/441 split at fca3f2e (rc 1) while the full run + could not see either value, so rc 1 meant "could not check this". 2. ORACLE -- the tool is RUN, at the revision its own docstring names, and each claim is compared against the value that run prints for that unit. @@ -146,10 +152,21 @@ FAILING ON THE CLASS IT CATCHES (`--fail`) The code means: 0 ran, and graded no fault (every claim adjudicated agrees) - 1 ran, and graded a fault (an INTERNAL SPLIT or a STALE claim) + 1 ran, and graded a fault (a VERIFIED INTERNAL SPLIT, or STALE) 2 could not grade (usage, a --tools filter that matches nothing tracked, unreadable input) + THE TWO MODES GRADE THE SAME CLASS. `--no-run` runs nothing, so it has no + oracle and cannot prove any split is a census unit: it prints every split as + ADVISORY with its values, and `--fail --no-run` exits 0. That is the ruling + of reply 22 (2026-10-18), and it fixes a real asymmetry: `--fail --no-run` + returned 1 on `citation_resolve_check.py` at fca3f2e (471@82 / 441@89) while + the same units were advisory under a full run. Measured after the repair at + the same rev, one mode apart: `--fail --no-run` rc 0 with the pair printed as + ADVISORY; `--fail` (full) rc 1 on the same split, now CHECKABLE because the + run prints `UNRESOLVED 471` -- plus the STALE claim at :89. The teeth are + kept where they can be verified, which is the only place they mean anything. + rc 2 keeps its meaning from the tools/ contract (6ee8fce) and is decided before the gate runs, so "could not grade" never reads as "graded clean" or as "graded a fault". A LEG FAILURE (RUN_FAILED) is reported in its own column @@ -159,7 +176,7 @@ FAILING ON THE CLASS IT CATCHES (`--fail`) USAGE python3 doc_census_check.py [--repo DIR] [--rev HEAD] [--tools NAME...] - [--json OUT] [--no-run] [--fail] [--selftest] # selftest: 52 cells, one per claim class + [--json OUT] [--no-run] [--fail] [--selftest] # selftest: 60 cells, one per claim class """ import argparse import ast @@ -759,12 +776,48 @@ def _fixture_checks(): check("gate: the two columns sum across tools", gate_faults([stale_rep, split_rep]), (1, 1)) + # ---- the mode symmetry (reply 22, 2026-10-18). A split is graded only when + # a leg verified its unit; without a run there is no oracle, so the + # same docstring that fails a full run must NOT fail `--no-run`. Both + # halves are pinned: the advisory side, and the teeth that survive. + check("mode: --no-run can only label a split, never verify it", + unit_scope("UNRESOLVED", [], True), "LABEL_ONLY") + check("mode: a unit no leg prints is not a census unit", + unit_scope("WOMBATS", [({"widgets": {7: [1]}}, {}, "")], False), + "NOT_A_CENSUS_UNIT") + with tempfile.TemporaryDirectory(prefix="doccensus-mode-") as td: + split_stub = os.path.join(td, "stub_split.py") + with open(split_stub, "w") as fh: + fh.write('"""A stub whose docstring splits one unit.\n' + "\n471 UNRESOLVED at the head.\n" + "pad " * 80 + "\n" + '441 UNRESOLVED at the tail.\n"""\n' + "print('UNRESOLVED: 471')\n") + rep_norun = check_tool(td, split_stub, "stub_split.py", {}, run=False, + tmpdir=td) + rep_run = check_tool(td, split_stub, "stub_split.py", {}, run=True, + tmpdir=td) + check("mode: --no-run prints the split, graded as none", + ([s["unit"] for s in rep_norun["splits"]], + [(s["unit"], s["how"]) for s in rep_norun["advisory"]]), + ([], [("UNRESOLVED", "LABEL_ONLY")])) + check("mode: the same split is CHECKABLE once a leg prints the unit", + [(s["unit"], s["how"]) for s in rep_run["splits"]], + [("UNRESOLVED", "CHECKABLE")]) + check("mode: --fail cannot fail on a split it could not verify", + gate_rc([rep_norun], True), 0) + check("mode: and still fails on the verified one", gate_rc([rep_run], True), 1) + with tempfile.TemporaryDirectory(prefix="doccensus-gate-") as td: os.makedirs(os.path.join(td, "tools")) for name, claim in (("stub_stale.py", 99), ("stub_ok.py", 7)): with open(os.path.join(td, "tools", name), "w") as fh: fh.write(f'"""A stub whose docstring says {claim} widgets."""\n' "print('widgets: 7')\n") + with open(os.path.join(td, "tools", "stub_split.py"), "w") as fh: + fh.write('"""A stub whose docstring splits one unit.\n' + "\n471 UNRESOLVED at the head.\n" + "pad " * 80 + "\n" + '441 UNRESOLVED at the tail.\n"""\n' + "print('UNRESOLVED: 471')\n") genv = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@t", GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@t") for sub in (["init", "-q"], ["add", "-A"], ["commit", "-qm", "fixture"]): @@ -782,6 +835,12 @@ def _fixture_checks(): run_gate("--tools", "tools/stub_ok.py", "--fail"), 0) check("gate e2e: a --tools filter matching nothing is rc 2, not 1", run_gate("--tools", "tools/nope.py", "--fail"), 2) + # the mode pair, as PROCESS exit codes: same tree, same split, the two + # modes must not disagree about what they verified + check("gate e2e: --fail --no-run on a split is rc 0 (no oracle, advisory)", + run_gate("--tools", "tools/stub_split.py", "--no-run", "--fail"), 0) + check("gate e2e: --fail with the run grades that same split (rc 1)", + run_gate("--tools", "tools/stub_split.py", "--fail"), 1) print(f"selftest: {cells[0]} cells, {cells[0] - bad[0]} passed, {bad[0]} failed") return 0 if ok else 1 @@ -936,7 +995,7 @@ def check_tool(repo, script, name, cache, run=True, tmpdir=".", isolation=240): # 1. INTERNAL SPLIT: one unit, one revision scope, two values, far enough # apart to be a restatement rather than a neighbouring sub-count. - splits, unchecked = [], collections.Counter() + splits, advisory, unchecked = [], [], collections.Counter() by_unit = collections.defaultdict(list) for first, revs, cl, _b in blk_claims: scope = tuple(revs) @@ -956,13 +1015,25 @@ def check_tool(repo, script, name, cache, run=True, tmpdir=".", isolation=240): rev_for_oracle = (items[0][0], scope[0] if len(scope) == 1 else "HEAD") why = ("SELF_NOT_RUN" if is_self and run else unit_scope(unit, legs_for(*rev_for_oracle), not run)) - if why != "CHECKABLE" and why != "LABEL_ONLY": + row = {"unit": unit, "revs": list(scope), "values": sorted(vals), + "how": why, "rows": [{"line": ln, "value": v, "block": b} + for b, ln, v, _ in items]} + # THE ONLY DOOR INTO `splits`. A split is graded only when a leg ran and + # printed its unit with exactly one value (CHECKABLE): then the docstring + # and the output are talking about one number and a disagreement is a + # defect the gate can verify. Every other reason -- no oracle at all + # (LABEL_ONLY, --no-run), a unit no leg prints (NOT_A_CENSUS_UNIT), a + # unit the tool itself prints per section (SECTION_SPLIT_BY_TOOL), this + # tool reading itself (SELF_NOT_RUN) -- is reported with its values and + # never reaches the gate. Decided 2026-10-18 (reply 22) after + # `--fail --no-run` returned 1 on the 471/441 split at fca3f2e while the + # full run returned 0 for the same units it could not see: rc 1 must + # never mean "I could not check this". + if why != "CHECKABLE": unchecked[why] += 1 + advisory.append(row) continue - splits.append({"unit": unit, "revs": list(scope), - "values": sorted(vals), "how": why, - "rows": [{"line": ln, "value": v, "block": b} - for b, ln, v, _ in items]}) + splits.append(row) # 2. ORACLE: every claim compared against what the tool prints, leg 1 first. if run: @@ -1003,7 +1074,8 @@ def check_tool(repo, script, name, cache, run=True, tmpdir=".", isolation=240): verdicts.append({"file": name, "block": first, "line": ln, "rev": rev, "unit": unit, "claim": val, "token": tok, "verdict": v, "detail": detail}) - return {"tool": name, "splits": splits, "unchecked": dict(unchecked), + return {"tool": name, "splits": splits, "advisory": advisory, + "unchecked": dict(unchecked), "failures": failures, "verdicts": verdicts, "self": is_self and run, "selftest_leg": (not run) or bool(selftest_leg and selftest_leg[0]), "blocks_with_claims": len(blk_claims)} @@ -1077,10 +1149,13 @@ def main(): ap.add_argument("--tools", nargs="*") ap.add_argument("--json") ap.add_argument("--no-run", action="store_true", - help="internal-split verdicts only; no subprocess") + help="label-only: no subprocess, no oracle, so every split " + "it prints is ADVISORY and --fail cannot fail on one -- " + "rc 1 would mean 'could not check this'") ap.add_argument("--fail", action="store_true", - help="rc 1 if any tool reports an INTERNAL SPLIT or a STALE " - "claim; leg failures are reported, never spent as " + help="rc 1 if any tool reports an INTERNAL SPLIT a leg " + "VERIFIED (CHECKABLE) or a STALE claim; unverified " + "splits and leg failures are reported, never spent as " "defects (rc 2 stays 'could not grade')") ap.add_argument("--selftest", action="store_true") a = ap.parse_args() @@ -1135,6 +1210,15 @@ def main(): print(f" SPLIT {s['unit']} [{scope}] ({s['how']}): {vals}") for k, c in sorted(rep["unchecked"].items()): print(f" (not flagged: {c} split(s) in units {k})") + # A --no-run split is the one advisory class a reader of the fast mode + # actually needs to SEE: it is what the old build failed the gate on. + # Printed with its values, labelled advisory, never counted above. + for s in rep.get("advisory") or []: + if s["how"] != "LABEL_ONLY": + continue + vals = ", ".join(f"{r['value']}@{r['line']}" for r in s["rows"]) + print(f" ADVISORY {s['unit']} (LABEL_ONLY, no oracle): {vals}" + f" -- reported, not graded") for v in bad: print(f" {v['verdict']:10s} {v['file']}:{v.get('line', '-')} " f"{v['unit']} claimed {v['claim']} at {v['rev']} -- {v['detail']}") @@ -1159,7 +1243,9 @@ def main(): + (" (internal-split only: --no-run)" if a.no_run else "")) return 1 print(f"\nOK: 0 INTERNAL SPLIT, 0 STALE claim(s) at {a.rev}" - + (" (internal-split only: --no-run)" if a.no_run else "")) + + (" (no oracle in this mode: nothing was verified, so nothing is " + "graded -- ADVISORY rows above are labels, not faults)" + if a.no_run else "")) return gate_rc(reports, a.fail)