diff --git a/tools/doc_census_check.py b/tools/doc_census_check.py index 4ff2185..5b42a60 100644 --- a/tools/doc_census_check.py +++ b/tools/doc_census_check.py @@ -172,9 +172,20 @@ FAILING ON THE CLASS IT CATCHES (`--fail`) A CACHED VOCABULARY PUTS THE TEETH BACK IN THE FAST MODE, without asking it to invent an oracle. `--save-vocab PATH` (full run) writes the census - vocabulary the run OBSERVED -- every unit key some leg printed, per tool -- - and `--no-run --vocab PATH` grades a split in a unit that vocabulary names as - CACHE_CHECKABLE. What the cache proves is narrow and enough: the unit is a + vocabulary the run OBSERVED -- every unit key some leg printed AND that a + claim in that tool's own docstring can spend, per tool -- and + `--no-run --vocab PATH` grades a split in a unit that vocabulary names as + CACHE_CHECKABLE. The two halves are both required: the leg proves the key is + something the tool actually prints, the docstring proves some claim could + ever ask about it. A key no claim can spend (`keys_for`'s expansion of every + claimed unit) is dropped, because a permission list is only auditable when + every entry in it can be exercised -- 422 observed keys at 30c8120, 24 + spendable, the other 398 unspendable by construction. That scoping is + measured to be outcome-neutral (see THE FAST MODE REPORTS WHAT IT CANNOT + GRADE); it does NOT close the cache's one over-reach, which is a word the + docstring really does claim twice (below). + + What the cache proves is narrow and enough: the unit is a census unit for that tool (a leg printed it at ``), and the disagreement being graded is INTERNAL to the docstring, so the leg never has to supply the value. What it cannot prove is a value, so a STALE claim is @@ -187,6 +198,18 @@ FAILING ON THE CLASS IT CATCHES (`--fail`) label is; it is NOT required to equal the revision being checked (unit-hood is a property of the unit word, and the graded disagreement is internal). + THE FAST MODE REPORTS WHAT IT CANNOT GRADE. Before 2026-10-18 a runless split + in a lowercase word vanished: `unit_scope` answered UNCHECKED_NO_ORACLE, and + the printer showed only the LABEL_ONLY class, so a reader of `--no-run` saw + nothing at all for it (`vocab-precision` case E: rc 0, no line). It is now + printed as ADVISORY with its values and never graded -- silent was the only + part of that behaviour that was wrong, and silence is not a verdict. Scoping + the saved vocabulary to spendable keys is measured OUTCOME-NEUTRAL on the + same fixture (A..E still [1,1,0,0,0]) and on the tree (fast gate rc 0 before + and after; the fca3f2e tooth still rc 1 from the cache), which is the honest + description: it makes the permission list auditable, it does not change a + verdict, and it does not close the over-reach this paragraph opened with. + rc 2 keeps its meaning from the tools/ contract (6ee8fce) and is decided before the gate runs, so "could not grade" never reads as "graded clean" or as "graded a fault". A LEG FAILURE (RUN_FAILED) is reported in its own column @@ -197,7 +220,7 @@ FAILING ON THE CLASS IT CATCHES (`--fail`) USAGE python3 doc_census_check.py [--repo DIR] [--rev HEAD] [--tools NAME...] [--json OUT] [--fail] [--no-run] [--vocab PATH] [--save-vocab PATH] - [--selftest] # selftest: 72 cells, one per claim class + [--selftest] # selftest: 80 cells, one per claim class """ import argparse import ast @@ -916,6 +939,59 @@ def _fixture_checks(): run_gate("--tools", "tools/stub_split.py", "--no-run", "--vocab", vrt, "--fail"), 1) + # ---- the saved vocabulary is SCOPED to what a claim can spend, and the + # fast mode reports the splits it cannot grade (2026-10-18). The + # first three cells pin the matcher identity that makes the scoping + # safe; the last six pin the process, because a helper returning a + # set proves nothing about the file a runner reads. + check("vocab scope: a claimed unit expands the way keys_for reads it", + sorted(spendable_keys(["citations"])), ["citation", "citations"]) + check("vocab scope: every key it admits is one vocab_has can spend", + all(vocab_has("citations", {k}) + for k in spendable_keys(["citations"])), True) + check("vocab scope: a digest no docstring claims is not spendable", + "abcdef12" in spendable_keys(["UNRESOLVED", "citations"]), False) + with open(os.path.join(td, "tools", "stub_vocab.py"), "w") as fh: + fh.write('"""A stub that prints its census and a digest.\n\n' + 'It found 7 widgets.\n"""\n' + "print('widgets: 7')\nprint('abcdef12: 3')\n") + with open(os.path.join(td, "tools", "stub_lower.py"), "w") as fh: + fh.write('"""A stub whose docstring splits a lowercase word.\n\n' + "We did 5 above the fold.\n" + "pad " * 80 + "\n" + 'Then 7 above the fold.\n"""\nprint(\'above: 3\')\n') + for sub in (["add", "-A"], ["commit", "-qm", "vocab scope"]): + subprocess.run(["git", "-C", td] + sub, capture_output=True, env=genv) + vscope = os.path.join(td, "vocab-scope.json") + check("gate e2e: --save-vocab on a stub printing a digest (rc 0)", + run_gate("--tools", "tools/stub_vocab.py", "--save-vocab", vscope, + "--fail"), 0) + with open(vscope) as fh: + check("gate e2e: the saved vocabulary holds the claimed unit only", + json.load(fh)["tools"].get("tools/stub_vocab.py"), ["widgets"]) + vlow = os.path.join(td, "vocab-lower.json") + with open(vlow, "w") as fh: + json.dump({"rev": "abc1234", + "tools": {"tools/stub_lower.py": ["above"]}}, fh) + check("gate e2e: a claimed lowercase word IS spendable -- the over-reach", + run_gate("--tools", "tools/stub_lower.py", "--no-run", + "--vocab", vlow, "--fail"), 1) + + def run_gate_out(*extra): + p = subprocess.run([sys.executable, os.path.abspath(__file__), + "--repo", td, "--rev", "HEAD"] + list(extra), + capture_output=True, env=genv) + return p.returncode, p.stdout.decode("utf-8", "replace") + + rc_low, out_low = run_gate_out("--tools", "tools/stub_lower.py", + "--no-run", "--fail") + check("gate e2e: a runless lowercase split is rc 0 (nothing verified)", + rc_low, 0) + check("gate e2e: and it is PRINTED as advisory, not swallowed", + [l.strip() for l in out_low.splitlines() + if l.strip().startswith("ADVISORY above (no census label")], + ["ADVISORY above (no census label and no cached vocabulary): " + "5@3, 7@5 -- reported, not graded"]) + print(f"selftest: {cells[0]} cells, {cells[0] - bad[0]} passed, {bad[0]} failed") return 0 if ok else 1 @@ -1027,6 +1103,23 @@ def vocab_has(unit, vocab): return bool(keys_for(unit, {u: [0] for u in (vocab or ())})) +def spendable_keys(units): + """The vocabulary keys a claim on one of `units` can actually spend. + + `vocab_has` answers `keys_for(unit, ...)`, so the set of keys any claim in a + docstring can reach is fixed: exactly the four spellings `keys_for` tries, + lowercased, for the units that docstring claims. A saved vocabulary entry + outside that set can never be consulted -- it is a permission nobody can + use -- so `--save-vocab` writes the intersection of what the legs observed + with this set. Same matcher on both sides, no second rule invented. + """ + out = set() + for u in units or (): + for c in (u, u.lower(), u.lower().rstrip("s"), u.upper()): + out.add(c.lower()) + return out + + def check_tool(repo, script, name, cache, run=True, tmpdir=".", isolation=240, vocab=None, vocab_rev=None): text = open(script, encoding="utf-8", errors="replace").read() @@ -1178,6 +1271,9 @@ def check_tool(repo, script, name, cache, run=True, tmpdir=".", isolation=240, "token": tok, "verdict": v, "detail": detail}) return {"tool": name, "splits": splits, "advisory": advisory, "unchecked": dict(unchecked), + "claim_units": sorted({(u.upper() if u.isupper() else u.lower()) + for _f, _r, cl, _b in blk_claims + for _ln, _v, u, _t, _p in cl}), "failures": failures, "verdicts": verdicts, "self": is_self and run, "selftest_leg": (not run) or bool(selftest_leg and selftest_leg[0]), "blocks_with_claims": len(blk_claims)} @@ -1346,15 +1442,29 @@ def main(): if s.get("oracle_rev") else "") print(f" SPLIT {s['unit']} [{scope}] ({s['how']}{prov}): {vals}") for k, c in sorted(rep["unchecked"].items()): + # in the fast mode the UNCHECKED_NO_ORACLE rows are printed one by + # one above, so the count would be a second, worse copy of them + if a.no_run and k == "UNCHECKED_NO_ORACLE": + continue print(f" (not flagged: {c} split(s) in units {k})") # A --no-run split is the one advisory class a reader of the fast mode # actually needs to SEE: it is what the old build failed the gate on. # Printed with its values, labelled advisory, never counted above. + # + # BOTH runless classes print (2026-10-18). The LABEL_ONLY class (an + # uppercase census label with no oracle) was shown here; a lowercase + # word answered UNCHECKED_NO_ORACLE and was shown NOWHERE, so a runless + # split in a lowercase word read as no split at all. Not graded either + # way -- the fix is to the reporting, not the verdict. for s in rep.get("advisory") or []: - if s["how"] != "LABEL_ONLY": + if s["how"] == "LABEL_ONLY": + why = "LABEL_ONLY, no oracle" + elif s["how"] == "UNCHECKED_NO_ORACLE" and a.no_run: + why = "no census label and no cached vocabulary" + else: continue vals = ", ".join(f"{r['value']}@{r['line']}" for r in s["rows"]) - print(f" ADVISORY {s['unit']} (LABEL_ONLY, no oracle): {vals}" + print(f" ADVISORY {s['unit']} ({why}): {vals}" f" -- reported, not graded") for v in bad: print(f" {v['verdict']:10s} {v['file']}:{v.get('line', '-')} " @@ -1377,17 +1487,32 @@ def main(): # keys `parse_printed` saw in a leg's stdout count as census units, so a # tool whose leg failed contributes nothing and cannot hand the fast # mode a vocabulary it never printed. + # + # AND IT IS SCOPED TO WHAT A CLAIM CAN SPEND (2026-10-18). The leg says + # the key exists; the tool's own docstring says whether any claim will + # ever ask about it. Digests (`abc1234`) and diffstat words (`added`, + # `gone`) come off a leg's stdout but no docstring claims them, so they + # were 398 of 422 permissions nobody could use. Dropping them changes no + # verdict -- see spendable_keys and the offline check in the artifact -- + # and it makes the file a list a reader can audit line by line. + per_tool = {r["tool"]: r for r in reports if r} units = collections.defaultdict(set) + seen = collections.defaultdict(set) for (name, rev), entry in cache.items(): if rev == "" or not entry[0]: continue - units[name].update(entry[0][0].keys()) + seen[name].update(entry[0][0]) + keep = spendable_keys((per_tool.get(name) or {}).get("claim_units")) + units[name].update(k for k in entry[0][0] if k in keep) + observed = sum(len(v) for v in seen.values()) + kept = sum(len(v) for v in units.values()) with open(a.save_vocab, "w", encoding="utf-8") as fh: json.dump({"rev": a.rev, "repo": a.repo, "tools": {k: sorted(v) for k, v in units.items()}}, fh, indent=1, sort_keys=True) print(f"wrote vocabulary {a.save_vocab}: {len(units)} tool(s), " - f"{sum(len(v) for v in units.values())} unit(s) at {a.rev}") + f"{kept} unit(s) at {a.rev} ({observed} key(s) observed, " + f"{observed - kept} unspendable by any claim)") if a.fail: nsplit, nstale = gate_faults(reports) if nsplit or nstale: