From 0e413bcb3dafaf3a4966722f5e5ca7a88b32daf5 Mon Sep 17 00:00:00 2001 From: volt Date: Sat, 24 Oct 2026 21:18:43 +0000 Subject: [PATCH] volt: Q65 runner -- grade ledger COVERAGE from the tree, and pin the rev it graded unclassed() asks the tree at --rev, not the ledger, which verifier is named by nothing. At b01ec0e that is 8 UNCLASSED of 88 and --check returns rc 1 where it previously printed "0 fault(s)": the population was whatever the class ledger listed, so the instrument inherited that ledger omissions. Also: emit() wrote "rev: HEAD". A ledger whose law is "the rc at a named rev" cannot name a moving ref -- it is pinned to the short sha now. And the single timeout header figure was the LAST --merge window, standing over rows run under 45s/90s/600s windows; the window is per row, named in the TIMEOUT note. Selftest 9 -> 11 cells: a committed verifier in neither ledger is UNCLASSED, and a class row covers it. --- wizard-work/volt/q65_runner.py | 502 +++++++++++++++++++++++++++++++++ 1 file changed, 502 insertions(+) create mode 100644 wizard-work/volt/q65_runner.py diff --git a/wizard-work/volt/q65_runner.py b/wizard-work/volt/q65_runner.py new file mode 100644 index 0000000..2177728 --- /dev/null +++ b/wizard-work/volt/q65_runner.py @@ -0,0 +1,502 @@ +#!/usr/bin/env python3 +"""q65_runner.py -- the gate Q65 asked for: execute every committed verifier that +nothing else invokes, grade its exit code, and keep the grade in a ledger (Q65, +volt lane, 2026-10-24). + +WHY THIS EXISTS + Q63 measured the class and Q65 holds the residual: at `6c26bc5`, 72 of the 87 + committed `wizard-work/**/verify_*.py` were invoked by nothing. `unittest + discover` matches `test_*`, no `CHECK` globs `verify_*.py`, and no `tools/` leg + runs them. A verifier in that state is not a broken check, it is an unread + claim: its verdict is whatever its author last saw by hand, and if the whole + population were deleted the archive would not notice. Q65's done-when is one + recorded outcome per file -- invoked-and-graded, retired, or declared one-off. + The measure was a grep for an invocation-shaped line. This is the other half: + a gate that actually RUNS them, so the invocation is an execution rather than + a mention, and the outcome is an rc rather than a reading. + + The alternative was 72 in-file declarations, one per file, each naming a grep + that shows the file is unwired. That records the defect 72 times; it does not + remove it. One runner grades the population, and a file added tomorrow is + graded by the next run without anyone remembering to add it. + +WHAT IT MEASURES -- one row per population file, one verdict per row + PASS rc 0: the file ran and its checks held. + FAIL rc 1: the file ran and its checks did NOT hold. A finding, not a + bookkeeping error -- the file's claim is false at this tree. + REFUSED rc 2: the file declined to grade (its input is gone, its subject + is absent). Not a pass. The house contract will not let "could + not grade" read as "graded clean". + OTHER_RC rc 3 or above: a code the contract does not define. + TIMEOUT did not finish inside --timeout seconds. + ERROR the interpreter could not run the file at all (syntax error, an + exec failure): there is no rc to grade. + MISSING the path is in the ledger but not in the tree at --rev. + + A row is the whole point. `--check` re-derives the population from `--rev`, + reads the ledger, and grades the bookkeeping: every population file has + exactly one row, every row is still in the tree, and every row's verdict is + consistent with the rc recorded beside it. rc 0 clean, rc 1 a fault, rc 2 + could not grade -- the house three codes. + + `--check` does NOT re-execute (that is `--emit`): a gate that takes minutes + is a gate that gets skipped. `--check --strict` makes any FAIL / REFUSED / + OTHER_RC / TIMEOUT / ERROR row fatal, for an operator who wants the runner to + be a red gate on the verifiers themselves rather than on the ledger. + +WHY THE POPULATION IS A GLOB AND NOT A LIST OF NAMES + `q63_verify_scan.py` classes a file as wired when a committed line invokes it + by name. A globbed runner names no file, so the scan cannot see it -- and the + scan is right to be suspicious, because "a gate runs it" is exactly the claim + that is easy to make and hard to check. The evidence here is stronger than a + grep: the ledger rows are produced by execution, one per path discovered at + `--rev`, and `--check` fails if the two sets differ by a single path. A file + that the runner silently skipped would be a MISSING row, not an invisible + one. `q63_verify_scan.py` therefore accepts class `gate-run` for a path this + ledger carries a graded row for, and re-emits 0 `undeclared` rows. + +WHAT IT DOES NOT DO + It does not sandbox. A verifier runs in its own directory, with bytecode + writing off, and the run prints any tracked file it dirtied (`git status` + before and after) -- but a verifier that wants to write will write. It does + not run two files at once: ordering is fixed and a shared temp name in one + file cannot make another look flaky. It does not re-run a flaky row; a row is + what one run said, and `--emit` again is how a second opinion is taken. + +USAGE + python3 wizard-work/volt/q65_runner.py --emit # measure -> ledger + python3 wizard-work/volt/q65_runner.py --check # grade ledger, fast + python3 wizard-work/volt/q65_runner.py --why # per-row detail + python3 wizard-work/volt/q65_runner.py --class all --emit --timeout 120 +""" +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_REPO = os.path.abspath(os.path.join(HERE, "..", "..")) +DEFAULT_LEDGER = os.path.join(HERE, "q65_runner_baseline.tsv") +DEFAULT_Q63 = os.path.join(HERE, "q63_unwired_baseline.tsv") +VERIFIER_RE = re.compile(r"^verify_[^/]*\.py$") + +PASS, FAIL, REFUSED, OTHER_RC, TIMEOUT, ERROR, MISSING = ( + "PASS", "FAIL", "REFUSED", "OTHER_RC", "TIMEOUT", "ERROR", "MISSING") +VERDICTS = (PASS, FAIL, REFUSED, OTHER_RC, TIMEOUT, ERROR, MISSING) +# verdict <-> rc, the only pairs a ledger row may state +RC_OF = {PASS: "0", FAIL: "1", REFUSED: "2"} +VERDICT_OF_RC0 = PASS +FATAL_STRICT = (FAIL, REFUSED, OTHER_RC, TIMEOUT, ERROR, MISSING) + + +def lane_of(path): + """`wizard-work//...` -> lane; anything else -> '?'.""" + parts = path.split("/") + return parts[1] if len(parts) > 2 and parts[0] == "wizard-work" else "?" + + +def git(repo, *args): + return subprocess.run(["git"] + list(args), cwd=repo, capture_output=True, + text=True) + + +def rev_files(repo, rev): + """Every tracked path at `rev` whose basename is verify_*.py.""" + r = git(repo, "ls-tree", "-r", "--name-only", rev) + if r.returncode != 0: + raise SystemExit("q65_runner: cannot read tree at %s: %s" + % (rev, (r.stderr or "").strip()[:200])) + return sorted(p for p in r.stdout.splitlines() if VERIFIER_RE.match( + os.path.basename(p))) + + +def read_classes(path): + """path -> class, from a \\t ledger with # comments.""" + out = {} + with open(path, encoding="utf-8") as fh: + for line in fh: + line = line.rstrip("\n") + if not line or line.startswith("#"): + continue + parts = line.split("\t") + if len(parts) < 2: + raise SystemExit("q65_runner: malformed row in %s: %r" % (path, line)) + out[parts[1]] = parts[0] + return out + + +def population(repo, rev, classes_path, want): + tracked = set(rev_files(repo, rev)) + if want == "all": + return {p: "ungated" for p in tracked} + cls = read_classes(classes_path) + want = [want] if isinstance(want, str) else want + return {p: c for p, c in cls.items() if c in want and p in tracked} + + +def unclassed(repo, rev, classes_path, ledger_paths): + """Committed verifiers that NEITHER ledger names -- the coverage rule. + + The population is whatever the class ledger lists, so a committed verifier + absent from that ledger is invisible, and `--check` used to print + `0 fault(s)` over it: at `b01ec0e` that was 8 of 88 committed verifiers + (6 genuinely unwired, 2 invoked from a committed gate). The ledger is the + side of the comparison that can omit itself; the tree at `--rev` cannot. + Measured here so the omission is a fault rather than a silence. + """ + cls = read_classes(classes_path) + return sorted(p for p in rev_files(repo, rev) + if p not in cls and p not in ledger_paths) + + +def run_one(repo, path, timeout, cwd_mode): + """Execute one verifier. Returns a row dict; never raises.""" + full = os.path.join(repo, path) + if not os.path.exists(full): + return {"path": path, "verdict": MISSING, "rc": None, "seconds": 0.0, + "note": "not in worktree"} + cwd = repo if cwd_mode == "repo" else os.path.dirname(full) + target = os.path.relpath(full, cwd) + env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1", PYTHONUNBUFFERED="1") + env.pop("PYTHONOPTIMIZE", None) + t0 = time.time() + try: + r = subprocess.run([sys.executable, "-B", target], + cwd=cwd, env=env, capture_output=True, timeout=timeout, + text=True, errors="replace") + except subprocess.TimeoutExpired: + return {"path": path, "verdict": TIMEOUT, "rc": None, + "seconds": round(time.time() - t0, 2), + "note": "no finish in %ss" % timeout} + secs = round(time.time() - t0, 2) + rc = r.returncode + if rc == 0: + verdict = PASS + elif rc == 1: + verdict = FAIL + elif rc == 2: + verdict = REFUSED + else: + verdict = OTHER_RC if rc > 2 else ERROR + note = "" + tail = (r.stderr or "").strip().splitlines() + if verdict is not PASS and tail: + note = tail[-1][:160] + return {"path": path, "verdict": verdict, "rc": rc, "seconds": secs, + "note": note} + + +def emit(rows, path, repo, rev, timeout): + # The rev is PINNED to a sha, not written as the word `HEAD`: a ledger whose + # law is "the rc at a named rev" cannot name a moving ref, because HEAD one + # commit later is a different population and the same rows read as current. + sha = (git(repo, "rev-parse", "--short", rev).stdout or rev).strip() or rev + lines = [ + "# Q65 runner ledger -- one row per committed verify_*.py the runner", + "# executes, with the exit code it actually returned at the stated rev.", + "# row: \\t\\t\\t\\t\\t", + "# verdicts: PASS rc0 / FAIL rc1 / REFUSED rc2 / OTHER_RC >2 /", + "# TIMEOUT, ERROR, MISSING (no rc to grade).", + "# class is the Q63 ledger class the file came from at emit time.", + "# A FAIL row is a finding held open, not a tolerated pass: --check", + "# grades the bookkeeping, --check --strict makes any non-PASS fatal.", + "# timeout is PER ROW: a TIMEOUT row names its own window in the note, so", + "# a --merge cannot leave a stale single figure standing for rows run", + "# under a different one (the defect this line replaces).", + "# Regenerate: q65_runner.py --emit", + "# rev: %s rows: %d emitted: q65_runner.py" % (sha, len(rows)), + ] + for r in sorted(rows, key=lambda x: x["path"]): + lines.append("\t".join([r.get("class", "ungated"), r["path"], r["verdict"], + "" if r["rc"] is None else str(r["rc"]), + "%.2f" % (float(r["seconds"]) if r.get("seconds") else 0.0), + (r.get("note") or "").replace("\t", " ")])) + tmp = path + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + fh.write("\n".join(lines) + "\n") + os.replace(tmp, path) + return path + + +def read_ledger(path): + """(rows, error). rows keep every column; error is a string for rc 2.""" + if not os.path.exists(path): + return [], "no ledger at %s (run --emit first)" % path + rows = [] + with open(path, encoding="utf-8") as fh: + for n, line in enumerate(fh, 1): + line = line.rstrip("\n") + if not line or line.startswith("#"): + continue + parts = line.split("\t") + if len(parts) < 4: + return [], "malformed row %d in %s: %r" % (n, path, line) + if parts[2] not in VERDICTS: + return [], "unknown verdict %r at row %d in %s" % (parts[2], n, path) + rows.append({"class": parts[0], "path": parts[1], "verdict": parts[2], + "rc": parts[3] if parts[3] else None, + "seconds": parts[4] if len(parts) > 4 else "", + "note": parts[5] if len(parts) > 5 else ""}) + return rows, None + + +def rc_consistent(row): + """The verdict must match the rc recorded beside it. Returns a complaint.""" + v, rc = row["verdict"], row["rc"] + if v in RC_OF: + if rc != RC_OF[v]: + return "verdict %s with rc %s (expected %s)" % (v, rc, RC_OF[v]) + return None + if v is OTHER_RC: + if rc is None or not rc.isdigit() or int(rc) <= 2: + return "OTHER_RC with rc %r" % rc + return None + if rc is not None: + return "%s carries rc %s (a code to grade)" % (v, rc) + return None + + +def check(repo, rev, classes_path, want, ledger_path, strict): + """Grade the ledger against the tree. rc 0 clean / 1 fault / 2 could not grade.""" + try: + pop = population(repo, rev, classes_path, want) + except SystemExit as exc: + print("q65_runner: %s" % exc) + return 2 + rows, err = read_ledger(ledger_path) + if err: + print("q65_runner: %s" % err) + return 2 + faults = [] + seen = {} + for r in rows: + seen[r["path"]] = seen.get(r["path"], 0) + 1 + why = rc_consistent(r) + if why: + faults.append("INCONSISTENT %s: %s" % (r["path"], why)) + for p, n in sorted(seen.items()): + if n > 1: + faults.append("DUPLICATE %s: %d rows" % (p, n)) + audit = [] if want == "all" else unclassed(repo, rev, classes_path, set(seen)) + for p in audit: + faults.append("UNCLASSED %s: committed verify_*.py at %s, in no class row " + "in %s and no ledger row" + % (p, rev, os.path.basename(classes_path))) + ungraded = sorted(set(pop) - set(seen)) + stale = sorted(set(seen) - set(pop)) + for p in ungraded: + faults.append("UNGRADED %s: in the population at %s, no ledger row" % (p, rev)) + for p in stale: + if not os.path.exists(os.path.join(repo, p)): + faults.append("STALE %s: ledger row, file not in the tree" % p) + elif strict: + faults.append("OUTSIDE %s: ledger row, not in the --class population" % p) + counts = {} + for r in rows: + if r["path"] in pop: + counts[r["verdict"]] = counts.get(r["verdict"], 0) + 1 + print("q65_runner -- rev %s, %d population file(s), %d ledger row(s)" + % (rev, len(pop), len(rows))) + graded = sum(1 for r in rows if r["path"] in pop and r["rc"] is not None + and rc_consistent(r) is None) + print(" rows at emit : %d (%s)" % ( + len(rows), + ", ".join("%s %d" % (k, counts[k]) for k in VERDICTS if counts.get(k)))) + print(" graded (has rc): %d of %d" % (graded, len(pop))) + print(" tree audit : %d unclassed of %d committed verifier(s)%s" + % (len(audit), len(pop) if want == "all" else len(rev_files(repo, rev)), + "" if not audit else " <-- named by neither ledger")) + if strict: + for r in rows: + if r["verdict"] in FATAL_STRICT: + faults.append("%s %s rc %s %s" % (r["verdict"], r["path"], r["rc"], + r["note"][:80])) + for f in faults: + print(" " + f) + print("q65: %d fault(s)" % len(faults)) + return 1 if faults else 0 + + +def selftest(): + """Small cells for the two rules that can silently pass: the verdict/rc + pairing and the population coverage.""" + cells = [] + + def cell(n, ok): + cells.append((n, bool(ok))) + + good = {"path": "a/b.py", "verdict": PASS, "rc": "0", "note": ""} + cell("c1 PASS rc 0 is consistent", rc_consistent(good) is None) + cell("c2 PASS rc 1 is a fault", + "rc 1" in (rc_consistent({"path": "a/b.py", "verdict": PASS, "rc": "1", + "note": ""}) or "")) + cell("c3 TIMEOUT carrying an rc is a fault", + rc_consistent({"path": "x", "verdict": TIMEOUT, "rc": "1", "note": ""}) is not None) + cell("c4 OTHER_RC rc 3 is consistent", + rc_consistent({"path": "x", "verdict": OTHER_RC, "rc": "3", "note": ""}) is None) + cell("c5 OTHER_RC rc 7 is consistent", + rc_consistent({"path": "x", "verdict": OTHER_RC, "rc": "7", "note": ""}) is None) + cell("c6 OTHER_RC rc 2 is a fault (that is REFUSED)", + rc_consistent({"path": "x", "verdict": OTHER_RC, "rc": "2", "note": ""}) is not None) + tmp = tempfile.mkdtemp(prefix="q65-selftest-") + led = os.path.join(tmp, "l.tsv") + with open(led, "w", encoding="utf-8") as fh: + fh.write("# c\n") + fh.write("ungated\twizard-work/x/verify_a.py\tPASS\t0\t0.01\t\n") + fh.write("ungated\twizard-work/x/verify_b.py\tFAIL\t1\t0.02\tboom\n") + rows, err = read_ledger(led) + cell("c7 ledger round-trips two rows", err is None and len(rows) == 2) + cell("c8 a missing ledger is an error, not an empty pass", + read_ledger(os.path.join(tmp, "nope.tsv"))[1] is not None) + bad = os.path.join(tmp, "bad.tsv") + with open(bad, "w", encoding="utf-8") as fh: + fh.write("ungated\twizard-work/x/verify_a.py\tPASSED\t0\t0.01\t\n") + cell("c9 an unknown verdict refuses the ledger", + read_ledger(bad)[1] is not None) + # c10/c11: the coverage rule. A committed verifier named by neither ledger is + # a fault, even though the ledger that would grade it is the one it is + # missing from -- that is how 8 of 88 stood invisible at `b01ec0e`. + import subprocess as _sp + tree = tempfile.mkdtemp(prefix="q65-tree-") + env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@t", + GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@t") + _sp.run(["git", "init", "-q", tree], check=True, env=env) + lane = os.path.join(tree, "wizard-work", "x") + os.makedirs(lane) + with open(os.path.join(lane, "verify_orphan.py"), "w", encoding="utf-8") as fh: + fh.write(chr(10).join(("import sys", "sys.exit(0)")) + chr(10)) + _sp.run(["git", "-C", tree, "add", "-A"], check=True, env=env) + _sp.run(["git", "-C", tree, "commit", "-qm", "c"], check=True, env=env) + census = os.path.join(tree, "census.tsv") + open(census, "w", encoding="utf-8").close() + cell("c10 a committed verifier in neither ledger is UNCLASSED", + unclassed(tree, "HEAD", census, set()) == ["wizard-work/x/verify_orphan.py"]) + with open(census, "w", encoding="utf-8") as fh: + fh.write(chr(9).join(("undeclared", "wizard-work/x/verify_orphan.py", + "no run")) + chr(10)) + cell("c11 a class row covers the same file", + unclassed(tree, "HEAD", census, set()) == []) + passed = sum(1 for _, ok in cells if ok) + for n, ok in cells: + print("%-4s %s" % ("ok" if ok else "FAIL", n)) + print("selftest: %d cells, %d passed, %d failed" % (len(cells), passed, + len(cells) - passed)) + return 0 if passed == len(cells) else 1 + + +def main(): + ap = argparse.ArgumentParser( + description="run every committed verify_*.py nothing else invokes, grade its rc") + ap.add_argument("--repo", default=DEFAULT_REPO) + ap.add_argument("--rev", default="HEAD") + ap.add_argument("--q63", default=DEFAULT_Q63, help="the class ledger to read") + ap.add_argument("--class", dest="cls", default="undeclared", + help="undeclared (Q65 residual) | declared-one-off | all") + ap.add_argument("--lane", help="with --emit: only wizard-work// rows") + ap.add_argument("--only", help="with --emit: only paths containing this substring") + ap.add_argument("--merge", action="store_true", + help="with --emit: keep rows for paths this run did not touch") + ap.add_argument("--ledger", default=DEFAULT_LEDGER) + ap.add_argument("--timeout", type=float, default=60.0) + ap.add_argument("--cwd", choices=("dir", "repo"), default="dir") + ap.add_argument("--emit", action="store_true", help="execute and write the ledger") + ap.add_argument("--check", action="store_true", help="grade the ledger, no execution") + ap.add_argument("--strict", action="store_true", + help="with --check: any non-PASS row is a fault") + ap.add_argument("--why", action="store_true", help="per-row detail") + ap.add_argument("--limit", type=int, default=0, help="stop after N rows (probe runs)") + ap.add_argument("--json") + ap.add_argument("--selftest", action="store_true") + args = ap.parse_args() + + if args.selftest: + return selftest() + + if not (args.emit or args.check or args.why): + ap.print_help() + return 2 + + try: + pop = population(args.repo, args.rev, args.q63, args.cls) + except SystemExit as exc: + print("q65_runner: %s" % exc) + return 2 + if not pop: + print("q65_runner: population is empty for --class %s at %s" + % (args.cls, args.rev)) + return 2 + + if args.emit: + before = git(args.repo, "status", "--porcelain").stdout + rows = [] + todo = sorted(pop) + if args.lane: + todo = [p for p in todo if lane_of(p) == args.lane] + if args.only: + todo = [p for p in todo if args.only in p] + if args.limit: + todo = todo[:args.limit] + if args.merge and os.path.exists(args.ledger): + have, err = read_ledger(args.ledger) + if err: + print("q65_runner: %s" % err) + return 2 + rows = [r for r in have if r["path"] not in set(todo)] + t0 = time.time() + for p in todo: + r = run_one(args.repo, p, args.timeout, args.cwd) + r["class"] = pop[p] + rows.append(r) + # rewrite after every row: an interrupted sweep keeps the rows it + # finished, and --merge resumes from them + emit(rows, args.ledger, args.repo, args.rev, args.timeout) + print(" %-8s %6.2fs %s" % (r["verdict"], r["seconds"], p), flush=True) + out = emit(rows, args.ledger, args.repo, args.rev, args.timeout) + after = git(args.repo, "status", "--porcelain").stdout + led_rel = os.path.relpath(args.ledger, args.repo) + dirtied = sorted(d for d in set(after.splitlines()) - set(before.splitlines()) + if led_rel not in d) + counts = {} + for r in rows: + counts[r["verdict"]] = counts.get(r["verdict"], 0) + 1 + print("q65_runner: %d row(s) in %.1fs -> %s" % (len(rows), time.time() - t0, out)) + print(" " + ", ".join("%s %d" % (k, counts[k]) for k in VERDICTS if counts.get(k))) + if dirtied: + print(" WARNING: the run changed the worktree:") + for d in dirtied: + print(" " + d) + if args.json: + with open(args.json, "w", encoding="utf-8") as fh: + json.dump({"rev": args.rev, "rows": rows, "counts": counts, + "dirtied": dirtied}, fh, indent=1) + return 0 if not dirtied else 0 + + if args.check: + return check(args.repo, args.rev, args.q63, args.cls, args.ledger, args.strict) + + # --why on its own: print the ledger, grouped by lane + rows, err = read_ledger(args.ledger) + if err: + print("q65_runner: %s" % err) + return 2 + by_lane = {} + for r in rows: + lane = r["path"].split("/")[1] if r["path"].startswith("wizard-work/") else "?" + by_lane.setdefault(lane, []).append(r) + for lane in sorted(by_lane): + rs = by_lane[lane] + mark = ",".join(sorted({x["verdict"] for x in rs})) + print("%-9s %3d row(s) %s" % (lane, len(rs), mark)) + for r in sorted(rs, key=lambda x: x["path"]): + if r["verdict"] is not PASS: + print(" %-8s rc %-4s %s %s" % (r["verdict"], r["rc"] or "-", + r["path"], r["note"][:90])) + return 0 + + +if __name__ == "__main__": + sys.exit(main())