#!/usr/bin/env python3 """queue_append_audit.py -- was the queue's index current IN THE TREE of every commit that touched the queue? WHY THIS EXISTS `docs/queue-index.md` is rendered from `WORK-QUEUE.md` and pins it by sha256, so a commit that appends to the queue and does not re-render the index is STALE by construction -- there is no "might be stale" case left to argue. Nothing obliged a commit to re-render, and the ledger measured the cost twice: `f2c3067` and `067afdf` each left the gate red until a later repair commit, the second for 2h37m until `de3dcdc`. The rule the ledger now states is "append and re-render in the SAME commit; `--check` must read rc 0 at the commit it lands in". Enforcing that rule by hand cost the control plane three wrong-slice counts in one week, and its own 15:16Z correction names the cause: a `queue_index --check` run in a working tree at HEAD says NOTHING about a cited revision -- "my first probe loop reported rc 0 'at 067afdf' while grading HEAD's tree". This tool takes the hand out of the loop. Each commit's own tree is materialised with `git archive` and that tree's OWN `tools/queue_index.py` runs inside it, so every row is a measurement of the commit it names and of the gate as it existed there. WHAT IT MEASURES -- one row per commit that touches the queue in the range verdict PASS the commit's own tree passes its own --check (rc 0) STALE --check graded a stale rendered document (rc 1) UNGRADABLE the gate did not exist in that tree (no tools/queue_index.py, or no index document), or the rev's own tool refused with rc 2. A commit that cannot be graded is NOT reported as stale, and is counted separately -- folding the two into one bucket is the exact defect this tool was built after. ERROR any other exit, a crash, or a timeout. rc 1 without the tool's `STALE:` line is a crash, not a fault. queue_add/del numstat for the queue in that commit. index_changed whether the same commit also changed the index document -- the mechanical shape of the rule, reported BESIDE the verdict and never instead of it: a commit can touch the index and still be STALE, and a commit can leave the index alone and still be current (an append that lands outside every section's extent would be such a case, and the verdict would say so). Precedence is deliberate: the verdict comes from a checkout of the commit, never from its message, and never from a run at HEAD. WHAT IT DOES NOT DO It repairs nothing, decides nothing about whether an UNGRADABLE commit should have had a gate, and adjudicates no queue content -- closure is `tools/queue_claim_check.py`'s verdict. `--guard` is the same measurement restricted to the NEWEST queue-touching commit, which is the shape a tick's `check.sh` can call: rc 0 means the commit about to be pushed left the index current in its own tree. EXIT CODES (house contract, decided 2026-10-17 at 6ee8fce) 0 ran; nothing graded faulty (UNGRADABLE rows are reported, not faults) 1 ran and graded a fault: at least one STALE or ERROR row 2 could not grade: not a git repository, an unknown revision, no commit in the range touches the queue, or the queue is not tracked at the target rev 3 retired, never returned USAGE python3 queue_append_audit.py --repo /root/troupe-standup python3 queue_append_audit.py --repo . --from 1423c4d --to HEAD --json out.json python3 queue_append_audit.py --repo . --guard # newest queue-touching commit python3 queue_append_audit.py --selftest # 25 cells, a synthetic history """ from __future__ import annotations import argparse import io import json import os import shutil import subprocess import sys import tempfile QUEUE_DEFAULT = "WORK-QUEUE.md" INDEX_DEFAULT = "docs/queue-index.md" GATE_TOOL = "tools/queue_index.py" HERE = os.path.dirname(os.path.abspath(__file__)) FATAL = ("STALE", "ERROR") # ---------------------------------------------------------------- git plumbing def git(repo, *args, timeout=120): """(rc, stdout, stderr) -- never a traceback for a git-level refusal.""" try: p = subprocess.run(["git", "-C", repo] + list(args), capture_output=True, timeout=timeout) except FileNotFoundError: return 127, b"", b"git not found on PATH" except subprocess.TimeoutExpired: return 124, b"", ("git %s timed out after %ss" % (args[0], timeout)).encode() return p.returncode, p.stdout, p.stderr def out_text(rc, out, err): s = (out or b"").decode("utf-8", "replace") e = (err or b"").decode("utf-8", "replace") return s, e def is_repo(repo): rc, _, _ = git(repo, "rev-parse", "--git-dir") return rc == 0 def resolve(repo, rev): rc, out, _ = git(repo, "rev-parse", "--verify", "--quiet", rev + "^{commit}") s, _ = out_text(rc, out, b"") return s.strip() if rc == 0 else None def exists(repo, rev, path): rc, _, _ = git(repo, "cat-file", "-e", "%s:%s" % (rev, path)) return rc == 0 def commit_meta(repo, rev): rc, out, _ = git(repo, "show", "-s", "--format=%h%x00%ad%x00%s", "--date=iso", rev) s, _ = out_text(rc, out, b"") parts = s.rstrip("\n").split("\x00") if s else [] while len(parts) < 3: parts.append("") return {"short": parts[0][:7] or rev[:7], "date": parts[1], "subject": parts[2]} def numstat(repo, rev, queue, index): """(queue_add, queue_del, index_changed) for one commit.""" rc, out, _ = git(repo, "show", "--numstat", "--format=", rev, "--", queue, index) s, _ = out_text(rc, out, b"") add = delete = None touched_index = False for line in s.splitlines(): parts = line.split("\t") if len(parts) != 3: continue a, d, path = parts if path == queue: add = int(a) if a.isdigit() else a delete = int(d) if d.isdigit() else d elif path == index: touched_index = True return add, delete, touched_index def materialise(repo, rev, paths, dest): """Extract exactly `paths` from `rev` into `dest`. Returns (ok, error).""" rc, out, err = git(repo, "archive", "--format=tar", rev, "--", *paths) if rc != 0: s, e = out_text(rc, out, err) return False, (e or s or "git archive rc %d" % rc).strip().splitlines()[0] try: p = subprocess.run(["tar", "-xf", "-", "-C", dest], input=out, capture_output=True, timeout=120) except (FileNotFoundError, subprocess.TimeoutExpired) as exc: return False, "could not unpack: %s" % exc if p.returncode != 0: return False, "tar rc %d: %s" % (p.returncode, p.stderr.decode("utf-8", "replace").strip()) return True, "" # ------------------------------------------------------------------- one commit def audit_commit(repo, rev, queue, index, timeout): """Grade one commit IN ITS OWN TREE. Never raises.""" row = {"rev": rev} row.update(commit_meta(repo, rev)) add, delete, touched_index = numstat(repo, rev, queue, index) row.update({"queue_add": add, "queue_del": delete, "index_changed": touched_index, "rc": None, "detail": ""}) if not exists(repo, rev, queue): row.update(verdict="UNGRADABLE", detail="%s does not exist at this rev" % queue) return row # The gate as it existed AT THAT REV, not as it exists today: a revision # where queue_index.py had not been written yet has no gate, and no claim # about its index can be true or false. if not exists(repo, rev, GATE_TOOL): row.update(verdict="UNGRADABLE", detail="no %s at this rev (the gate did not exist yet)" % GATE_TOOL) return row if not exists(repo, rev, index): row.update(verdict="UNGRADABLE", detail="no %s at this rev (nothing to compare against)" % index) return row td = tempfile.mkdtemp(prefix="qaa-") try: ok, why = materialise(repo, rev, [queue, index, GATE_TOOL], td) if not ok: row.update(verdict="ERROR", detail="could not materialise: %s" % why) return row tool = os.path.join(td, GATE_TOOL) cmd = [sys.executable, tool, "--repo", td, "--queue", os.path.join(td, queue), "--check", os.path.join(td, index)] try: p = subprocess.run(cmd, cwd=td, capture_output=True, timeout=timeout) except subprocess.TimeoutExpired: row.update(verdict="ERROR", detail="--check timed out after %ss" % timeout) return row s = p.stdout.decode("utf-8", "replace") e = p.stderr.decode("utf-8", "replace") first = (s.strip() or e.strip() or "").splitlines() row["detail"] = first[0][:180] if first else "" row["rc"] = p.returncode if p.returncode == 0: row["verdict"] = "PASS" elif p.returncode == 1 and "STALE:" in s: row["verdict"] = "STALE" elif p.returncode == 2: row["verdict"] = "UNGRADABLE" else: # rc 1 with no STALE line is a crash wearing the verdict code; rc 3 # is retired. Neither is a graded fault, and neither is a pass. row["verdict"] = "ERROR" return row finally: shutil.rmtree(td, ignore_errors=True) # ------------------------------------------------------------------ the range def queue_revs(repo, queue, rev_from, rev_to): args = ["rev-list", "--reverse", rev_to] if rev_from: args = ["rev-list", "--reverse", "%s..%s" % (rev_from, rev_to)] rc, out, err = git(repo, *(args + ["--", queue])) s, e = out_text(rc, out, err) return rc, [h for h in s.split() if h], e def audit(repo, queue, index, rev_from, rev_to, guard=False, timeout=30, all_history=False): """(rc, payload). The only place exit codes are decided. Range rules, in order: --from REV grade exactly REV..TO (REV excluded), graded or not. --all grade every commit that ever touched the queue, including the hundreds that predate the index. Each of those is UNGRADABLE, which is the honest answer and a useless table. (default) begin at the commit that INTRODUCED the index: no commit before it can be graded, and listing 328 rows that all say so buries the 9 that can. The count left out is printed, never silently dropped. """ payload = {"repo": repo, "queue": queue, "index": index, "from": None, "to": None, "guard": bool(guard), "auto_from": False, "pre_gate": 0, "rows": [], "counts": {}, "graded": 0, "rc": None} def _grade_error(msg): """Attach a grade error to the payload and return the (2, payload) verdict. Every precondition failure in this function is the same three-statement shape: set the message, set rc=2, return. Six inline copies were the smell; this helper is the single source of that shape. """ payload["error"] = msg payload["rc"] = 2 return 2, payload if not os.path.isdir(repo): return _grade_error("not a directory: %s" % repo) if not is_repo(repo): return _grade_error("not a git repository: %s" % repo) to_sha = resolve(repo, rev_to) if to_sha is None: return _grade_error("unknown revision: %s" % rev_to) payload["to"] = to_sha from_sha = None if rev_from: from_sha = resolve(repo, rev_from) if from_sha is None: return _grade_error("unknown revision: %s" % rev_from) elif not all_history: rc, out, _ = git(repo, "log", "--diff-filter=A", "--format=%H", "-1", to_sha, "--", index) intro = out_text(rc, out, b"")[0].strip() if intro: # Include the introducing commit itself: it may also carry a queue # append, and a commit that adds both is exactly the shape to grade. rc2, out2, _ = git(repo, "rev-parse", intro + "^") from_sha = out2.decode("utf-8", "replace").strip() if rc2 == 0 else intro payload["auto_from"] = True payload["from"] = from_sha rc, revs, err = queue_revs(repo, queue, from_sha, to_sha) if rc != 0: return _grade_error("git rev-list failed: %s" % ((err.strip().splitlines() or [""])[0])) if guard: revs = revs[-1:] if not revs: where = ("%s..%s" % (rev_from, rev_to) if rev_from else rev_to) return _grade_error("no commit in %s touches %s, so there is no index " "currency to grade" % (where, queue)) if payload["auto_from"] and not guard: rc_n, out_n, _ = git(repo, "rev-list", "--count", to_sha, "--", queue) try: payload["pre_gate"] = max(0, int(out_n.decode().strip()) - len(revs)) except (ValueError, AttributeError): payload["pre_gate"] = 0 rows = [audit_commit(repo, r, queue, index, timeout) for r in revs] counts = {} for r in rows: counts[r["verdict"]] = counts.get(r["verdict"], 0) + 1 payload["rows"] = rows payload["counts"] = counts payload["graded"] = counts.get("PASS", 0) + counts.get("STALE", 0) + counts.get("ERROR", 0) payload["rc"] = 1 if any(r["verdict"] in FATAL for r in rows) else 0 return payload["rc"], payload # --------------------------------------------------------------------- report def report(payload, stream=sys.stdout): w = stream.write if payload.get("error"): w("queue_append_audit: %s\n" % payload["error"]) return frm, to = payload.get("from"), payload.get("to") or payload["to"] if payload["guard"]: where = "newest queue-touching commit at or below %s" % (to or "")[:7] elif payload["auto_from"]: where = "%s..%s (auto: from the commit that introduced %s)" % ( (frm or "")[:7], (to or "")[:7], payload["index"]) elif frm: where = "%s..%s" % (frm[:7], (to or "")[:7]) else: where = "%s (all history)" % (to or "")[:7] w("queue_append_audit -- queue: %s index: %s range: %s%s\n" % (payload["queue"], payload["index"], where, " [guard]" if payload["guard"] else "")) if payload.get("pre_gate"): w(" %d earlier commit(s) touch the queue before the gate existed; they " "cannot be graded and are not listed (--all lists them)\n" % payload["pre_gate"]) w(" %d commit(s) touch the queue\n\n" % len(payload["rows"])) for r in payload["rows"]: add, dele = r["queue_add"], r["queue_del"] q = "q %s/%s" % (add if add is not None else "-", dele if dele is not None else "-") w(" %-10s %-8s %-25s %-9s index:%-3s %s\n" % (r["verdict"], r["short"], r["date"][:25], q, "yes" if r["index_changed"] else "no", r["detail"])) c = payload["counts"] w("\nsummary: %d commit(s) -- %s\n" % (len(payload["rows"]), ", ".join("%d %s" % (c[k], k) for k in ("PASS", "STALE", "UNGRADABLE", "ERROR") if k in c) or "nothing graded")) ungradable = c.get("UNGRADABLE", 0) w("graded: %d of %d%s\n" % (payload["graded"], len(payload["rows"]), (" (%d could not be graded -- reported, NOT counted as stale)" % ungradable) if ungradable else "")) stale = [r["short"] for r in payload["rows"] if r["verdict"] == "STALE"] if stale: w("stale: %s\n" % " ".join(stale)) bad = [r["short"] for r in payload["rows"] if r["verdict"] == "ERROR"] if bad: w("error: %s\n" % " ".join(bad)) if payload["rc"] == 0: w("verdict: every graded commit left the index current in its own tree\n") else: w("verdict: %d commit(s) did NOT leave the index current in their own tree -- " "rc 1\n" % (len(stale) + len(bad))) # ------------------------------------------------------------------- selftest def _sh(cwd, *args): return subprocess.run(args, cwd=cwd, capture_output=True, text=True) def _fixture(base): """A five-commit history with one commit of each kind. Returns (repo, revs). A: queue only, no gate tool -> UNGRADABLE B: appends, and adds tools/queue_index.py + its render -> PASS C: appends to the queue, no re-render -> STALE D: appends AND re-renders -> PASS E: touches an unrelated file -> not selected at all """ repo = os.path.join(base, "repo") os.makedirs(repo) _sh(base, "git", "init", "-q", repo) _sh(repo, "git", "config", "user.name", "Selftest") _sh(repo, "git", "config", "user.email", "selftest@invalid") _sh(repo, "git", "config", "commit.gpgsign", "false") queue = os.path.join(repo, "WORK-QUEUE.md") with open(queue, "w", encoding="utf-8") as fh: fh.write("## Q1 -- alpha\n\nStatus: open\n") _sh(repo, "git", "add", "-A") _sh(repo, "git", "commit", "-q", "-m", "A: queue only") os.makedirs(os.path.join(repo, "tools")) os.makedirs(os.path.join(repo, "docs")) with open(os.path.join(repo, GATE_TOOL), "w", encoding="utf-8") as fh: fh.write(open(os.path.join(HERE, "queue_index.py"), encoding="utf-8").read()) with open(queue, "a", encoding="utf-8") as fh: fh.write("- beta appended\n") _sh(repo, sys.executable, GATE_TOOL, "--repo", ".", "--write", INDEX_DEFAULT) _sh(repo, "git", "add", "-A") _sh(repo, "git", "commit", "-q", "-m", "B: append + gate + index") with open(queue, "a", encoding="utf-8") as fh: fh.write("- gamma appended\n") _sh(repo, "git", "add", "-A") _sh(repo, "git", "commit", "-q", "-m", "C: append, no re-render") with open(queue, "a", encoding="utf-8") as fh: fh.write("- delta appended\n") _sh(repo, sys.executable, GATE_TOOL, "--repo", ".", "--write", INDEX_DEFAULT) _sh(repo, "git", "add", "-A") _sh(repo, "git", "commit", "-q", "-m", "D: append + re-render") with open(os.path.join(repo, "notes.txt"), "w", encoding="utf-8") as fh: fh.write("unrelated\n") _sh(repo, "git", "add", "-A") _sh(repo, "git", "commit", "-q", "-m", "E: unrelated file") revs = _sh(repo, "git", "rev-list", "--reverse", "HEAD").stdout.split() return repo, revs def selftest(): cells = [] def check(name, got, want): cells.append((name, got == want, got, want)) base = tempfile.mkdtemp(prefix="qaa-selftest-") try: repo, revs = _fixture(base) a, b, c, d, e = revs rc, p = audit(repo, QUEUE_DEFAULT, INDEX_DEFAULT, None, "HEAD", all_history=True) rows = {r["rev"]: r for r in p["rows"]} check("the audit selects the 4 queue-touching commits, not the 5th", len(p["rows"]), 4) check("a commit with no gate tool is UNGRADABLE, not STALE", rows[a]["verdict"], "UNGRADABLE") check("the commit that added the gate and its index is PASS", rows[b]["verdict"], "PASS") check("an append with no re-render is STALE", rows[c]["verdict"], "STALE") check("an append that re-renders in the same commit is PASS", rows[d]["verdict"], "PASS") check("STALE row says the index was not touched", rows[c]["index_changed"], False) check("PASS row says the index was touched", rows[d]["index_changed"], True) check("the stale row's numstat is the append", (rows[c]["queue_add"], rows[c]["queue_del"]), (1, 0)) check("a range with a STALE row exits 1", rc, 1) check("counts keep UNGRADABLE out of the STALE bucket", p["counts"].get("STALE"), 1) check("graded counts PASS+STALE+ERROR only", p["graded"], 3) check("the STALE detail carries the tool's own word", "STALE:" in rows[c]["detail"], True) rc_g, p_g = audit(repo, QUEUE_DEFAULT, INDEX_DEFAULT, None, "HEAD", guard=True) check("--guard on a clean head exits 0", rc_g, 0) check("--guard reports exactly one commit", len(p_g["rows"]), 1) rc_s, _ = audit(repo, QUEUE_DEFAULT, INDEX_DEFAULT, None, c, guard=True) check("--guard on the stale commit exits 1", rc_s, 1) # Default range: begin where the gate does. B introduced the index, so # the default table starts at B's parent and A is counted, not listed. rc_a, p_a = audit(repo, QUEUE_DEFAULT, INDEX_DEFAULT, None, "HEAD") check("the default range begins at the commit that introduced the index", (p_a["auto_from"], len(p_a["rows"])), (True, 3)) check("pre-gate commits are counted, not silently dropped", p_a["pre_gate"], 1) check("the default range still grades the stale commit", rc_a, 1) rc_e, p_e = audit(repo, QUEUE_DEFAULT, INDEX_DEFAULT, d, d) check("an empty range is rc 2, not a clean 0", (rc_e, p_e.get("error") is not None), (2, True)) rc_r, _ = audit(os.path.join(base, "no-such-repo"), QUEUE_DEFAULT, INDEX_DEFAULT, None, "HEAD") check("a missing repo is rc 2", rc_r, 2) rc_b, p_b = audit(repo, QUEUE_DEFAULT, INDEX_DEFAULT, None, "no-such-rev") check("an unknown revision is rc 2", (rc_b, p_b.get("error") is not None), (2, True)) rc_q, p_q = audit(repo, "NOT-THE-QUEUE.md", INDEX_DEFAULT, None, "HEAD") check("a queue no commit touches is rc 2", (rc_q, "touches" in (p_q.get("error") or "")), (2, True)) json_path = os.path.join(base, "out.json") buf = io.StringIO() rc_j, p_j = audit(repo, QUEUE_DEFAULT, INDEX_DEFAULT, None, "HEAD", all_history=True) with open(json_path, "w", encoding="utf-8") as fh: json.dump(p_j, fh, indent=1) report(p_j, stream=buf) with open(json_path, encoding="utf-8") as fh: loaded = json.load(fh) check("--json round-trips the same 4 rows", len(loaded["rows"]), 4) check("the report names the stale commit by short sha", c[:7] in buf.getvalue(), True) check("a report with a fault never prints a clean verdict", "every graded commit" in buf.getvalue(), False) finally: shutil.rmtree(base, ignore_errors=True) bad = [c for c in cells if not c[1]] for name, ok, got, want in bad: print("FAIL %s\n got: %r\n want: %r" % (name, got, want)) print("selftest: %d cells, %d passed, %d failed" % (len(cells), len(cells) - len(bad), len(bad))) return 0 if not bad else 1 # ----------------------------------------------------------------------- main def main(argv=None): ap = argparse.ArgumentParser( description="Did every commit that touched the work queue leave its " "index current IN ITS OWN TREE?") ap.add_argument("--repo", default=".") ap.add_argument("--queue", default=QUEUE_DEFAULT) ap.add_argument("--index", default=INDEX_DEFAULT) ap.add_argument("--from", dest="rev_from", default=None, help="exclusive lower bound; default: the commit that " "introduced the index document") ap.add_argument("--to", dest="rev_to", default="HEAD") ap.add_argument("--all", dest="all_history", action="store_true", help="start at the root of history, listing the pre-gate " "commits as UNGRADABLE rows too") ap.add_argument("--guard", action="store_true", help="grade only the newest queue-touching commit") ap.add_argument("--timeout", type=int, default=30, help="seconds per commit's --check (default 30)") ap.add_argument("--json", metavar="PATH", help="write the rows here") ap.add_argument("--selftest", action="store_true") args = ap.parse_args(argv) if args.selftest: return selftest() rc, payload = audit(args.repo, args.queue, args.index, args.rev_from, args.rev_to, args.guard, args.timeout, args.all_history) report(payload) if args.json: try: with open(args.json, "w", encoding="utf-8") as fh: json.dump(payload, fh, indent=1) except OSError as exc: print("cannot write json: %s" % exc, file=sys.stderr) return 2 return rc if __name__ == "__main__": sys.exit(main())