diff --git a/tools/queue_index.py b/tools/queue_index.py index 99d50e1..5a4924c 100644 --- a/tools/queue_index.py +++ b/tools/queue_index.py @@ -44,8 +44,13 @@ WHAT IT DOES NOT DO It does not grade the work, does not adjudicate closure, and does not read git history. It answers "what is in this file and where", and it keeps the generated copy of that answer honest: `--write` renders `docs/queue-index.md` - with the parse pinned inside a machine block, and `--check` re-derives the - parse and goes RED the moment the document and the queue disagree. + with the parse pinned inside a machine block, and `--check` re-renders the + whole document -- prose table included, compared byte for byte -- and goes RED + the moment the two disagree. (The pin alone was not enough: the first + whole-gate run found the table above the pin drifted while the pin still + matched, because the document had begun listing itself in its own corpus. + The rendered file is excluded from that corpus, which makes `--write` a fixed + point, and the bytes are now compared whole.) `--check` exit codes: 0 current, 1 stale (with the delta printed), 2 the document carries no parseable pinned block. @@ -73,6 +78,7 @@ import sys import tempfile QUEUE_DEFAULT = "WORK-QUEUE.md" +DOC_PATH = "docs/queue-index.md" BEGIN = "" END = "" @@ -215,29 +221,44 @@ def parse(text): return payload -def outside_ids(repo, queue_path, exclude_tools=True): - """Ids referenced by the prose corpus that the queue never mentions.""" +def outside_ids(repo, queue_path, exclude=(), exclude_tools=True): + """Ids referenced by the prose corpus that the queue never mentions. + + `exclude` is the rendered document itself: a generated file that names ids + would otherwise join the corpus that generates it, so `--write` would not be + a fixed point -- measured once, the first whole-gate run went red on exactly + that (the document added itself to Q63's file list). + """ try: out = subprocess.check_output(["git", "-C", repo, "ls-files"], text=True) except (subprocess.CalledProcessError, FileNotFoundError): return None + skip = {os.path.realpath(p) for p in exclude if p} files = [f for f in out.split("\n") if f.endswith(".md")] if exclude_tools: files = [f for f in files if not f.startswith("tools/")] + # The generated document is never part of the corpus it summarizes, whatever + # path it is written to -- otherwise a render into any other filename would + # count the committed copy and the output would cease to be a fixed point. + files = [f for f in files if f != DOC_PATH] queue_abs = os.path.normpath(os.path.join(repo, queue_path)) queue_text = open(queue_abs, encoding="utf-8", errors="replace").read() in_queue = set(ID_TOKEN.findall(queue_text)) hits = {} + read = 0 for f in files: path = os.path.join(repo, f) + if os.path.realpath(path) in skip: + continue try: txt = open(path, encoding="utf-8", errors="replace").read() except OSError: continue + read += 1 for m in ID_TOKEN.finditer(txt): hits.setdefault(m.group(0), set()).add(f) return {k: sorted(v) for k, v in sorted( - hits.items(), key=lambda kv: int(kv[0][1:])) if k not in in_queue}, len(files) + hits.items(), key=lambda kv: int(kv[0][1:])) if k not in in_queue}, read def summary(payload): @@ -390,15 +411,30 @@ def run(args): if pinned is None: print("UNVERIFIABLE: no parseable pinned block in %s" % args.check) return 2 - if json.dumps(pinned, sort_keys=True) == json.dumps(payload, sort_keys=True): - print("index current: %s" % summary(payload)) - return 0 - print("STALE: %s" % delta(pinned, payload)) - print("regenerate with: python3 tools/queue_index.py --repo %s --write %s" - % (repo, args.check)) - return 1 + outside, corpus = outside_ids(repo, args.queue, exclude=(args.check,)) + render = build_doc(payload, outside, corpus or 0) + if render != doc: + # The rendered prose is graded, not only the pin. A pin-only gate + # stayed green while the table above it had drifted; that was the + # first whole-gate run's finding, so the bytes are compared whole. + rl = render.split("\n") + dl = doc.split("\n") + first = next((i for i in range(min(len(rl), len(dl))) if rl[i] != dl[i]), + min(len(rl), len(dl))) + print("STALE: rendered document differs at line %d" % (first + 1)) + if first < len(dl): + print(" document: %s" % dl[first][:160]) + if first < len(rl): + print(" render: %s" % rl[first][:160]) + if pinned is not None: + print(" pin: %s" % delta(pinned, payload)) + print("regenerate with: python3 tools/queue_index.py --repo %s --write %s" + % (repo, args.check)) + return 1 + print("index current: %s" % summary(payload)) + return 0 - outside, corpus = outside_ids(repo, args.queue) + outside, corpus = outside_ids(repo, args.queue, exclude=(args.write,)) if args.write: with open(args.write, "w", encoding="utf-8") as fh: fh.write(build_doc(payload, outside, corpus or 0))