diff --git a/wizard-work/ledger/check_queue_extract.py b/wizard-work/ledger/check_queue_extract.py new file mode 100644 index 0000000..efe0ee3 --- /dev/null +++ b/wizard-work/ledger/check_queue_extract.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""check_queue_extract.py -- gate for the queue census extraction artifact. + +Verdict (exit 0 == correct, nonzero == wrong) is the exit code of this script. +Checks, in order, with no fallbacks: + 1. Round-trip determinism: re-extracting WORK-QUEUE.md with queue-extract.py + reproduces the committed artifact TSV byte for byte. + 2. Structural agreement: every item in the lane's pinned parse (docs/queue-index.md) + matches the corresponding TSV row on (id, line) for end_line, lines, bytes, role. + 3. Byte accounting: the sum of the TSV's byte column equals the queue's own + sha256-derived byte count (2,149,154) and the file's line count (13,555). + 4. No gap: items tile WORK-QUEUE.md with no overlap and no gap. +""" +import json +import re +import subprocess +import sys + +ROOT = "/root/troupe-standup" +EXTRACT = ROOT + "/wizard-work/ledger/queue-extract.py" +TSV = ROOT + "/wizard-work/ledger/queue-extracted.tsv" +QUEUE = ROOT + "/WORK-QUEUE.md" +DOC = ROOT + "/docs/queue-index.md" + +BEGIN = "" +END = "" + +FAILS = [] + + +def check(name, ok, detail=""): + if ok: + print("PASS %s" % name) + else: + print("FAIL %s %s" % (name, detail)) + FAILS.append(name) + + +def read_pinned(): + doc = open(DOC, encoding="utf-8").read() + body = doc.split(BEGIN, 1)[1].split(END, 1)[0] + m = re.search(r"```json\n(.*?)\n```", body, re.S) + if not m: + check("pinned block present", False, "no json fence") + sys.exit(2) + return json.loads(m.group(1)) + + +def read_tsv(): + rows = [] + for ln in open(TSV, encoding="utf-8"): + ln = ln.rstrip("\n") + if ln.startswith("id\t"): + continue + if ln == "---\n": + continue + parts = ln.split("\t") + if len(parts) < 11: + check("tsv row width", False, ln[:80]) + continue + rows.append({ + "id": parts[0], "kind": parts[1], "line": int(parts[2]), + "end_line": int(parts[3]), "lines": int(parts[4]), + "bytes": int(parts[5]), "role": parts[6], "role_detail": parts[7], + "status": parts[8], "status_line": int(parts[9]) if parts[9] not in ("None", "") else None, + "title": parts[10], + }) + return rows + + +def main(): + # 1. round-trip determinism + p1 = subprocess.run([sys.executable, EXTRACT], capture_output=True, text=True) + first = p1.stdout + p2 = subprocess.run([sys.executable, EXTRACT], capture_output=True, text=True) + check("re-extract rc=0", p1.returncode == 0 and p2.returncode == 0, + "rc=%s,%s" % (p1.returncode, p2.returncode)) + check("re-extract deterministic", first == p2.stdout) + + tsv = read_tsv() + pinned = read_pinned() + + # 2. structural agreement against the pinned parse + by_key = {} + for it in pinned["items"]: + by_key[(it["id"], it["line"])] = it + check("pinned item count", len(pinned["items"]) == len(tsv), + "pinned=%d tsv=%d" % (len(pinned["items"]), len(tsv))) + matched = 0 + for t in tsv: + k = (t["id"], t["line"]) + p = by_key.get(k) + if p is None: + continue + matched += 1 + same_end = t["end_line"] == p["end_line"] + same_lines = t["lines"] == p["lines"] + same_bytes = t["bytes"] == p["bytes"] + same_role = t["role"] == p["role"] + if not (same_end and same_lines and same_bytes and same_role): + check("item %s@%d" % k, + False, + "tsv(end=%d,li=%d,by=%d,ro=%s) pin(end=%d,li=%d,by=%d,ro=%s)" + % (t["end_line"], t["lines"], t["bytes"], t["role"], + p["end_line"], p["lines"], p["bytes"], p["role"])) + check("all pinned items matched in TSV", matched == len(pinned["items"]), + "matched=%d" % matched) + + # 3. byte accounting + total_bytes = sum(t["bytes"] for t in tsv) + content = open(QUEUE, encoding="utf-8").read() + total_lines = len(content.split("\n")) - (1 if content.endswith("\n") else 0) + check("tsv byte sum", total_bytes == 2149154, "sum=%d" % total_bytes) + check("queue line count", total_lines == pinned["queue_lines"], + "lines=%d pinned=%d" % (total_lines, pinned["queue_lines"])) + + # 4. items tile WORK-QUEUE.md with no gap/overlap + tsv_sorted = sorted(tsv, key=lambda t: (t["line"], t["id"])) + ok_tile = True + prev_end = 0 + for t in tsv_sorted: + if t["line"] != prev_end + 1: + ok_tile = False + break + prev_end = t["end_line"] + check("items tile the queue", ok_tile and prev_end == total_lines - 1, + "first_gap_after=%d last_end=%d" % (prev_end, prev_end)) + + # top-level summary numbers echoed into the TSV + summ = {} + for ln in first.split("---\n"): + if ln.startswith("items="): + summ["items"] = ln.split("=", 1)[1].split("\n")[0] + check("artifact header count", summ.get("items") == str(len(tsv)), + "header=%s tsv=%d" % (summ.get("items"), len(tsv))) + + if FAILS: + print("VERDICT: %d FAIL(S)" % len(FAILS)) + sys.exit(1) + print("VERDICT: correct") + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/wizard-work/ledger/check_workqueue_qcensus.py b/wizard-work/ledger/check_workqueue_qcensus.py new file mode 100644 index 0000000..c243edb --- /dev/null +++ b/wizard-work/ledger/check_workqueue_qcensus.py @@ -0,0 +1,165 @@ +import hashlib +import pathlib +import re +import sys + +SRC = '/root/troupe-standup/WORK-QUEUE.md' +TSV = '/root/troupe-standup/wizard-work/ledger/workqueue-q-census.tsv' + +def _strip_decorations(s): + s = s.rstrip() + start = s.rfind('*(') + if start == -1: + return s + i = start + 1 + depth = 0 + for j in range(i, len(s)): + if s[j] == '(': + depth += 1 + elif s[j] == ')': + depth -= 1 + if depth == 0: + return s[:start] + s[j + 2:].strip() + return s + +def parse_sections(text): + lines = text.splitlines() + sections = [] + counts = {} + cur = None + for raw in lines: + m = re.match(r'^## (Q\d+)(.*)$', raw) + if m: + if cur is not None: + sections.append(cur) + qid = m.group(1) + rest = m.group(2).strip() + is_closure = rest.startswith('closure') + if not is_closure: + counts[qid] = counts.get(qid, 0) + 1 + qid_out = qid if counts[qid] == 1 else f'{qid} #{counts[qid]}' + else: + qid_out = qid + ' closure' + title = clean_title(_strip_decorations(rest)) + cur = { + 'q_id': qid_out, + 'rest': rest, + 'title': title, + 'is_closure': is_closure, + 'status_raw': None, + 'body_lines': 0, + } + continue + if cur is not None: + cur['body_lines'] += 1 + sm = re.match(r'\*\*Status:\s*(.*)$', raw) + if sm and cur['status_raw'] is None: + s = sm.group(1).strip() + if s.endswith('**'): + s = s[:-2].rstrip() + cur['status_raw'] = s + if cur is not None: + sections.append(cur) + return sections + +def clean_title(title): + return title.lstrip('—-').strip() + +def main(): + text = pathlib.Path(SRC).read_text() + sections = parse_sections(text) + + errors = [] + + # 1. Section count + if len(sections) != 97: + errors.append(f'expected 97 sections, got {len(sections)}') + + # 2. Closure count + closures = [s for s in sections if s['is_closure']] + if len(closures) != 8: + errors.append(f'expected 8 closures, got {len(closures)}') + + # 3. Deterministic regeneration: recompute TSV content and diff md5 vs committed + rows = [] + for s in sections: + rows.append('\t'.join([ + s['q_id'], s['rest'].rstrip(), s['status_raw'] or '', + re.search(r'[a-f0-9]{7,}', s['rest'] + ' ' + s['q_id']).group(0) if re.search(r'[a-f0-9]{7,}', s['rest'] + ' ' + s['q_id']) else '', + str(s['body_lines']), + '1' if s['is_closure'] else '0', + ])) + expected_tsv = '#q_id\tannex\tstatus_raw\tkey_hash\tbody_lines\tclosure\n' + for r in rows: + expected_tsv += r + '\n' + expected_md5 = hashlib.md5(expected_tsv.encode('utf-8')).hexdigest() + actual_md5 = hashlib.md5(pathlib.Path(TSV).read_bytes()).hexdigest() + if expected_md5 != actual_md5: + errors.append(f'md5 mismatch: expected {expected_md5}, got {actual_md5}') + + # 4. Anchor checks via a lookup map + lookup = {} + for s in sections: + lookup[s['q_id']] = s + + def norm(s): + if s is None: + return '' + return s + + def check_attr(qid, attr, expected_contains): + if qid not in lookup: + errors.append(f'{qid} missing') + return + v = norm(lookup[qid].get(attr)) + if expected_contains not in v: + errors.append(f'{qid}.{attr} mismatch: {v!r} does not contain {expected_contains!r}') + + # 5. Exact title anchor + if 'Q1' not in lookup or lookup['Q1']['rest'].lstrip('—-').strip() != 'Recover the fifteen silent rooms': + errors.append('Q1 title mismatch') + + def status_of(qid): + return norm(lookup.get(qid, {}).get('status_raw')) + + def hash_of(qid): + h = re.search(r'[a-f0-9]{7,}', lookup.get(qid, {}).get('rest', '') + ' ' + qid) + return h.group(0) if h else '' + + if 'CLOSED — 0/15 recovered' not in norm(status_of('Q1')): + errors.append('Q1 status mismatch') + if 'DONE — recovered.' not in norm(status_of('Q2')): + errors.append('Q2 status mismatch') + if 'OPEN — awaiting Alexander' not in norm(status_of('Q45')): + errors.append('Q45 status mismatch') + if 'open and unclaimed' not in norm(status_of('Q75')): + errors.append('Q75 status mismatch') + if 'open and unclaimed' not in norm(status_of('Q88')): + errors.append('Q88 status mismatch') + if 'DONE at `9878f12`' not in norm(status_of('Q93 closure')): + errors.append('Q93 closure status mismatch') + if 'DONE' not in norm(status_of('Q15')): + errors.append('Q15 status mismatch') + + if hash_of('Q75') != 'b82b023': + errors.append(f'Q75 hash {hash_of("Q75")!r}') + if hash_of('Q88') != 'b0a2886': + errors.append(f'Q88 hash {hash_of("Q88")!r}') + if hash_of('Q93 closure') != 'd56105a': + errors.append(f'Q93 closure hash {hash_of("Q93 closure")!r}') + + # 6. Uniqueness of q_ids + qids = [s['q_id'] for s in sections] + if len(set(qids)) != len(qids): + errors.append('duplicate q_ids present') + + if errors: + print('FAIL') + for e in errors: + print(' -', e) + sys.exit(1) + print('PASS: 97 sections, 8 closures, regeneration md5 match, all anchors verified') + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/wizard-work/ledger/diag.py b/wizard-work/ledger/diag.py new file mode 100644 index 0000000..cb7f40b --- /dev/null +++ b/wizard-work/ledger/diag.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""diagnose parse divergence against the pinned ground truth.""" +import importlib.util +import json, re, sys + +QUEUE = "/root/troupe-standup/WORK-QUEUE.md" +SPEC = importlib.util.spec_from_file_location( + "queue_extract", "/root/troupe-standup/wizard-work/ledger/queue-extract.py") +qe = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(qe) + +QUEUE = "/root/troupe-standup/WORK-QUEUE.md" +BEGIN = "" +END = "" +doc = open("/root/troupe-standup/docs/queue-index.md", encoding="utf-8").read() +body = doc.split(BEGIN, 1)[1].split(END, 1)[0] +m = re.search(r"```json\n(.*?)\n```", body, re.S) +pinned = json.loads(m.group(1)) + +mine, _, _ = qe.extract() + +mine_by_key = {(it["id"], it["line"]): it for it in mine} +pin_by_key = {(it["id"], it["line"]): it for it in pinned["items"]} + +print("mine=%d pinned=%d" % (len(mine), len(pinned["items"]))) +print("kind mine:", dict((k, sum(1 for v in mine if v["kind"] == k)) for k in ("heading", "record"))) +print("kind pin:", dict((k, sum(1 for v in pinned["items"] if v["kind"] == k)) for k in ("heading", "record"))) +print("role mine:", dict((k, sum(1 for v in mine if v["role"] == k)) for k in ("opening", "continuation", "record"))) +print("role pin:", dict((k, sum(1 for v in pinned["items"] if v["role"] == k)) for k in ("opening", "continuation", "record"))) + +only_mine = [k for k in mine_by_key if k not in pin_by_key] +only_pin = [k for k in pin_by_key if k not in mine_by_key] +print("only in mine:", len(only_mine), only_mine) +print("only in pin:", len(only_pin), only_pin[:40]) + +# shared keys: role mismatch +role_mismatch = [] +for k in set(mine_by_key) & set(pin_by_key): + a, b = mine_by_key[k], pin_by_key[k] + if a["role"] != b["role"] or a["role_detail"] != b["role_detail"]: + role_mismatch.append((k, a, b)) +print("role mismatches:", len(role_mismatch)) +for k, a, b in role_mismatch[:40]: + print(" %s mine_role=%s pin_role=%s mine_det=%r pin_det=%r mine_off=%r pin_off=%r" % + (k, a["role"], b["role"], a["role_detail"], b["role_detail"], a.get("status_line"), b.get("status_line"))) + +# size discrepancy beyond 195/227: compare byte sums +print("mine byte sum:", sum(it["bytes"] for it in mine)) +print("pin byte sum:", sum(it["bytes"] for it in pinned["items"])) + +# sample of mine-only items and pin-only items +print("\n--- mine-only sample ---") +for k in only_mine[:20]: + print(" ", k, mine_by_key[k]) +print("--- pin-only sample ---") +for k in only_pin[:20]: + print(" ", k, pin_by_key[k]) diff --git a/wizard-work/ledger/diag2.py b/wizard-work/ledger/diag2.py new file mode 100644 index 0000000..68c44c7 --- /dev/null +++ b/wizard-work/ledger/diag2.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Precise (id,line) -> end_line/bytes delta between my parse and the pin.""" +import importlib.util +import json +import re +import sys + +BEGIN = "" +END = "" +doc = open("/root/troupe-standup/docs/queue-index.md", encoding="utf-8").read() +body = doc.split(BEGIN, 1)[1].split(END, 1)[0] +m = re.search(r"```json\n(.*?)\n```", body, re.S) +pinned = json.loads(m.group(1)) + +SPEC = importlib.util.spec_from_file_location( + "qe", "/root/troupe-standup/wizard-work/ledger/queue-extract.py") +qm = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(qm) +mine, _, _ = qm.extract() + +pin_by_key = {(it["id"], it["line"]): it for it in pinned["items"]} +mine_by_key = {(it["id"], it["line"]): it for it in mine} + +delta = [] +for k in sorted(set(pin_by_key) & set(mine_by_key), key=lambda t: (t[1], t[0])): + pa, ma = pin_by_key[k], mine_by_key[k] + if pa["end_line"] != ma["end_line"] or pa["lines"] != ma["lines"] or pa["bytes"] != ma["bytes"]: + delta.append((k, pa, ma)) +print("total shared keys:", len(pin_by_key)) +print("keys with end_line/lines/bytes divergence:", len(delta)) +for k, pa, ma in delta: + print(" %s mine(end=%-5d lines=%-4d by=%-6d) pin(end=%-5d lines=%-4d by=%-6d)" % + (k, ma["end_line"], ma["lines"], ma["bytes"], pa["end_line"], pa["lines"], pa["bytes"])) + +content = open("/root/troupe-standup/WORK-QUEUE.md", encoding="utf-8").read() +lines, trailing = qm.split_lines(content) +fenced = qm.fence_mask(lines) +items, end_of = qm.build_items(lines, fenced) +sizes2 = [qm.line_bytes(ln, i == len(lines) - 1, trailing) for i, ln in enumerate(lines)] +spans = sorted([(it["line"], end_of(it["line"])) for it in items]) +overlap = 0 +prev_end = 0 +for (a, b) in spans: + if a < prev_end + 1: + overlap += 1 + prev_end = max(prev_end, b) +print("items:", len(items), "first:", items[0]["line"], "last:", items[-1]["end_line"]) +print("overlap count:", overlap) +print("end of last:", items[-1]["end_line"], "file lines:", len(lines)) +print("my item byte sum:", sum(it["bytes"] for it in items)) diff --git a/wizard-work/ledger/diag3.py b/wizard-work/ledger/diag3.py new file mode 100644 index 0000000..ba573c4 --- /dev/null +++ b/wizard-work/ledger/diag3.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +import importlib.util +import json + +SPEC = importlib.util.spec_from_file_location( + "qe", "/root/troupe-standup/wizard-work/ledger/queue-extract.py") +qm = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(qm) +mine, _, _ = qm.extract() + +from collections import Counter +c = Counter((it["id"], it["line"]) for it in mine) +dups = {k: v for k, v in c.items() if v > 1} +print("duplicate (id,line) keys:", len(dups)) +for k, v in sorted(dups.items())[:40]: + # show the two items + matches = [it for it in mine if (it["id"], it["line"]) == k] + print(" %s x%d:" % (k, v)) + for it in matches: + print(" ", it["kind"], it["role"], "end=%d lines=%d by=%d det=%r status=%r" % + (it["end_line"], it["lines"], it["bytes"], it["role_detail"], it["status"])) diff --git a/wizard-work/ledger/probe.py b/wizard-work/ledger/probe.py new file mode 100644 index 0000000..3f32aad --- /dev/null +++ b/wizard-work/ledger/probe.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +import importlib.util, re +SPEC = importlib.util.spec_from_file_location("qe", "/root/troupe-standup/wizard-work/ledger/queue-extract.py") +qm = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(qm) +content = open("/root/troupe-standup/WORK-QUEUE.md", encoding="utf-8").read() +lines, trailing = qm.split_lines(content) +fenced = qm.fence_mask(lines) + +HEADING = qm.HEADING +ID_TOKEN = qm.ID_TOKEN +record_form = qm.record_form + +# replicate first_record + later + boundaries +first_record = {} +for i, ln in enumerate(lines, 1): + if i in fenced: + continue + for m in ID_TOKEN.finditer(ln): + qid = m.group(0) + if qid not in first_record: + frm = record_form(ln, qid) + if frm: + first_record[qid] = (i, frm) +first_record_keys = {(qid, line) for qid, (line, frm) in first_record.items()} +print("first_record:") +for k, v in sorted(first_record.items()): + print(" ", k, v) +later = [] +for i, ln in enumerate(lines, 1): + if i in fenced or not ln.startswith("**"): + continue + prev = lines[i - 2].strip() if i >= 2 else "" + if prev != "" and prev != "---" and not prev.startswith("