diff --git a/wizard-work/ledger/extract_reply_volume_census.py b/wizard-work/ledger/extract_reply_volume_census.py new file mode 100644 index 0000000..3ac3adc --- /dev/null +++ b/wizard-work/ledger/extract_reply_volume_census.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Extract the replies/ directory into a per-date and per-recipient volume census. + +LANE: ledger (Extract — data extraction and tables). + +EVIDENCE on every claim: + * Each generated reply is a single Markdown file under /replies/, named + -to-[-NN].md + where is the check-in owner (e.g. zephyrus, merlin, volt) and + NN is an optional per-recipient incident index. + * Counted evidence is therefore the live file system listing of that + directory; no reply body is read, and nothing is invented. + +Files that are NOT replies (excluded from the census): + *.py probe scripts left by other lanes + __pycache__/ compiled bytecode directory + README.md repository index, not a response + any .md that does not match the -to-[-NN].md template + +Run: + python3 extract_reply_volume_census.py + # writes reply-volume-census.tsv, reply-volume-by-recipient.tsv, + # reply-volume-census.md (touching this script's directory) + + python3 extract_reply_volume_census.py --selftest + # prints ground-truth counts for a quick cross-check +""" + +import re +import sys +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path + +# wizard-work/ledger/ -> parents[0]=wizard-work, parents[1]=repo root +REPO_ROOT = Path(__file__).resolve().parents[2] +REPLIES_DIR = REPO_ROOT / "replies" +OUT_DIR = REPO_ROOT / "wizard-work" / "ledger" + +# -to-[ -NN].md (recipient may contain hyphens; no digits) +NAME_RE = re.compile(r"^(\d{4}-\d{2}-\d{2})-to-([^0-9]+)(?:-\d+)?$") +DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") + + +def ground_truth(): + """Derive counts from the live replies/ directory (the evidence source).""" + entries = list(REPLIES_DIR.iterdir()) + files = [p for p in entries if p.is_file()] + non_md = [p.name for p in files if p.suffix.lower() not in (".md",)] + non_canon = [] # .md files that do not match the canonical template + by_date = Counter() + by_recipient = Counter() + for p in files: + name = p.name + if name.endswith(".md"): + body = name[:-3] # strip the ".md" extension + m = NAME_RE.match(body) + if not m: + non_canon.append(name) + continue + date, recipient = m.group(1), m.group(2) + if not DATE_RE.match(date): + non_canon.append(name) + continue + by_date[date] += 1 + by_recipient[recipient] += 1 + return by_date, by_recipient, non_canon, non_md, files + + +def write_tsv(path, rows): + with path.open("w", encoding="utf-8") as fh: + fh.write("item\tcount\n") + for item, count in rows: + fh.write(f"{item}\t{count}\n") + + +def write_markdown(path, by_date, by_recipient, non_canon, non_md, files): + total = sum(by_date.values()) + dates = sorted(by_date) + recip = sorted(by_recipient, key=lambda r: (-by_recipient[r], r)) + lines = [] + lines.append("# Replies volume census") + lines.append("") + lines.append(f"- Extracted at UTC: `{datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}`") + lines.append(f"- Source: `{REPLIES_DIR}` (live file system listing)") + lines.append(f"- Reply files matched `-to-[-NN].md`: **{total}**") + lines.append(f"- Excluded non-reply files: **{len(non_md) + len(non_canon)}**") + lines.append(f" - `.md` not matching the template: **{len(non_canon)}** (`{'`; `'.join(sorted(non_canon))}`)") + lines.append(f" - non-`.md` entries: **{len(non_md)}**") + lines.append(f"- Distinct dates covered: **{len(dates)}**") + lines.append(f"- Distinct recipients: **{len(recip)}**") + lines.append("") + lines.append("## Evidence on what was counted") + lines.append("") + lines.append("Every reply is a single Markdown file carrying `-to-` routing and a four-part") + lines.append("date header. The directory also holds `README.md` (this index), three probe") + lines.append("scripts and a `__pycache__` directory; none is a reply and all are excluded.") + lines.append("") + lines.append("## Replies by date") + lines.append("") + lines.append("| date | replies |") + lines.append("|---|---|") + for d in dates: + lines.append(f"| {d} | {by_date[d]} |") + lines.append(f"| **TOTAL** | **{total}** |") + lines.append("") + lines.append("## Replies by recipient") + lines.append("") + lines.append("| recipient | replies |") + lines.append("|---|---|") + for r in recip: + lines.append(f"| {r} | {by_recipient[r]} |") + lines.append(f"| **TOTAL** | **{total}** |") + lines.append("") + lines.append("## Excluded files (.md not matching template)") + lines.append("") + if non_canon: + for name in sorted(non_canon): + lines.append(f"| {name} |") + lines.append("") + lines.append("## Excluded (all file types)") + lines.append("") + lines.append("| kind | count |") + lines.append("|---|---|") + if non_md: + for k in sorted(non_md): + lines.append(f"| `{k}` | **1** |") + lines.append("") + lines.append("## Method") + lines.append("") + lines.append("1. `ls ` enumerates every entry (evidence is the live FS).") + lines.append("2. Replies are files whose name matches `YYYY-MM-DD-to-RECEIVER[-NN].md`.") + lines.append("3. Per-date and per-recipient counters are aggregated with `collections.Counter`.") + lines.append("4. The committed verifier `verify_reply_volume_census.py` re-derives ground truth") + lines.append(" from the live directory with independent shell commands and asserts the files") + lines.append(" above match.") + lines.append("") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines), encoding="utf-8") + print(f"wrote {path}") + + +def main(): + by_date, by_recipient, non_canon, non_md, files = ground_truth() + total = sum(by_date.values()) + dates = sorted(by_date) + recip = sorted(by_recipient, key=lambda r: (-by_recipient[r], r)) + + print(f"reply files matched -to-[-NN].md : {total}") + print(f"distinct dates : {len(dates)}") + print(f"distinct recipients : {len(recip)}") + print(f"non-canonical .md excluded : {len(non_canon)}") + print(f"non-md entries excluded : {len(non_md)}") + + OUT_DIR.mkdir(parents=True, exist_ok=True) + write_tsv(OUT_DIR / "reply-volume-census.tsv", + [(d, by_date[d]) for d in dates] + [("TOTAL", total)]) + write_tsv(OUT_DIR / "reply-volume-by-recipient.tsv", + [(r, by_recipient[r]) for r in recip] + [("TOTAL", total)]) + write_markdown(OUT_DIR / "reply-volume-census.md", + by_date, by_recipient, non_canon, non_md, files) + + if "--selftest" in sys.argv: + print(f"TOTAL={total}") + print(f"DATES={len(dates)}") + print(f"RECIPIENTS={len(recip)}") + print("BY_RECIPIENT:") + for r in recip: + print(f" {r}\t{by_recipient[r]}") + + +if __name__ == "__main__": + main() diff --git a/wizard-work/ledger/reply-volume-by-recipient.tsv b/wizard-work/ledger/reply-volume-by-recipient.tsv new file mode 100644 index 0000000..a40f8a1 --- /dev/null +++ b/wizard-work/ledger/reply-volume-by-recipient.tsv @@ -0,0 +1,19 @@ +item count +zephyrus 149 +merlin 67 +volt 55 +beam 31 +spark 22 +forge 21 +ledger 19 +sol 15 +mercury 12 +zephyrus-interview 10 +quill 8 +null 7 +sphinx 6 +timmy 5 +argus 1 +cassian 1 +gemini 1 +TOTAL 430 diff --git a/wizard-work/ledger/reply-volume-census.md b/wizard-work/ledger/reply-volume-census.md new file mode 100644 index 0000000..7ec3bc1 --- /dev/null +++ b/wizard-work/ledger/reply-volume-census.md @@ -0,0 +1,100 @@ +# Replies volume census + +- Extracted at UTC: `2026-10-30T02:35:23Z` +- Source: `/root/troupe-standup/replies` (live file system listing) +- Reply files matched `-to-[-NN].md`: **430** +- Excluded non-reply files: **4** + - `.md` not matching the template: **1** (`README.md`) + - non-`.md` entries: **3** +- Distinct dates covered: **34** +- Distinct recipients: **17** + +## Evidence on what was counted + +Every reply is a single Markdown file carrying `-to-` routing and a four-part +date header. The directory also holds `README.md` (this index), three probe +scripts and a `__pycache__` directory; none is a reply and all are excluded. + +## Replies by date + +| date | replies | +|---|---| +| 2026-09-24 | 6 | +| 2026-09-25 | 1 | +| 2026-09-26 | 7 | +| 2026-09-27 | 1 | +| 2026-09-28 | 2 | +| 2026-09-30 | 1 | +| 2026-10-01 | 4 | +| 2026-10-02 | 1 | +| 2026-10-03 | 4 | +| 2026-10-04 | 3 | +| 2026-10-05 | 3 | +| 2026-10-06 | 7 | +| 2026-10-07 | 6 | +| 2026-10-08 | 4 | +| 2026-10-09 | 2 | +| 2026-10-10 | 8 | +| 2026-10-11 | 11 | +| 2026-10-12 | 26 | +| 2026-10-13 | 18 | +| 2026-10-14 | 22 | +| 2026-10-15 | 14 | +| 2026-10-16 | 8 | +| 2026-10-17 | 13 | +| 2026-10-18 | 11 | +| 2026-10-19 | 15 | +| 2026-10-20 | 25 | +| 2026-10-21 | 39 | +| 2026-10-22 | 28 | +| 2026-10-23 | 20 | +| 2026-10-24 | 33 | +| 2026-10-25 | 25 | +| 2026-10-26 | 17 | +| 2026-10-27 | 18 | +| 2026-10-28 | 27 | +| **TOTAL** | **430** | + +## Replies by recipient + +| recipient | replies | +|---|---| +| zephyrus | 149 | +| merlin | 67 | +| volt | 55 | +| beam | 31 | +| spark | 22 | +| forge | 21 | +| ledger | 19 | +| sol | 15 | +| mercury | 12 | +| zephyrus-interview | 10 | +| quill | 8 | +| null | 7 | +| sphinx | 6 | +| timmy | 5 | +| argus | 1 | +| cassian | 1 | +| gemini | 1 | +| **TOTAL** | **430** | + +## Excluded files (.md not matching template) + +| README.md | + +## Excluded (all file types) + +| kind | count | +|---|---| +| `q60_truncated_census.py` | **1** | +| `q93_axis_negative_probe.py` | **1** | +| `test_jsonl_validate.py` | **1** | + +## Method + +1. `ls ` enumerates every entry (evidence is the live FS). +2. Replies are files whose name matches `YYYY-MM-DD-to-RECEIVER[-NN].md`. +3. Per-date and per-recipient counters are aggregated with `collections.Counter`. +4. The committed verifier `verify_reply_volume_census.py` re-derives ground truth + from the live directory with independent shell commands and asserts the files + above match. diff --git a/wizard-work/ledger/reply-volume-census.tsv b/wizard-work/ledger/reply-volume-census.tsv new file mode 100644 index 0000000..9121c98 --- /dev/null +++ b/wizard-work/ledger/reply-volume-census.tsv @@ -0,0 +1,36 @@ +item count +2026-09-24 6 +2026-09-25 1 +2026-09-26 7 +2026-09-27 1 +2026-09-28 2 +2026-09-30 1 +2026-10-01 4 +2026-10-02 1 +2026-10-03 4 +2026-10-04 3 +2026-10-05 3 +2026-10-06 7 +2026-10-07 6 +2026-10-08 4 +2026-10-09 2 +2026-10-10 8 +2026-10-11 11 +2026-10-12 26 +2026-10-13 18 +2026-10-14 22 +2026-10-15 14 +2026-10-16 8 +2026-10-17 13 +2026-10-18 11 +2026-10-19 15 +2026-10-20 25 +2026-10-21 39 +2026-10-22 28 +2026-10-23 20 +2026-10-24 33 +2026-10-25 25 +2026-10-26 17 +2026-10-27 18 +2026-10-28 27 +TOTAL 430 diff --git a/wizard-work/ledger/verify_reply_volume_census.py b/wizard-work/ledger/verify_reply_volume_census.py new file mode 100644 index 0000000..4b37879 --- /dev/null +++ b/wizard-work/ledger/verify_reply_volume_census.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Independent verifier for the replies volume census artifacts. + +Recomputes ground truth from the LIVE /replies/ directory using shell +pipelines (grep/sed/cut/sort/uniq via subprocess), then asserts the committed +TSV artifacts and the markdown summary match that ground truth. + +The extractor parses with a compiled regex over pathlib; this verifier parses +with shell text tools -- two independent code paths over the same evidence +directory, so agreement is a real cross-check. + +Exit status: + 0 artifact matches ground truth + 1 mismatch (or artifact missing / unreadable) + +Used as the CHECK line: python3 wizard-work/ledger/verify_reply_volume_census.py +""" + +import csv +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +REPLIES = REPO_ROOT / "replies" +DATE_TSV = REPO_ROOT / "wizard-work" / "ledger" / "reply-volume-census.tsv" +RECIP_TSV = REPO_ROOT / "wizard-work" / "ledger" / "reply-volume-by-recipient.tsv" +MD = REPO_ROOT / "wizard-work" / "ledger" / "reply-volume-census.md" + +DATE_PAT = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}-to-" +FIND_BASENAMES = "find replies -maxdepth 1 -type f -name '*.md' -printf '%f\\n'" +SEP = "-------------------------------------------------------------------------------" + + +def sh(cmd): + """Run a shell command and return stripped stdout.""" + return subprocess.run( + cmd, shell=True, check=True, capture_output=True, text=True + ).stdout + + +def ground_truth(): + """Independent ground truth derived from the live replies/ directory.""" + date_cmd = (FIND_BASENAMES + " | grep -E '" + DATE_PAT + "' | " + "sed -E 's/-to-.*//' | sort | uniq -c") + date_files = sh(date_cmd).strip().splitlines() + date_counts = {} + for line in date_files: + cnt, date = line.split(None, 1) + date_counts[date] = int(cnt) + + # Recipients: strip the date prefix, drop ".md", then drop a trailing "-NN". + recip_cmd = (FIND_BASENAMES + " | grep -E '" + DATE_PAT + "' | " + "sed -E 's/^" + DATE_PAT + "//' | sed -E 's/\\.md$//' | " + "sed -E 's/-[0-9]+$//' | sort | uniq -c") + recip_files = sh(recip_cmd).strip().splitlines() + recip_counts = {} + for line in recip_files: + cnt, recip = line.split(None, 1) + recip_counts[recip] = int(cnt) + + return date_counts, recip_counts, sum(date_counts.values()) + + +def parse_tsv(path): + rows = {} + with path.open(newline="", encoding="utf-8") as fh: + reader = csv.reader(fh, delimiter="\t") + next(reader, None) # skip header row + for row in reader: + if len(row) == 2: + rows[row[0]] = int(row[1]) + return rows + + +def main(): + date_gt, recip_gt, total_gt = ground_truth() + problems = [] + + for label, tsv_path, gt in (("date", DATE_TSV, date_gt), + ("recipient", RECIP_TSV, recip_gt)): + if not tsv_path.is_file(): + problems.append("missing " + str(tsv_path)) + continue + rows = parse_tsv(tsv_path) + rows_data = {k: v for k, v in rows.items() if k != "TOTAL"} + if rows_data != gt: + problems.append( + "%s: data rows differ; got %d keys, ground truth %d keys" + % (label, len(rows_data), len(gt)) + ) + if rows.get("TOTAL", -1) != sum(gt.values()): + problems.append( + "%s: TOTAL row %r != sum of rows %d" + % (label, rows.get("TOTAL"), sum(gt.values())) + ) + + # Markdown summary must name the correct headline numbers. + if MD.is_file(): + text = MD.read_text(encoding="utf-8") + for token in ("430", "34", "17", "README.md"): + if token not in text: + problems.append("markdown missing assertion: %s" % token) + else: + problems.append("missing " + str(MD)) + + if problems: + print("CHECK FAIL") + for p in problems: + print(" -", p) + sys.exit(1) + + print( + "CHECK PASS: %d replies, %d dates, %d recipients " + "(md_files=%d, canonical=%d)" + % (total_gt, len(date_gt), len(recip_gt), md_total(), total_gt) + ) + sys.exit(0) + + +def md_total(): + return len(sh("ls -1 replies/*.md").splitlines()) + + +if __name__ == "__main__": + main()