troupe-standup/wizard-work/ledger/extract_checkin_census_2026-10-27.py

#!/usr/bin/env python3
"""extract_checkin_census_2026-10-27.py -- check-in census table, from a PIN.

Extract lane (data extraction + tables). Counts `checkins/*.md` at a named
revision, not on whatever the directory holds when the script runs, and emits:

  * 2026-10-27-checkin-census.files  the pin: one header line, then every dated
                                    filename at that revision, sorted
  * 2026-10-27-checkin-census.tsv   wizard<TAB>checkin_files, count desc then name
  * 2026-10-27-checkin-census.md    the same table, a per-date appendix, provenance

Why a pin. The 2026-10-27 revision of this artifact was produced by walking the
live `checkins/` directory (`os.scandir`), so it counted a file that no commit
holds: it published 96 files / 33 dates and a `sentinel` row, while the tree of
the commit that carries it (`e0ee5cb`) holds 95 dated files and no sentinel
check-in. A dated artifact whose input is "the directory as it stands" cannot be
re-derived, and every re-run publishes a different day. The pin makes the input
a committed name list at a fixed revision (Q88).

Commands (each is exercised by `--help`):

    python3 extract_checkin_census_2026-10-27.py --pin e0ee5cb
    python3 extract_checkin_census_2026-10-27.py --pin e0ee5cb --write
    python3 extract_checkin_census_2026-10-27.py --manifest 2026-10-27-checkin-census.files --write
    python3 extract_checkin_census_2026-10-27.py --outdir /tmp/scratch --write

Without `--write` nothing is written: the derivation is printed so a reader can
compare it with the committed tables. Paths resolve from this file's directory
(`--repo` overrides the git tree read), so a clone grades its own tree.
"""
import argparse
import collections
import os
import re
import subprocess
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
DATED = re.compile(r"^(\d{4}-\d{2}-\d{2})-(.+)\.md$")
SUFFIX = re.compile(r"-\d+$")
HEADER = re.compile(r"^# pin checkins@([0-9a-f]{40}) tree=([0-9a-f]{40})$")
MANIFEST_NAME = "2026-10-27-checkin-census.files"


def git(repo, *args):
    out = subprocess.run(["git", "-C", repo] + list(args),
                         capture_output=True, text=True)
    if out.returncode != 0:
        sys.exit(f"git {' '.join(args)} failed: {out.stderr.strip()}")
    return out.stdout


def names_from_pin(repo, rev):
    commit = git(repo, "rev-parse", "--verify", f"{rev}^{{commit}}").strip()
    tree = git(repo, "rev-parse", f"{commit}:checkins").strip()
    listing = git(repo, "ls-tree", "-r", "--name-only", tree)
    names = [line.rsplit("/", 1)[-1] for line in listing.splitlines() if line.strip()]
    return commit, tree, names


def names_from_manifest(path):
    with open(path, encoding="utf-8") as fh:
        lines = [line.rstrip("\n") for line in fh if line.strip()]
    head = HEADER.match(lines[0]) if lines else None
    if not head:
        sys.exit(f"manifest header not understood: {path}")
    return head.group(1), head.group(2), lines[1:]


def derive(names):
    """(per_date, per_wizard, excluded, dated) from filenames."""
    per_date, per_wizard, excluded, dated = (collections.Counter(), collections.Counter(),
                                             [], [])
    for name in sorted(names):
        if not name.endswith(".md"):
            excluded.append(f"{name} (not .md)")
            continue
        m = DATED.match(name)
        if not m:
            excluded.append(name[:-3])
            continue
        dated.append(name)
        per_date[m.group(1)] += 1
        per_wizard[SUFFIX.sub("", m.group(2))] += 1
    return per_date, per_wizard, excluded, dated


def rows_of(per_wizard):
    return sorted(per_wizard.items(), key=lambda kv: (-kv[1], kv[0]))


def render_tsv(per_wizard):
    out = ["wizard\tcheckin_files"]
    out += [f"{w}\t{n}" for w, n in rows_of(per_wizard)]
    return "\n".join(out) + "\n"


def render_md(per_date, per_wizard, excluded, commit, tree, dated):
    total_files, total_dates = len(dated), len(per_date)
    latest = max(per_date)
    lines = [
        "# Check-in census",
        "",
        "Extract lane table: per-wizard tally of `checkins/*.md` at a pinned",
        f"revision (README.md and any other undated file excluded). The latest day",
        f"at the pin is {latest} ({per_date[latest]} files).",
        "",
        "| wizard | checkin_files |",
        "|---|---|",
    ]
    lines += [f"| {w} | {n} |" for w, n in rows_of(per_wizard)]
    lines += [
        "",
        f"**Total files:** {total_files} across {total_dates} dates "
        "(undated files excluded).",
        "",
        "## Per-date appendix",
        "",
        "| date | files |",
        "|---|---|",
    ]
    lines += [f"| {d} | {n} |" for d, n in sorted(per_date.items())]
    lines += [
        "",
        "## Provenance",
        "",
        f"- Pin: `checkins/` at commit `{commit}`, tree `{tree}` "
        f"(`{MANIFEST_NAME}` beside this file holds the sorted name list).",
        f"- Date-prefixed files at the pin: {total_files}; distinct dates: {total_dates}; "
        f"files excluded: {len(excluded)}"
        + (f" ({', '.join(excluded)})." if excluded else " (none)."),
        "- `wizard` = filename with leading `YYYY-MM-DD-` and trailing `-{digits}` stripped.",
        "- Correction: the `e0ee5cb` revision of this artifact counted the live "
        "directory and published 96 files / 33 dates; no revision of `checkins/` "
        "holds that `sentinel` file, and the numbers here are recomputed from the pin.",
        "- Check: `CHECK-2026-10-27-checkin-census` recomputes the table above, "
        "the total line and the per-date appendix from the pinned name list and "
        "diffs all three; it never reads today's directory.",
    ]
    return "\n".join(lines) + "\n"


def main(argv=None):
    ap = argparse.ArgumentParser(
        description="Check-in census from a pinned revision of checkins/.")
    ap.add_argument("--pin", metavar="REV",
                    help="pin to this revision's checkins/ tree (e.g. e0ee5cb)")
    ap.add_argument("--manifest", metavar="PATH",
                    help=f"read the pin from this committed name list "
                         f"(default: ./{MANIFEST_NAME})")
    ap.add_argument("--repo", default=os.path.dirname(os.path.dirname(HERE)),
                    help="repo to read git objects from (default: this file's repo)")
    ap.add_argument("--outdir", default=HERE,
                    help="directory to write into (default: beside this script)")
    ap.add_argument("--write", action="store_true",
                    help="write the manifest and both tables (default: print only)")
    args = ap.parse_args(argv)

    if args.pin:
        commit, tree, names = names_from_pin(args.repo, args.pin)
    else:
        manifest = args.manifest or os.path.join(HERE, MANIFEST_NAME)
        if not os.path.exists(manifest):
            ap.error(f"no --pin given and no manifest at {manifest}")
        commit, tree, names = names_from_manifest(manifest)

    per_date, per_wizard, excluded, dated = derive(names)
    tsv, md = render_tsv(per_wizard), render_md(per_date, per_wizard, excluded,
                                                commit, tree, dated)
    print(f"pin commit={commit} tree={tree}")
    print(f"dated_files={len(dated)} dates={len(per_date)} "
          f"wizards={len(per_wizard)} excluded={len(excluded)}")
    for w, n in rows_of(per_wizard):
        print(f"  {w}\t{n}")

    if not args.write:
        print("dry run (pass --write to write the manifest and tables)")
        return 0

    os.makedirs(args.outdir, exist_ok=True)
    manifest_path = os.path.join(args.outdir, MANIFEST_NAME)
    with open(manifest_path, "w", encoding="utf-8") as fh:
        fh.write(f"# pin checkins@{commit} tree={tree}\n")
        fh.write("".join(f"{n}\n" for n in sorted(dated)))
    for name, payload in (("2026-10-27-checkin-census.tsv", tsv),
                          ("2026-10-27-checkin-census.md", md)):
        with open(os.path.join(args.outdir, name), "w", encoding="utf-8") as fh:
            fh.write(payload)
    print(f"wrote {manifest_path}, 2026-10-27-checkin-census.tsv, "
          f"2026-10-27-checkin-census.md")
    return 0


if __name__ == "__main__":
    sys.exit(main())