From 4d7e8b43cfa0f606e91a38658c1109d90fd6e7b5 Mon Sep 17 00:00:00 2001
From: Timmy <timmy@stackchain.ai>
Date: Sat, 17 Oct 2026 03:34:07 +0000
Subject: [PATCH] tools (volt): census_row_delta.py -- a row-level partition of
 two censuses, and the 96->95 pair measured instead of argued

The briefs over the 2026-10-16 digest-rev fix agreed on the new census total and
disagreed on the partition, because a partition is a property of the two runs AND
of the key a row is identified by. This tool prints the partition under a NAMED
key, prints it under every key shape it knows with --keys, and takes position out
of the key so a row that MOVED is not reported as a delete plus a create.

Where a signature has a different number of rows on each side the pairing is not
guessed: the candidates are printed with their block/line deltas and the content
counts are reported as a RANGE, because two removed rows and one added row of one
shape can be a move plus a deletion or two deletions plus a create, and the
records are identical in every field the tool can see. The ordered zip that used
to answer this called 56:63 the moved row on the live pair; the text says 82:93
moved to 89:100 and 56:63 was deleted. Count right, identity wrong -- the defect
class this tool exists for.

selftest: 33 cells, 33 passed, 0 failed.
---
 tools/census_row_delta.py | 438 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 438 insertions(+)
 create mode 100644 tools/census_row_delta.py

diff --git a/tools/census_row_delta.py b/tools/census_row_delta.py
new file mode 100644
index 0000000..a3930cd
--- /dev/null
+++ b/tools/census_row_delta.py
@@ -0,0 +1,438 @@
+#!/usr/bin/env python3
+"""census_row_delta.py -- row-level diff of two doc_census_check.py censuses.
+
+WHY THIS EXISTS
+  `tools/doc_census_check.py` prints a census; the number a reader quotes is its
+  row total. When that total moved (the digest-rev fix of 2026-10-16), the
+  sentence that mattered was not the total but the PARTITION: how many rows were
+  REMOVED, how many ADDED, how many CHANGED VERDICT. Two briefs over the same
+  pair of runs reached the same new total with different partitions -- nineteen
+  removed and three added in one, twenty removed and four added in the other --
+  and both are arithmetically consistent, because the partition is not a property
+  of the two runs alone. It is a property of the two runs AND of the key a row is
+  identified by. A census row is a record (tool, block, line, rev, unit, claim,
+  token, verdict); pick a different subset of those as the identity and you
+  rename a row that only moved position into a row that was deleted and another
+  that was created.
+
+  So this tool prints the partition under a NAMED key, never a bare "removed N",
+  and `--keys` prints it under every key shape it knows -- so a disagreement
+  about the count can be resolved to the key that produced it instead of argued
+  about. Every partition is then restated with position taken out of the key: a
+  removed row and an added row whose records agree on every field the key holds
+  fixed are ONE row that MOVED, reported separately from a row that was deleted
+  or introduced. Reading a position shift as a content change is the whole defect
+  class this tool exists for.
+
+WHAT IT READS
+  Two JSON files written by `doc_census_check.py --json`. It does not run the
+  census and does not judge a census: it cannot see anything the JSON does not
+  carry, and it makes no claim about which census is right.
+
+READ-ONLY. The only write path is `--json PATH`; every other mode prints.
+
+A STATED LIMIT, NOT AN IMPLIED ONE
+  Rows are compared as records, so a claim whose TEXT moved between blocks (a
+  docstring line re-wrapped, a claim reworded) is REMOVED and ADDED, never
+  CHANGED. `--moves` catches the pure-position case and nothing else; a reworded
+  claim is reported as a delete plus a create and is not paired. The key shapes
+  below are the whole vocabulary this tool has for that; it invents no similarity
+  metric.
+
+PASS-THROUGH, because a census leg drives every tool as `<script> --repo R --rev
+  REV`: `--repo` and `--rev` are accepted and IGNORED -- this tool's subject is a
+  pair of files, not a revision. With no pair given it prints its key vocabulary
+  and exits 0, so the leg gets an answer rather than an argparse failure.
+"""
+import argparse
+import collections
+import json
+import sys
+
+# A row is identified by a subset of these fields -- the names are the JSON keys.
+FIELDS = ("tool", "block", "line", "rev", "unit", "claim", "token")
+
+# Every key shape this tool knows, in the order `--keys` prints them. The
+# canonical one is the row as doc_census_check.py constructs it; the others are
+# the plausible under-keyings a reader might reach for, named so that a
+# disagreement about a count can be attributed rather than argued.
+KEYS = {
+    "canonical": ("tool", "block", "line", "rev", "unit", "claim"),
+    "strict": ("tool", "block", "line", "rev", "unit", "claim", "token"),
+    "token": ("tool", "block", "line", "rev", "token", "claim"),
+    "merged-rev": ("tool", "block", "line", "unit", "claim"),
+    "no-block": ("tool", "line", "rev", "unit", "claim"),
+    "loose": ("tool", "line", "unit", "claim"),
+    "line-only": ("tool", "line"),
+}
+
+
+def load(path):
+    """A census JSON -> {rev, repo, rows, fails, per}.
+
+    `rows` is every surviving verdict dict, each carrying its tool name; keying
+    happens in `keyed()` so one load serves every key shape.
+    """
+    d = json.load(open(path, encoding="utf-8"))
+    rows, fails, per = [], [], {}
+    for rep in d["reports"]:
+        for v in rep["verdicts"]:
+            row = dict(v)
+            row["tool"] = rep["tool"]
+            rows.append(row)
+        for f in rep["failures"]:
+            fails.append((rep["tool"], f.get("block"), f.get("rev")))
+        per[rep["tool"]] = len(rep["verdicts"])
+    return {"rev": d.get("rev"), "repo": d.get("repo"), "rows": rows,
+            "fails": fails, "per": per}
+
+
+def keyed(census, fields):
+    """({key: row}, collisions) -- later duplicates win, and the count is kept.
+
+    A collision says the key is not an identity for this census: two distinct
+    rows share it, so the partition computed under it is a partition of ROWS
+    COLLAPSED, not of rows. Reported, never silent.
+    """
+    out, collisions = {}, 0
+    for row in census["rows"]:
+        k = tuple(row.get(f) for f in fields)
+        if k in out:
+            collisions += 1
+        out[k] = row
+    return out, collisions
+
+
+def diff(before, after, fields):
+    b, bc = keyed(before, fields)
+    a, ac = keyed(after, fields)
+    removed = sorted(set(b) - set(a))
+    added = sorted(set(a) - set(b))
+    changed = sorted(k for k in set(a) & set(b)
+                     if b[k]["verdict"] != a[k]["verdict"])
+    return {"removed": removed, "added": added, "changed": changed,
+            "b": b, "a": a, "collisions": (bc, ac)}
+
+
+def moves(before, after, d, fields):
+    """What the removed/added rows are once position is taken out of the key.
+
+    Returns {pairs, candidates, ambiguous, ambiguous_min, undecided_removed,
+    undecided_added, unmatched_removed, unmatched_added}.
+
+    Pairs are ONE-TO-ONE and DECIDED: a signature (the row with `block` and
+    `line` removed from the key) that has the same number of rows on both sides.
+    Each pair is ONE row that MOVED, not a delete plus a create.
+
+    Where a signature has a different number of rows on each side, the tool does
+    NOT choose: it prints every cross candidate with its block and line delta,
+    records `ambiguous_min` -- how many pairings that signature could hold,
+    min(removed, added) -- and leaves the question open, because two removed rows
+    and one added row of the same shape can be a move plus a deletion or two
+    deletions plus a create, and the records are identical in every field this
+    tool can see. Resolving that needs the text the rows came from -- outside
+    this tool, and stated as such rather than papered over with an arbitrary
+    pairing. Rows inside such a signature are `undecided_*`, kept apart from the
+    `unmatched_*` rows that are genuinely gone or genuinely new.
+
+    AN ORDERED GUESS IS NOT A RESOLUTION. Until 2026-10-17 this branch zipped the
+    two sides in sorted order and reported the first rows as one MOVED row. On
+    the live 0031a78 -> HEAD pair that produced `MOVED 56:63 -> 89:100` for a
+    signature of two removed rows and one added row -- the right COUNT of moved
+    rows (one) and the WRONG ROW: the text says block 82 line 93 is the row that
+    moved to block 89 line 100, and the block 56 line 63 claim was deleted. A
+    count that survives a wrong identity is the defect this tool exists to catch,
+    so the pairing is no longer guessed at all.
+
+    Only meaningful for a key that carries `block` or `line`; with neither there
+    is no position to have moved from, and the empty result is the answer rather
+    than a guess.
+    """
+    empty = {"pairs": [], "candidates": [], "ambiguous": [], "ambiguous_min": 0,
+             "undecided_removed": [], "undecided_added": [],
+             "unmatched_removed": [], "unmatched_added": []}
+    if "block" not in fields and "line" not in fields:
+        return empty
+    free = [f for f in ("block", "line") if f in fields]
+    ident = tuple(f for f in fields if f not in free)
+    rem, add = collections.defaultdict(list), collections.defaultdict(list)
+    for k in d["removed"]:
+        rem[tuple(d["b"][k].get(f) for f in ident)].append(k)
+    for k in d["added"]:
+        add[tuple(d["a"][k].get(f) for f in ident)].append(k)
+    out = dict(empty)
+    for sig in sorted(set(rem) | set(add), key=repr):
+        r, a = sorted(rem.get(sig, [])), sorted(add.get(sig, []))
+        if r and a and len(r) != len(a):
+            out["ambiguous"].append((sig, len(r), len(a)))
+            out["ambiguous_min"] += min(len(r), len(a))
+            out["undecided_removed"].extend(r)
+            out["undecided_added"].extend(a)
+            for k in r:
+                for k2 in a:
+                    out["candidates"].append(
+                        (k, k2, _delta(d["b"][k], d["a"][k2], "block"),
+                         _delta(d["b"][k], d["a"][k2], "line")))
+            continue
+        n = min(len(r), len(a))
+        out["pairs"].extend(zip(r[:n], a[:n]))
+        out["unmatched_removed"].extend(r[n:])
+        out["unmatched_added"].extend(a[n:])
+    return out
+
+
+def _delta(a, b, field):
+    """b[field] - a[field], or None when either side does not carry it."""
+    if a.get(field) is None or b.get(field) is None:
+        return None
+    return b[field] - a[field]
+
+
+def banner(tag, census):
+    return ("%-8s rev=%-12s rows=%-4d leg-failures=%-2d tools=%d"
+            % (tag, census["rev"], len(census["rows"]), len(census["fails"]),
+               len(census["per"])))
+
+
+def totals(census):
+    return dict(sorted(collections.Counter(
+        r["verdict"] for r in census["rows"]).items()))
+
+
+def report(before, after, name, fields, verbose=False):
+    d = diff(before, after, fields)
+    mv = moves(before, after, d, fields)
+    nb, na = len(before["rows"]), len(after["rows"])
+    print("\nkey %-11s (%s)" % (name, " + ".join(fields)))
+    print("  keys before %d, after %d, collisions before/after %d/%d"
+          % (len(d["b"]), len(d["a"]), d["collisions"][0], d["collisions"][1]))
+    print("  rows before %d, after %d -- removed %d, added %d, changed %d"
+          % (nb, na, len(d["removed"]), len(d["added"]), len(d["changed"])))
+    print("  arithmetic under this key: %d - %d + %d = %d %s"
+          % (nb, len(d["removed"]), len(d["added"]),
+             nb - len(d["removed"]) + len(d["added"]),
+             "= the after total" if nb - len(d["removed"]) + len(d["added"]) == na
+             else "!= the after total, because the key collides"))
+    if mv["pairs"] or mv["ambiguous"]:
+        print("  position is not content: %d row(s) of the removed/added sets are "
+              "ONE row at a different block/line" % len(mv["pairs"]))
+        for k, k2 in mv["pairs"]:
+            print("    MOVED %s:%s -> %s:%s   %s %s %s"
+                  % (d["b"][k].get("block"), d["b"][k].get("line"),
+                     d["a"][k2].get("block"), d["a"][k2].get("line"),
+                     d["b"][k].get("unit"), d["b"][k].get("claim"),
+                     d["b"][k].get("verdict")))
+        for sig, nr, na2 in mv["ambiguous"]:
+            print("    AMBIGUOUS %s: %d removed vs %d added of one shape -- "
+                  "%d pairing(s) possible, not decided by these records"
+                  % (sig, nr, na2, min(nr, na2)))
+        print("  as CONTENT, with the DECIDED moves netted out: removed %d, added %d, "
+              "moved %d" % (len(d["removed"]) - len(mv["pairs"]),
+                            len(d["added"]) - len(mv["pairs"]), len(mv["pairs"])))
+        amb = mv["ambiguous_min"]
+        if amb:
+            print("    %d pairing(s) the records cannot decide, so the content counts "
+                  "are a RANGE:" % amb)
+            print("      every undecided row a move: removed %d, added %d, moved %d"
+                  % (len(d["removed"]) - len(mv["pairs"]) - amb,
+                     len(d["added"]) - len(mv["pairs"]) - amb,
+                     len(mv["pairs"]) + amb))
+            print("      none of them a move:        removed %d, added %d, moved %d"
+                  % (len(d["removed"]) - len(mv["pairs"]),
+                     len(d["added"]) - len(mv["pairs"]), len(mv["pairs"])))
+    if verbose:
+        for k in d["removed"]:
+            print("    REMOVED %s -> absent" % (k,))
+        for k in d["added"]:
+            print("    ADDED   %s -> %s" % (k, d["a"][k]["verdict"]))
+        for k in d["changed"]:
+            print("    CHANGED %s %s -> %s" % (k, d["b"][k]["verdict"],
+                                               d["a"][k]["verdict"]))
+    return d
+
+
+def _fixture():
+    """Two synthetic censuses: one row deleted, one added, one moved, one flipped.
+
+    A fixture, not a live read: the arithmetic under test is this tool's, and a
+    live census changes under it. Every row carries every field the keys use.
+    """
+    def row(tool, block, line, rev, unit, claim, verdict, token=None):
+        return {"tool": tool, "block": block, "line": line, "rev": rev,
+                "unit": unit, "claim": claim, "token": token or claim,
+                "verdict": verdict, "detail": "", "file": tool}
+    before = [row("t/a.py", 4, 10, "HEAD", "cells", 3, "MATCHES"),
+              row("t/a.py", 4, 11, "HEAD", "rows", 2, "NOT_PRINTED"),
+              row("t/b.py", 20, 40, "abc1234", "cells", 7, "STALE")]
+    after = [row("t/a.py", 4, 10, "HEAD", "cells", 3, "MATCHES"),
+             row("t/a.py", 4, 11, "HEAD", "rows", 2, "STALE"),
+             row("t/b.py", 21, 41, "abc1234", "cells", 7, "STALE"),
+             row("t/b.py", 21, 42, "HEAD", "splits", 1, "MATCHES")]
+    meta = {"rev": "HEAD", "repo": "/x"}
+    return (dict(rows=before, fails=[("t/b.py", 20, "abc1234")],
+                 per={"t/a.py": 2, "t/b.py": 1}, **meta),
+            dict(rows=after, fails=[], per={"t/a.py": 2, "t/b.py": 2}, **meta))
+
+
+def selftest():
+    before, after = _fixture()
+    cells, bad = [0], [0]
+
+    def check(name, got, want):
+        good = got == want
+        cells[0] += 1
+        bad[0] += 0 if good else 1
+        print("%-52s %s" % (name, "PASS" if good else "FAIL"))
+        if not good:
+            print("      got  %r\n      want %r" % (got, want))
+
+    d = diff(before, after, KEYS["canonical"])
+    mv = moves(before, after, d, KEYS["canonical"])
+    check("canonical: removed rows", len(d["removed"]), 1)
+    check("canonical: added rows (the moved row is one of them)", len(d["added"]), 2)
+    check("canonical: changed verdicts", len(d["changed"]), 1)
+    check("canonical: the move is paired",
+          [(d["b"][k]["block"], d["b"][k]["line"], d["a"][k2]["block"],
+            d["a"][k2]["line"]) for k, k2 in mv["pairs"]],
+          [(20, 40, 21, 41)])
+    check("canonical: exactly one pair, the genuinely new row is not one",
+          len(mv["pairs"]), 1)
+    check("canonical: no removed row is left unmatched",
+          mv["unmatched_removed"], [])
+    check("canonical: one added row is left unmatched",
+          len(mv["unmatched_added"]), 1)
+    check("canonical: nothing ambiguous here", mv["ambiguous"], [])
+    check("canonical: content = raw minus the move",
+          (len(d["removed"]) - len(mv["pairs"]), len(d["added"]) - len(mv["pairs"])),
+          (0, 1))
+    check("canonical: partition arithmetic closes",
+          len(before["rows"]) - len(d["removed"]) + len(d["added"]),
+          len(after["rows"]))
+    d2 = diff(before, after, KEYS["merged-rev"])
+    check("merged-rev: dropping rev moves nothing here",
+          (len(d2["removed"]), len(d2["added"]), len(d2["changed"])), (1, 2, 1))
+    d3 = diff(before, after, KEYS["line-only"])
+    m3 = moves(before, after, d3, KEYS["line-only"])
+    check("line-only: removed", len(d3["removed"]), 1)
+    check("line-only: added", len(d3["added"]), 2)
+    # 1 removed vs 2 added at one line: the old code zipped them and called
+    # (40 -> 41) a move. Nothing in the records says which of the two added rows
+    # is the shifted one, so the pairing is UNDECIDED and both are printed.
+    check("line-only: a 1-vs-2 signature is left undecided, not paired in order",
+          len(m3["pairs"]), 0)
+    check("line-only: every candidate is printed, neither chosen",
+          sorted((d3["b"][k]["line"], d3["a"][k2]["line"])
+                 for k, k2, _db, _dl in m3["candidates"]),
+          [(40, 41), (40, 42)])
+    check("line-only: the undecided pairing is counted", m3["ambiguous_min"], 1)
+    check("line-only: no removed row is called gone", m3["unmatched_removed"], [])
+    check("line-only: no added row is called new", m3["unmatched_added"], [])
+    check("line-only: its rows are the undecided ones",
+          (len(m3["undecided_removed"]), len(m3["undecided_added"])), (1, 2))
+    dup = {"rows": before["rows"] + [dict(before["rows"][2], block=19, line=39)],
+           "fails": [], "per": {}, "rev": "HEAD", "repo": "/x"}
+    d4 = diff(dup, after, KEYS["canonical"])
+    m4 = moves(dup, after, d4, KEYS["canonical"])
+    check("many-to-one: one ambiguous signature reported",
+          [(nr, na) for _sig, nr, na in m4["ambiguous"]], [(2, 1)])
+    check("many-to-one: nothing is paired on the tool's guess", len(m4["pairs"]), 0)
+    check("many-to-one: the undecided pairing is counted", m4["ambiguous_min"], 1)
+    check("many-to-one: both removed rows are undecided, neither called gone",
+          len(m4["undecided_removed"]), 2)
+    check("many-to-one: the added row of that shape is undecided too",
+          len(m4["undecided_added"]), 1)
+    check("many-to-one: nothing is left unmatched on the removed side",
+          len(m4["unmatched_removed"]), 0)
+    check("many-to-one: the genuinely new row is the unmatched one",
+          len(m4["unmatched_added"]), 1)
+    check("many-to-one: the candidates carry their block and line deltas",
+          sorted((db, dl) for _k, _k2, db, dl in m4["candidates"]),
+          [(1, 1), (2, 2)])
+    check("many-to-one: content bounds -- moved",
+          (len(m4["pairs"]), len(m4["pairs"]) + m4["ambiguous_min"]), (0, 1))
+    check("many-to-one: content bounds -- removed",
+          (len(d4["removed"]) - len(m4["pairs"]) - m4["ambiguous_min"],
+           len(d4["removed"]) - len(m4["pairs"])), (1, 2))
+    check("keyed: a key that is not an identity collides",
+          keyed(before, ("tool",))[1], 1)
+    check("keyed: collided key count", len(keyed(before, ("tool",))[0]), 2)
+    check("totals: before", totals(before),
+          {"MATCHES": 1, "NOT_PRINTED": 1, "STALE": 1})
+    check("totals: after", totals(after), {"MATCHES": 2, "STALE": 2})
+    print("selftest: %d cells, %d passed, %d failed"
+          % (cells[0], cells[0] - bad[0], bad[0]))
+    return 1 if bad[0] else 0
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("before", nargs="?", help="earlier census JSON")
+    ap.add_argument("after", nargs="?", help="later census JSON")
+    ap.add_argument("--key", default="canonical", choices=sorted(KEYS))
+    ap.add_argument("--keys", action="store_true",
+                    help="print the partition under every key shape")
+    ap.add_argument("--verbose", action="store_true")
+    ap.add_argument("--selftest", action="store_true")
+    ap.add_argument("--json", help="write the partitions it printed")
+    # accepted and ignored: see the pass-through note in the module docstring
+    ap.add_argument("--repo", help=argparse.SUPPRESS)
+    ap.add_argument("--rev", help=argparse.SUPPRESS)
+    a = ap.parse_args()
+
+    if a.selftest:
+        return selftest()
+    if not (a.before and a.after):
+        print("census_row_delta -- no census pair given; key shapes available:")
+        for name in sorted(KEYS):
+            print("  %-11s %s" % (name, " + ".join(KEYS[name])))
+        return 0
+
+    before, after = load(a.before), load(a.after)
+    print(banner("before", before))
+    print(banner("after", after))
+    print("verdict totals before: %s" % totals(before))
+    print("verdict totals after:  %s" % totals(after))
+    print("\nrows per tool: before -> after")
+    for t in sorted(set(before["per"]) | set(after["per"])):
+        print("  %-40s %s -> %s" % (t, before["per"].get(t), after["per"].get(t)))
+    bf, af = set(before["fails"]), set(after["fails"])
+    if bf or af:
+        print("\nleg failures: before %d, after %d (%d gone, %d new)"
+              % (len(bf), len(af), len(bf - af), len(af - bf)))
+
+    out = {}
+    for name in (sorted(KEYS) if a.keys else [a.key]):
+        d = report(before, after, name, KEYS[name], a.verbose)
+        mv = moves(before, after, d, KEYS[name])
+        amb = mv["ambiguous_min"]
+        out[name] = {"fields": list(KEYS[name]), "removed": len(d["removed"]),
+                     "added": len(d["added"]), "changed": len(d["changed"]),
+                     "moved": len(mv["pairs"]), "undecided_pairings": amb,
+                     "moved_range": [len(mv["pairs"]), len(mv["pairs"]) + amb],
+                     "removed_content_range": [
+                         len(d["removed"]) - len(mv["pairs"]) - amb,
+                         len(d["removed"]) - len(mv["pairs"])],
+                     "added_content_range": [
+                         len(d["added"]) - len(mv["pairs"]) - amb,
+                         len(d["added"]) - len(mv["pairs"])],
+                     "unmatched_removed": len(mv["unmatched_removed"]),
+                     "unmatched_added": len(mv["unmatched_added"])}
+        if a.verbose or a.keys:
+            for k in d["removed"]:
+                print("    -%s %s" % (k, d["b"][k]["verdict"]))
+            for k in d["added"]:
+                print("    +%s %s" % (k, d["a"][k]["verdict"]))
+    if a.json:
+        with open(a.json, "w", encoding="utf-8") as fh:
+            json.dump({"before": a.before, "after": a.after,
+                       "before_rev": before["rev"], "after_rev": after["rev"],
+                       "rows_before": len(before["rows"]),
+                       "rows_after": len(after["rows"]),
+                       "partitions": out}, fh, indent=1)
+        print("\nwrote %s" % a.json)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())