#!/usr/bin/env python3
"""census_row_delta.py -- row-level diff of two doc_census_check.py censuses.

WHY THIS EXISTS
  `tools/doc_census_check.py` prints a census; the number a reader quotes is its
  row total. When that total moved (the digest-rev fix of 2026-10-16), the
  sentence that mattered was not the total but the PARTITION: how many rows were
  REMOVED, how many ADDED, how many CHANGED VERDICT. Two briefs over the same
  pair of runs reached the same new total with different partitions -- nineteen
  removed and three added in one, twenty removed and four added in the other --
  and both are arithmetically consistent, because the partition is not a property
  of the two runs alone. It is a property of the two runs AND of the key a row is
  identified by. A census row is a record (tool, block, line, rev, unit, claim,
  token, verdict); pick a different subset of those as the identity and you
  rename a row that only moved position into a row that was deleted and another
  that was created.

  So this tool prints the partition under a NAMED key, never a bare "removed N",
  and `--keys` prints it under every key shape it knows -- so a disagreement
  about the count can be resolved to the key that produced it instead of argued
  about. Every partition is then restated with position taken out of the key: a
  removed row and an added row whose records agree on every field the key holds
  fixed are ONE row that MOVED, reported separately from a row that was deleted
  or introduced. Reading a position shift as a content change is the whole defect
  class this tool exists for.

WHAT IT READS
  Two JSON files written by `doc_census_check.py --json`. It does not run the
  census and does not judge a census: it cannot see anything the JSON does not
  carry, and it makes no claim about which census is right.

READ-ONLY. The only write path is `--json PATH`; every other mode prints.

A STATED LIMIT, NOT AN IMPLIED ONE
  Rows are compared as records, so a claim whose TEXT moved between blocks (a
  docstring line re-wrapped, a claim reworded) is REMOVED and ADDED, never
  CHANGED. `--moves` catches the pure-position case and nothing else; a reworded
  claim is reported as a delete plus a create and is not paired. The key shapes
  below are the whole vocabulary this tool has for that; it invents no similarity
  metric.

PASS-THROUGH, because a census leg drives every tool as `<script> --repo R --rev
  REV`: `--repo` and `--rev` are accepted and IGNORED -- this tool's subject is a
  pair of files, not a revision. It still prints its key vocabulary for that
  shape, because a reader who mistyped a path needs the shapes, but it no longer
  reports the shape as a verdict. A run that was given no pair, or only one half
  of one, graded nothing, and this tree's exit contract reserves 0 for a run
  that graded and found nothing to report (`tools/verify.py`: 0 clean, 1 graded
  fault, 2 could not grade). Exiting 0 there was the defect this paragraph used
  to document: reply 18 (`replies/2026-10-17-to-volt-18.md:3`) records a probe
  that passed ONE path and "read a spurious clean exit" until the parser was
  read by hand. The refusal is now a code, not a footnote -- no pair rc 2, half
  a pair rc 2 naming the half, and `--repo`/`--rev` alone rc 2 as well.

USAGE
  python3 census_row_delta.py A.json B.json [--moves] [--json OUT]
  python3 census_row_delta.py --selftest          # 37 cells, keys and partitions
"""
import argparse
import collections
import contextlib
import io
import json
import sys


class ReadError(Exception):
    """Input this tool could not read or could not recognise: rc 2, never rc 1.

    rc 1 is the tool's verdict code (a partition that does not close). A file
    that cannot be opened, decoded or parsed has produced no verdict at all, so
    it must not borrow the code that means "the grading found a fault".
    """


# A row is identified by a subset of these fields -- the names are the JSON keys.
FIELDS = ("tool", "block", "line", "rev", "unit", "claim", "token")

# Every key shape this tool knows, in the order `--keys` prints them. The
# canonical one is the row as doc_census_check.py constructs it; the others are
# the plausible under-keyings a reader might reach for, named so that a
# disagreement about a count can be attributed rather than argued.
KEYS = {
    "canonical": ("tool", "block", "line", "rev", "unit", "claim"),
    "strict": ("tool", "block", "line", "rev", "unit", "claim", "token"),
    "token": ("tool", "block", "line", "rev", "token", "claim"),
    "merged-rev": ("tool", "block", "line", "unit", "claim"),
    "no-block": ("tool", "line", "rev", "unit", "claim"),
    "loose": ("tool", "line", "unit", "claim"),
    "line-only": ("tool", "line"),
}


def load(path):
    """A census JSON -> {rev, repo, rows, fails, per}.

    `rows` is every surviving verdict dict, each carrying its tool name; keying
    happens in `keyed()` so one load serves every key shape.

    An unreadable or non-census file is a PRECONDITION failure, not a traceback
    and not a verdict: this used to raise the json decoder's error or a bare
    `KeyError: 'reports'`, with rc 1 -- the code that means "the grading found a
    fault". `ReadError` is turned into rc 2 in main(); the message names the
    file and what was expected instead.
    """
    try:
        with open(path, encoding="utf-8") as fh:
            d = json.load(fh)
    except (OSError, ValueError) as exc:
        raise ReadError("cannot read census %s: %s" % (path, exc))
    if not isinstance(d, dict) or not isinstance(d.get("reports"), list):
        raise ReadError("not a doc_census_check census (no 'reports' list): %s" % path)
    rows, fails, per = [], [], {}
    for rep in d["reports"]:
        for v in rep["verdicts"]:
            row = dict(v)
            row["tool"] = rep["tool"]
            rows.append(row)
        for f in rep["failures"]:
            fails.append((rep["tool"], f.get("block"), f.get("rev")))
        per[rep["tool"]] = len(rep["verdicts"])
    return {"rev": d.get("rev"), "repo": d.get("repo"), "rows": rows,
            "fails": fails, "per": per}


def keyed(census, fields):
    """({key: row}, collisions) -- later duplicates win, and the count is kept.

    A collision says the key is not an identity for this census: two distinct
    rows share it, so the partition computed under it is a partition of ROWS
    COLLAPSED, not of rows. Reported, never silent.
    """
    out, collisions = {}, 0
    for row in census["rows"]:
        k = tuple(row.get(f) for f in fields)
        if k in out:
            collisions += 1
        out[k] = row
    return out, collisions


def diff(before, after, fields):
    b, bc = keyed(before, fields)
    a, ac = keyed(after, fields)
    removed = sorted(set(b) - set(a))
    added = sorted(set(a) - set(b))
    changed = sorted(k for k in set(a) & set(b)
                     if b[k]["verdict"] != a[k]["verdict"])
    return {"removed": removed, "added": added, "changed": changed,
            "b": b, "a": a, "collisions": (bc, ac)}


def moves(before, after, d, fields):
    """What the removed/added rows are once position is taken out of the key.

    Returns {pairs, candidates, ambiguous, ambiguous_min, undecided_removed,
    undecided_added, unmatched_removed, unmatched_added}.

    Pairs are ONE-TO-ONE and DECIDED: a signature (the row with `block` and
    `line` removed from the key) that has the same number of rows on both sides.
    Each pair is ONE row that MOVED, not a delete plus a create.

    Where a signature has a different number of rows on each side, the tool does
    NOT choose: it prints every cross candidate with its block and line delta,
    records `ambiguous_min` -- how many pairings that signature could hold,
    min(removed, added) -- and leaves the question open, because two removed rows
    and one added row of the same shape can be a move plus a deletion or two
    deletions plus a create, and the records are identical in every field this
    tool can see. Resolving that needs the text the rows came from -- outside
    this tool, and stated as such rather than papered over with an arbitrary
    pairing. Rows inside such a signature are `undecided_*`, kept apart from the
    `unmatched_*` rows that are genuinely gone or genuinely new.

    AN ORDERED GUESS IS NOT A RESOLUTION. Until 2026-10-17 this branch zipped the
    two sides in sorted order and reported the first rows as one MOVED row. On
    the live 0031a78 -> HEAD pair that produced `MOVED 56:63 -> 89:100` for a
    signature of two removed rows and one added row -- the right COUNT of moved
    rows (one) and the WRONG ROW: the text says block 82 line 93 is the row that
    moved to block 89 line 100, and the block 56 line 63 claim was deleted. A
    count that survives a wrong identity is the defect this tool exists to catch,
    so the pairing is no longer guessed at all.

    Only meaningful for a key that carries `block` or `line`; with neither there
    is no position to have moved from, and the empty result is the answer rather
    than a guess.
    """
    empty = {"pairs": [], "candidates": [], "ambiguous": [], "ambiguous_min": 0,
             "undecided_removed": [], "undecided_added": [],
             "unmatched_removed": [], "unmatched_added": []}
    if "block" not in fields and "line" not in fields:
        return empty
    free = [f for f in ("block", "line") if f in fields]
    ident = tuple(f for f in fields if f not in free)
    rem, add = collections.defaultdict(list), collections.defaultdict(list)
    for k in d["removed"]:
        rem[tuple(d["b"][k].get(f) for f in ident)].append(k)
    for k in d["added"]:
        add[tuple(d["a"][k].get(f) for f in ident)].append(k)
    out = dict(empty)
    for sig in sorted(set(rem) | set(add), key=repr):
        r, a = sorted(rem.get(sig, [])), sorted(add.get(sig, []))
        if r and a and len(r) != len(a):
            out["ambiguous"].append((sig, len(r), len(a)))
            out["ambiguous_min"] += min(len(r), len(a))
            out["undecided_removed"].extend(r)
            out["undecided_added"].extend(a)
            for k in r:
                for k2 in a:
                    out["candidates"].append(
                        (k, k2, _delta(d["b"][k], d["a"][k2], "block"),
                         _delta(d["b"][k], d["a"][k2], "line")))
            continue
        n = min(len(r), len(a))
        out["pairs"].extend(zip(r[:n], a[:n]))
        out["unmatched_removed"].extend(r[n:])
        out["unmatched_added"].extend(a[n:])
    return out


def _delta(a, b, field):
    """b[field] - a[field], or None when either side does not carry it."""
    if a.get(field) is None or b.get(field) is None:
        return None
    return b[field] - a[field]


def banner(tag, census):
    return ("%-8s rev=%-12s rows=%-4d leg-failures=%-2d tools=%d"
            % (tag, census["rev"], len(census["rows"]), len(census["fails"]),
               len(census["per"])))


def totals(census):
    return dict(sorted(collections.Counter(
        r["verdict"] for r in census["rows"]).items()))


def report(before, after, name, fields, verbose=False):
    d = diff(before, after, fields)
    mv = moves(before, after, d, fields)
    nb, na = len(before["rows"]), len(after["rows"])
    print("\nkey %-11s (%s)" % (name, " + ".join(fields)))
    print("  keys before %d, after %d, collisions before/after %d/%d"
          % (len(d["b"]), len(d["a"]), d["collisions"][0], d["collisions"][1]))
    print("  rows before %d, after %d -- removed %d, added %d, changed %d"
          % (nb, na, len(d["removed"]), len(d["added"]), len(d["changed"])))
    print("  arithmetic under this key: %d - %d + %d = %d %s"
          % (nb, len(d["removed"]), len(d["added"]),
             nb - len(d["removed"]) + len(d["added"]),
             "= the after total" if nb - len(d["removed"]) + len(d["added"]) == na
             else "!= the after total, because the key collides"))
    if mv["pairs"] or mv["ambiguous"]:
        print("  position is not content: %d row(s) of the removed/added sets are "
              "ONE row at a different block/line" % len(mv["pairs"]))
        for k, k2 in mv["pairs"]:
            print("    MOVED %s:%s -> %s:%s   %s %s %s"
                  % (d["b"][k].get("block"), d["b"][k].get("line"),
                     d["a"][k2].get("block"), d["a"][k2].get("line"),
                     d["b"][k].get("unit"), d["b"][k].get("claim"),
                     d["b"][k].get("verdict")))
        for sig, nr, na2 in mv["ambiguous"]:
            print("    AMBIGUOUS %s: %d removed vs %d added of one shape -- "
                  "%d pairing(s) possible, not decided by these records"
                  % (sig, nr, na2, min(nr, na2)))
        print("  as CONTENT, with the DECIDED moves netted out: removed %d, added %d, "
              "moved %d" % (len(d["removed"]) - len(mv["pairs"]),
                            len(d["added"]) - len(mv["pairs"]), len(mv["pairs"])))
        amb = mv["ambiguous_min"]
        if amb:
            print("    %d pairing(s) the records cannot decide, so the content counts "
                  "are a RANGE:" % amb)
            print("      every undecided row a move: removed %d, added %d, moved %d"
                  % (len(d["removed"]) - len(mv["pairs"]) - amb,
                     len(d["added"]) - len(mv["pairs"]) - amb,
                     len(mv["pairs"]) + amb))
            print("      none of them a move:        removed %d, added %d, moved %d"
                  % (len(d["removed"]) - len(mv["pairs"]),
                     len(d["added"]) - len(mv["pairs"]), len(mv["pairs"])))
    if verbose:
        for k in d["removed"]:
            print("    REMOVED %s -> absent" % (k,))
        for k in d["added"]:
            print("    ADDED   %s -> %s" % (k, d["a"][k]["verdict"]))
        for k in d["changed"]:
            print("    CHANGED %s %s -> %s" % (k, d["b"][k]["verdict"],
                                               d["a"][k]["verdict"]))
    return d, mv


def _fixture():
    """Two synthetic censuses: one row deleted, one added, one moved, one flipped.

    A fixture, not a live read: the arithmetic under test is this tool's, and a
    live census changes under it. Every row carries every field the keys use.
    """
    def row(tool, block, line, rev, unit, claim, verdict, token=None):
        return {"tool": tool, "block": block, "line": line, "rev": rev,
                "unit": unit, "claim": claim, "token": token or claim,
                "verdict": verdict, "detail": "", "file": tool}
    before = [row("t/a.py", 4, 10, "HEAD", "cells", 3, "MATCHES"),
              row("t/a.py", 4, 11, "HEAD", "rows", 2, "NOT_PRINTED"),
              row("t/b.py", 20, 40, "abc1234", "cells", 7, "STALE")]
    after = [row("t/a.py", 4, 10, "HEAD", "cells", 3, "MATCHES"),
             row("t/a.py", 4, 11, "HEAD", "rows", 2, "STALE"),
             row("t/b.py", 21, 41, "abc1234", "cells", 7, "STALE"),
             row("t/b.py", 21, 42, "HEAD", "splits", 1, "MATCHES")]
    meta = {"rev": "HEAD", "repo": "/x"}
    return (dict(rows=before, fails=[("t/b.py", 20, "abc1234")],
                 per={"t/a.py": 2, "t/b.py": 1}, **meta),
            dict(rows=after, fails=[], per={"t/a.py": 2, "t/b.py": 2}, **meta))


def selftest():
    before, after = _fixture()
    cells, bad = [0], [0]

    def check(name, got, want):
        good = got == want
        cells[0] += 1
        bad[0] += 0 if good else 1
        print("%-52s %s" % (name, "PASS" if good else "FAIL"))
        if not good:
            print("      got  %r\n      want %r" % (got, want))

    d = diff(before, after, KEYS["canonical"])
    mv = moves(before, after, d, KEYS["canonical"])
    check("canonical: removed rows", len(d["removed"]), 1)
    check("canonical: added rows (the moved row is one of them)", len(d["added"]), 2)
    check("canonical: changed verdicts", len(d["changed"]), 1)
    check("canonical: the move is paired",
          [(d["b"][k]["block"], d["b"][k]["line"], d["a"][k2]["block"],
            d["a"][k2]["line"]) for k, k2 in mv["pairs"]],
          [(20, 40, 21, 41)])
    check("canonical: exactly one pair, the genuinely new row is not one",
          len(mv["pairs"]), 1)
    check("canonical: no removed row is left unmatched",
          mv["unmatched_removed"], [])
    check("canonical: one added row is left unmatched",
          len(mv["unmatched_added"]), 1)
    check("canonical: nothing ambiguous here", mv["ambiguous"], [])
    check("canonical: content = raw minus the move",
          (len(d["removed"]) - len(mv["pairs"]), len(d["added"]) - len(mv["pairs"])),
          (0, 1))
    check("canonical: partition arithmetic closes",
          len(before["rows"]) - len(d["removed"]) + len(d["added"]),
          len(after["rows"]))
    d2 = diff(before, after, KEYS["merged-rev"])
    check("merged-rev: dropping rev moves nothing here",
          (len(d2["removed"]), len(d2["added"]), len(d2["changed"])), (1, 2, 1))
    d3 = diff(before, after, KEYS["line-only"])
    m3 = moves(before, after, d3, KEYS["line-only"])
    check("line-only: removed", len(d3["removed"]), 1)
    check("line-only: added", len(d3["added"]), 2)
    # 1 removed vs 2 added at one line: the old code zipped them and called
    # (40 -> 41) a move. Nothing in the records says which of the two added rows
    # is the shifted one, so the pairing is UNDECIDED and both are printed.
    check("line-only: a 1-vs-2 signature is left undecided, not paired in order",
          len(m3["pairs"]), 0)
    check("line-only: every candidate is printed, neither chosen",
          sorted((d3["b"][k]["line"], d3["a"][k2]["line"])
                 for k, k2, _db, _dl in m3["candidates"]),
          [(40, 41), (40, 42)])
    check("line-only: the undecided pairing is counted", m3["ambiguous_min"], 1)
    check("line-only: no removed row is called gone", m3["unmatched_removed"], [])
    check("line-only: no added row is called new", m3["unmatched_added"], [])
    check("line-only: its rows are the undecided ones",
          (len(m3["undecided_removed"]), len(m3["undecided_added"])), (1, 2))
    dup = {"rows": before["rows"] + [dict(before["rows"][2], block=19, line=39)],
           "fails": [], "per": {}, "rev": "HEAD", "repo": "/x"}
    d4 = diff(dup, after, KEYS["canonical"])
    m4 = moves(dup, after, d4, KEYS["canonical"])
    check("many-to-one: one ambiguous signature reported",
          [(nr, na) for _sig, nr, na in m4["ambiguous"]], [(2, 1)])
    check("many-to-one: nothing is paired on the tool's guess", len(m4["pairs"]), 0)
    check("many-to-one: the undecided pairing is counted", m4["ambiguous_min"], 1)
    check("many-to-one: both removed rows are undecided, neither called gone",
          len(m4["undecided_removed"]), 2)
    check("many-to-one: the added row of that shape is undecided too",
          len(m4["undecided_added"]), 1)
    check("many-to-one: nothing is left unmatched on the removed side",
          len(m4["unmatched_removed"]), 0)
    check("many-to-one: the genuinely new row is the unmatched one",
          len(m4["unmatched_added"]), 1)
    check("many-to-one: the candidates carry their block and line deltas",
          sorted((db, dl) for _k, _k2, db, dl in m4["candidates"]),
          [(1, 1), (2, 2)])
    check("many-to-one: content bounds -- moved",
          (len(m4["pairs"]), len(m4["pairs"]) + m4["ambiguous_min"]), (0, 1))
    check("many-to-one: content bounds -- removed",
          (len(d4["removed"]) - len(m4["pairs"]) - m4["ambiguous_min"],
           len(d4["removed"]) - len(m4["pairs"])), (1, 2))
    check("keyed: a key that is not an identity collides",
          keyed(before, ("tool",))[1], 1)
    check("keyed: collided key count", len(keyed(before, ("tool",))[0]), 2)
    check("totals: before", totals(before),
          {"MATCHES": 1, "NOT_PRINTED": 1, "STALE": 1})
    check("totals: after", totals(after), {"MATCHES": 2, "STALE": 2})

    # -- the no-pair refusal is a code, not a printed footnote -----------------
    # Each cell drives main() itself, so the exit code is the one a caller
    # reads, not a re-statement of it. `--repo`/`--rev` alone is the shape a
    # census leg uses; it must refuse too, because it names no pair either.
    def refusal(argv):
        out, err = io.StringIO(), io.StringIO()
        with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
            rc = main(argv)
        return rc, out.getvalue(), err.getvalue()

    rc, out, _err = refusal([])
    check("no pair: rc 2, not a clean exit", rc, 2)
    check("no pair: the key vocabulary is still printed",
          "key shapes available" in out, True)
    rc, _out, err = refusal(["only-before.json"])
    check("half a pair: rc 2 and the half is named in stderr",
          (rc, "only-before.json" in err), (2, True))
    rc, _out, _err = refusal(["--repo", "/x", "--rev", "HEAD"])
    check("pass-through shape (--repo/--rev only): rc 2, it graded nothing",
          rc, 2)

    print("selftest: %d cells, %d passed, %d failed"
          % (cells[0], cells[0] - bad[0], bad[0]))
    return 1 if bad[0] else 0


def main(argv=None):
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("before", nargs="?", help="earlier census JSON")
    ap.add_argument("after", nargs="?", help="later census JSON")
    ap.add_argument("--key", default="canonical", choices=sorted(KEYS))
    ap.add_argument("--keys", action="store_true",
                    help="print the partition under every key shape")
    ap.add_argument("--verbose", action="store_true")
    ap.add_argument("--selftest", action="store_true")
    ap.add_argument("--json", help="write the partitions it printed")
    # accepted and ignored: see the pass-through note in the module docstring
    ap.add_argument("--repo", help=argparse.SUPPRESS)
    ap.add_argument("--rev", help=argparse.SUPPRESS)
    a = ap.parse_args(argv)

    if a.selftest:
        return selftest()
    if not (a.before and a.after):
        given = [p for p in (a.before, a.after) if p]
        if given:
            print("cannot grade: a census pair is required; got only %s"
                  % " ".join(given), file=sys.stderr)
        else:
            print("cannot grade: no census pair given", file=sys.stderr)
        print("census_row_delta -- no census pair given; key shapes available:")
        for name in sorted(KEYS):
            print("  %-11s %s" % (name, " + ".join(KEYS[name])))
        return 2

    try:
        before, after = load(a.before), load(a.after)
    except ReadError as exc:
        print(str(exc), file=sys.stderr)
        return 2
    print(banner("before", before))
    print(banner("after", after))
    print("verdict totals before: %s" % totals(before))
    print("verdict totals after:  %s" % totals(after))
    print("\nrows per tool: before -> after")
    for t in sorted(set(before["per"]) | set(after["per"])):
        print("  %-40s %s -> %s" % (t, before["per"].get(t), after["per"].get(t)))
    bf, af = set(before["fails"]), set(after["fails"])
    if bf or af:
        print("\nleg failures: before %d, after %d (%d gone, %d new)"
              % (len(bf), len(af), len(bf - af), len(af - bf)))

    out = {}
    for name in (sorted(KEYS) if a.keys else [a.key]):
        d, mv = report(before, after, name, KEYS[name], a.verbose)
        amb = mv["ambiguous_min"]
        out[name] = {"fields": list(KEYS[name]), "removed": len(d["removed"]),
                     "added": len(d["added"]), "changed": len(d["changed"]),
                     "moved": len(mv["pairs"]), "undecided_pairings": amb,
                     "moved_range": [len(mv["pairs"]), len(mv["pairs"]) + amb],
                     "removed_content_range": [
                         len(d["removed"]) - len(mv["pairs"]) - amb,
                         len(d["removed"]) - len(mv["pairs"])],
                     "added_content_range": [
                         len(d["added"]) - len(mv["pairs"]) - amb,
                         len(d["added"]) - len(mv["pairs"])],
                     "unmatched_removed": len(mv["unmatched_removed"]),
                     "unmatched_added": len(mv["unmatched_added"])}
        if a.verbose or a.keys:
            for k in d["removed"]:
                print("    -%s %s" % (k, d["b"][k]["verdict"]))
            for k in d["added"]:
                print("    +%s %s" % (k, d["a"][k]["verdict"]))
    if a.json:
        try:
            with open(a.json, "w", encoding="utf-8") as fh:
                json.dump({"before": a.before, "after": a.after,
                           "before_rev": before["rev"], "after_rev": after["rev"],
                           "rows_before": len(before["rows"]),
                           "rows_after": len(after["rows"]),
                           "partitions": out}, fh, indent=1)
        except OSError as exc:
            print("cannot write --json " + a.json + ": " + str(exc),
                  file=sys.stderr)
            return 2
        print("\nwrote %s" % a.json)
    return 0


if __name__ == "__main__":
    sys.exit(main())