511 lines
25 KiB
Python
511 lines
25 KiB
Python
#!/usr/bin/env python3
|
|
"""census_row_delta.py -- row-level diff of two doc_census_check.py censuses.
|
|
|
|
WHY THIS EXISTS
|
|
`tools/doc_census_check.py` prints a census; the number a reader quotes is its
|
|
row total. When that total moved (the digest-rev fix of 2026-10-16), the
|
|
sentence that mattered was not the total but the PARTITION: how many rows were
|
|
REMOVED, how many ADDED, how many CHANGED VERDICT. Two briefs over the same
|
|
pair of runs reached the same new total with different partitions -- nineteen
|
|
removed and three added in one, twenty removed and four added in the other --
|
|
and both are arithmetically consistent, because the partition is not a property
|
|
of the two runs alone. It is a property of the two runs AND of the key a row is
|
|
identified by. A census row is a record (tool, block, line, rev, unit, claim,
|
|
token, verdict); pick a different subset of those as the identity and you
|
|
rename a row that only moved position into a row that was deleted and another
|
|
that was created.
|
|
|
|
So this tool prints the partition under a NAMED key, never a bare "removed N",
|
|
and `--keys` prints it under every key shape it knows -- so a disagreement
|
|
about the count can be resolved to the key that produced it instead of argued
|
|
about. Every partition is then restated with position taken out of the key: a
|
|
removed row and an added row whose records agree on every field the key holds
|
|
fixed are ONE row that MOVED, reported separately from a row that was deleted
|
|
or introduced. Reading a position shift as a content change is the whole defect
|
|
class this tool exists for.
|
|
|
|
WHAT IT READS
|
|
Two JSON files written by `doc_census_check.py --json`. It does not run the
|
|
census and does not judge a census: it cannot see anything the JSON does not
|
|
carry, and it makes no claim about which census is right.
|
|
|
|
READ-ONLY. The only write path is `--json PATH`; every other mode prints.
|
|
|
|
A STATED LIMIT, NOT AN IMPLIED ONE
|
|
Rows are compared as records, so a claim whose TEXT moved between blocks (a
|
|
docstring line re-wrapped, a claim reworded) is REMOVED and ADDED, never
|
|
CHANGED. `--moves` catches the pure-position case and nothing else; a reworded
|
|
claim is reported as a delete plus a create and is not paired. The key shapes
|
|
below are the whole vocabulary this tool has for that; it invents no similarity
|
|
metric.
|
|
|
|
PASS-THROUGH, because a census leg drives every tool as `<script> --repo R --rev
|
|
REV`: `--repo` and `--rev` are accepted and IGNORED -- this tool's subject is a
|
|
pair of files, not a revision. It still prints its key vocabulary for that
|
|
shape, because a reader who mistyped a path needs the shapes, but it no longer
|
|
reports the shape as a verdict. A run that was given no pair, or only one half
|
|
of one, graded nothing, and this tree's exit contract reserves 0 for a run
|
|
that graded and found nothing to report (`tools/verify.py`: 0 clean, 1 graded
|
|
fault, 2 could not grade). Exiting 0 there was the defect this paragraph used
|
|
to document: reply 18 (`replies/2026-10-17-to-volt-18.md:3`) records a probe
|
|
that passed ONE path and "read a spurious clean exit" until the parser was
|
|
read by hand. The refusal is now a code, not a footnote -- no pair rc 2, half
|
|
a pair rc 2 naming the half, and `--repo`/`--rev` alone rc 2 as well.
|
|
|
|
USAGE
|
|
python3 census_row_delta.py A.json B.json [--moves] [--json OUT]
|
|
python3 census_row_delta.py --selftest # 37 cells, keys and partitions
|
|
"""
|
|
import argparse
|
|
import collections
|
|
import contextlib
|
|
import io
|
|
import json
|
|
import sys
|
|
|
|
|
|
class ReadError(Exception):
|
|
"""Input this tool could not read or could not recognise: rc 2, never rc 1.
|
|
|
|
rc 1 is the tool's verdict code (a partition that does not close). A file
|
|
that cannot be opened, decoded or parsed has produced no verdict at all, so
|
|
it must not borrow the code that means "the grading found a fault".
|
|
"""
|
|
|
|
|
|
# A row is identified by a subset of these fields -- the names are the JSON keys.
|
|
FIELDS = ("tool", "block", "line", "rev", "unit", "claim", "token")
|
|
|
|
# Every key shape this tool knows, in the order `--keys` prints them. The
|
|
# canonical one is the row as doc_census_check.py constructs it; the others are
|
|
# the plausible under-keyings a reader might reach for, named so that a
|
|
# disagreement about a count can be attributed rather than argued.
|
|
KEYS = {
|
|
"canonical": ("tool", "block", "line", "rev", "unit", "claim"),
|
|
"strict": ("tool", "block", "line", "rev", "unit", "claim", "token"),
|
|
"token": ("tool", "block", "line", "rev", "token", "claim"),
|
|
"merged-rev": ("tool", "block", "line", "unit", "claim"),
|
|
"no-block": ("tool", "line", "rev", "unit", "claim"),
|
|
"loose": ("tool", "line", "unit", "claim"),
|
|
"line-only": ("tool", "line"),
|
|
}
|
|
|
|
|
|
def load(path):
|
|
"""A census JSON -> {rev, repo, rows, fails, per}.
|
|
|
|
`rows` is every surviving verdict dict, each carrying its tool name; keying
|
|
happens in `keyed()` so one load serves every key shape.
|
|
|
|
An unreadable or non-census file is a PRECONDITION failure, not a traceback
|
|
and not a verdict: this used to raise the json decoder's error or a bare
|
|
`KeyError: 'reports'`, with rc 1 -- the code that means "the grading found a
|
|
fault". `ReadError` is turned into rc 2 in main(); the message names the
|
|
file and what was expected instead.
|
|
"""
|
|
try:
|
|
with open(path, encoding="utf-8") as fh:
|
|
d = json.load(fh)
|
|
except (OSError, ValueError) as exc:
|
|
raise ReadError("cannot read census %s: %s" % (path, exc))
|
|
if not isinstance(d, dict) or not isinstance(d.get("reports"), list):
|
|
raise ReadError("not a doc_census_check census (no 'reports' list): %s" % path)
|
|
rows, fails, per = [], [], {}
|
|
for rep in d["reports"]:
|
|
for v in rep["verdicts"]:
|
|
row = dict(v)
|
|
row["tool"] = rep["tool"]
|
|
rows.append(row)
|
|
for f in rep["failures"]:
|
|
fails.append((rep["tool"], f.get("block"), f.get("rev")))
|
|
per[rep["tool"]] = len(rep["verdicts"])
|
|
return {"rev": d.get("rev"), "repo": d.get("repo"), "rows": rows,
|
|
"fails": fails, "per": per}
|
|
|
|
|
|
def keyed(census, fields):
|
|
"""({key: row}, collisions) -- later duplicates win, and the count is kept.
|
|
|
|
A collision says the key is not an identity for this census: two distinct
|
|
rows share it, so the partition computed under it is a partition of ROWS
|
|
COLLAPSED, not of rows. Reported, never silent.
|
|
"""
|
|
out, collisions = {}, 0
|
|
for row in census["rows"]:
|
|
k = tuple(row.get(f) for f in fields)
|
|
if k in out:
|
|
collisions += 1
|
|
out[k] = row
|
|
return out, collisions
|
|
|
|
|
|
def diff(before, after, fields):
|
|
b, bc = keyed(before, fields)
|
|
a, ac = keyed(after, fields)
|
|
removed = sorted(set(b) - set(a))
|
|
added = sorted(set(a) - set(b))
|
|
changed = sorted(k for k in set(a) & set(b)
|
|
if b[k]["verdict"] != a[k]["verdict"])
|
|
return {"removed": removed, "added": added, "changed": changed,
|
|
"b": b, "a": a, "collisions": (bc, ac)}
|
|
|
|
|
|
def moves(before, after, d, fields):
|
|
"""What the removed/added rows are once position is taken out of the key.
|
|
|
|
Returns {pairs, candidates, ambiguous, ambiguous_min, undecided_removed,
|
|
undecided_added, unmatched_removed, unmatched_added}.
|
|
|
|
Pairs are ONE-TO-ONE and DECIDED: a signature (the row with `block` and
|
|
`line` removed from the key) that has the same number of rows on both sides.
|
|
Each pair is ONE row that MOVED, not a delete plus a create.
|
|
|
|
Where a signature has a different number of rows on each side, the tool does
|
|
NOT choose: it prints every cross candidate with its block and line delta,
|
|
records `ambiguous_min` -- how many pairings that signature could hold,
|
|
min(removed, added) -- and leaves the question open, because two removed rows
|
|
and one added row of the same shape can be a move plus a deletion or two
|
|
deletions plus a create, and the records are identical in every field this
|
|
tool can see. Resolving that needs the text the rows came from -- outside
|
|
this tool, and stated as such rather than papered over with an arbitrary
|
|
pairing. Rows inside such a signature are `undecided_*`, kept apart from the
|
|
`unmatched_*` rows that are genuinely gone or genuinely new.
|
|
|
|
AN ORDERED GUESS IS NOT A RESOLUTION. Until 2026-10-17 this branch zipped the
|
|
two sides in sorted order and reported the first rows as one MOVED row. On
|
|
the live 0031a78 -> HEAD pair that produced `MOVED 56:63 -> 89:100` for a
|
|
signature of two removed rows and one added row -- the right COUNT of moved
|
|
rows (one) and the WRONG ROW: the text says block 82 line 93 is the row that
|
|
moved to block 89 line 100, and the block 56 line 63 claim was deleted. A
|
|
count that survives a wrong identity is the defect this tool exists to catch,
|
|
so the pairing is no longer guessed at all.
|
|
|
|
Only meaningful for a key that carries `block` or `line`; with neither there
|
|
is no position to have moved from, and the empty result is the answer rather
|
|
than a guess.
|
|
"""
|
|
empty = {"pairs": [], "candidates": [], "ambiguous": [], "ambiguous_min": 0,
|
|
"undecided_removed": [], "undecided_added": [],
|
|
"unmatched_removed": [], "unmatched_added": []}
|
|
if "block" not in fields and "line" not in fields:
|
|
return empty
|
|
free = [f for f in ("block", "line") if f in fields]
|
|
ident = tuple(f for f in fields if f not in free)
|
|
rem, add = collections.defaultdict(list), collections.defaultdict(list)
|
|
for k in d["removed"]:
|
|
rem[tuple(d["b"][k].get(f) for f in ident)].append(k)
|
|
for k in d["added"]:
|
|
add[tuple(d["a"][k].get(f) for f in ident)].append(k)
|
|
out = dict(empty)
|
|
for sig in sorted(set(rem) | set(add), key=repr):
|
|
r, a = sorted(rem.get(sig, [])), sorted(add.get(sig, []))
|
|
if r and a and len(r) != len(a):
|
|
out["ambiguous"].append((sig, len(r), len(a)))
|
|
out["ambiguous_min"] += min(len(r), len(a))
|
|
out["undecided_removed"].extend(r)
|
|
out["undecided_added"].extend(a)
|
|
for k in r:
|
|
for k2 in a:
|
|
out["candidates"].append(
|
|
(k, k2, _delta(d["b"][k], d["a"][k2], "block"),
|
|
_delta(d["b"][k], d["a"][k2], "line")))
|
|
continue
|
|
n = min(len(r), len(a))
|
|
out["pairs"].extend(zip(r[:n], a[:n]))
|
|
out["unmatched_removed"].extend(r[n:])
|
|
out["unmatched_added"].extend(a[n:])
|
|
return out
|
|
|
|
|
|
def _delta(a, b, field):
|
|
"""b[field] - a[field], or None when either side does not carry it."""
|
|
if a.get(field) is None or b.get(field) is None:
|
|
return None
|
|
return b[field] - a[field]
|
|
|
|
|
|
def banner(tag, census):
|
|
return ("%-8s rev=%-12s rows=%-4d leg-failures=%-2d tools=%d"
|
|
% (tag, census["rev"], len(census["rows"]), len(census["fails"]),
|
|
len(census["per"])))
|
|
|
|
|
|
def totals(census):
|
|
return dict(sorted(collections.Counter(
|
|
r["verdict"] for r in census["rows"]).items()))
|
|
|
|
|
|
def report(before, after, name, fields, verbose=False):
|
|
d = diff(before, after, fields)
|
|
mv = moves(before, after, d, fields)
|
|
nb, na = len(before["rows"]), len(after["rows"])
|
|
print("\nkey %-11s (%s)" % (name, " + ".join(fields)))
|
|
print(" keys before %d, after %d, collisions before/after %d/%d"
|
|
% (len(d["b"]), len(d["a"]), d["collisions"][0], d["collisions"][1]))
|
|
print(" rows before %d, after %d -- removed %d, added %d, changed %d"
|
|
% (nb, na, len(d["removed"]), len(d["added"]), len(d["changed"])))
|
|
print(" arithmetic under this key: %d - %d + %d = %d %s"
|
|
% (nb, len(d["removed"]), len(d["added"]),
|
|
nb - len(d["removed"]) + len(d["added"]),
|
|
"= the after total" if nb - len(d["removed"]) + len(d["added"]) == na
|
|
else "!= the after total, because the key collides"))
|
|
if mv["pairs"] or mv["ambiguous"]:
|
|
print(" position is not content: %d row(s) of the removed/added sets are "
|
|
"ONE row at a different block/line" % len(mv["pairs"]))
|
|
for k, k2 in mv["pairs"]:
|
|
print(" MOVED %s:%s -> %s:%s %s %s %s"
|
|
% (d["b"][k].get("block"), d["b"][k].get("line"),
|
|
d["a"][k2].get("block"), d["a"][k2].get("line"),
|
|
d["b"][k].get("unit"), d["b"][k].get("claim"),
|
|
d["b"][k].get("verdict")))
|
|
for sig, nr, na2 in mv["ambiguous"]:
|
|
print(" AMBIGUOUS %s: %d removed vs %d added of one shape -- "
|
|
"%d pairing(s) possible, not decided by these records"
|
|
% (sig, nr, na2, min(nr, na2)))
|
|
print(" as CONTENT, with the DECIDED moves netted out: removed %d, added %d, "
|
|
"moved %d" % (len(d["removed"]) - len(mv["pairs"]),
|
|
len(d["added"]) - len(mv["pairs"]), len(mv["pairs"])))
|
|
amb = mv["ambiguous_min"]
|
|
if amb:
|
|
print(" %d pairing(s) the records cannot decide, so the content counts "
|
|
"are a RANGE:" % amb)
|
|
print(" every undecided row a move: removed %d, added %d, moved %d"
|
|
% (len(d["removed"]) - len(mv["pairs"]) - amb,
|
|
len(d["added"]) - len(mv["pairs"]) - amb,
|
|
len(mv["pairs"]) + amb))
|
|
print(" none of them a move: removed %d, added %d, moved %d"
|
|
% (len(d["removed"]) - len(mv["pairs"]),
|
|
len(d["added"]) - len(mv["pairs"]), len(mv["pairs"])))
|
|
if verbose:
|
|
for k in d["removed"]:
|
|
print(" REMOVED %s -> absent" % (k,))
|
|
for k in d["added"]:
|
|
print(" ADDED %s -> %s" % (k, d["a"][k]["verdict"]))
|
|
for k in d["changed"]:
|
|
print(" CHANGED %s %s -> %s" % (k, d["b"][k]["verdict"],
|
|
d["a"][k]["verdict"]))
|
|
return d, mv
|
|
|
|
|
|
def _fixture():
|
|
"""Two synthetic censuses: one row deleted, one added, one moved, one flipped.
|
|
|
|
A fixture, not a live read: the arithmetic under test is this tool's, and a
|
|
live census changes under it. Every row carries every field the keys use.
|
|
"""
|
|
def row(tool, block, line, rev, unit, claim, verdict, token=None):
|
|
return {"tool": tool, "block": block, "line": line, "rev": rev,
|
|
"unit": unit, "claim": claim, "token": token or claim,
|
|
"verdict": verdict, "detail": "", "file": tool}
|
|
before = [row("t/a.py", 4, 10, "HEAD", "cells", 3, "MATCHES"),
|
|
row("t/a.py", 4, 11, "HEAD", "rows", 2, "NOT_PRINTED"),
|
|
row("t/b.py", 20, 40, "abc1234", "cells", 7, "STALE")]
|
|
after = [row("t/a.py", 4, 10, "HEAD", "cells", 3, "MATCHES"),
|
|
row("t/a.py", 4, 11, "HEAD", "rows", 2, "STALE"),
|
|
row("t/b.py", 21, 41, "abc1234", "cells", 7, "STALE"),
|
|
row("t/b.py", 21, 42, "HEAD", "splits", 1, "MATCHES")]
|
|
meta = {"rev": "HEAD", "repo": "/x"}
|
|
return (dict(rows=before, fails=[("t/b.py", 20, "abc1234")],
|
|
per={"t/a.py": 2, "t/b.py": 1}, **meta),
|
|
dict(rows=after, fails=[], per={"t/a.py": 2, "t/b.py": 2}, **meta))
|
|
|
|
|
|
def selftest():
|
|
before, after = _fixture()
|
|
cells, bad = [0], [0]
|
|
|
|
def check(name, got, want):
|
|
good = got == want
|
|
cells[0] += 1
|
|
bad[0] += 0 if good else 1
|
|
print("%-52s %s" % (name, "PASS" if good else "FAIL"))
|
|
if not good:
|
|
print(" got %r\n want %r" % (got, want))
|
|
|
|
d = diff(before, after, KEYS["canonical"])
|
|
mv = moves(before, after, d, KEYS["canonical"])
|
|
check("canonical: removed rows", len(d["removed"]), 1)
|
|
check("canonical: added rows (the moved row is one of them)", len(d["added"]), 2)
|
|
check("canonical: changed verdicts", len(d["changed"]), 1)
|
|
check("canonical: the move is paired",
|
|
[(d["b"][k]["block"], d["b"][k]["line"], d["a"][k2]["block"],
|
|
d["a"][k2]["line"]) for k, k2 in mv["pairs"]],
|
|
[(20, 40, 21, 41)])
|
|
check("canonical: exactly one pair, the genuinely new row is not one",
|
|
len(mv["pairs"]), 1)
|
|
check("canonical: no removed row is left unmatched",
|
|
mv["unmatched_removed"], [])
|
|
check("canonical: one added row is left unmatched",
|
|
len(mv["unmatched_added"]), 1)
|
|
check("canonical: nothing ambiguous here", mv["ambiguous"], [])
|
|
check("canonical: content = raw minus the move",
|
|
(len(d["removed"]) - len(mv["pairs"]), len(d["added"]) - len(mv["pairs"])),
|
|
(0, 1))
|
|
check("canonical: partition arithmetic closes",
|
|
len(before["rows"]) - len(d["removed"]) + len(d["added"]),
|
|
len(after["rows"]))
|
|
d2 = diff(before, after, KEYS["merged-rev"])
|
|
check("merged-rev: dropping rev moves nothing here",
|
|
(len(d2["removed"]), len(d2["added"]), len(d2["changed"])), (1, 2, 1))
|
|
d3 = diff(before, after, KEYS["line-only"])
|
|
m3 = moves(before, after, d3, KEYS["line-only"])
|
|
check("line-only: removed", len(d3["removed"]), 1)
|
|
check("line-only: added", len(d3["added"]), 2)
|
|
# 1 removed vs 2 added at one line: the old code zipped them and called
|
|
# (40 -> 41) a move. Nothing in the records says which of the two added rows
|
|
# is the shifted one, so the pairing is UNDECIDED and both are printed.
|
|
check("line-only: a 1-vs-2 signature is left undecided, not paired in order",
|
|
len(m3["pairs"]), 0)
|
|
check("line-only: every candidate is printed, neither chosen",
|
|
sorted((d3["b"][k]["line"], d3["a"][k2]["line"])
|
|
for k, k2, _db, _dl in m3["candidates"]),
|
|
[(40, 41), (40, 42)])
|
|
check("line-only: the undecided pairing is counted", m3["ambiguous_min"], 1)
|
|
check("line-only: no removed row is called gone", m3["unmatched_removed"], [])
|
|
check("line-only: no added row is called new", m3["unmatched_added"], [])
|
|
check("line-only: its rows are the undecided ones",
|
|
(len(m3["undecided_removed"]), len(m3["undecided_added"])), (1, 2))
|
|
dup = {"rows": before["rows"] + [dict(before["rows"][2], block=19, line=39)],
|
|
"fails": [], "per": {}, "rev": "HEAD", "repo": "/x"}
|
|
d4 = diff(dup, after, KEYS["canonical"])
|
|
m4 = moves(dup, after, d4, KEYS["canonical"])
|
|
check("many-to-one: one ambiguous signature reported",
|
|
[(nr, na) for _sig, nr, na in m4["ambiguous"]], [(2, 1)])
|
|
check("many-to-one: nothing is paired on the tool's guess", len(m4["pairs"]), 0)
|
|
check("many-to-one: the undecided pairing is counted", m4["ambiguous_min"], 1)
|
|
check("many-to-one: both removed rows are undecided, neither called gone",
|
|
len(m4["undecided_removed"]), 2)
|
|
check("many-to-one: the added row of that shape is undecided too",
|
|
len(m4["undecided_added"]), 1)
|
|
check("many-to-one: nothing is left unmatched on the removed side",
|
|
len(m4["unmatched_removed"]), 0)
|
|
check("many-to-one: the genuinely new row is the unmatched one",
|
|
len(m4["unmatched_added"]), 1)
|
|
check("many-to-one: the candidates carry their block and line deltas",
|
|
sorted((db, dl) for _k, _k2, db, dl in m4["candidates"]),
|
|
[(1, 1), (2, 2)])
|
|
check("many-to-one: content bounds -- moved",
|
|
(len(m4["pairs"]), len(m4["pairs"]) + m4["ambiguous_min"]), (0, 1))
|
|
check("many-to-one: content bounds -- removed",
|
|
(len(d4["removed"]) - len(m4["pairs"]) - m4["ambiguous_min"],
|
|
len(d4["removed"]) - len(m4["pairs"])), (1, 2))
|
|
check("keyed: a key that is not an identity collides",
|
|
keyed(before, ("tool",))[1], 1)
|
|
check("keyed: collided key count", len(keyed(before, ("tool",))[0]), 2)
|
|
check("totals: before", totals(before),
|
|
{"MATCHES": 1, "NOT_PRINTED": 1, "STALE": 1})
|
|
check("totals: after", totals(after), {"MATCHES": 2, "STALE": 2})
|
|
|
|
# -- the no-pair refusal is a code, not a printed footnote -----------------
|
|
# Each cell drives main() itself, so the exit code is the one a caller
|
|
# reads, not a re-statement of it. `--repo`/`--rev` alone is the shape a
|
|
# census leg uses; it must refuse too, because it names no pair either.
|
|
def refusal(argv):
|
|
out, err = io.StringIO(), io.StringIO()
|
|
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
|
rc = main(argv)
|
|
return rc, out.getvalue(), err.getvalue()
|
|
|
|
rc, out, _err = refusal([])
|
|
check("no pair: rc 2, not a clean exit", rc, 2)
|
|
check("no pair: the key vocabulary is still printed",
|
|
"key shapes available" in out, True)
|
|
rc, _out, err = refusal(["only-before.json"])
|
|
check("half a pair: rc 2 and the half is named in stderr",
|
|
(rc, "only-before.json" in err), (2, True))
|
|
rc, _out, _err = refusal(["--repo", "/x", "--rev", "HEAD"])
|
|
check("pass-through shape (--repo/--rev only): rc 2, it graded nothing",
|
|
rc, 2)
|
|
|
|
print("selftest: %d cells, %d passed, %d failed"
|
|
% (cells[0], cells[0] - bad[0], bad[0]))
|
|
return 1 if bad[0] else 0
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
ap.add_argument("before", nargs="?", help="earlier census JSON")
|
|
ap.add_argument("after", nargs="?", help="later census JSON")
|
|
ap.add_argument("--key", default="canonical", choices=sorted(KEYS))
|
|
ap.add_argument("--keys", action="store_true",
|
|
help="print the partition under every key shape")
|
|
ap.add_argument("--verbose", action="store_true")
|
|
ap.add_argument("--selftest", action="store_true")
|
|
ap.add_argument("--json", help="write the partitions it printed")
|
|
# accepted and ignored: see the pass-through note in the module docstring
|
|
ap.add_argument("--repo", help=argparse.SUPPRESS)
|
|
ap.add_argument("--rev", help=argparse.SUPPRESS)
|
|
a = ap.parse_args(argv)
|
|
|
|
if a.selftest:
|
|
return selftest()
|
|
if not (a.before and a.after):
|
|
given = [p for p in (a.before, a.after) if p]
|
|
if given:
|
|
print("cannot grade: a census pair is required; got only %s"
|
|
% " ".join(given), file=sys.stderr)
|
|
else:
|
|
print("cannot grade: no census pair given", file=sys.stderr)
|
|
print("census_row_delta -- no census pair given; key shapes available:")
|
|
for name in sorted(KEYS):
|
|
print(" %-11s %s" % (name, " + ".join(KEYS[name])))
|
|
return 2
|
|
|
|
try:
|
|
before, after = load(a.before), load(a.after)
|
|
except ReadError as exc:
|
|
print(str(exc), file=sys.stderr)
|
|
return 2
|
|
print(banner("before", before))
|
|
print(banner("after", after))
|
|
print("verdict totals before: %s" % totals(before))
|
|
print("verdict totals after: %s" % totals(after))
|
|
print("\nrows per tool: before -> after")
|
|
for t in sorted(set(before["per"]) | set(after["per"])):
|
|
print(" %-40s %s -> %s" % (t, before["per"].get(t), after["per"].get(t)))
|
|
bf, af = set(before["fails"]), set(after["fails"])
|
|
if bf or af:
|
|
print("\nleg failures: before %d, after %d (%d gone, %d new)"
|
|
% (len(bf), len(af), len(bf - af), len(af - bf)))
|
|
|
|
out = {}
|
|
for name in (sorted(KEYS) if a.keys else [a.key]):
|
|
d, mv = report(before, after, name, KEYS[name], a.verbose)
|
|
amb = mv["ambiguous_min"]
|
|
out[name] = {"fields": list(KEYS[name]), "removed": len(d["removed"]),
|
|
"added": len(d["added"]), "changed": len(d["changed"]),
|
|
"moved": len(mv["pairs"]), "undecided_pairings": amb,
|
|
"moved_range": [len(mv["pairs"]), len(mv["pairs"]) + amb],
|
|
"removed_content_range": [
|
|
len(d["removed"]) - len(mv["pairs"]) - amb,
|
|
len(d["removed"]) - len(mv["pairs"])],
|
|
"added_content_range": [
|
|
len(d["added"]) - len(mv["pairs"]) - amb,
|
|
len(d["added"]) - len(mv["pairs"])],
|
|
"unmatched_removed": len(mv["unmatched_removed"]),
|
|
"unmatched_added": len(mv["unmatched_added"])}
|
|
if a.verbose or a.keys:
|
|
for k in d["removed"]:
|
|
print(" -%s %s" % (k, d["b"][k]["verdict"]))
|
|
for k in d["added"]:
|
|
print(" +%s %s" % (k, d["a"][k]["verdict"]))
|
|
if a.json:
|
|
try:
|
|
with open(a.json, "w", encoding="utf-8") as fh:
|
|
json.dump({"before": a.before, "after": a.after,
|
|
"before_rev": before["rev"], "after_rev": after["rev"],
|
|
"rows_before": len(before["rows"]),
|
|
"rows_after": len(after["rows"]),
|
|
"partitions": out}, fh, indent=1)
|
|
except OSError as exc:
|
|
print("cannot write --json " + a.json + ": " + str(exc),
|
|
file=sys.stderr)
|
|
return 2
|
|
print("\nwrote %s" % a.json)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|