Files
troupe-standup/wizard-work/volt/q143-check-sweep/retake.py
T

509 lines
21 KiB
Python

#!/usr/bin/env python3
"""retake.py -- re-take named rows of the CHECK entry-point baseline.
WHY THIS EXISTS
`check_sweep.py` grades the tree against `check_entry_baseline.tsv` and fails
in both directions: a code that moved `0 -> 1` is a revert, and a code that
moved `1 -> 0` is "IMPROVED ... re-take the baseline". The second direction is
a chore done by hand in Q145/Q146/Q147 -- copy the path, run the entry point,
paste the code and the first fault line into the right row, and write a
provenance comment so a reader knows which tick moved it. Done by hand it is
slow, it is typed rather than measured, and a mistyped code is invisible: the
baseline's whole contract is that every number in it was measured.
This tool is that chore, with the contract imported rather than re-implemented:
measurement is `check_sweep.run_entry` and `check_sweep.classify`, the baseline
reader is `check_sweep.read_baseline`, so a row written here is a reading the
sweep would take. It measures, reports, and (only with `--write`) rewrites the
rows whose code moved, leaving every other byte of the file alone.
WHAT IT MEASURES
By default, the rows whose recorded code is not 0 -- the audit that matters,
because a green row can only degrade and a red row is the one that might have
been repaired since. `--all` measures every row; `--path` (repeatable,
substring match) narrows to named rows.
EXIT CODES -- the tree's convention
0 no row moved (dry run), or every moved row was written (`--write`)
1 a row moved and nothing was written (dry run), or a write was refused
because the baseline names a path the tree does not have
2 could not grade: no baseline, unreadable or malformed baseline, no repo
`--selftest` runs 12 cells in a fixture tree: a green control, a moved code,
a write, a write refused for a missing path, a malformed baseline, the note
cut, the fault-hint rule, the imported contract, a baseline that does not name
every entry point (it must not invent rows), a one-shot sign of a path the
baseline does not name, a host-path grep of this file, and a note that carries
a tab (it must stay one cell and read back whole). A green reading is never
the only thing this tool can print.
NOT CLAIMED
That a green row measures the right thing -- that is each entry point's own
selftest. Nor that a re-take is stable: a row that is a function of host load
or of a live API can move between two takes, which is why the provenance
comment records the timeout, the job count and the rev.
"""
import argparse
import concurrent.futures
import datetime
import os
import re
import subprocess
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import check_sweep # noqa: E402
run_entry = check_sweep.run_entry
classify = check_sweep.classify
DEFAULT_BASELINE = os.path.join(HERE, "check_entry_baseline.tsv")
NOTE_CUT = 160
def repo_of(path):
"""The git toplevel that owns `path`, or the tree above this tool."""
d = os.path.dirname(os.path.abspath(path))
try:
r = subprocess.run(["git", "-C", d, "rev-parse", "--show-toplevel"],
capture_output=True, text=True, timeout=60)
if r.returncode == 0 and r.stdout.strip():
return r.stdout.strip()
except (OSError, subprocess.SubprocessError):
pass
return os.path.dirname(os.path.dirname(HERE))
def head_rev(repo):
try:
r = subprocess.run(["git", "-C", repo, "rev-parse", "--short", "HEAD"],
capture_output=True, text=True, timeout=60)
if r.returncode == 0:
return r.stdout.strip() or "unknown"
except (OSError, subprocess.SubprocessError):
pass
return "unknown"
class BaselineError(Exception):
"""The baseline cannot be graded: a parse fault, not a row fault."""
def parse_baseline(path):
"""-> (lines, rows); rows are dicts with the physical line index.
Comments and blank lines are kept verbatim in `lines` so a rewrite can be
expressed as "replace these three field values on these line numbers".
"""
if not os.path.isfile(path):
raise BaselineError("no baseline at %s" % path)
try:
with open(path, "r", encoding="utf-8") as fh:
lines = fh.read().split("\n")
except OSError as exc:
raise BaselineError("cannot read %s: %s" % (path, exc))
rows = []
for i, line in enumerate(lines):
if not line.strip() or line.startswith("#"):
continue
parts = line.split("\t")
if len(parts) < 2:
raise BaselineError("line %d has no tab: %r" % (i + 1, line))
if not parts[1].strip().isdigit():
raise BaselineError("line %d rc is not an integer: %r"
% (i + 1, line))
rows.append({"line": i, "path": parts[0], "rc": int(parts[1].strip()),
"note": parts[2] if len(parts) > 2 else ""})
if not rows:
raise BaselineError("baseline has no rows: %s" % path)
return lines, rows
def select(rows, want_all, only):
out = []
for r in rows:
if only and not any(s in r["path"] for s in only):
continue
if want_all or only or r["rc"] != 0:
out.append(r)
return out
def measure(repo, rows, timeout, jobs):
"""One reading per named row, in sorted path order, `jobs` at a time."""
todo = sorted(rows, key=lambda r: r["path"])
missing = [r for r in todo if not os.path.isfile(os.path.join(repo, r["path"]))]
present = [r for r in todo if r not in missing]
def one(r):
rc, note = run_entry(repo, r["path"], timeout)
return {"path": r["path"], "was": r["rc"], "now": rc,
"note": clean_note(note)[:NOTE_CUT], "state": classify(rc)}
if jobs < 2 or len(present) < 2:
got = [one(r) for r in present]
else:
got = []
with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool:
futures = {pool.submit(one, r): r for r in present}
for fut in concurrent.futures.as_completed(futures):
got.append(fut.result())
got.sort(key=lambda g: g["path"])
return got, [r["path"] for r in missing]
def moved(readings):
return [g for g in readings if g["was"] != g["now"]]
def clean_note(note):
"""A note safe for a tab-separated row: tabs and runs of whitespace become
one space, because a tab inside the note adds a field and the reader that
grades the row would see a truncated note while the writer reported success."""
return " ".join(note.replace("\t", " ").split())
def provenance(count, moved_rows, rev, timeout, jobs, stamp):
"""The comment block a rewrite leaves behind, one line per moved row."""
head = ("# row-level re-take (q148 via retake.py, %s): %d row(s) measured,"
" --timeout %d --jobs %d, at HEAD %s; %d code(s) moved."
% (stamp, count, timeout, jobs, rev, len(moved_rows)))
body = ["# %s %d -> %d (%s)" % (m["path"], m["was"], m["now"], m["state"])
for m in moved_rows]
return [head] + body
def rewrite(lines, rows, readings, rev, timeout, jobs, stamp):
"""-> new list of lines: changed rows updated in place, one comment block
inserted after the leading comment block. Order and every other byte kept."""
line_of = {r["path"]: r["line"] for r in rows}
by_line = {line_of[g["path"]]: g for g in readings if g["path"] in line_of}
out = []
inserted = False
block = provenance(len(readings), moved(readings), rev, timeout, jobs, stamp)
first_row = next((i for i, l in enumerate(lines)
if l.strip() and not l.startswith("#")), len(lines))
for i, line in enumerate(lines):
if i == first_row and not inserted:
out.extend(block)
inserted = True
r = by_line.get(i)
if r is None:
out.append(line)
continue
parts = line.split("\t")
parts[1] = str(r["now"])
note = clean_note(r["note"])
if len(parts) > 2:
parts[2] = note
else:
parts.append(note)
out.append("\t".join(parts))
if not inserted:
out.extend(block)
return out
def sign(baseline, lines, rows, repo, paths, timeout, jobs):
"""Measure paths the baseline does not name and append their rows.
The sweep's OTHER fault direction: an entry point that landed after the last
take is `NEW` until a tick measures and signs it. This is that signing, with
the same `run_entry`, appended after the last data row.
"""
known = {r["path"] for r in rows}
out = []
for p in paths:
if p in known:
print("retake: refusing to sign %s -- the baseline already names it"
" (a code change is a re-take, not a sign)" % p)
return 1
if not os.path.isfile(os.path.join(repo, p)):
print("CANNOT GRADE: %s is not in the tree at %s" % (p, repo))
return 2
out.append(p)
rev = head_rev(repo)
stamped = []
for p in out:
rc, note = run_entry(repo, p, timeout)
note = clean_note(note)[:NOTE_CUT]
stamped.append((p, rc, note[:NOTE_CUT], classify(rc)))
print("SIGNED %s %d (%s) %s" % (p, rc, classify(rc), note[:NOTE_CUT]))
stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
block = ["# rows signed (q148 via retake.py, %s): %d entry point(s) measured,"
" --timeout %d --jobs %d, at HEAD %s"
% (stamp, len(stamped), timeout, jobs, rev)]
body = lines[:]
tail = body.pop() if body and body[-1] == "" else ""
body.extend(block)
body.extend("%s\t%d\t%s" % (p, rc, note) for p, rc, note, _s in stamped)
if tail == "":
body.append("")
tmp = baseline + ".sign.tmp"
with open(tmp, "w", encoding="utf-8", newline="\n") as fh:
fh.write("\n".join(body))
os.replace(tmp, baseline)
_l, after = parse_baseline(baseline)
disk = {r["path"]: r["rc"] for r in after}
bad = [p for p, rc, _n, _s in stamped if disk.get(p) != rc]
if bad:
print("retake: SIGNED WRONG ROWS (re-read disagrees): %s" % ", ".join(bad))
return 1
print("retake: %d row(s) signed into %s" % (len(stamped), baseline))
return 0
def main(argv=None):
ap = argparse.ArgumentParser(
description="Re-take named rows of the CHECK entry-point baseline.")
ap.add_argument("--baseline", default=DEFAULT_BASELINE)
ap.add_argument("--repo", default=None)
ap.add_argument("--path", action="append", default=[],
help="substring of a baseline path; repeatable")
ap.add_argument("--all", action="store_true",
help="measure every row, not only the red ones")
ap.add_argument("--timeout", type=int, default=120)
ap.add_argument("--jobs", type=int, default=4)
ap.add_argument("--write", action="store_true")
ap.add_argument("--sign", action="append", default=[], metavar="PATH",
help="measure a path the baseline does not name and append "
"it; repeatable, and a write")
ap.add_argument("--verbose", action="store_true")
ap.add_argument("--selftest", action="store_true")
a = ap.parse_args(argv)
if a.selftest:
return selftest(a.verbose)
baseline = os.path.abspath(a.baseline)
repo = os.path.abspath(a.repo) if a.repo else repo_of(baseline)
try:
lines, rows = parse_baseline(baseline)
except BaselineError as exc:
print("CANNOT GRADE: %s" % exc)
return 2
if not os.path.isdir(repo):
print("CANNOT GRADE: no repo at %s" % repo)
return 2
if a.sign:
return sign(baseline, lines, rows, repo, a.sign, a.timeout, a.jobs)
targets = select(rows, a.all, a.path)
if not targets:
print("retake: nothing to measure (0 row(s) selected)")
return 0
rev = head_rev(repo)
readings, missing = measure(repo, targets, a.timeout, a.jobs)
moves = moved(readings)
print("# retake %s" % baseline)
print("# repo %s at HEAD %s; timeout %ds, jobs %d; %d row(s) measured"
% (repo, rev, a.timeout, a.jobs, len(readings)))
for p in missing:
print("MISSING %s -- the baseline names it, the tree does not have it" % p)
for m in moves:
print("MOVED %s %d -> %d (%s) %s"
% (m["path"], m["was"], m["now"], m["state"], m["note"]))
if a.verbose:
for g in readings:
if g["was"] == g["now"]:
print("SAME %s %d (%s)" % (g["path"], g["now"], g["state"]))
if not a.write:
print("retake: %d measured, %d moved, %d unchanged%s"
% (len(readings), len(moves), len(readings) - len(moves),
"; nothing written (dry run)" if moves else ""))
return 1 if (moves or missing) else 0
if missing:
print("retake: refusing to write -- %d named row(s) missing from the "
"tree; a re-take that skips a path would claim coverage it does "
"not have" % len(missing))
return 1
if not moves:
print("retake: %d measured, 0 moved; nothing written" % len(readings))
return 0
stamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d")
new_lines = rewrite(lines, rows, readings, rev, a.timeout, a.jobs, stamp)
body = "\n".join(new_lines)
tmp = baseline + ".retake.tmp"
with open(tmp, "w", encoding="utf-8", newline="\n") as fh:
fh.write(body)
os.replace(tmp, baseline)
# Read it back: the rows this tool says it wrote must be the rows on disk.
_lines, after = parse_baseline(baseline)
disk = {r["path"]: r["rc"] for r in after}
bad = [m["path"] for m in moves if disk.get(m["path"]) != m["now"]]
if bad:
print("retake: WROTE WRONG ROWS (re-read disagrees): %s" % ", ".join(bad))
return 1
print("retake: %d measured, %d moved and written into %s"
% (len(readings), len(moves), baseline))
return 0
# --------------------------------------------------------------------- selftest
FIXTURES = {
"CHECK-a": "#!/bin/bash\necho 'PASS a is green'\nexit 0\n",
"CHECK-b": "#!/bin/bash\necho 'warming up'\necho 'FAIL b is red'\nexit 1\n",
"CHECK-loud": "#!/bin/bash\nprintf 'ok %0300d\\n' 0\nexit 0\n",
"CHECK-tabbed": "#!/bin/bash\nprintf 'FAIL tabbed\\tfaultline after the tab\\n'\nexit 1\n",
}
def selftest(verbose=False):
import tempfile
cell_failed = 0
cells = 0
def cell(name, ok, detail=""):
nonlocal cell_failed, cells
cells += 1
if not ok:
cell_failed += 1
print(" %-24s %s %s" % (name, "PASS" if ok else "FAIL", detail if
(verbose or not ok) else ""))
def run(argv):
"""main() with its report captured, so cell lines stay readable."""
import contextlib
import io
buf = io.StringIO()
with contextlib.redirect_stdout(buf):
rc = main(argv)
return rc, buf.getvalue()
work = tempfile.mkdtemp(prefix="retake-selftest.")
try:
tree = os.path.join(work, "tree")
os.makedirs(tree)
for name, body in FIXTURES.items():
p = os.path.join(tree, name)
with open(p, "w", encoding="utf-8") as fh:
fh.write(body)
os.chmod(p, 0o755)
base = os.path.join(work, "baseline.tsv")
green = ("# path\trc\tnote -- fixture baseline\n"
"# a comment that must survive a rewrite\n"
"CHECK-a\t0\tPASS a is green\n"
"CHECK-b\t1\tFAIL b is red\n")
with open(base, "w", encoding="utf-8") as fh:
fh.write(green)
# 1 -- green control: a baseline that agrees with the tree moves nothing
rc, _o = run(["--baseline", base, "--repo", tree, "--all"])
cell("green-control", rc == 0, "rc=%d" % rc)
# 2 -- a moved code is named, and the dry run writes nothing
with open(base, "w", encoding="utf-8") as fh:
fh.write(green.replace("CHECK-b\t1\tFAIL b is red",
"CHECK-b\t0\tFAIL b is red"))
before = open(base, encoding="utf-8").read()
rc, _o = run(["--baseline", base, "--repo", tree, "--all"])
after = open(base, encoding="utf-8").read()
cell("moved-code-named", rc == 1 and after == before,
"rc=%d bytes-unchanged=%s" % (rc, after == before))
# 3 -- the write moves exactly that row and keeps the comments
rc, _o = run(["--baseline", base, "--repo", tree, "--all", "--write"])
text = open(base, encoding="utf-8").read()
_l, rows = parse_baseline(base)
rc2, _o2 = run(["--baseline", base, "--repo", tree, "--all"])
code = {r["path"]: r["rc"] for r in rows}
cell("write-updates-row", rc == 0 and code["CHECK-b"] == 1
and "a comment that must survive" in text
and "retake.py" in text and rc2 == 0,
"rc=%d b=%d idempotent_rc=%d" % (rc, code["CHECK-b"], rc2))
# 4 -- a baseline row with no file in the tree is a refusal, not a skip
with open(base, "w", encoding="utf-8") as fh:
fh.write(green + "CHECK-gone\t0\tnothing\n")
rc, _o = run(["--baseline", base, "--repo", tree, "--all", "--write"])
text = open(base, encoding="utf-8").read()
cell("missing-row-refused", rc == 1 and "CHECK-gone\t0" in text,
"rc=%d untouched=%s" % (rc, "CHECK-gone\t0" in text))
# 5 -- a malformed baseline cannot be graded (2, never a pass)
with open(base, "w", encoding="utf-8") as fh:
fh.write(green + "CHECK-a\tnot-an-int\tx\n")
rc, _o = run(["--baseline", base, "--repo", tree, "--all"])
cell("malformed-ungradable", rc == 2, "rc=%d" % rc)
# 6 -- the note is cut at the same width the sweep uses
with open(base, "w", encoding="utf-8") as fh:
fh.write("# path\trc\tnote\nCHECK-loud\t1\tstale\n")
rc, _o = run(["--baseline", base, "--repo", tree, "--all", "--write"])
_l, rows = parse_baseline(base)
note = rows[0]["note"] if rows else ""
cell("note-cut", rc == 0 and 0 < len(note) <= NOTE_CUT,
"len=%d" % len(note))
# 7 -- the fault-hint rule is the sweep's: the FAIL line, not the last
rc, note = run_entry(tree, "CHECK-b", 30)
cell("fault-hint-parity", rc == 1 and note.startswith("FAIL b is red"),
"note=%r" % note)
# 8 -- the contract is imported, not re-implemented
cell("contract-imported", run_entry is check_sweep.run_entry
and classify is check_sweep.classify, "")
# 9 -- a baseline that does not name every entry point invents no rows
with open(base, "w", encoding="utf-8") as fh:
fh.write("# path\trc\tnote\nCHECK-a\t0\tPASS a is green\n")
rc, _o = run(["--baseline", base, "--repo", tree, "--all", "--write"])
_l, rows = parse_baseline(base)
cell("invents-no-rows", rc == 0 and len(rows) == 1
and rows[0]["path"] == "CHECK-a", "rows=%d" % len(rows))
# 10 -- this file names no host path
src = open(os.path.abspath(__file__), encoding="utf-8").read()
cell("no-host-path", ("/" + "root/") not in src, "")
# 11 -- a path the baseline does not name is measured and signed once
with open(base, "w", encoding="utf-8") as fh:
fh.write("# path\trc\tnote\nCHECK-a\t0\tPASS a is green\n")
rc, _o = run(["--baseline", base, "--repo", tree, "--sign", "CHECK-b"])
_l, rows = parse_baseline(base)
signed = {r["path"]: r["rc"] for r in rows}
rc2, _o2 = run(["--baseline", base, "--repo", tree, "--sign", "CHECK-b"])
cell("sign-once", rc == 0 and signed.get("CHECK-b") == 1 and rc2 == 1
and len(rows) == 2,
"rc=%d b=%s again_rc=%d rows=%d"
% (rc, signed.get("CHECK-b"), rc2, len(rows)))
# 12 -- a fault line carrying a tab stays one note cell, and the sweep's
# own reader reads the whole line back. Written verbatim it became four
# fields and read back as "FAIL tabbed": a writer that reports success
# and a reader that grades a truncated string.
with open(base, "w", encoding="utf-8") as fh:
fh.write("# path\trc\tnote\nCHECK-tabbed\t0\tstale\n")
rc, _o = run(["--baseline", base, "--repo", tree, "--all", "--write"])
raw = [l for l in open(base, encoding="utf-8").read().split("\n")
if l.startswith("CHECK-tabbed")]
nf = len(raw[0].split("\t")) if raw else 0
_r, srows = check_sweep.read_baseline(base)
got = srows[0][2] if srows else ""
want = "FAIL tabbed faultline after the tab"
cell("note-sanitised", rc == 0 and nf == 3 and got == want,
"rc=%d fields=%d note=%r" % (rc, nf, got))
finally:
import shutil
shutil.rmtree(work, ignore_errors=True)
print("selftest: %d cell(s), %d passed, %d failed"
% (cells, cells - cell_failed, cell_failed))
return 0 if cell_failed == 0 else 1
if __name__ == "__main__":
raise SystemExit(main())