ledger
6b1e24fec4
sweep.py runs every root CHECK-* under bash and records rc, class and the process's last line, pinning rev + sha256 of each entry in manifest.json. report.md is the table; TRIAGE.md groups the 12 reds into four causes. check-root-sweep.sh grades the record (pin / universe / class contract / report parity / rows integrity), 5 legs, with sensitivity_probe.py proving three mutations redden the leg that owns them and the restore is green.
205 lines
8.4 KiB
Python
205 lines
8.4 KiB
Python
#!/usr/bin/env python3
|
|
"""verify_sweep.py -- grade the root CHECK-* sweep artifact against itself.
|
|
|
|
WHAT THIS IS
|
|
`sweep.py` runs every root `CHECK-*` entry point and classifies its exit
|
|
code. Its verdict is only worth the bytes it measured, so this verifier
|
|
grades the artifact, not the repository: the pin (rev, timestamp, timeout),
|
|
the universe (which entry points, and their sha256), the class mapping
|
|
(rc -> class, the repo's published contract), the report (one table row per
|
|
entry, counts and rev stated), and the integrity of the row file itself.
|
|
|
|
It does NOT re-run the entry points -- that is the sweep's job and it costs
|
|
minutes. It answers "is this artifact a faithful record of a run over the
|
|
universe it names", which is the question a reader of a table actually has.
|
|
|
|
EXIT
|
|
0 every leg passed; 1 a fault; 2 the artifact is not there to grade.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as _dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
ART = os.path.dirname(os.path.abspath(__file__))
|
|
MANIFEST = os.path.join(ART, "manifest.json")
|
|
RESULTS = os.path.join(ART, "results.tsv")
|
|
REPORT = os.path.join(ART, "report.md")
|
|
|
|
CLASS = {0: "OK", 1: "FAULT", 2: "COULD_NOT_GRADE"}
|
|
HEX40 = re.compile(r"^[0-9a-f]{40}$")
|
|
UTC = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$")
|
|
|
|
passed = 0
|
|
faults: list[str] = []
|
|
|
|
|
|
def leg(name: str, problems: list[str], detail: str) -> None:
|
|
global passed
|
|
if problems:
|
|
print(f" {name:<10} FAULT {'; '.join(problems[:4])}")
|
|
faults.extend(problems)
|
|
else:
|
|
print(f" {name:<10} PASS {detail}")
|
|
passed += 1
|
|
|
|
|
|
def sha256_file(path: str) -> str:
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as fh:
|
|
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def git(*args: str) -> str:
|
|
p = subprocess.run(["git", "-C", ART, *args], capture_output=True,
|
|
text=True, errors="replace")
|
|
return p.stdout.strip() if p.returncode == 0 else ""
|
|
|
|
|
|
def git_ok(*args: str) -> bool:
|
|
p = subprocess.run(["git", "-C", ART, *args], capture_output=True,
|
|
text=True, errors="replace")
|
|
return p.returncode == 0
|
|
|
|
|
|
def read_rows() -> list[dict]:
|
|
rows = []
|
|
with open(RESULTS, encoding="utf-8") as fh:
|
|
header = fh.readline().rstrip("\n").split("\t")
|
|
want = ["entry", "rc", "secs", "class", "traceback",
|
|
"last_stdout", "last_stderr"]
|
|
if header != want:
|
|
raise SystemExit(f"results.tsv header {header!r} != {want!r}")
|
|
for line in fh:
|
|
parts = line.rstrip("\n").split("\t")
|
|
if len(parts) != 7:
|
|
raise SystemExit(f"results.tsv row has {len(parts)} fields: {line!r}")
|
|
rows.append(dict(zip(want, parts)))
|
|
return rows
|
|
|
|
|
|
def main() -> int:
|
|
for path in (MANIFEST, RESULTS, REPORT):
|
|
if not os.path.isfile(path):
|
|
print(f"verify_sweep: no {path} -- cannot grade", file=sys.stderr)
|
|
return 2
|
|
manifest = json.load(open(MANIFEST, encoding="utf-8"))
|
|
rows = read_rows()
|
|
report = open(REPORT, encoding="utf-8").read()
|
|
|
|
# 1 -- the pin: a full rev that resolves, a real UTC stamp, a coherent count.
|
|
problems = []
|
|
rev = str(manifest.get("rev", ""))
|
|
if not HEX40.match(rev):
|
|
problems.append(f"rev is not a full 40-hex sha: {rev!r}")
|
|
elif not git_ok("cat-file", "-e", f"{rev}^{{commit}}"):
|
|
problems.append(f"rev {rev} does not resolve in this repo")
|
|
stamp = str(manifest.get("generated_utc", ""))
|
|
if not UTC.match(stamp):
|
|
problems.append(f"generated_utc is not YYYY-MM-DDTHH:MM:SSZ: {stamp!r}")
|
|
else:
|
|
try:
|
|
_dt.datetime.fromisoformat(stamp.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
problems.append(f"generated_utc does not parse: {stamp!r}")
|
|
entries = manifest.get("entries") or []
|
|
if manifest.get("entry_count") != len(entries):
|
|
problems.append(f"entry_count {manifest.get('entry_count')} != "
|
|
f"{len(entries)} manifest entries")
|
|
if len(entries) != len(rows):
|
|
problems.append(f"manifest names {len(entries)} entries, results has {len(rows)} rows")
|
|
if not isinstance(manifest.get("timeout_s"), int) or manifest["timeout_s"] <= 0:
|
|
problems.append(f"timeout_s is not a positive int: {manifest.get('timeout_s')!r}")
|
|
leg("pin", problems, f"{len(rows)} rows at {rev[:12]} {stamp}")
|
|
|
|
# 2 -- the universe: same names, same order, same bytes, same trackedness.
|
|
problems = []
|
|
names = [r["entry"] for r in rows]
|
|
if names != [e.get("name") for e in entries]:
|
|
problems.append("results row order/names differ from manifest entries")
|
|
root = git("rev-parse", "--show-toplevel") or os.path.dirname(ART)
|
|
tracked = set(git("ls-tree", "-r", "--full-tree", "--name-only", rev).splitlines()) if HEX40.match(rev) else set()
|
|
for e in entries:
|
|
path = os.path.join(root, e.get("name", ""))
|
|
if not os.path.isfile(path):
|
|
problems.append(f"{e.get('name')}: file absent")
|
|
continue
|
|
if sha256_file(path) != e.get("sha256"):
|
|
problems.append(f"{e.get('name')}: sha256 moved since the sweep")
|
|
if bool(e.get("tracked_at_rev")) != (e.get("name") in tracked):
|
|
problems.append(f"{e.get('name')}: tracked_at_rev disagrees with ls-tree {rev[:8]}")
|
|
leg("universe", problems, f"{len(entries)} entry files byte-identical to the manifest pin")
|
|
|
|
# 3 -- the class mapping: rc -> class is the contract, not a label.
|
|
problems = []
|
|
for r in rows:
|
|
try:
|
|
rc = int(r["rc"])
|
|
except ValueError:
|
|
problems.append(f"{r['entry']}: rc {r['rc']!r} is not an int")
|
|
continue
|
|
want = CLASS.get(rc, "UNEXPECTED")
|
|
if r["class"] != want:
|
|
problems.append(f"{r['entry']}: rc {rc} labelled {r['class']}, contract says {want}")
|
|
if r["traceback"] not in ("0", "1"):
|
|
problems.append(f"{r['entry']}: traceback flag {r['traceback']!r}")
|
|
try:
|
|
if float(r["secs"]) < 0:
|
|
problems.append(f"{r['entry']}: negative seconds")
|
|
except ValueError:
|
|
problems.append(f"{r['entry']}: secs {r['secs']!r}")
|
|
leg("classes", problems, "every rc maps to its class under the published contract")
|
|
|
|
# 4 -- the report: one table row per entry, counts and rev as measured.
|
|
problems = []
|
|
counts: dict[str, int] = {}
|
|
for r in rows:
|
|
counts[r["class"]] = counts.get(r["class"], 0) + 1
|
|
table = re.findall(r"^\| `([^`]+)` \| ([^|]+?) \| ([^|]+?) \| ([A-Z_]+) \|", report, re.M)
|
|
if [t[0] for t in table] != names:
|
|
problems.append(f"report table names {len(table)} rows, results has {len(names)}")
|
|
if f"- entry points run: **{len(rows)}**" not in report:
|
|
problems.append("report does not state entry points run")
|
|
for klass, count in counts.items():
|
|
if f"- {klass}: **{count}**" not in report:
|
|
problems.append(f"report does not state {klass} count {count}")
|
|
if rev and rev not in report:
|
|
problems.append("report does not name the pinned rev")
|
|
leg("report", problems, f"{len(table)} table rows, counts and rev agree with the rows file")
|
|
|
|
# 5 -- substance: the rows file is the file the manifest hashed and no row
|
|
# is silent, so a green row is a process that said something.
|
|
problems = []
|
|
if sha256_file(RESULTS) != manifest.get("results_sha256"):
|
|
problems.append("results.tsv sha256 differs from the manifest")
|
|
for r in rows:
|
|
if r["class"] in ("OK", "FAULT") and r["last_stdout"].strip() in ("", "-") \
|
|
and r["last_stderr"].strip() in ("", "-"):
|
|
problems.append(f"{r['entry']}: {r['class']} row carries no output at all")
|
|
bad = sorted(r["entry"] for r in rows if r["class"] != "OK")
|
|
bullets = re.findall(r"^- `([^`]+)` rc=", report, re.M)
|
|
if sorted(bullets) != bad:
|
|
problems.append(f"Not-OK bullets {sorted(bullets)} != rows {bad}")
|
|
if not any(r["class"] == "OK" for r in rows):
|
|
problems.append("no OK row -- a sweep that cannot pass anything is not evidence")
|
|
leg("substance", problems, f"{len(bad)} non-OK row(s) enumerated, every row carries output")
|
|
|
|
print()
|
|
if faults:
|
|
print(f"sweep-check: {passed} passed, {len(faults)} fault(s)")
|
|
return 1
|
|
print(f"sweep-check: 5 leg(s), all passed")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|