ledger
6b1e24fec4
sweep.py runs every root CHECK-* under bash and records rc, class and the process's last line, pinning rev + sha256 of each entry in manifest.json. report.md is the table; TRIAGE.md groups the 12 reds into four causes. check-root-sweep.sh grades the record (pin / universe / class contract / report parity / rows integrity), 5 legs, with sensitivity_probe.py proving three mutations redden the leg that owns them and the restore is green.
233 lines
8.7 KiB
Python
233 lines
8.7 KiB
Python
#!/usr/bin/env python3
|
|
"""sweep.py -- run every root CHECK-* entry point and classify its exit code.
|
|
|
|
WHY
|
|
The repo gate (`tools/verify.py --fast`) runs a fixed set of legs; none of
|
|
them runs the committed `CHECK-*` entry points sitting at the repository
|
|
root. Each of those was written as a deliverable's own verification command,
|
|
so the question "does the tree still satisfy the checks it shipped with?"
|
|
has no answer from the gate. This sweep answers it by running each one under
|
|
the interpreter every caller in this tree uses (`bash <path>`) and
|
|
classifying the exit code against this repo's published contract:
|
|
|
|
rc 0 OK the check graded clean
|
|
rc 1 FAULT the check graded a fault (a real, reportable red)
|
|
rc 2 COULD_NOT_GRADE the check could not grade its subject
|
|
other UNEXPECTED a shell/driver error (126, 127, signal, ...)
|
|
timeout TIMEOUT killed after --timeout seconds
|
|
|
|
Nothing here is a stub: every row is a real process with a real exit code,
|
|
and stdout/stderr tails are the process's own bytes.
|
|
|
|
WHAT IS PINNED
|
|
A verdict is only worth the tree it was measured on, so every run writes
|
|
`manifest.json` beside its rows: the full HEAD sha and subject, the UTC
|
|
timestamp, the timeout, and the sha256 of every swept entry-point file, plus
|
|
a flag for whether that path exists at HEAD. The report prints the same rev.
|
|
The bytes that were run are therefore recoverable, not implied.
|
|
|
|
An interrupted run is partial evidence, not no evidence: rows are flushed as
|
|
they finish, so a sweep killed at the wall clock leaves every completed row.
|
|
|
|
USAGE
|
|
python3 sweep.py --out <dir> [--repo <path>] [--timeout 300] [--json]
|
|
|
|
Writes <dir>/results.tsv, <dir>/manifest.json and <dir>/report.md, and
|
|
prints one summary line. Exit code of the sweep itself: 0 if every row
|
|
graded 0, 1 if any row graded 1 or worse, 2 if the sweep could not run (no
|
|
entry points found, or a manifest could not be captured).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as _dt
|
|
import glob
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
CLASS = {0: "OK", 1: "FAULT", 2: "COULD_NOT_GRADE"}
|
|
|
|
|
|
def discover(repo: str) -> list[str]:
|
|
rows = []
|
|
for path in sorted(glob.glob(os.path.join(repo, "CHECK*"))):
|
|
if os.path.isfile(path):
|
|
rows.append(os.path.basename(path))
|
|
return rows
|
|
|
|
|
|
def tail(text: str, limit: int = 160) -> str:
|
|
text = (text or "").strip().replace("\t", " ").replace("\r", " ")
|
|
lines = [ln for ln in text.splitlines() if ln.strip()]
|
|
if not lines:
|
|
return "-"
|
|
return lines[-1].strip()[:limit]
|
|
|
|
|
|
def run_one(repo: str, name: str, timeout: int) -> dict:
|
|
started = time.time()
|
|
try:
|
|
proc = subprocess.run(
|
|
["bash", name],
|
|
cwd=repo,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
errors="replace",
|
|
)
|
|
rc, out, err = proc.returncode, proc.stdout, proc.stderr
|
|
except subprocess.TimeoutExpired as exc:
|
|
rc = "timeout"
|
|
out = (exc.stdout or "") if isinstance(exc.stdout, str) else ""
|
|
err = (exc.stderr or "") if isinstance(exc.stderr, str) else ""
|
|
secs = round(time.time() - started, 2)
|
|
klass = "TIMEOUT" if rc == "timeout" else CLASS.get(rc, "UNEXPECTED")
|
|
return {
|
|
"name": name,
|
|
"rc": rc,
|
|
"secs": secs,
|
|
"class": klass,
|
|
"last_out": tail(out),
|
|
"last_err": tail(err),
|
|
"traceback": "Traceback (most recent call last)" in (err or ""),
|
|
}
|
|
|
|
|
|
def git_out(repo: str, *args: str) -> str:
|
|
try:
|
|
p = subprocess.run(["git", "-C", repo, *args], capture_output=True,
|
|
text=True, timeout=30, errors="replace")
|
|
except (OSError, subprocess.SubprocessError):
|
|
return ""
|
|
return p.stdout.strip() if p.returncode == 0 else ""
|
|
|
|
|
|
def sha256_file(path: str) -> str:
|
|
h = hashlib.sha256()
|
|
with open(path, "rb") as fh:
|
|
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def capture_manifest(repo: str, names: list[str], timeout: int) -> dict:
|
|
rev = git_out(repo, "rev-parse", "HEAD")
|
|
if len(rev) != 40:
|
|
raise RuntimeError(f"cannot resolve HEAD in {repo!r}")
|
|
subject = git_out(repo, "log", "-1", "--format=%s", "HEAD")
|
|
tracked = set(git_out(repo, "ls-tree", "-r", "--name-only", "HEAD")
|
|
.splitlines())
|
|
entries = []
|
|
for name in names:
|
|
path = os.path.join(repo, name)
|
|
entries.append({
|
|
"name": name,
|
|
"sha256": sha256_file(path),
|
|
"bytes": os.path.getsize(path),
|
|
"tracked_at_rev": name in tracked,
|
|
})
|
|
dirty = [ln for ln in git_out(repo, "status", "--porcelain").splitlines()
|
|
if ln.strip()]
|
|
return {
|
|
"tool": "wizard-work/volt/root-check-sweep/sweep.py",
|
|
"generated_utc": _dt.datetime.now(_dt.timezone.utc)
|
|
.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"repo": os.path.basename(repo.rstrip("/")),
|
|
"rev": rev,
|
|
"rev_subject": subject,
|
|
"worktree_dirty_paths": len(dirty),
|
|
"timeout_s": timeout,
|
|
"entry_count": len(entries),
|
|
"entries": entries,
|
|
}
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--repo", default=os.path.dirname(os.path.abspath(__file__)))
|
|
ap.add_argument("--out", required=True)
|
|
ap.add_argument("--timeout", type=int, default=300)
|
|
ap.add_argument("--json", action="store_true")
|
|
args = ap.parse_args(argv)
|
|
|
|
repo = os.path.abspath(args.repo)
|
|
names = discover(repo)
|
|
if not names:
|
|
print(f"sweep: no CHECK* entry points under {repo}")
|
|
return 2
|
|
try:
|
|
manifest = capture_manifest(repo, names, args.timeout)
|
|
except (OSError, RuntimeError) as exc:
|
|
print(f"sweep: cannot capture manifest: {exc}", file=sys.stderr)
|
|
return 2
|
|
|
|
rows: list[dict] = []
|
|
os.makedirs(args.out, exist_ok=True)
|
|
tsv = os.path.join(args.out, "results.tsv")
|
|
with open(tsv, "w", encoding="utf-8") as fh:
|
|
fh.write("entry\trc\tsecs\tclass\ttraceback\tlast_stdout\tlast_stderr\n")
|
|
fh.flush()
|
|
for name in names:
|
|
r = run_one(repo, name, args.timeout)
|
|
rows.append(r)
|
|
fh.write(
|
|
f"{r['name']}\t{r['rc']}\t{r['secs']}\t{r['class']}\t"
|
|
f"{int(r['traceback'])}\t{r['last_out']}\t{r['last_err']}\n"
|
|
)
|
|
fh.flush()
|
|
print(f" {r['class']:<16} rc={r['rc']:<8} {r['secs']:>7.2f}s {r['name']}",
|
|
flush=True)
|
|
|
|
counts: dict[str, int] = {}
|
|
for r in rows:
|
|
counts[r["class"]] = counts.get(r["class"], 0) + 1
|
|
bad = [r for r in rows if r["class"] != "OK"]
|
|
|
|
manifest["results_sha256"] = sha256_file(tsv)
|
|
manifest["counts"] = dict(sorted(counts.items()))
|
|
with open(os.path.join(args.out, "manifest.json"), "w", encoding="utf-8") as fh:
|
|
json.dump(manifest, fh, indent=2, sort_keys=True)
|
|
fh.write("\n")
|
|
|
|
report = os.path.join(args.out, "report.md")
|
|
with open(report, "w", encoding="utf-8") as fh:
|
|
fh.write("# Root CHECK-* entry-point sweep\n\n")
|
|
fh.write(f"- measured at: `{manifest['generated_utc']}`\n")
|
|
fh.write(f"- rev: `{manifest['rev']}` -- {manifest['rev_subject']}\n")
|
|
fh.write(f"- worktree: {manifest['worktree_dirty_paths']} path(s) differ "
|
|
f"from that rev (modified + untracked); the worktree is what "
|
|
f"was graded\n")
|
|
fh.write(f"- entry points run: **{len(rows)}**\n")
|
|
for k in sorted(counts):
|
|
fh.write(f"- {k}: **{counts[k]}**\n")
|
|
fh.write(f"- total wall time: **{round(sum(r['secs'] for r in rows), 1)} s**\n\n")
|
|
fh.write("| entry | rc | s | class | last line |\n|---|---|---|---|---|\n")
|
|
for r in rows:
|
|
last = r["last_err"] if r["last_err"] != "-" else r["last_out"]
|
|
fh.write(
|
|
f"| `{r['name']}` | {r['rc']} | {r['secs']} | {r['class']} | "
|
|
f"{last.replace('|', '/')} |\n"
|
|
)
|
|
if bad:
|
|
fh.write("\n## Not OK\n\n")
|
|
for r in bad:
|
|
fh.write(f"- `{r['name']}` rc={r['rc']} ({r['class']}): "
|
|
f"{r['last_err'] if r['last_err'] != '-' else r['last_out']}\n")
|
|
|
|
if args.json:
|
|
print(json.dumps({"manifest": manifest, "rows": rows}, indent=2))
|
|
else:
|
|
detail = ", ".join(f"{k} {counts[k]}" for k in sorted(counts))
|
|
print(f"sweep: {len(rows)} entry point(s) -- {detail} "
|
|
f"({round(sum(r['secs'] for r in rows), 1)} s) at {manifest['rev'][:12]}")
|
|
return 0 if not bad else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|