ledger
6b1e24fec4
sweep.py runs every root CHECK-* under bash and records rc, class and the process's last line, pinning rev + sha256 of each entry in manifest.json. report.md is the table; TRIAGE.md groups the 12 reds into four causes. check-root-sweep.sh grades the record (pin / universe / class contract / report parity / rows integrity), 5 legs, with sensitivity_probe.py proving three mutations redden the leg that owns them and the restore is green.
108 lines
4.1 KiB
Python
108 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
|
"""sensitivity_probe.py -- prove the sweep verifier can print a red.
|
|
|
|
A verifier that has only ever printed PASS is not evidence: it may be a print
|
|
statement with a five-leg shape. This probe mutates a copy of the artifact in
|
|
three independent places -- a class label, a pinned sha256, a stated count --
|
|
and requires `check-root-sweep.sh` to exit 1 and name the leg that owns the
|
|
mutated field. Every mutation is restored from a pristine copy, and the last
|
|
assertion is that the restored artifact is green again.
|
|
|
|
Run: python3 sensitivity_probe.py (exit 0 = all three mutations reddened the
|
|
leg that owns them and the restore is green)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
ART = os.path.dirname(os.path.abspath(__file__))
|
|
ROOT = subprocess.run(["git", "-C", ART, "rev-parse", "--show-toplevel"],
|
|
capture_output=True, text=True).stdout.strip()
|
|
CHECK = os.path.join(ROOT, "check-root-sweep.sh")
|
|
FILES = ["results.tsv", "manifest.json", "report.md"]
|
|
|
|
|
|
def run_check() -> tuple[int, str]:
|
|
p = subprocess.run(["bash", CHECK], capture_output=True, text=True,
|
|
errors="replace")
|
|
return p.returncode, p.stdout + p.stderr
|
|
|
|
|
|
def leg_line(out: str, leg: str) -> str:
|
|
for line in out.splitlines():
|
|
if line.strip().startswith(leg + " "):
|
|
return line.strip()
|
|
return ""
|
|
|
|
|
|
def mutate(name: str, path: str, old: str, new: str) -> None:
|
|
text = open(path, encoding="utf-8").read()
|
|
if text.count(old) != 1:
|
|
raise SystemExit(f"{name}: mutation site occurs {text.count(old)}x, want 1")
|
|
open(path, "w", encoding="utf-8").write(text.replace(old, new))
|
|
|
|
|
|
def main() -> int:
|
|
pristine = tempfile.mkdtemp(prefix="sweep-pristine-")
|
|
for f in FILES:
|
|
shutil.copy2(os.path.join(ART, f), os.path.join(pristine, f))
|
|
|
|
rows = []
|
|
try:
|
|
# 1 -- a class label that contradicts its rc.
|
|
name = "class-label"
|
|
path = os.path.join(ART, "results.tsv")
|
|
first_ok = open(path, encoding="utf-8").read().splitlines()[1]
|
|
cols = first_ok.split("\t")
|
|
mutate(name, path, "\t".join(cols), "\t".join(
|
|
cols[:3] + ["FAULT"] + cols[4:]))
|
|
rc, out = run_check()
|
|
rows.append((name, "classes", rc, leg_line(out, "classes")))
|
|
shutil.copy2(os.path.join(pristine, "results.tsv"), path)
|
|
|
|
# 2 -- a pinned sha256 that no longer describes the file.
|
|
name = "pinned-sha"
|
|
mpath = os.path.join(ART, "manifest.json")
|
|
manifest = json.load(open(mpath, encoding="utf-8"))
|
|
good = manifest["entries"][0]["sha256"]
|
|
mutate(name, mpath, good, "0" * 64)
|
|
rc, out = run_check()
|
|
rows.append((name, "universe", rc, leg_line(out, "universe")))
|
|
shutil.copy2(os.path.join(pristine, "manifest.json"), mpath)
|
|
|
|
# 3 -- a stated count that disagrees with the rows file.
|
|
name = "report-count"
|
|
rpath = os.path.join(ART, "report.md")
|
|
mutate(name, rpath, f"- entry points run: **{len(manifest['entries'])}**",
|
|
f"- entry points run: **{len(manifest['entries']) - 1}**")
|
|
rc, out = run_check()
|
|
rows.append((name, "report", rc, leg_line(out, "report")))
|
|
shutil.copy2(os.path.join(pristine, "report.md"), rpath)
|
|
|
|
# 4 -- the restore: the artifact as shipped must be green again.
|
|
rc, out = run_check()
|
|
rows.append(("restored", "-", rc, out.strip().splitlines()[-1]))
|
|
finally:
|
|
shutil.rmtree(pristine, ignore_errors=True)
|
|
|
|
ok = True
|
|
for name, leg, rc, line in rows:
|
|
if name == "restored":
|
|
good = rc == 0
|
|
else:
|
|
good = rc == 1 and leg in line and "FAULT" in line
|
|
ok = ok and good
|
|
print(f" {name:<13} {'PASS' if good else 'FAULT'} rc={rc} {line}")
|
|
print(f"\nsensitivity: {sum(1 for r in rows if (r[0] == 'restored' and r[2] == 0) or (r[0] != 'restored' and r[2] == 1))}/4 cell(s) as expected")
|
|
return 0 if ok else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|