#!/usr/bin/env python3 """sensitivity_probe.py -- prove the sweep verifier can print a red. A verifier that has only ever printed PASS is not evidence: it may be a print statement with a five-leg shape. This probe mutates a copy of the artifact in three independent places -- a class label, a pinned sha256, a stated count -- and requires `check-root-sweep.sh` to exit 1 and name the leg that owns the mutated field. Every mutation is restored from a pristine copy, and the last assertion is that the restored artifact is green again. Run: python3 sensitivity_probe.py (exit 0 = all three mutations reddened the leg that owns them and the restore is green) """ from __future__ import annotations import json import os import shutil import subprocess import sys import tempfile ART = os.path.dirname(os.path.abspath(__file__)) ROOT = subprocess.run(["git", "-C", ART, "rev-parse", "--show-toplevel"], capture_output=True, text=True).stdout.strip() CHECK = os.path.join(ROOT, "check-root-sweep.sh") FILES = ["results.tsv", "manifest.json", "report.md"] def run_check() -> tuple[int, str]: p = subprocess.run(["bash", CHECK], capture_output=True, text=True, errors="replace") return p.returncode, p.stdout + p.stderr def leg_line(out: str, leg: str) -> str: for line in out.splitlines(): if line.strip().startswith(leg + " "): return line.strip() return "" def mutate(name: str, path: str, old: str, new: str) -> None: text = open(path, encoding="utf-8").read() if text.count(old) != 1: raise SystemExit(f"{name}: mutation site occurs {text.count(old)}x, want 1") open(path, "w", encoding="utf-8").write(text.replace(old, new)) def main() -> int: pristine = tempfile.mkdtemp(prefix="sweep-pristine-") for f in FILES: shutil.copy2(os.path.join(ART, f), os.path.join(pristine, f)) rows = [] try: # 1 -- a class label that contradicts its rc. name = "class-label" path = os.path.join(ART, "results.tsv") first_ok = open(path, encoding="utf-8").read().splitlines()[1] cols = first_ok.split("\t") mutate(name, path, "\t".join(cols), "\t".join( cols[:3] + ["FAULT"] + cols[4:])) rc, out = run_check() rows.append((name, "classes", rc, leg_line(out, "classes"))) shutil.copy2(os.path.join(pristine, "results.tsv"), path) # 2 -- a pinned sha256 that no longer describes the file. name = "pinned-sha" mpath = os.path.join(ART, "manifest.json") manifest = json.load(open(mpath, encoding="utf-8")) good = manifest["entries"][0]["sha256"] mutate(name, mpath, good, "0" * 64) rc, out = run_check() rows.append((name, "universe", rc, leg_line(out, "universe"))) shutil.copy2(os.path.join(pristine, "manifest.json"), mpath) # 3 -- a stated count that disagrees with the rows file. name = "report-count" rpath = os.path.join(ART, "report.md") mutate(name, rpath, f"- entry points run: **{len(manifest['entries'])}**", f"- entry points run: **{len(manifest['entries']) - 1}**") rc, out = run_check() rows.append((name, "report", rc, leg_line(out, "report"))) shutil.copy2(os.path.join(pristine, "report.md"), rpath) # 4 -- the restore: the artifact as shipped must be green again. rc, out = run_check() rows.append(("restored", "-", rc, out.strip().splitlines()[-1])) finally: shutil.rmtree(pristine, ignore_errors=True) ok = True for name, leg, rc, line in rows: if name == "restored": good = rc == 0 else: good = rc == 1 and leg in line and "FAULT" in line ok = ok and good print(f" {name:<13} {'PASS' if good else 'FAULT'} rc={rc} {line}") print(f"\nsensitivity: {sum(1 for r in rows if (r[0] == 'restored' and r[2] == 0) or (r[0] != 'restored' and r[2] == 1))}/4 cell(s) as expected") return 0 if ok else 1 if __name__ == "__main__": sys.exit(main())