70 lines
2.6 KiB
Python
70 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""verify.py -- one command, one exit code, for the whole proveit battery.
|
|
|
|
Exit 0 == every stage passed, and the judge demonstrated teeth on 15 planted defects.
|
|
Exit 1 == a stage failed.
|
|
Exit 2 == the battery itself broke (a stage file is missing).
|
|
|
|
Stages, each run as a subprocess whose own exit code is the verdict:
|
|
1. proveit.py --selfcheck the artifact's own meta-test (8/8)
|
|
2. _judge.py external contract judge (11 contracts)
|
|
3. _probe_mutations.py judge sensitivity: 15 planted defects, all caught
|
|
4. _probe_teeth.py the judge rejects a deliberately lying artifact
|
|
|
|
This file is not a tautology. Stage 3 plants real regressions (a swapped
|
|
PASS/FAIL derivation, a neutered run loop, a hardcoded `measured = True`, ...)
|
|
and requires the judge to exit nonzero on every one, so a judge that always
|
|
exits 0 cannot pass this battery. Stage 3's success token also pins the count
|
|
("15/15"), so a new planted defect that slips past the judge fails here rather
|
|
than silently widening the blind spot.
|
|
"""
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
|
|
STAGES = [
|
|
("proveit.py --selfcheck",
|
|
[sys.executable, os.path.join(HERE, "proveit.py"), "--selfcheck"],
|
|
"SELFCHECK OK"),
|
|
("_judge.py",
|
|
[sys.executable, os.path.join(HERE, "_judge.py")],
|
|
"all contract checks hold"),
|
|
("_probe_mutations.py",
|
|
[sys.executable, os.path.join(HERE, "_probe_mutations.py")],
|
|
"15/15 planted defects caught"),
|
|
("_probe_teeth.py",
|
|
[sys.executable, os.path.join(HERE, "_probe_teeth.py")],
|
|
"TEETH OK"),
|
|
]
|
|
|
|
|
|
def main() -> int:
|
|
failures = []
|
|
for label, cmd, token in STAGES:
|
|
stage_file = cmd[1]
|
|
if not os.path.exists(stage_file):
|
|
print("[FAIL] %-24s stage file missing: %s" % (label, stage_file))
|
|
failures.append(label + " (missing)")
|
|
continue
|
|
p = subprocess.run(cmd, capture_output=True, text=True, cwd=HERE)
|
|
ok = p.returncode == 0 and token in p.stdout
|
|
print("[%s] %-24s exit=%d" % ("ok " if ok else "FAIL", label, p.returncode))
|
|
if not ok:
|
|
failures.append(label)
|
|
for line in (p.stdout + p.stderr).strip().splitlines()[-12:]:
|
|
print(" " + line)
|
|
|
|
print()
|
|
if failures:
|
|
print("VERIFY: %d stage(s) failed: %s" % (len(failures), ", ".join(failures)))
|
|
return 1
|
|
print("VERIFY: 4/4 stages passed "
|
|
"(selfcheck 8/8, judge 11 contracts, 15/15 mutation sensitivity, teeth).")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|