#!/usr/bin/env python3 """Does the judge actually FAIL when the artifact lies? A verdict that cannot go nonzero is a tautology. This plants a real defect -- rewriting the runner's PASS to UNVERIFIED -- and asserts the judge exits nonzero. Exit 0 == the judge has teeth. """ import os import subprocess import sys import tempfile ART = os.path.join(os.path.dirname(os.path.abspath(__file__)), "proveit.py") JUDGE = os.path.abspath(__file__) src = open(ART, encoding="utf-8").read() # Plant the lie: the status domain check must now see UNVERIFIED. mutated = src.replace('status="PASS" if (ok and measured) else "FAIL"', 'status="UNVERIFIED" if (ok and measured) else "FAIL"') assert mutated != src, "mutation did not apply -- test is invalid" tmp = tempfile.mkdtemp(prefix="proveit-mut-") bad = os.path.join(tmp, "proveit.py") open(bad, "w", encoding="utf-8").write(mutated) # run the judge against the mutated artifact by pointing ART at it judge_src = open(JUDGE, encoding="utf-8").read().replace( 'ART = "/root/.hermes/wizard-outputs/null/proveit.py"', 'ART = %r' % bad) badjudge = os.path.join(tmp, "_judge.py") open(badjudge, "w", encoding="utf-8").write(judge_src) p = subprocess.run([sys.executable, badjudge], capture_output=True, text=True) print("mutated-artifact judge exit code:", p.returncode) print("--- judge output (tail) ---") print("\n".join(p.stdout.strip().splitlines()[-6:])) if p.returncode == 0: print() print("TOOTHLESS: judge passed a mutated artifact -> verdict is a tautology") sys.exit(1) print() print("TEETH OK: judge rejected the mutated artifact (exit %d)" % p.returncode) sys.exit(0)