44 lines
1.6 KiB
Python
44 lines
1.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Does the judge actually FAIL when the artifact lies?
|
|
|
|
A verdict that cannot go nonzero is a tautology. This plants a real defect --
|
|
rewriting the runner's PASS to UNVERIFIED -- and asserts the judge exits
|
|
nonzero. Exit 0 == the judge has teeth.
|
|
"""
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
ART = os.path.join(os.path.dirname(os.path.abspath(__file__)), "proveit.py")
|
|
JUDGE = os.path.abspath(__file__)
|
|
|
|
src = open(ART, encoding="utf-8").read()
|
|
# Plant the lie: the status domain check must now see UNVERIFIED.
|
|
mutated = src.replace('status="PASS" if (ok and measured) else "FAIL"',
|
|
'status="UNVERIFIED" if (ok and measured) else "FAIL"')
|
|
assert mutated != src, "mutation did not apply -- test is invalid"
|
|
|
|
tmp = tempfile.mkdtemp(prefix="proveit-mut-")
|
|
bad = os.path.join(tmp, "proveit.py")
|
|
open(bad, "w", encoding="utf-8").write(mutated)
|
|
|
|
# run the judge against the mutated artifact by pointing ART at it
|
|
judge_src = open(JUDGE, encoding="utf-8").read().replace(
|
|
'ART = "/root/.hermes/wizard-outputs/null/proveit.py"', 'ART = %r' % bad)
|
|
badjudge = os.path.join(tmp, "_judge.py")
|
|
open(badjudge, "w", encoding="utf-8").write(judge_src)
|
|
|
|
p = subprocess.run([sys.executable, badjudge], capture_output=True, text=True)
|
|
print("mutated-artifact judge exit code:", p.returncode)
|
|
print("--- judge output (tail) ---")
|
|
print("\n".join(p.stdout.strip().splitlines()[-6:]))
|
|
|
|
if p.returncode == 0:
|
|
print()
|
|
print("TOOTHLESS: judge passed a mutated artifact -> verdict is a tautology")
|
|
sys.exit(1)
|
|
print()
|
|
print("TEETH OK: judge rejected the mutated artifact (exit %d)" % p.returncode)
|
|
sys.exit(0)
|