61 lines
2.2 KiB
Python
61 lines
2.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Does the judge actually FAIL when the artifact lies?
|
|
|
|
A verdict that cannot go nonzero is a tautology. This plants a real defect --
|
|
rewriting the runner's PASS to UNVERIFIED -- and asserts the judge exits
|
|
nonzero. Exit 0 == the judge has teeth.
|
|
|
|
The judge discovers its artifact relative to its own location, so the honest
|
|
way to point it at a broken artifact is to copy both files into a scratch dir
|
|
and run the copy. An earlier version of this probe instead did a string
|
|
replace on a hardcoded absolute path that no longer exists in _judge.py; the
|
|
replace silently matched nothing, so the probe's own premise was never
|
|
checked. It is now asserted: if the mutation does not apply, the probe fails
|
|
loudly instead of printing TEETH OK over an unmutated artifact.
|
|
"""
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ART = os.path.join(HERE, "proveit.py")
|
|
JUDGE = os.path.join(HERE, "_judge.py")
|
|
|
|
OLD = 'status="PASS" if (ok and measured) else "FAIL"'
|
|
NEW = 'status="UNVERIFIED" if (ok and measured) else "FAIL"'
|
|
|
|
src = open(ART, encoding="utf-8").read()
|
|
mutated = src.replace(OLD, NEW, 1)
|
|
if mutated == src:
|
|
# The anchor is gone or no longer unique: the premise of this probe is
|
|
# false, so the probe must not report success.
|
|
print("TOOTHLESS: mutation anchor %r not found in %s" % (OLD, ART))
|
|
sys.exit(1)
|
|
|
|
tmp = tempfile.mkdtemp(prefix="proveit-teeth-")
|
|
try:
|
|
bad = os.path.join(tmp, "proveit.py")
|
|
with open(bad, "w", encoding="utf-8") as fh:
|
|
fh.write(mutated)
|
|
shutil.copy2(JUDGE, os.path.join(tmp, "_judge.py"))
|
|
|
|
p = subprocess.run([sys.executable, os.path.join(tmp, "_judge.py")],
|
|
capture_output=True, text=True, cwd=tmp)
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
print("mutated-artifact judge exit code: %d" % p.returncode)
|
|
print("--- judge output (tail) ---")
|
|
for line in p.stdout.strip().splitlines()[-6:]:
|
|
print(line)
|
|
|
|
if p.returncode == 0:
|
|
print()
|
|
print("TOOTHLESS: judge passed a mutated artifact -> verdict is a tautology")
|
|
sys.exit(1)
|
|
print()
|
|
print("TEETH OK: judge rejected the mutated artifact (exit %d)" % p.returncode)
|
|
sys.exit(0)
|