#!/usr/bin/env python3 """Does the judge actually FAIL when the artifact lies? A verdict that cannot go nonzero is a tautology. This plants a real defect -- rewriting the runner's PASS to UNVERIFIED -- and asserts the judge exits nonzero. Exit 0 == the judge has teeth. The judge discovers its artifact relative to its own location, so the honest way to point it at a broken artifact is to copy both files into a scratch dir and run the copy. An earlier version of this probe instead did a string replace on a hardcoded absolute path that no longer exists in _judge.py; the replace silently matched nothing, so the probe's own premise was never checked. It is now asserted: if the mutation does not apply, the probe fails loudly instead of printing TEETH OK over an unmutated artifact. """ import os import shutil import subprocess import sys import tempfile HERE = os.path.dirname(os.path.abspath(__file__)) ART = os.path.join(HERE, "proveit.py") JUDGE = os.path.join(HERE, "_judge.py") OLD = 'status="PASS" if (ok and measured) else "FAIL"' NEW = 'status="UNVERIFIED" if (ok and measured) else "FAIL"' src = open(ART, encoding="utf-8").read() mutated = src.replace(OLD, NEW, 1) if mutated == src: # The anchor is gone or no longer unique: the premise of this probe is # false, so the probe must not report success. print("TOOTHLESS: mutation anchor %r not found in %s" % (OLD, ART)) sys.exit(1) tmp = tempfile.mkdtemp(prefix="proveit-teeth-") try: bad = os.path.join(tmp, "proveit.py") with open(bad, "w", encoding="utf-8") as fh: fh.write(mutated) shutil.copy2(JUDGE, os.path.join(tmp, "_judge.py")) p = subprocess.run([sys.executable, os.path.join(tmp, "_judge.py")], capture_output=True, text=True, cwd=tmp) finally: shutil.rmtree(tmp, ignore_errors=True) print("mutated-artifact judge exit code: %d" % p.returncode) print("--- judge output (tail) ---") for line in p.stdout.strip().splitlines()[-6:]: print(line) if p.returncode == 0: print() print("TOOTHLESS: judge passed a mutated artifact -> verdict is a tautology") sys.exit(1) print() print("TEETH OK: judge rejected the mutated artifact (exit %d)" % p.returncode) sys.exit(0)