115 lines
5.0 KiB
Python
115 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
|
"""_hunt_blindspots.py -- find regressions the judge cannot see.
|
|
|
|
The mutation battery in _probe_mutations.py proves the judge catches a *known*
|
|
set of defects. This file asks the open question: what plausible regressions
|
|
slip past the judge entirely? It applies each candidate to a throwaway copy of
|
|
proveit.py, runs the judge against that copy, and prints which ones were MISSED.
|
|
|
|
Measured on 2026-10-20 (before contracts 12-15 were added), the judge exit code
|
|
was 0 -- i.e. the defect was invisible -- for each of these:
|
|
|
|
truthy non-bool accepted as PASS (contract 12)
|
|
unrelated raise counts as controlled (contract 13)
|
|
controlled-exception check uses type() not isinstance()(contract 14)
|
|
harness fault counter never incremented (contract 15)
|
|
claim drops the error text from detail (no contract; detail
|
|
is not a documented
|
|
claim -- out of scope)
|
|
seconds always zero (no contract; timing is
|
|
not a documented
|
|
claim -- out of scope)
|
|
status ignores measured (equivalent: measured
|
|
is False only when a
|
|
raise set ok=False)
|
|
selfcheck always returns OK (unmeasurable from
|
|
outside: it is the
|
|
meta-test's own
|
|
verdict; reported, not
|
|
faked)
|
|
|
|
The four in-scope holes are now pinned by contracts 12-15 and by four matching
|
|
entries in _probe_mutations.py. Exit code: 0 == no in-scope hole remains
|
|
unpinned; 1 == a hole listed as pinned is still being missed (the pin broke).
|
|
"""
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
ART = os.path.join(HERE, "proveit.py")
|
|
JUDGE = os.path.join(HERE, "_judge.py")
|
|
|
|
# (name, old, new, in_scope)
|
|
CANDIDATES = [
|
|
("truthy non-bool accepted as PASS",
|
|
"ok = isinstance(value, bool) and value is True",
|
|
"ok = bool(value)", True),
|
|
("unrelated raise counts as controlled",
|
|
"ok = isinstance(exc, expect_exception) if expect_exception else False",
|
|
"ok = True if expect_exception else False", True),
|
|
("controlled-exception check uses type() not isinstance()",
|
|
"ok = isinstance(exc, expect_exception) if expect_exception else False",
|
|
"ok = type(exc) is expect_exception if expect_exception else False", True),
|
|
("harness fault counter never incremented",
|
|
" FAILURES += 1",
|
|
" pass # mutation", True),
|
|
("claim drops the error text from detail",
|
|
"detail=last_error or repr(value),",
|
|
"detail=repr(value),", False),
|
|
("seconds always zero",
|
|
"seconds=elapsed,",
|
|
"seconds=0.0,", False),
|
|
("status ignores measured",
|
|
'status="PASS" if (ok and measured) else "FAIL"',
|
|
'status="PASS" if ok else "FAIL"', False),
|
|
("selfcheck always returns OK",
|
|
" if problems:\n print(\"SELFCHECK FAILED:\")",
|
|
" if False:\n print(\"SELFCHECK FAILED:\")", False),
|
|
]
|
|
|
|
|
|
def run(cmd, cwd=None):
|
|
p = subprocess.run(cmd, capture_output=True, text=True, cwd=cwd)
|
|
return p.returncode, p.stdout, p.stderr
|
|
|
|
|
|
def main():
|
|
with open(ART, "r", encoding="utf-8") as fh:
|
|
src = fh.read()
|
|
broken_pins = []
|
|
for name, old, new, in_scope in CANDIDATES:
|
|
if src.count(old) != 1:
|
|
print(" ANCHOR? %-52s count=%d" % (name, src.count(old)))
|
|
if in_scope:
|
|
broken_pins.append(name + " (anchor not unique)")
|
|
continue
|
|
d = tempfile.mkdtemp(prefix="hunt-")
|
|
try:
|
|
with open(os.path.join(d, "proveit.py"), "w", encoding="utf-8") as fh:
|
|
fh.write(src.replace(old, new, 1))
|
|
shutil.copy2(JUDGE, os.path.join(d, "_judge.py"))
|
|
rc, out, _ = run([sys.executable, os.path.join(d, "_judge.py")], cwd=d)
|
|
finally:
|
|
shutil.rmtree(d, ignore_errors=True)
|
|
if rc == 0:
|
|
tag = "MISSED (hole)" if in_scope else "slips (out of scope)"
|
|
print(" %-18s %s" % (tag, name))
|
|
if in_scope:
|
|
broken_pins.append(name)
|
|
else:
|
|
print(" %-18s %s" % ("caught", name))
|
|
print()
|
|
if broken_pins:
|
|
print("HUNT: %d pinned defect(s) still slip past the judge: %s"
|
|
% (len(broken_pins), ", ".join(broken_pins)))
|
|
return 1
|
|
print("HUNT OK: every in-scope candidate regression is caught by the judge.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|