127 lines
4.8 KiB
Python
127 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Mutation harness for deporder: does the test suite actually have teeth?
|
|
|
|
Each mutation is a literal source edit to deporder.py, applied in an isolated
|
|
temp directory holding a copy of deporder.py + test_deporder.py. A mutation is
|
|
"caught" when the suite exits non-zero on the mutant.
|
|
|
|
Exit code is 0 only when every mutation labelled expect="caught" is caught and
|
|
every mutation labelled expect="equivalent" survives (a documented redundant
|
|
guard, not a hole in the tests).
|
|
|
|
python3 mutate_teeth.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
SRC = os.path.join(HERE, "deporder.py")
|
|
TESTS = os.path.join(HERE, "test_deporder.py")
|
|
|
|
# (label, old, new, expectation, why)
|
|
MUTATIONS = [
|
|
("waves: drop the +1 level bump",
|
|
"level[n] = 1 + max(level[d] for d in deps[n])",
|
|
"level[n] = max(level[d] for d in deps[n])",
|
|
"caught", "every task collapses into wave 0"),
|
|
("order: pop the heap tail instead of the min",
|
|
"n = heapq.heappop(ready)",
|
|
"n = ready.pop()",
|
|
"caught", "lexicographic tie-break lost in Kahn"),
|
|
("cycles: disable the residual-node check",
|
|
"if len(order) != len(deps):",
|
|
"if False:",
|
|
"caught", "cycle silently yields a partial order"),
|
|
("cycle path: report a single node",
|
|
"return path[path.index(nxt):]",
|
|
"return [nxt]",
|
|
"caught", "reported 'cycle' is not a cycle"),
|
|
("unknown deps: stop validating",
|
|
"if unknown:",
|
|
"if False:",
|
|
"caught", "ghost dependency accepted"),
|
|
("spec: allow est == 0",
|
|
"if not (est > 0.0) or est == INF:",
|
|
"if not (est >= 0.0) or est == INF:",
|
|
"caught", "zero-duration task accepted"),
|
|
("critical path: break the predecessor tie-break",
|
|
"if best_d is None or dist[d] > best or (dist[d] == best and d < best_d):",
|
|
"if best_d is None or dist[d] >= best:",
|
|
"caught", "path no longer lexicographically canonical"),
|
|
("schedule: release completions strictly before now",
|
|
"while running and running[0][0] <= now:",
|
|
"while running and running[0][0] < now:",
|
|
"caught", "worker never freed / clock stalls"),
|
|
("schedule: double every task duration",
|
|
"end = now + est[name]",
|
|
"end = now + est[name] * 2",
|
|
"caught", "makespan and critical-path bound violated"),
|
|
("schedule: pin the clock to itself (kill the stall guard)",
|
|
"nxt = running[0][0] if running else None",
|
|
"nxt = now",
|
|
"caught", "guard fires: stalled scheduler must not spin"),
|
|
("graph: drop the redundant set() in build_graph",
|
|
'deps = {n: tuple(sorted(set(tasks[n]["deps"]))) for n in names}',
|
|
'deps = {n: tuple(sorted(tasks[n]["deps"])) for n in names}',
|
|
"equivalent", "parse_spec already deduplicates; guard is redundant"),
|
|
]
|
|
|
|
|
|
def run_mutant(label, old, new, timeout=300):
|
|
tmp = tempfile.mkdtemp(prefix="deporder_mut_")
|
|
try:
|
|
src = open(SRC, "r", encoding="utf-8").read()
|
|
hits = src.count(old)
|
|
if hits == 0:
|
|
return None, "ANCHOR MISSING"
|
|
mutated = src.replace(old, new)
|
|
with open(os.path.join(tmp, "deporder.py"), "w", encoding="utf-8") as fh:
|
|
fh.write(mutated)
|
|
shutil.copy(TESTS, os.path.join(tmp, "test_deporder.py"))
|
|
try:
|
|
proc = subprocess.run([sys.executable, "test_deporder.py"], cwd=tmp,
|
|
capture_output=True, text=True, timeout=timeout)
|
|
except subprocess.TimeoutExpired:
|
|
return True, "timeout/hang -> caught"
|
|
if proc.returncode == 0:
|
|
return False, "suite PASSED on mutant (not caught)"
|
|
tail = [ln for ln in proc.stderr.strip().splitlines() if ln.strip()]
|
|
detail = tail[-1] if tail else "nonzero exit"
|
|
return True, "exit %d | %s" % (proc.returncode, detail[:90])
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
|
|
def main():
|
|
print("deporder mutation harness: %d mutations" % len(MUTATIONS))
|
|
print("-" * 100)
|
|
bad = 0
|
|
caught = 0
|
|
for label, old, new, expect, why in MUTATIONS:
|
|
got, detail = run_mutant(label, old, new)
|
|
if got is None:
|
|
bad += 1
|
|
status = "BROKEN (%s)" % detail
|
|
else:
|
|
ok = (got and expect == "caught") or (not got and expect == "equivalent")
|
|
bad += 0 if ok else 1
|
|
caught += 1 if got else 0
|
|
status = ("CAUGHT " if got else "SURVIVED ") + detail
|
|
if not ok:
|
|
status += " <-- UNEXPECTED (expect=%s)" % expect
|
|
print("%-46s %s" % (label, status))
|
|
print("%-46s why: %s" % ("", why))
|
|
print("-" * 100)
|
|
print("caught %d/%d mutants that must die; %d problem(s)" % (caught, len(MUTATIONS), bad))
|
|
return 1 if bad else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|