143 lines
5.3 KiB
Python
143 lines
5.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
check_schedkit.py -- one command, one verdict for the schedkit artifact.
|
|
|
|
Exit 0 == the committed schedkit.py is the fixed build, the spec that grades it
|
|
is green, and that spec bites.
|
|
Exit 1 == otherwise (the failing leg is printed).
|
|
|
|
Legs
|
|
1 spec on the committed module must exit 0
|
|
2 spec on four sampled mutants each must exit non-zero
|
|
3 cost differential, one 2,000-node chain, both builds, same machine:
|
|
the fixed build must be fast and the pre-fix build slow -- the perf claim
|
|
graded by measurement, not by assertion
|
|
|
|
Usage: python3 check_schedkit.py [--mutant-budget SECONDS]
|
|
"""
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, HERE)
|
|
|
|
import mutate_teeth as teeth # noqa: E402
|
|
|
|
# Four single-substitution bugs the spec must kill. Indices into teeth.MUTANTS;
|
|
# all four are pure-behaviour mutants, so each run costs one full spec pass.
|
|
SAMPLE = [0, 1, 7, 9]
|
|
|
|
TIMING_PROBE = '''
|
|
import importlib.util, sys, time
|
|
path, n = sys.argv[1], int(sys.argv[2])
|
|
spec = importlib.util.spec_from_file_location("probe_mod", path)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
g = mod.Graph()
|
|
for i in range(n - 1):
|
|
g.add_edge("d%06d" % i, "d%06d" % (i + 1))
|
|
t0 = time.monotonic()
|
|
cyc = mod.find_cycles(g)
|
|
print("%.4f %d" % (time.monotonic() - t0, len(cyc)))
|
|
'''
|
|
|
|
|
|
def main() -> int:
|
|
budget = 300.0
|
|
if "--mutant-budget" in sys.argv:
|
|
budget = float(sys.argv[sys.argv.index("--mutant-budget") + 1])
|
|
|
|
fails = []
|
|
print("check_schedkit -- %s" % HERE)
|
|
|
|
# -------------------------------------------------- leg 1: the spec is green
|
|
tmp = tempfile.mkdtemp(prefix="check-control-")
|
|
try:
|
|
shutil.copy(teeth.MODULE, os.path.join(tmp, "schedkit.py"))
|
|
shutil.copy(teeth.SPEC, os.path.join(tmp, "spec_schedkit.py"))
|
|
rc, out, dt = teeth.run_spec(tmp, budget)
|
|
print("1 spec on committed module rc=%s %.1fs %s" % (rc, dt, teeth.summary(out)))
|
|
if rc != 0:
|
|
fails.append("leg 1: the spec did not pass on the committed module")
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
# ------------------------------------------------- leg 2: the spec has teeth
|
|
src = open(teeth.MODULE).read()
|
|
for idx in SAMPLE:
|
|
label, old, new = teeth.MUTANTS[idx]
|
|
if src.count(old) != 1:
|
|
fails.append("leg 2: mutant %d anchor is not unique" % idx)
|
|
print("2 mutant %-2d ANCHOR MISSING %s" % (idx, label))
|
|
continue
|
|
tmp = tempfile.mkdtemp(prefix="check-m%02d-" % idx)
|
|
try:
|
|
with open(os.path.join(tmp, "schedkit.py"), "w") as fh:
|
|
fh.write(src.replace(old, new))
|
|
shutil.copy(teeth.SPEC, os.path.join(tmp, "spec_schedkit.py"))
|
|
rc, out, dt = teeth.run_spec(tmp, budget)
|
|
ok = rc is None or rc != 0
|
|
print(
|
|
"2 mutant %-2d %-8s %.1fs %s"
|
|
% (idx, "CAUGHT" if ok else "SURVIVED", dt, label)
|
|
)
|
|
if not ok:
|
|
fails.append("leg 2: mutant %d survived: %s" % (idx, label))
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
# ------------------------------------- leg 3: measured cost of both builds
|
|
tmp = tempfile.mkdtemp(prefix="check-timing-")
|
|
try:
|
|
probe = os.path.join(tmp, "probe.py")
|
|
with open(probe, "w") as fh:
|
|
fh.write(TIMING_PROBE)
|
|
n = 2000
|
|
times = {}
|
|
for name, mod in (("fixed", teeth.MODULE), ("pre-fix", teeth.BASELINE)):
|
|
if not os.path.exists(mod):
|
|
fails.append("leg 3: missing module %s" % mod)
|
|
continue
|
|
p = subprocess.run(
|
|
[sys.executable, probe, mod, str(n)],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT,
|
|
timeout=300,
|
|
text=True,
|
|
)
|
|
if p.returncode != 0:
|
|
fails.append("leg 3: probe on %s failed: %s" % (name, p.stdout.strip()))
|
|
continue
|
|
secs, cycles = p.stdout.split()
|
|
times[name] = float(secs)
|
|
print("3 %-7s chain n=%d find_cycles %.4fs, %s cycles" % (name, n, float(secs), cycles))
|
|
if cycles != "0":
|
|
fails.append("leg 3: %s reported %s cycles on an acyclic chain" % (name, cycles))
|
|
if "fixed" in times and times["fixed"] >= 0.5:
|
|
fails.append("leg 3: fixed build took %.3fs on a 2,000-node chain" % times["fixed"])
|
|
if "pre-fix" in times and times["pre-fix"] <= 1.0:
|
|
fails.append(
|
|
"leg 3: pre-fix build was fast (%.3fs) -- the quadratic claim is not reproducible"
|
|
% times["pre-fix"]
|
|
)
|
|
if len(times) == 2:
|
|
print(" ratio: %.0fx slower before the fix" % (times["pre-fix"] / times["fixed"]))
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
if fails:
|
|
print("\nCHECK FAILED (%d):" % len(fails))
|
|
for f in fails:
|
|
print(" - %s" % f)
|
|
return 1
|
|
print("\nCHECK OK: spec green on the committed module, 4/4 sampled mutants killed, cost differential reproduced")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|