ledger
232c8af304
marker + .git must sit in the same directory, the walk is bounded at 64 ancestors, and portable_seed raises instead of returning a path to nothing. Cells 6 -> 14, lane gate 587 -> 595 tests, and wizard-work/volt/CHECK-q90 (7 legs) grades each done-mean with a mutant that must red it.
168 lines
6.1 KiB
Python
168 lines
6.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Q90 instrument: grade tree_root.py's four done-means, and prove each cell bites.
|
|
|
|
The shared tree is never mutated. Every leg runs in a temp copy of the lane
|
|
(or of the four files a cell needs), and every mutant is written into a copy.
|
|
|
|
Legs
|
|
1. control small tree -- tree_root.py's 14 cells all pass under
|
|
run_lane_tests.py, rc 0, printed count == defs
|
|
2. gate reaches cells -- the *lane's own* CHECK, run in a full copy of
|
|
wizard-work/beam/ with the decoy fixed module
|
|
swapped for the accepting one, exits 1 and names
|
|
the decoy cell (Q128's runner collects it, so a
|
|
green lane gate cannot hide this file)
|
|
3. decoy marker -- mutant: marker alone accepted again -> rc 1
|
|
4. depth bound -- mutant: depth test removed -> rc 1
|
|
5. seed existence -- mutant: isfile raise removed -> rc 1
|
|
6. import-time read -- mutant: a real module-scope Path.read_text()
|
|
appended to tree_root.py -> rc 1
|
|
7. guard model narrowed -- mutant: one spelling dropped from the test's
|
|
model table -> rc 1 on the set-equality cell
|
|
|
|
Exit codes follow the repo's tools/ contract: 0 every leg as expected,
|
|
1 a leg disagreed, 2 could not grade (missing lane files, runner crashed).
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
REPO = os.path.dirname(os.path.dirname(HERE))
|
|
LANE = os.path.join(REPO, "wizard-work", "beam")
|
|
SMALL = ("tree_root.py", "test_tree_root.py", "beam_assert.py", "run_lane_tests.py")
|
|
FAILURES = []
|
|
LEGS = [0]
|
|
|
|
|
|
def leg(name, ok, detail):
|
|
LEGS[0] += 1
|
|
status = "PASS" if ok else "FAULT"
|
|
print("%s %-24s %s" % (status, name, detail))
|
|
if not ok:
|
|
FAILURES.append(name)
|
|
|
|
|
|
def run(cwd, argv, timeout=300):
|
|
p = subprocess.run(argv, cwd=cwd, capture_output=True, text=True,
|
|
timeout=timeout)
|
|
return p.returncode, p.stdout + p.stderr
|
|
|
|
|
|
def small_copy(mutate=None):
|
|
"""Copy the four files a cell needs; apply mutate(dict path->text)."""
|
|
d = tempfile.mkdtemp(prefix="q90-")
|
|
for n in SMALL:
|
|
shutil.copyfile(os.path.join(LANE, n), os.path.join(d, n))
|
|
if mutate:
|
|
for name, fn in mutate.items():
|
|
path = os.path.join(d, name)
|
|
with open(path) as fh:
|
|
text = fh.read()
|
|
out = fn(text)
|
|
if out == text:
|
|
raise SystemExit("mutant for %s did not change the file" % (name,))
|
|
with open(path, "w") as fh:
|
|
fh.write(out)
|
|
return d
|
|
|
|
|
|
def lane_copy(mutate=None):
|
|
d = tempfile.mkdtemp(prefix="q90-lane-")
|
|
dest = os.path.join(d, "beam")
|
|
shutil.copytree(LANE, dest, ignore=shutil.ignore_patterns("__pycache__"))
|
|
if mutate:
|
|
for name, fn in mutate.items():
|
|
path = os.path.join(dest, name)
|
|
with open(path) as fh:
|
|
text = fh.read()
|
|
with open(path, "w") as fh:
|
|
fh.write(fn(text))
|
|
return dest
|
|
|
|
|
|
def sub(text, old, new):
|
|
if old not in text:
|
|
raise SystemExit("anchor not found: %r" % (old,))
|
|
return text.replace(old, new, 1)
|
|
|
|
|
|
def defs(path):
|
|
with open(path) as fh:
|
|
return re.findall(r"^def (test_\w+)\(\):", fh.read(), re.M)
|
|
|
|
|
|
def main():
|
|
for n in SMALL:
|
|
if not os.path.isfile(os.path.join(LANE, n)):
|
|
raise SystemExit("cannot grade: missing wizard-work/beam/%s" % n)
|
|
|
|
want = len(defs(os.path.join(LANE, "test_tree_root.py")))
|
|
if want < 10:
|
|
raise SystemExit("cannot grade: test_tree_root.py exposes %d cells" % want)
|
|
|
|
# 1. control
|
|
d = small_copy()
|
|
rc, out = run(d, [sys.executable, "run_lane_tests.py"])
|
|
m = re.search(r"(\d+) bare case\(s\)", out)
|
|
got = int(m.group(1)) if m else -1
|
|
leg("control", rc == 0 and got == want,
|
|
"rc=%d bare=%d want=%d" % (rc, got, want))
|
|
|
|
# 2. the lane's own gate must reach these cells
|
|
d = lane_copy({"tree_root.py": lambda t: sub(
|
|
t,
|
|
'if not require_git or os.path.exists(os.path.join(cur, ".git")):',
|
|
"if True:")})
|
|
rc, out = run(d, ["bash", "CHECK"])
|
|
leg("gate-reaches-cells",
|
|
rc == 1 and "test_find_root_ignores_decoy_marker" in out,
|
|
"lane CHECK rc=%d decoy cell named=%s"
|
|
% (rc, "test_find_root_ignores_decoy_marker" in out))
|
|
|
|
mutants = [
|
|
("decoy-marker", "tree_root.py", lambda t: sub(
|
|
t,
|
|
'if not require_git or os.path.exists(os.path.join(cur, ".git")):',
|
|
"if True:"), "test_find_root_ignores_decoy_marker"),
|
|
("depth-bound", "tree_root.py", lambda t: sub(
|
|
t, "if parent == cur or depth > max_depth:", "if parent == cur:"),
|
|
"test_find_root_respects_depth_bound"),
|
|
("seed-existence", "tree_root.py", lambda t: sub(
|
|
t, "if not os.path.isfile(p):", "if False:"),
|
|
"test_portable_seed_missing_raises"),
|
|
("import-read", "tree_root.py", lambda t: (
|
|
t + "\nfrom pathlib import Path as _ProbePath\n"
|
|
"_ProbePath(__file__).read_text()\n"), "test_import_no_io"),
|
|
("model-narrowed", "test_tree_root.py", lambda t: sub(
|
|
t, ' "Path.read_text": "from pathlib import Path\\n'
|
|
'Path(__file__).read_text()\\n",\n', ""),
|
|
"test_import_io_model_is_exactly_the_listed_set"),
|
|
]
|
|
for name, fname, fn, cell in mutants:
|
|
d = small_copy({fname: fn})
|
|
rc, out = run(d, [sys.executable, "run_lane_tests.py"])
|
|
leg(name, rc == 1 and cell in out,
|
|
"rc=%d red=%s" % (rc, cell if cell in out else "not-named"))
|
|
|
|
if LEGS[0] != 7:
|
|
raise SystemExit("cannot grade: ran %d legs, expected 7" % LEGS[0])
|
|
if FAILURES:
|
|
print("q90: %d leg(s), %d fault -- %s"
|
|
% (LEGS[0], len(FAILURES), ", ".join(FAILURES)))
|
|
return 1
|
|
print("q90: %d leg(s), 0 fault -- OK" % LEGS[0])
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.exit(main())
|
|
except subprocess.TimeoutExpired:
|
|
print("q90: could not grade -- a leg timed out")
|
|
sys.exit(2)
|