volt: Q90 -- tree_root.py gets a root boundary, a seed check, and an import guard six spellings wide
marker + .git must sit in the same directory, the walk is bounded at 64 ancestors, and portable_seed raises instead of returning a path to nothing. Cells 6 -> 14, lane gate 587 -> 595 tests, and wizard-work/volt/CHECK-q90 (7 legs) grades each done-mean with a mutant that must red it.
This commit is contained in:
ledger committed
1 parent
d0ba06e7de
commit
232c8af304
7 files changed
+481
-45
No files matched your search
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
# Q90 CHECK -- tree_root.py's root boundary, its seed existence, and the
|
||||
# import-time-read model, each graded by a mutant that must red it.
|
||||
#
|
||||
# rc 0 = every leg as expected; 1 = a graded fault; 2 = could not grade.
|
||||
# The work is done by wizard-work/volt/q90_verify.py beside this file; every
|
||||
# leg runs in a temp copy of the lane, so the tree this instrument ships in is
|
||||
# never mutated. Run it from any checkout.
|
||||
set -u
|
||||
REPO="$(cd "$(dirname "$0")/../.." && pwd)" || exit 2
|
||||
if ! cd "$REPO"; then
|
||||
echo "q90: could not grade -- cannot cd to $REPO"
|
||||
exit 2
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
echo "q90: could not grade -- no python3 on PATH"
|
||||
exit 2
|
||||
fi
|
||||
exec python3 wizard-work/volt/q90_verify.py
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Q90 instrument: grade tree_root.py's four done-means, and prove each cell bites.
|
||||
|
||||
The shared tree is never mutated. Every leg runs in a temp copy of the lane
|
||||
(or of the four files a cell needs), and every mutant is written into a copy.
|
||||
|
||||
Legs
|
||||
1. control small tree -- tree_root.py's 14 cells all pass under
|
||||
run_lane_tests.py, rc 0, printed count == defs
|
||||
2. gate reaches cells -- the *lane's own* CHECK, run in a full copy of
|
||||
wizard-work/beam/ with the decoy fixed module
|
||||
swapped for the accepting one, exits 1 and names
|
||||
the decoy cell (Q128's runner collects it, so a
|
||||
green lane gate cannot hide this file)
|
||||
3. decoy marker -- mutant: marker alone accepted again -> rc 1
|
||||
4. depth bound -- mutant: depth test removed -> rc 1
|
||||
5. seed existence -- mutant: isfile raise removed -> rc 1
|
||||
6. import-time read -- mutant: a real module-scope Path.read_text()
|
||||
appended to tree_root.py -> rc 1
|
||||
7. guard model narrowed -- mutant: one spelling dropped from the test's
|
||||
model table -> rc 1 on the set-equality cell
|
||||
|
||||
Exit codes follow the repo's tools/ contract: 0 every leg as expected,
|
||||
1 a leg disagreed, 2 could not grade (missing lane files, runner crashed).
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
REPO = os.path.dirname(os.path.dirname(HERE))
|
||||
LANE = os.path.join(REPO, "wizard-work", "beam")
|
||||
SMALL = ("tree_root.py", "test_tree_root.py", "beam_assert.py", "run_lane_tests.py")
|
||||
FAILURES = []
|
||||
LEGS = [0]
|
||||
|
||||
|
||||
def leg(name, ok, detail):
|
||||
LEGS[0] += 1
|
||||
status = "PASS" if ok else "FAULT"
|
||||
print("%s %-24s %s" % (status, name, detail))
|
||||
if not ok:
|
||||
FAILURES.append(name)
|
||||
|
||||
|
||||
def run(cwd, argv, timeout=300):
|
||||
p = subprocess.run(argv, cwd=cwd, capture_output=True, text=True,
|
||||
timeout=timeout)
|
||||
return p.returncode, p.stdout + p.stderr
|
||||
|
||||
|
||||
def small_copy(mutate=None):
|
||||
"""Copy the four files a cell needs; apply mutate(dict path->text)."""
|
||||
d = tempfile.mkdtemp(prefix="q90-")
|
||||
for n in SMALL:
|
||||
shutil.copyfile(os.path.join(LANE, n), os.path.join(d, n))
|
||||
if mutate:
|
||||
for name, fn in mutate.items():
|
||||
path = os.path.join(d, name)
|
||||
with open(path) as fh:
|
||||
text = fh.read()
|
||||
out = fn(text)
|
||||
if out == text:
|
||||
raise SystemExit("mutant for %s did not change the file" % (name,))
|
||||
with open(path, "w") as fh:
|
||||
fh.write(out)
|
||||
return d
|
||||
|
||||
|
||||
def lane_copy(mutate=None):
|
||||
d = tempfile.mkdtemp(prefix="q90-lane-")
|
||||
dest = os.path.join(d, "beam")
|
||||
shutil.copytree(LANE, dest, ignore=shutil.ignore_patterns("__pycache__"))
|
||||
if mutate:
|
||||
for name, fn in mutate.items():
|
||||
path = os.path.join(dest, name)
|
||||
with open(path) as fh:
|
||||
text = fh.read()
|
||||
with open(path, "w") as fh:
|
||||
fh.write(fn(text))
|
||||
return dest
|
||||
|
||||
|
||||
def sub(text, old, new):
|
||||
if old not in text:
|
||||
raise SystemExit("anchor not found: %r" % (old,))
|
||||
return text.replace(old, new, 1)
|
||||
|
||||
|
||||
def defs(path):
|
||||
with open(path) as fh:
|
||||
return re.findall(r"^def (test_\w+)\(\):", fh.read(), re.M)
|
||||
|
||||
|
||||
def main():
|
||||
for n in SMALL:
|
||||
if not os.path.isfile(os.path.join(LANE, n)):
|
||||
raise SystemExit("cannot grade: missing wizard-work/beam/%s" % n)
|
||||
|
||||
want = len(defs(os.path.join(LANE, "test_tree_root.py")))
|
||||
if want < 10:
|
||||
raise SystemExit("cannot grade: test_tree_root.py exposes %d cells" % want)
|
||||
|
||||
# 1. control
|
||||
d = small_copy()
|
||||
rc, out = run(d, [sys.executable, "run_lane_tests.py"])
|
||||
m = re.search(r"(\d+) bare case\(s\)", out)
|
||||
got = int(m.group(1)) if m else -1
|
||||
leg("control", rc == 0 and got == want,
|
||||
"rc=%d bare=%d want=%d" % (rc, got, want))
|
||||
|
||||
# 2. the lane's own gate must reach these cells
|
||||
d = lane_copy({"tree_root.py": lambda t: sub(
|
||||
t,
|
||||
'if not require_git or os.path.exists(os.path.join(cur, ".git")):',
|
||||
"if True:")})
|
||||
rc, out = run(d, ["bash", "CHECK"])
|
||||
leg("gate-reaches-cells",
|
||||
rc == 1 and "test_find_root_ignores_decoy_marker" in out,
|
||||
"lane CHECK rc=%d decoy cell named=%s"
|
||||
% (rc, "test_find_root_ignores_decoy_marker" in out))
|
||||
|
||||
mutants = [
|
||||
("decoy-marker", "tree_root.py", lambda t: sub(
|
||||
t,
|
||||
'if not require_git or os.path.exists(os.path.join(cur, ".git")):',
|
||||
"if True:"), "test_find_root_ignores_decoy_marker"),
|
||||
("depth-bound", "tree_root.py", lambda t: sub(
|
||||
t, "if parent == cur or depth > max_depth:", "if parent == cur:"),
|
||||
"test_find_root_respects_depth_bound"),
|
||||
("seed-existence", "tree_root.py", lambda t: sub(
|
||||
t, "if not os.path.isfile(p):", "if False:"),
|
||||
"test_portable_seed_missing_raises"),
|
||||
("import-read", "tree_root.py", lambda t: (
|
||||
t + "\nfrom pathlib import Path as _ProbePath\n"
|
||||
"_ProbePath(__file__).read_text()\n"), "test_import_no_io"),
|
||||
("model-narrowed", "test_tree_root.py", lambda t: sub(
|
||||
t, ' "Path.read_text": "from pathlib import Path\\n'
|
||||
'Path(__file__).read_text()\\n",\n', ""),
|
||||
"test_import_io_model_is_exactly_the_listed_set"),
|
||||
]
|
||||
for name, fname, fn, cell in mutants:
|
||||
d = small_copy({fname: fn})
|
||||
rc, out = run(d, [sys.executable, "run_lane_tests.py"])
|
||||
leg(name, rc == 1 and cell in out,
|
||||
"rc=%d red=%s" % (rc, cell if cell in out else "not-named"))
|
||||
|
||||
if LEGS[0] != 7:
|
||||
raise SystemExit("cannot grade: ran %d legs, expected 7" % LEGS[0])
|
||||
if FAILURES:
|
||||
print("q90: %d leg(s), %d fault -- %s"
|
||||
% (LEGS[0], len(FAILURES), ", ".join(FAILURES)))
|
||||
return 1
|
||||
print("q90: %d leg(s), 0 fault -- OK" % LEGS[0])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except subprocess.TimeoutExpired:
|
||||
print("q90: could not grade -- a leg timed out")
|
||||
sys.exit(2)
|
||||
Reference in new issue
Block a user