178 lines
6.9 KiB
Python
178 lines
6.9 KiB
Python
#!/usr/bin/env python3
|
|
"""q97_check.py -- the Q97 gate: a refusal must be about the INPUT.
|
|
|
|
The defect this grades was filed against `tools/selftest_sweep.py`: a probe
|
|
that is refused at rc 2 with `usage: ... / error: unrecognized arguments: ...`
|
|
is an argparse failure to parse the INVOCATION, not a refusal of the input.
|
|
The tool never read anything, and the malformed-input axis -- the one place CLI
|
|
drift could not hide -- read clean.
|
|
|
|
Four legs, all measured on this tree:
|
|
|
|
1. the sweep's own selftest is green and its printed cell count equals the
|
|
pinned row in `tools/selftest_baseline.tsv` (a new cell without a re-pin
|
|
is a PIN_DRIFT, so the gate cannot pass on a cell nobody counted);
|
|
2. the live axes on this checkout are clean -- 0 faults on read-refusal and
|
|
malformed-input -- so the new rule does not redden a tool that refuses
|
|
correctly (28 + 27 probes at the landing head);
|
|
3. MUTANT: `tools/citation_hash_class.py` copied to a scratch file with its
|
|
`--repo` flag renamed to `--tree` (a rename that is NOT a prefix of the
|
|
old one: argparse abbreviates long options, so `--repo` -> `--repo-dir`
|
|
still parses and this check refuses to pretend otherwise). Both probes
|
|
that invoke it with `--repo` must read ECHOES_USAGE, and the axis must be
|
|
red, rc 1;
|
|
4. CONTROL: the same copy unmutated reads no ECHOES_USAGE at all and every
|
|
probe is OK -- so leg 3 is the mutation, not a file that was already red.
|
|
|
|
Exit codes follow the repo contract: 0 clean, 1 graded fault, 2 could not
|
|
grade (no sweep module, no tool copy, no python3).
|
|
|
|
Run: python3 wizard-work/volt/q97_check.py
|
|
"""
|
|
import importlib.util
|
|
import io
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from contextlib import redirect_stdout
|
|
|
|
HERE = os.path.dirname(os.path.realpath(__file__))
|
|
ROOT = os.path.dirname(os.path.dirname(HERE))
|
|
TOOLS = os.path.join(ROOT, "tools")
|
|
SWEEP = os.path.join(TOOLS, "selftest_sweep.py")
|
|
BASELINE = os.path.join(TOOLS, "selftest_baseline.tsv")
|
|
SUBJECT = os.path.join(TOOLS, "citation_hash_class.py")
|
|
|
|
failures = []
|
|
|
|
|
|
def note(ok, line):
|
|
print(("PASS " if ok else "FAIL ") + line)
|
|
if not ok:
|
|
failures.append(line)
|
|
|
|
|
|
def leg_selftest():
|
|
"""The sweep's own cells, and the pin that counts them."""
|
|
proc = subprocess.run([sys.executable, SWEEP, "--selftest"],
|
|
capture_output=True, text=True, timeout=300)
|
|
out = (proc.stdout or "").splitlines()
|
|
cells = [l for l in out if l.startswith("selftest:")]
|
|
printed = cells[-1] if cells else "no count line"
|
|
note(proc.returncode == 0 and bool(cells),
|
|
"leg 1a sweep --selftest rc %d: %s"
|
|
% (proc.returncode, printed))
|
|
pinned = None
|
|
with open(BASELINE, encoding="utf-8") as fh:
|
|
for line in fh:
|
|
parts = line.split("\t")
|
|
if len(parts) == 3 and parts[0] == "selftest_sweep.py":
|
|
pinned = int(parts[1])
|
|
if pinned is None:
|
|
note(False, "leg 1b no pinned row for selftest_sweep.py -- cannot grade")
|
|
return
|
|
seen = None
|
|
if cells:
|
|
try:
|
|
seen = int(cells[-1].split()[1].replace(",", ""))
|
|
except (IndexError, ValueError):
|
|
seen = None
|
|
note(seen == pinned,
|
|
"leg 1b printed %s cells, pinned %s" % (seen, pinned))
|
|
|
|
|
|
_SWEEP = []
|
|
|
|
|
|
def sweep_module():
|
|
"""`tools/selftest_sweep.py`, loaded by path so this gate grades the tree
|
|
it sits in and not a copy on sys.path."""
|
|
if not _SWEEP:
|
|
spec = importlib.util.spec_from_file_location("q97_selftest_sweep", SWEEP)
|
|
if spec is None or spec.loader is None:
|
|
raise RuntimeError("cannot load %s" % SWEEP)
|
|
mod = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(mod)
|
|
_SWEEP.append(mod)
|
|
return _SWEEP[0]
|
|
|
|
|
|
def axis_for(path):
|
|
"""The rows for one `citation_hash_class.py` copy, through the real probe
|
|
path. Rows for other tools are dropped: with a one-row table every other
|
|
probe is NO_TOOL, which is noise about the table, not about the copy."""
|
|
sweep = sweep_module()
|
|
rows = sweep.read_axis([("citation_hash_class.py", path)], 60)
|
|
return sweep, [r for r in rows if r["tool"] == "citation_hash_class.py"]
|
|
|
|
|
|
def leg_mutant_and_control():
|
|
sweep, control = axis_for(SUBJECT)
|
|
echo_c = [r for r in control if r["verdict"] == "ECHOES_USAGE"]
|
|
bad_c = [r for r in control if r["verdict"] in sweep.FATAL_READ]
|
|
note(not echo_c and not bad_c,
|
|
"leg 4 control: clean copy reads %d probe(s), 0 ECHOES_USAGE, "
|
|
"0 fault" % len(control))
|
|
|
|
td = tempfile.mkdtemp(prefix="q97-")
|
|
try:
|
|
mutant = os.path.join(td, "citation_hash_class.py")
|
|
shutil.copyfile(SUBJECT, mutant)
|
|
with open(mutant, encoding="utf-8") as fh:
|
|
src = fh.read()
|
|
old = "ap.add_argument('--repo', default=DEFAULT_REPO)"
|
|
new = "ap.add_argument('--tree', dest='repo', default=DEFAULT_REPO)"
|
|
if src.count(old) != 1:
|
|
note(False, "leg 3 cannot build the mutant: %r appears %d time(s)"
|
|
% (old, src.count(old)))
|
|
return
|
|
with open(mutant, "w", encoding="utf-8") as fh:
|
|
fh.write(src.replace(old, new, 1))
|
|
sweep2, rows = axis_for(mutant)
|
|
echo_m = [r for r in rows if r["verdict"] == "ECHOES_USAGE"]
|
|
named = sorted((r["probe"] for r in echo_m))
|
|
note(named == ["missing", "not-a-repo"],
|
|
"leg 3 mutant (--repo -> --tree): ECHOES_USAGE on %s" % (named,))
|
|
buf = io.StringIO()
|
|
with redirect_stdout(buf):
|
|
rc, hard, soft = sweep2.report_read(rows, quiet=True)
|
|
line = buf.getvalue().strip().splitlines()
|
|
summary = line[-1] if line else "no summary"
|
|
note(rc == 1 and (hard, soft) == (1, 1)
|
|
and "ECHOES_USAGE citation_hash_class.py/not-a-repo" in buf.getvalue(),
|
|
"leg 3b mutant axis %s" % summary)
|
|
finally:
|
|
shutil.rmtree(td, ignore_errors=True)
|
|
|
|
|
|
def leg_live_axes():
|
|
sweep = sweep_module()
|
|
live, _source, _untracked = sweep.discover(ROOT, None)
|
|
axis = sweep.read_axis(live, 60)
|
|
bad = [r for r in axis if r["verdict"] in sweep.FATAL_READ]
|
|
echo = [r for r in axis if r["verdict"] == "ECHOES_USAGE"]
|
|
note(not bad and not echo,
|
|
"leg 2 live axes: %d probe(s), 0 fault, 0 ECHOES_USAGE" % len(axis))
|
|
|
|
|
|
def main():
|
|
if not (os.path.isfile(SWEEP) and os.path.isfile(SUBJECT)):
|
|
print("cannot grade: tools/selftest_sweep.py or citation_hash_class.py "
|
|
"missing under %s" % ROOT, file=sys.stderr)
|
|
return 2
|
|
leg_selftest()
|
|
leg_live_axes()
|
|
leg_mutant_and_control()
|
|
if failures:
|
|
print("Q97 GATE: rc 1 -- %d leg(s) failed" % len(failures))
|
|
return 1
|
|
print("Q97 GATE: rc 0 -- a refusal that echoes the invocation is fatal on "
|
|
"both input axes, and a mutating flag rename is caught.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|