#!/usr/bin/env python3 """q97_check.py -- the Q97 gate: a refusal must be about the INPUT. The defect this grades was filed against `tools/selftest_sweep.py`: a probe that is refused at rc 2 with `usage: ... / error: unrecognized arguments: ...` is an argparse failure to parse the INVOCATION, not a refusal of the input. The tool never read anything, and the malformed-input axis -- the one place CLI drift could not hide -- read clean. Four legs, all measured on this tree: 1. the sweep's own selftest is green and its printed cell count equals the pinned row in `tools/selftest_baseline.tsv` (a new cell without a re-pin is a PIN_DRIFT, so the gate cannot pass on a cell nobody counted); 2. the live axes on this checkout are clean -- 0 faults on read-refusal and malformed-input -- so the new rule does not redden a tool that refuses correctly (28 + 27 probes at the landing head); 3. MUTANT: `tools/citation_hash_class.py` copied to a scratch file with its `--repo` flag renamed to `--tree` (a rename that is NOT a prefix of the old one: argparse abbreviates long options, so `--repo` -> `--repo-dir` still parses and this check refuses to pretend otherwise). Both probes that invoke it with `--repo` must read ECHOES_USAGE, and the axis must be red, rc 1; 4. CONTROL: the same copy unmutated reads no ECHOES_USAGE at all and every probe is OK -- so leg 3 is the mutation, not a file that was already red. Exit codes follow the repo contract: 0 clean, 1 graded fault, 2 could not grade (no sweep module, no tool copy, no python3). Run: python3 wizard-work/volt/q97_check.py """ import importlib.util import io import os import shutil import subprocess import sys import tempfile from contextlib import redirect_stdout HERE = os.path.dirname(os.path.realpath(__file__)) ROOT = os.path.dirname(os.path.dirname(HERE)) TOOLS = os.path.join(ROOT, "tools") SWEEP = os.path.join(TOOLS, "selftest_sweep.py") BASELINE = os.path.join(TOOLS, "selftest_baseline.tsv") SUBJECT = os.path.join(TOOLS, "citation_hash_class.py") failures = [] def note(ok, line): print(("PASS " if ok else "FAIL ") + line) if not ok: failures.append(line) def leg_selftest(): """The sweep's own cells, and the pin that counts them.""" proc = subprocess.run([sys.executable, SWEEP, "--selftest"], capture_output=True, text=True, timeout=300) out = (proc.stdout or "").splitlines() cells = [l for l in out if l.startswith("selftest:")] printed = cells[-1] if cells else "no count line" note(proc.returncode == 0 and bool(cells), "leg 1a sweep --selftest rc %d: %s" % (proc.returncode, printed)) pinned = None with open(BASELINE, encoding="utf-8") as fh: for line in fh: parts = line.split("\t") if len(parts) == 3 and parts[0] == "selftest_sweep.py": pinned = int(parts[1]) if pinned is None: note(False, "leg 1b no pinned row for selftest_sweep.py -- cannot grade") return seen = None if cells: try: seen = int(cells[-1].split()[1].replace(",", "")) except (IndexError, ValueError): seen = None note(seen == pinned, "leg 1b printed %s cells, pinned %s" % (seen, pinned)) _SWEEP = [] def sweep_module(): """`tools/selftest_sweep.py`, loaded by path so this gate grades the tree it sits in and not a copy on sys.path.""" if not _SWEEP: spec = importlib.util.spec_from_file_location("q97_selftest_sweep", SWEEP) if spec is None or spec.loader is None: raise RuntimeError("cannot load %s" % SWEEP) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) _SWEEP.append(mod) return _SWEEP[0] def axis_for(path): """The rows for one `citation_hash_class.py` copy, through the real probe path. Rows for other tools are dropped: with a one-row table every other probe is NO_TOOL, which is noise about the table, not about the copy.""" sweep = sweep_module() rows = sweep.read_axis([("citation_hash_class.py", path)], 60) return sweep, [r for r in rows if r["tool"] == "citation_hash_class.py"] def leg_mutant_and_control(): sweep, control = axis_for(SUBJECT) echo_c = [r for r in control if r["verdict"] == "ECHOES_USAGE"] bad_c = [r for r in control if r["verdict"] in sweep.FATAL_READ] note(not echo_c and not bad_c, "leg 4 control: clean copy reads %d probe(s), 0 ECHOES_USAGE, " "0 fault" % len(control)) td = tempfile.mkdtemp(prefix="q97-") try: mutant = os.path.join(td, "citation_hash_class.py") shutil.copyfile(SUBJECT, mutant) with open(mutant, encoding="utf-8") as fh: src = fh.read() old = "ap.add_argument('--repo', default=DEFAULT_REPO)" new = "ap.add_argument('--tree', dest='repo', default=DEFAULT_REPO)" if src.count(old) != 1: note(False, "leg 3 cannot build the mutant: %r appears %d time(s)" % (old, src.count(old))) return with open(mutant, "w", encoding="utf-8") as fh: fh.write(src.replace(old, new, 1)) sweep2, rows = axis_for(mutant) echo_m = [r for r in rows if r["verdict"] == "ECHOES_USAGE"] named = sorted((r["probe"] for r in echo_m)) note(named == ["missing", "not-a-repo"], "leg 3 mutant (--repo -> --tree): ECHOES_USAGE on %s" % (named,)) buf = io.StringIO() with redirect_stdout(buf): rc, hard, soft = sweep2.report_read(rows, quiet=True) line = buf.getvalue().strip().splitlines() summary = line[-1] if line else "no summary" note(rc == 1 and (hard, soft) == (1, 1) and "ECHOES_USAGE citation_hash_class.py/not-a-repo" in buf.getvalue(), "leg 3b mutant axis %s" % summary) finally: shutil.rmtree(td, ignore_errors=True) def leg_live_axes(): sweep = sweep_module() live, _source, _untracked = sweep.discover(ROOT, None) axis = sweep.read_axis(live, 60) bad = [r for r in axis if r["verdict"] in sweep.FATAL_READ] echo = [r for r in axis if r["verdict"] == "ECHOES_USAGE"] note(not bad and not echo, "leg 2 live axes: %d probe(s), 0 fault, 0 ECHOES_USAGE" % len(axis)) def main(): if not (os.path.isfile(SWEEP) and os.path.isfile(SUBJECT)): print("cannot grade: tools/selftest_sweep.py or citation_hash_class.py " "missing under %s" % ROOT, file=sys.stderr) return 2 leg_selftest() leg_live_axes() leg_mutant_and_control() if failures: print("Q97 GATE: rc 1 -- %d leg(s) failed" % len(failures)) return 1 print("Q97 GATE: rc 0 -- a refusal that echoes the invocation is fatal on " "both input axes, and a mutating flag rename is caught.") return 0 if __name__ == "__main__": sys.exit(main())