diff --git a/wizard-work/volt/q63_verify_scan.py b/wizard-work/volt/q63_verify_scan.py index 9a0d8a2..1970211 100644 --- a/wizard-work/volt/q63_verify_scan.py +++ b/wizard-work/volt/q63_verify_scan.py @@ -66,6 +66,7 @@ Usage: python3 wizard-work/volt/q63_verify_scan.py --selftest """ import argparse +import ast import json import os import re @@ -74,12 +75,23 @@ import sys DEFAULT_BASELINE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "q63_unwired_baseline.tsv") +# Q65: the runner ledger. A file this carries a GRADED row for was executed and +# its rc recorded -- see q65_runner.py. Class `gate-run` in the baseline below. +DEFAULT_GATE = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "q65_runner_baseline.tsv") +GATE_RC = {"PASS": "0", "FAIL": "1", "REFUSED": "2"} VERIFY_RE = re.compile(r"/verify_.*\.py$") ASSERT_RE = re.compile(r"(?m)^[ \t]*assert[ \t(]") SAFE_PATH_RE = re.compile(r"\b_check\(") RAISE_RE = re.compile(r"(?m)^[ \t]*raise\b") EXIT_RE = re.compile(r"\b(?:sys\.)?exit\(\s*([^)]*?)\s*\)") -CODE_PATHSPECS = ["*.py", "*.sh", "CHECK", "*.yml", "*.yaml"] +# `CHECK` as a bare pathspec is a LITERAL path: it matches ./CHECK only, so the +# four nested `wizard-work/*/CHECK` files were outside the wiring grep until +# 2026-10-24. Measured at `2bc2662` with `:(glob)**/CHECK`: the same 7 files are +# wired, because no nested CHECK invokes a verify_*.py (they run +# `unittest discover -p 'test_*.py'`). The defect was latent, not academic: +# a nested CHECK that ran a verifier would have been read as not running it. +CODE_PATHSPECS = ["*.py", "*.sh", ":(glob)**/CHECK", "*.yml", "*.yaml"] DECLARED_RE = re.compile(r"Q63", re.I) DECLARED_PHRASE = "no committed gate invokes" # The measuring instruments name every file in the class; naming is not running. @@ -115,10 +127,96 @@ def exit_safe(src): return False +# A basename that appears only inside a string literal is a MENTION unless the +# line also carries an exec-shaped call: a command quoted in a report string runs +# nothing, the same command passed to subprocess.run() is the invocation itself. +EXEC_RE = re.compile(r"\b(?:subprocess|Popen|runpy|run_path|run_module|" + r"check_call|check_output|os\.system|os\.exec\w*|executable)\b") +_KIND_CACHE = {} + + +def _docstring_rows(tree): + """Rows covered by a module/class/function docstring -- never executed.""" + rows = set() + for node in ast.walk(tree): + if not isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef, + ast.AsyncFunctionDef)): + continue + body = getattr(node, "body", []) + if body and isinstance(body[0], ast.Expr) and \ + isinstance(body[0].value, ast.Constant) and \ + isinstance(body[0].value.value, str): + first = body[0].value + rows |= set(range(first.lineno, (first.end_lineno or first.lineno) + 1)) + return rows + + +def py_line_kinds(src): + """line number -> 'dead' | 'string', for the lines of one Python referrer. + + 'dead' the line is inside a docstring or is a comment: the interpreter + never runs it, so no shape on that line invokes anything. + 'string' a string literal covers the line (a report line, a message) -- an + invocation is still possible there, but only if the line also + carries an exec-shaped call; the caller applies that second test. + + Returns None when the source does not parse: the caller then falls back to + the grep shape rather than guessing, which is the old behaviour. + """ + import tokenize, io as _io + try: + tree = ast.parse(src) + except SyntaxError: + return None + kinds = {} + for row in _docstring_rows(tree): + kinds[row] = "dead" + for node in ast.walk(tree): + if isinstance(node, ast.Constant) and isinstance(node.value, str): + last = node.end_lineno or node.lineno + for row in range(node.lineno, last + 1): + kinds.setdefault(row, "string") + try: + for tok in tokenize.generate_tokens(_io.StringIO(src).readline): + if tok.type == tokenize.COMMENT: + row, col = tok.start + line = src.splitlines()[row - 1] if row <= len(src.splitlines()) else "" + if not line[:col].strip(): # comment-only line + kinds[row] = "dead" + except (tokenize.TokenError, IndentationError, SyntaxError): + pass + return kinds + + +def ref_kinds(repo, rev, where): + """(kind map, error) for the referrer at `rev`; cached per path.""" + key = (repo, rev, where) + if key in _KIND_CACHE: + return _KIND_CACHE[key] + try: + if rev == "WORKTREE": + src = open(os.path.join(repo, where), encoding="utf-8", + errors="replace").read() + else: + src = git(repo, "show", "%s:%s" % (rev, where)) + except (OSError, RuntimeError): + _KIND_CACHE[key] = None + return None + if where.endswith(".py"): + kinds = py_line_kinds(src) + else: + kinds = {} + for n, line in enumerate(src.splitlines(), 1): + if line.lstrip().startswith("#"): + kinds[n] = "dead" # shell/yaml comment: never run + _KIND_CACHE[key] = kinds + return kinds + + def wiring_refs(repo, rev, path, base, stem): """(invocations, mentions) for one basename at one revision. - git grep gives the lines; each line is classified. Two accuracy rules, both + git grep gives the lines; each line is classified. Three accuracy rules, all paid for by measurement rather than assumed: * a reference counts only in an INVOCATION shape (python3 X, runpy/import, @@ -128,6 +226,18 @@ def wiring_refs(repo, rev, path, base, stem): sphinx archives; a reference from another archive's directory is a reference to ITS sibling, not to this file. So a wiring reference must either name this file's parent directory, or sit in the same directory. + * THE SHAPE IS TESTED ON THE LINE TEXT, NEVER ON `::`. The + first version tested `rest` -- the whole grep record -- so an interpreter + token could come from the REFERRER'S OWN FILENAME: the prose line + `- Verdict: the sibling verify_github_python_repos.py re-queries` was + read as `python ... verify_github_python_repos.py` and wired a verifier + nothing invokes, because the referring file is itself named + `extract_github_python_repos.py`. Measured at `b01ec0e`; the line was + WIRED before this rule and is a mention after it. + + And a shape found inside unexecuted text is not a shape: `ref_kinds` marks + docstring and comment lines dead, and a string-only line is a mention unless + it also carries an exec-shaped call. """ args = ["grep", "-F", "-n", "--", base] if rev != "WORKTREE": @@ -142,15 +252,25 @@ def wiring_refs(repo, rev, path, base, stem): for ln in lines: # git grep output is REV:path:line:text at a rev, path:line:text on disk rest = ln.split(":", 1)[1] if rev != "WORKTREE" else ln - where = rest.split(":", 2)[0] + where, _, tail = rest.partition(":") + num, _, text = tail.partition(":") if where == path: continue # the file's own text is not a reference if where.startswith(INSTRUMENT_DIR): continue # the instruments that measure this class - hit = any(re.search(p.format(base=re.escape(base), stem=re.escape(stem)), rest) + hit = any(re.search(p.format(base=re.escape(base), stem=re.escape(stem)), text) for p in INVOKE_PATTERNS) same_dir = os.path.dirname(where) == mydir - names_dir = mydir and (mydir + "/") in rest + names_dir = mydir and (mydir + "/") in text + kind = None + if hit: + kinds = ref_kinds(repo, rev, where) + if kinds is not None: + kind = kinds.get(int(num)) if num.isdigit() else None + if kind == "dead": + hit = False # docstring or comment: unexecuted + elif kind == "string" and not EXEC_RE.search(text): + hit = False # a command quoted in a report string if hit and (same_dir or names_dir): inv.append(ln) else: @@ -200,15 +320,46 @@ def read_baseline(path): return led -def grade(rows, ledger): +def read_gate(path): + """path -> verdict, for rows whose verdict and rc AGREE. + + Assumes nothing about the ledger's writer. The rule that makes a row + evidence of a run is the one q65_runner.py grades itself with: the verdict + must match the rc recorded beside it. A row that says PASS with rc 1, or + TIMEOUT with no rc, is not a grade -- there is no rc to grade -- so the file + stays unwired here and the fault is the one Q65 opened, not a tolerated row. + """ + gate, ungraded = {}, {} + if not os.path.isfile(path): + return gate, ungraded + for ln in open(path, encoding="utf-8"): + if not ln.strip() or ln.lstrip().startswith("#"): + continue + parts = ln.rstrip("\n").split("\t") + if len(parts) < 4: + continue + verdict, rc, p = parts[2], parts[3], parts[1] + if verdict in GATE_RC and rc == GATE_RC[verdict]: + gate[p] = verdict + elif verdict == "OTHER_RC" and rc.isdigit() and int(rc) > 2: + gate[p] = verdict + else: + ungraded[p] = verdict + return gate, ungraded + + +def grade(rows, ledger, gate=None): + gate = gate or {} stripped = [r for r in rows if "STRIPPED_ASSERT" in r["faults"]] - unwired = [r for r in rows if not r["wired_refs"]] + unwired = [r for r in rows if not r["wired_refs"] and r["path"] not in gate] new_unwired = [r for r in unwired if r["path"] not in ledger] stale = [p for p in ledger if p not in {r["path"] for r in rows}] return stripped, unwired, new_unwired, stale -def report(rows, ledger, rev, verbose=False): +def report(rows, ledger, rev, verbose=False, gate=None, ungraded=None): + gate = gate or {} + ungraded = ungraded or {} lanes = {} for r in rows: lanes[r["lane"]] = lanes.get(r["lane"], 0) + 1 @@ -219,17 +370,34 @@ def report(rows, ledger, rev, verbose=False): for r in rows: tag = ",".join(r["faults"]) or ("wired" if r["wired_refs"] else ledger.get(r["path"], "NEW-UNWIRED")) print(f" {'FAULT ' if r['faults'] else 'ok '}{r['asserts']:2d} assert {tag:18s} {r['path']}") - stripped, unwired, new_unwired, stale = grade(rows, ledger) + stripped, unwired, new_unwired, stale = grade(rows, ledger, gate) + ran = [r for r in rows if r["path"] in gate and not r["wired_refs"]] + by_verdict = {} + for r in ran: + by_verdict[gate[r["path"]]] = by_verdict.get(gate[r["path"]], 0) + 1 print(" stripped-assert : %d of %d (no _check/raise/non-zero exit; -O compiles `assert` away)" % (len(stripped), len(rows))) for r in stripped: print(f" STRIPPED_ASSERT {r['path']}") + print(" wired by name (invoke) : %d of %d" % (len(rows) - len([r for r in rows if not r['wired_refs']]), len(rows))) + print(" run by q65_runner : %d (%s)" + % (len(ran), ", ".join(f"{k} {v}" for k, v in sorted(by_verdict.items())) or "no graded row")) + for p, v in sorted(ungraded.items()): + print(f" NOT_GRADED {v:9s} {p} (ledger row without an rc to grade)") print(" unwired (code-grep) : %d of %d (`git grep -F -l %s -- %s`)" % (len(unwired), len(rows), rev, " ".join(CODE_PATHSPECS))) - print(" unwired, ledger-tolerated : %d (%s: %d declared one-off, %d undeclared)" - % (len(unwired) - len(new_unwired), os.path.basename(DEFAULT_BASELINE), - sum(1 for r in unwired if ledger.get(r["path"]) == "declared-one-off"), - sum(1 for r in unwired if ledger.get(r["path"]) == "undeclared"))) + tol = len(unwired) - len(new_unwired) + buckets = {} + for r in unwired: + k = ledger.get(r["path"], "NEW-UNWIRED") + buckets[k] = buckets.get(k, 0) + 1 + print(" unwired, ledger-tolerated : %d (%s: %s)" + % (tol, os.path.basename(DEFAULT_BASELINE), + ", ".join("%d %s" % (v, k) for k, v in sorted(buckets.items())) or "none")) + covered = sum(1 for r in rows if r["path"] in ledger) + print(" census coverage : %d of %d committed verifier(s) carry a ledger row%s" + % (covered, len(rows), "" if covered == len(rows) + else " <-- %d UNCLASSED, invisible to both instruments" % (len(rows) - covered))) print(" unwired, NEW (fault) : %d" % len(new_unwired)) for r in new_unwired: print(f" NEW_UNWIRED {r['path']}") @@ -240,21 +408,52 @@ def report(rows, ledger, rev, verbose=False): return faults -def emit_baseline(rows, path): - out = ["# Q63 unwired-verifier baseline -- the unwired population this archive", - "# already carried when the rule was written. One row per file: \\t.", - "# class declared-one-off = the file states in-file that no gate invokes it;", - "# class undeclared = measured unwired, no such statement (the Q65 residual).", - "# A row is a stated tolerance, not an exemption: an unwired file that is NOT", - "# listed makes q63_verify_scan.py rc 1, and a stripped-assert file always does.", - "# Regenerate: q63_verify_scan.py --repo . --emit-baseline"] +BASELINE_HEAD = [ + "# Q63 verifier census -- one row per COMMITTED verify_*.py at the rev the", + "# scan read, class plus the evidence the class rests on.", + "# row: TAB TAB (three tab-separated columns)", + "# class gate-run = q65_runner.py executed it and recorded an rc (Q65);", + "# class wired-invoke = a committed line invokes it (the ref is the evidence);", + "# class declared-one-off = the file states in-file that no gate invokes it;", + "# class undeclared = unwired, no statement and no graded run (the Q65 residual).", + "# ARITHMETIC: the row count must equal the committed verifier count at that rev.", + "# This file used to list only the UNWIRED files, so a verifier that appeared in", + "# no row was invisible and both instruments still read clean -- 8 of 88 at", + "# `b01ec0e`. A census makes the omission itself the fault: q65_runner.py --check", + "# reports UNCLASSED for a committed verifier this file does not name.", + "# Regenerate: q63_verify_scan.py --repo . --emit-baseline"] + + +def _short(line, limit=96): + s = " ".join(line.split()) + return s if len(s) <= limit else s[:limit - 3] + "..." + + +def emit_baseline(rows, path, gate=None): + """Write the census: one row per verifier the scan saw, with its evidence.""" + gate = gate or {} + out = list(BASELINE_HEAD) for r in sorted(rows, key=lambda r: r["path"]): - if r["wired_refs"]: - continue - cls = "declared-one-off" if r["declared_one_off"] else "undeclared" - out.append(f"{cls}\t{r['path']}") - open(path, "w", encoding="utf-8").write("\n".join(out) + "\n") - print(f"baseline written: {path} ({sum(1 for r in rows if not r['wired_refs'])} rows)") + if r["path"] in gate: + cls, ev = "gate-run", "q65:%s" % gate[r["path"]] + elif r["wired_refs"]: + cls, ev = "wired-invoke", _short(r["wired_refs"][0]) + elif r["declared_one_off"]: + cls, ev = "declared-one-off", "in-file declaration" + else: + cls, ev = "undeclared", "no invocation, no declaration, no graded run" + out.append(chr(9).join((cls, r["path"], ev))) + open(path, "w", encoding="utf-8").write(chr(10).join(out) + chr(10)) + tally = {} + for r in rows: + k = ("gate-run" if r["path"] in gate else "wired-invoke" if r["wired_refs"] + else "declared-one-off" if r["declared_one_off"] else "undeclared") + tally[k] = tally.get(k, 0) + 1 + print("census written: %s (%d rows: %s)" % ( + path, len(rows), ", ".join("%s %d" % (k, tally[k]) for k in + ("gate-run", "wired-invoke", "declared-one-off", + "undeclared") if tally.get(k)))) + return len(rows) def selftest(): @@ -306,6 +505,62 @@ def selftest(): led["wizard-work/x/verify_orphan.py"] = "undeclared" cells.append(("unwired file the ledger names is tolerated", len(grade(rows, led)[2]) == 0)) + # (5) Q65: a graded row in the runner ledger wires the file, no grep needed + gate = {"wizard-work/x/verify_orphan.py": "PASS"} + cells.append(("a graded gate-run row is not unwired", + len(grade(rows, led, gate)[1]) == 0)) + # (6) a row whose verdict and rc disagree is not evidence of a run + with open(os.path.join(d, "gate.tsv"), "w") as fh: + fh.write("ungated\twizard-work/x/verify_orphan.py\tPASS\t1\t0.1\t\n") + parsed, ungraded = read_gate(os.path.join(d, "gate.tsv")) + cells.append(("PASS-with-rc-1 is not a graded row", + parsed == {} and "wizard-work/x/verify_orphan.py" in ungraded + and [r["path"] for r in grade(rows, led, parsed)[1]] + == ["wizard-work/x/verify_orphan.py"])) + + # (7) prose is not an invocation. All four lines below are real shapes from + # the archive at `b01ec0e`, which the pre-fix scanner read as WIRED. + put("wizard-work/x/verify_prose.py", "import sys\nsys.exit(0)\n") + put("wizard-work/x/runner_docstring.py", + '"""Runner.\n\nRun as: python3 verify_prose.py\n"""\nprint("x")\n') + commit("c7") + rows = scan(d, "HEAD") + row = [r for r in rows if r["path"].endswith("verify_prose.py")][0] + cells.append(("a command quoted in a docstring wires nothing", + row["wired_refs"] == [] and row["mention_refs"] != [])) + + # (8) the same basename passed to subprocess.run() IS the invocation + put("wizard-work/x/verify_call.py", "import sys\nsys.exit(0)\n") + put("wizard-work/x/runner_call.py", + 'import subprocess, sys\n' + 'subprocess.run([sys.executable, "verify_call.py"], check=True)\n') + commit("c8") + rows = scan(d, "HEAD") + row = [r for r in rows if r["path"].endswith("verify_call.py")][0] + cells.append(("the basename passed to subprocess.run is an invocation", + len(row["wired_refs"]) == 1)) + + # (9) a report string in code, with no exec-shaped call on the line + put("wizard-work/x/verify_report.py", "import sys\nsys.exit(0)\n") + put("wizard-work/x/report_gen.py", + 'A("`python3 verify_report.py .` runs 8 checks (A-H) and exits 0")\n') + commit("c9") + rows = scan(d, "HEAD") + row = [r for r in rows if r["path"].endswith("verify_report.py")][0] + cells.append(("a command quoted in a report string wires nothing", + row["wired_refs"] == [])) + + # (10) the interpreter token may not come from the REFERRER's filename: + # `extract_python_thing.py` + a prose mention read as "python ... X". + put("wizard-work/x/verify_pathtok.py", "import sys\nsys.exit(0)\n") + put("wizard-work/x/extract_python_thing.py", + 'note = "the sibling verify_pathtok.py re-queries the API"\n') + commit("c10") + rows = scan(d, "HEAD") + row = [r for r in rows if r["path"].endswith("verify_pathtok.py")][0] + cells.append(("a token from the referrer's own path does not wire a file", + row["wired_refs"] == [])) + for name, passed in cells: print(f" {'PASS' if passed else 'FAIL'} {name}") ok &= passed @@ -318,6 +573,8 @@ def main(): ap.add_argument("--repo", default=".") ap.add_argument("--rev", default="HEAD") ap.add_argument("--baseline", default=DEFAULT_BASELINE) + ap.add_argument("--gate", default=DEFAULT_GATE, + help="q65_runner ledger: rows with a graded rc count as run") ap.add_argument("--emit-baseline", action="store_true") ap.add_argument("--json") ap.add_argument("--verbose", action="store_true") @@ -333,9 +590,11 @@ def main(): print(f"q63_verify_scan: cannot grade: {e}") return 2 if a.emit_baseline: - emit_baseline(rows, a.baseline) + gate, _ = read_gate(a.gate) + emit_baseline(rows, a.baseline, gate) return 0 - faults = report(rows, read_baseline(a.baseline), a.rev, a.verbose) + gate, ungraded = read_gate(a.gate) + faults = report(rows, read_baseline(a.baseline), a.rev, a.verbose, gate, ungraded) if a.why: print("\nwiring detail (what the count is made of):") for r in rows: