stackchain/troupe-standup
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

volt: Q63 census -- every committed verifier is a row, and prose is not an invocation

Browse Source 
Two defects, both measured at b01ec0e before the change.

1. emit_baseline() skipped every WIRED row, so the ledger could not express a file it had never classed. Eight of the 88 committed verify_*.py sat in no class and no ledger while both instruments printed clean. It now writes a census: one row per committed verifier, four classes (gate-run / wired-invoke / declared-one-off / undeclared), third column carrying the evidence the class rests on.

2. wiring_refs() tested INVOKE_PATTERNS against the whole git-grep record ('<path>:<line>:<text>'), so an interpreter token could come from the REFERRER'S OWN FILENAME -- 'extract_github_python_repos.py' supplied 'python' and a prose mention supplied the basename. The shape is now tested on the line TEXT only. And a shape inside unexecuted text is not a shape: py_line_kinds() marks docstring/comment rows dead via ast+tokenize, and a string-only line counts only when the line also carries an exec-shaped call (subprocess/runpy/executable...).

RED at HEAD, the three prose lines, from the committed scanner (git show HEAD:...): WIRED ledger/verify_arxiv_csv.py, WIRED ledger/verify_github_python_repos.py, WIRED sphinx/archive-remember-r1-figures-2026-10-23/verify_archive.py. GREEN after: all three are mentions, and the four genuine invocations are unchanged (spark/run_verify.sh, lc-coverage and lc-mm-dossier mutation_battery.py, transformer CHECK.py). Selftest 6 -> 10 cells, the four new ones built from these exact shapes.
This commit is contained in:
volt committed 2026-10-24 21:18:23 +00:00
1 parent d669515410
commit ff8b4da0fd
1 file changed
+288 -29
Download Patch File Download Diff File Expand all files Collapse all files
wizard-work/volt/q63_verify_scan.py
+288 -29
View File
@@ -66,6 +66,7 @@ Usage:
python3 wizard-work/volt/q63_verify_scan.py --selftest
"""
import argparse
import ast
import json
import os
import re
@@ -74,12 +75,23 @@ import sys
DEFAULT_BASELINE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"q63_unwired_baseline.tsv")
# Q65: the runner ledger. A file this carries a GRADED row for was executed and
# its rc recorded -- see q65_runner.py. Class `gate-run` in the baseline below.
DEFAULT_GATE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"q65_runner_baseline.tsv")
GATE_RC = {"PASS": "0", "FAIL": "1", "REFUSED": "2"}
VERIFY_RE = re.compile(r"/verify_.*\.py$")
ASSERT_RE = re.compile(r"(?m)^[ \t]*assert[ \t(]")
SAFE_PATH_RE = re.compile(r"\b_check\(")
RAISE_RE = re.compile(r"(?m)^[ \t]*raise\b")
EXIT_RE = re.compile(r"\b(?:sys\.)?exit\(\s*([^)]*?)\s*\)")
CODE_PATHSPECS = ["*.py", "*.sh", "CHECK", "*.yml", "*.yaml"]
# `CHECK` as a bare pathspec is a LITERAL path: it matches ./CHECK only, so the
# four nested `wizard-work/*/CHECK` files were outside the wiring grep until
# 2026-10-24. Measured at `2bc2662` with `:(glob)**/CHECK`: the same 7 files are
# wired, because no nested CHECK invokes a verify_*.py (they run
# `unittest discover -p 'test_*.py'`). The defect was latent, not academic:
# a nested CHECK that ran a verifier would have been read as not running it.
CODE_PATHSPECS = ["*.py", "*.sh", ":(glob)**/CHECK", "*.yml", "*.yaml"]
DECLARED_RE = re.compile(r"Q63", re.I)
DECLARED_PHRASE = "no committed gate invokes"
# The measuring instruments name every file in the class; naming is not running.
@@ -115,10 +127,96 @@ def exit_safe(src):
return False
# A basename that appears only inside a string literal is a MENTION unless the
# line also carries an exec-shaped call: a command quoted in a report string runs
# nothing, the same command passed to subprocess.run() is the invocation itself.
EXEC_RE = re.compile(r"\b(?:subprocess|Popen|runpy|run_path|run_module|"
r"check_call|check_output|os\.system|os\.exec\w*|executable)\b")
_KIND_CACHE = {}
def _docstring_rows(tree):
"""Rows covered by a module/class/function docstring -- never executed."""
rows = set()
for node in ast.walk(tree):
if not isinstance(node, (ast.Module, ast.ClassDef, ast.FunctionDef,
ast.AsyncFunctionDef)):
continue
body = getattr(node, "body", [])
if body and isinstance(body[0], ast.Expr) and \
isinstance(body[0].value, ast.Constant) and \
isinstance(body[0].value.value, str):
first = body[0].value
rows |= set(range(first.lineno, (first.end_lineno or first.lineno) + 1))
return rows
def py_line_kinds(src):
"""line number -> 'dead' | 'string', for the lines of one Python referrer.
'dead' the line is inside a docstring or is a comment: the interpreter
never runs it, so no shape on that line invokes anything.
'string' a string literal covers the line (a report line, a message) -- an
invocation is still possible there, but only if the line also
carries an exec-shaped call; the caller applies that second test.
Returns None when the source does not parse: the caller then falls back to
the grep shape rather than guessing, which is the old behaviour.
"""
import tokenize, io as _io
try:
tree = ast.parse(src)
except SyntaxError:
return None
kinds = {}
for row in _docstring_rows(tree):
kinds[row] = "dead"
for node in ast.walk(tree):
if isinstance(node, ast.Constant) and isinstance(node.value, str):
last = node.end_lineno or node.lineno
for row in range(node.lineno, last + 1):
kinds.setdefault(row, "string")
try:
for tok in tokenize.generate_tokens(_io.StringIO(src).readline):
if tok.type == tokenize.COMMENT:
row, col = tok.start
line = src.splitlines()[row - 1] if row <= len(src.splitlines()) else ""
if not line[:col].strip(): # comment-only line
kinds[row] = "dead"
except (tokenize.TokenError, IndentationError, SyntaxError):
pass
return kinds
def ref_kinds(repo, rev, where):
"""(kind map, error) for the referrer at `rev`; cached per path."""
key = (repo, rev, where)
if key in _KIND_CACHE:
return _KIND_CACHE[key]
try:
if rev == "WORKTREE":
src = open(os.path.join(repo, where), encoding="utf-8",
errors="replace").read()
else:
src = git(repo, "show", "%s:%s" % (rev, where))
except (OSError, RuntimeError):
_KIND_CACHE[key] = None
return None
if where.endswith(".py"):
kinds = py_line_kinds(src)
else:
kinds = {}
for n, line in enumerate(src.splitlines(), 1):
if line.lstrip().startswith("#"):
kinds[n] = "dead" # shell/yaml comment: never run
_KIND_CACHE[key] = kinds
return kinds
def wiring_refs(repo, rev, path, base, stem):
"""(invocations, mentions) for one basename at one revision.
git grep gives the lines; each line is classified. Two accuracy rules, both
git grep gives the lines; each line is classified. Three accuracy rules, all
paid for by measurement rather than assumed:
* a reference counts only in an INVOCATION shape (python3 X, runpy/import,
@@ -128,6 +226,18 @@ def wiring_refs(repo, rev, path, base, stem):
sphinx archives; a reference from another archive's directory is a
reference to ITS sibling, not to this file. So a wiring reference must
either name this file's parent directory, or sit in the same directory.
* THE SHAPE IS TESTED ON THE LINE TEXT, NEVER ON `<path>:<line>:`. The
first version tested `rest` -- the whole grep record -- so an interpreter
token could come from the REFERRER'S OWN FILENAME: the prose line
`- Verdict: the sibling verify_github_python_repos.py re-queries` was
read as `python ... verify_github_python_repos.py` and wired a verifier
nothing invokes, because the referring file is itself named
`extract_github_python_repos.py`. Measured at `b01ec0e`; the line was
WIRED before this rule and is a mention after it.
And a shape found inside unexecuted text is not a shape: `ref_kinds` marks
docstring and comment lines dead, and a string-only line is a mention unless
it also carries an exec-shaped call.
"""
args = ["grep", "-F", "-n", "--", base]
if rev != "WORKTREE":
@@ -142,15 +252,25 @@ def wiring_refs(repo, rev, path, base, stem):
for ln in lines:
# git grep output is REV:path:line:text at a rev, path:line:text on disk
rest = ln.split(":", 1)[1] if rev != "WORKTREE" else ln
where = rest.split(":", 2)[0]
where, _, tail = rest.partition(":")
num, _, text = tail.partition(":")
if where == path:
continue # the file's own text is not a reference
if where.startswith(INSTRUMENT_DIR):
continue # the instruments that measure this class
hit = any(re.search(p.format(base=re.escape(base), stem=re.escape(stem)), rest)
hit = any(re.search(p.format(base=re.escape(base), stem=re.escape(stem)), text)
for p in INVOKE_PATTERNS)
same_dir = os.path.dirname(where) == mydir
names_dir = mydir and (mydir + "/") in rest
names_dir = mydir and (mydir + "/") in text
kind = None
if hit:
kinds = ref_kinds(repo, rev, where)
if kinds is not None:
kind = kinds.get(int(num)) if num.isdigit() else None
if kind == "dead":
hit = False # docstring or comment: unexecuted
elif kind == "string" and not EXEC_RE.search(text):
hit = False # a command quoted in a report string
if hit and (same_dir or names_dir):
inv.append(ln)
else:
@@ -200,15 +320,46 @@ def read_baseline(path):
return led
def grade(rows, ledger):
def read_gate(path):
"""path -> verdict, for rows whose verdict and rc AGREE.
Assumes nothing about the ledger's writer. The rule that makes a row
evidence of a run is the one q65_runner.py grades itself with: the verdict
must match the rc recorded beside it. A row that says PASS with rc 1, or
TIMEOUT with no rc, is not a grade -- there is no rc to grade -- so the file
stays unwired here and the fault is the one Q65 opened, not a tolerated row.
"""
gate, ungraded = {}, {}
if not os.path.isfile(path):
return gate, ungraded
for ln in open(path, encoding="utf-8"):
if not ln.strip() or ln.lstrip().startswith("#"):
continue
parts = ln.rstrip("\n").split("\t")
if len(parts) < 4:
continue
verdict, rc, p = parts[2], parts[3], parts[1]
if verdict in GATE_RC and rc == GATE_RC[verdict]:
gate[p] = verdict
elif verdict == "OTHER_RC" and rc.isdigit() and int(rc) > 2:
gate[p] = verdict
else:
ungraded[p] = verdict
return gate, ungraded
def grade(rows, ledger, gate=None):
gate = gate or {}
stripped = [r for r in rows if "STRIPPED_ASSERT" in r["faults"]]
unwired = [r for r in rows if not r["wired_refs"]]
unwired = [r for r in rows if not r["wired_refs"] and r["path"] not in gate]
new_unwired = [r for r in unwired if r["path"] not in ledger]
stale = [p for p in ledger if p not in {r["path"] for r in rows}]
return stripped, unwired, new_unwired, stale
def report(rows, ledger, rev, verbose=False):
def report(rows, ledger, rev, verbose=False, gate=None, ungraded=None):
gate = gate or {}
ungraded = ungraded or {}
lanes = {}
for r in rows:
lanes[r["lane"]] = lanes.get(r["lane"], 0) + 1
@@ -219,17 +370,34 @@ def report(rows, ledger, rev, verbose=False):
for r in rows:
tag = ",".join(r["faults"]) or ("wired" if r["wired_refs"] else ledger.get(r["path"], "NEW-UNWIRED"))
print(f" {'FAULT ' if r['faults'] else 'ok '}{r['asserts']:2d} assert {tag:18s} {r['path']}")
stripped, unwired, new_unwired, stale = grade(rows, ledger)
stripped, unwired, new_unwired, stale = grade(rows, ledger, gate)
ran = [r for r in rows if r["path"] in gate and not r["wired_refs"]]
by_verdict = {}
for r in ran:
by_verdict[gate[r["path"]]] = by_verdict.get(gate[r["path"]], 0) + 1
print(" stripped-assert : %d of %d (no _check/raise/non-zero exit; -O compiles `assert` away)"
% (len(stripped), len(rows)))
for r in stripped:
print(f" STRIPPED_ASSERT {r['path']}")
print(" wired by name (invoke) : %d of %d" % (len(rows) - len([r for r in rows if not r['wired_refs']]), len(rows)))
print(" run by q65_runner : %d (%s)"
% (len(ran), ", ".join(f"{k} {v}" for k, v in sorted(by_verdict.items())) or "no graded row"))
for p, v in sorted(ungraded.items()):
print(f" NOT_GRADED {v:9s} {p} (ledger row without an rc to grade)")
print(" unwired (code-grep) : %d of %d (`git grep -F -l <basename> %s -- %s`)"
% (len(unwired), len(rows), rev, " ".join(CODE_PATHSPECS)))
print(" unwired, ledger-tolerated : %d (%s: %d declared one-off, %d undeclared)"
% (len(unwired) - len(new_unwired), os.path.basename(DEFAULT_BASELINE),
sum(1 for r in unwired if ledger.get(r["path"]) == "declared-one-off"),
sum(1 for r in unwired if ledger.get(r["path"]) == "undeclared")))
tol = len(unwired) - len(new_unwired)
buckets = {}
for r in unwired:
k = ledger.get(r["path"], "NEW-UNWIRED")
buckets[k] = buckets.get(k, 0) + 1
print(" unwired, ledger-tolerated : %d (%s: %s)"
% (tol, os.path.basename(DEFAULT_BASELINE),
", ".join("%d %s" % (v, k) for k, v in sorted(buckets.items())) or "none"))
covered = sum(1 for r in rows if r["path"] in ledger)
print(" census coverage : %d of %d committed verifier(s) carry a ledger row%s"
% (covered, len(rows), "" if covered == len(rows)
else " <-- %d UNCLASSED, invisible to both instruments" % (len(rows) - covered)))
print(" unwired, NEW (fault) : %d" % len(new_unwired))
for r in new_unwired:
print(f" NEW_UNWIRED {r['path']}")
@@ -240,21 +408,52 @@ def report(rows, ledger, rev, verbose=False):
return faults
def emit_baseline(rows, path):
out = ["# Q63 unwired-verifier baseline -- the unwired population this archive",
"# already carried when the rule was written. One row per file: <class>\\t<path>.",
"# class declared-one-off = the file states in-file that no gate invokes it;",
"# class undeclared = measured unwired, no such statement (the Q65 residual).",
"# A row is a stated tolerance, not an exemption: an unwired file that is NOT",
"# listed makes q63_verify_scan.py rc 1, and a stripped-assert file always does.",
"# Regenerate: q63_verify_scan.py --repo . --emit-baseline"]
BASELINE_HEAD = [
"# Q63 verifier census -- one row per COMMITTED verify_*.py at the rev the",
"# scan read, class plus the evidence the class rests on.",
"# row: <class> TAB <path> TAB <evidence> (three tab-separated columns)",
"# class gate-run = q65_runner.py executed it and recorded an rc (Q65);",
"# class wired-invoke = a committed line invokes it (the ref is the evidence);",
"# class declared-one-off = the file states in-file that no gate invokes it;",
"# class undeclared = unwired, no statement and no graded run (the Q65 residual).",
"# ARITHMETIC: the row count must equal the committed verifier count at that rev.",
"# This file used to list only the UNWIRED files, so a verifier that appeared in",
"# no row was invisible and both instruments still read clean -- 8 of 88 at",
"# `b01ec0e`. A census makes the omission itself the fault: q65_runner.py --check",
"# reports UNCLASSED for a committed verifier this file does not name.",
"# Regenerate: q63_verify_scan.py --repo . --emit-baseline"]
def _short(line, limit=96):
s = " ".join(line.split())
return s if len(s) <= limit else s[:limit - 3] + "..."
def emit_baseline(rows, path, gate=None):
"""Write the census: one row per verifier the scan saw, with its evidence."""
gate = gate or {}
out = list(BASELINE_HEAD)
for r in sorted(rows, key=lambda r: r["path"]):
if r["wired_refs"]:
continue
cls = "declared-one-off" if r["declared_one_off"] else "undeclared"
out.append(f"{cls}\t{r['path']}")
open(path, "w", encoding="utf-8").write("\n".join(out) + "\n")
print(f"baseline written: {path} ({sum(1 for r in rows if not r['wired_refs'])} rows)")
if r["path"] in gate:
cls, ev = "gate-run", "q65:%s" % gate[r["path"]]
elif r["wired_refs"]:
cls, ev = "wired-invoke", _short(r["wired_refs"][0])
elif r["declared_one_off"]:
cls, ev = "declared-one-off", "in-file declaration"
else:
cls, ev = "undeclared", "no invocation, no declaration, no graded run"
out.append(chr(9).join((cls, r["path"], ev)))
open(path, "w", encoding="utf-8").write(chr(10).join(out) + chr(10))
tally = {}
for r in rows:
k = ("gate-run" if r["path"] in gate else "wired-invoke" if r["wired_refs"]
else "declared-one-off" if r["declared_one_off"] else "undeclared")
tally[k] = tally.get(k, 0) + 1
print("census written: %s (%d rows: %s)" % (
path, len(rows), ", ".join("%s %d" % (k, tally[k]) for k in
("gate-run", "wired-invoke", "declared-one-off",
"undeclared") if tally.get(k))))
return len(rows)
def selftest():
@@ -306,6 +505,62 @@ def selftest():
led["wizard-work/x/verify_orphan.py"] = "undeclared"
cells.append(("unwired file the ledger names is tolerated", len(grade(rows, led)[2]) == 0))
# (5) Q65: a graded row in the runner ledger wires the file, no grep needed
gate = {"wizard-work/x/verify_orphan.py": "PASS"}
cells.append(("a graded gate-run row is not unwired",
len(grade(rows, led, gate)[1]) == 0))
# (6) a row whose verdict and rc disagree is not evidence of a run
with open(os.path.join(d, "gate.tsv"), "w") as fh:
fh.write("ungated\twizard-work/x/verify_orphan.py\tPASS\t1\t0.1\t\n")
parsed, ungraded = read_gate(os.path.join(d, "gate.tsv"))
cells.append(("PASS-with-rc-1 is not a graded row",
parsed == {} and "wizard-work/x/verify_orphan.py" in ungraded
and [r["path"] for r in grade(rows, led, parsed)[1]]
== ["wizard-work/x/verify_orphan.py"]))
# (7) prose is not an invocation. All four lines below are real shapes from
# the archive at `b01ec0e`, which the pre-fix scanner read as WIRED.
put("wizard-work/x/verify_prose.py", "import sys\nsys.exit(0)\n")
put("wizard-work/x/runner_docstring.py",
'"""Runner.\n\nRun as: python3 verify_prose.py\n"""\nprint("x")\n')
commit("c7")
rows = scan(d, "HEAD")
row = [r for r in rows if r["path"].endswith("verify_prose.py")][0]
cells.append(("a command quoted in a docstring wires nothing",
row["wired_refs"] == [] and row["mention_refs"] != []))
# (8) the same basename passed to subprocess.run() IS the invocation
put("wizard-work/x/verify_call.py", "import sys\nsys.exit(0)\n")
put("wizard-work/x/runner_call.py",
'import subprocess, sys\n'
'subprocess.run([sys.executable, "verify_call.py"], check=True)\n')
commit("c8")
rows = scan(d, "HEAD")
row = [r for r in rows if r["path"].endswith("verify_call.py")][0]
cells.append(("the basename passed to subprocess.run is an invocation",
len(row["wired_refs"]) == 1))
# (9) a report string in code, with no exec-shaped call on the line
put("wizard-work/x/verify_report.py", "import sys\nsys.exit(0)\n")
put("wizard-work/x/report_gen.py",
'A("`python3 verify_report.py .` runs 8 checks (A-H) and exits 0")\n')
commit("c9")
rows = scan(d, "HEAD")
row = [r for r in rows if r["path"].endswith("verify_report.py")][0]
cells.append(("a command quoted in a report string wires nothing",
row["wired_refs"] == []))
# (10) the interpreter token may not come from the REFERRER's filename:
# `extract_python_thing.py` + a prose mention read as "python ... X".
put("wizard-work/x/verify_pathtok.py", "import sys\nsys.exit(0)\n")
put("wizard-work/x/extract_python_thing.py",
'note = "the sibling verify_pathtok.py re-queries the API"\n')
commit("c10")
rows = scan(d, "HEAD")
row = [r for r in rows if r["path"].endswith("verify_pathtok.py")][0]
cells.append(("a token from the referrer's own path does not wire a file",
row["wired_refs"] == []))
for name, passed in cells:
print(f" {'PASS' if passed else 'FAIL'} {name}")
ok &= passed
@@ -318,6 +573,8 @@ def main():
ap.add_argument("--repo", default=".")
ap.add_argument("--rev", default="HEAD")
ap.add_argument("--baseline", default=DEFAULT_BASELINE)
ap.add_argument("--gate", default=DEFAULT_GATE,
help="q65_runner ledger: rows with a graded rc count as run")
ap.add_argument("--emit-baseline", action="store_true")
ap.add_argument("--json")
ap.add_argument("--verbose", action="store_true")
@@ -333,9 +590,11 @@ def main():
print(f"q63_verify_scan: cannot grade: {e}")
return 2
if a.emit_baseline:
emit_baseline(rows, a.baseline)
gate, _ = read_gate(a.gate)
emit_baseline(rows, a.baseline, gate)
return 0
faults = report(rows, read_baseline(a.baseline), a.rev, a.verbose)
gate, ungraded = read_gate(a.gate)
faults = report(rows, read_baseline(a.baseline), a.rev, a.verbose, gate, ungraded)
if a.why:
print("\nwiring detail (what the count is made of):")
for r in rows: