volt
17d0180cef
wizard-work/forge had 39 committed verify_*.py and no CHECK; tools/verify.py
names the lane zero times. verifier_gate.py runs them, keeps one row per file
bound to the git blob the rc was measured against, and grades that ledger in
under a second. First full run at 7b42050: PASS 21 (18 of them PASS* -- they
read an absolute host root), FAIL 18, all named with causes.
Five verifiers in this lane clean up with rmtree(os.path.dirname(tmp)), i.e.
they delete $TMPDIR itself; each file now runs under its own sandbox so the
harness cannot be measured instead of the lane.
196 lines
8.7 KiB
Python
196 lines
8.7 KiB
Python
#!/usr/bin/env python3
|
|
"""q87_verify.py -- the instrument behind CHECK-q87 (2026-10-31, volt lane).
|
|
|
|
WHAT IT GRADES
|
|
The forge lane gate (`wizard-work/forge/CHECK` -> `verifier_gate.py`) and the
|
|
ledger it grades. Seven legs, each of which can fail on its own:
|
|
|
|
1 selftest the gate's own 10 cells pass (a cell that silently passes on a
|
|
truthy string is the defect the first draft of the selftest had).
|
|
2 check `CHECK` in this tree exits 0 and prints the population and the
|
|
known-red count, so a green cannot hide a red.
|
|
3 strict the same gate with --strict exits 1 and names the non-PASS rows,
|
|
which is the switch from "books consistent" to "lane healthy".
|
|
4 drift in a throwaway clone, one byte appended to a PASS verifier makes
|
|
--check exit 1 with DRIFT naming that file, and restoring the
|
|
bytes makes it exit 0 again.
|
|
5 ungraded a new committed `verify_*.py` with no ledger row makes --check
|
|
exit 1 with UNGRADED naming it (a file added tomorrow is seen).
|
|
6 causeless blanking the note of a FAIL row makes --check exit 1 with
|
|
CAUSELESS (a red must carry its reason).
|
|
7 portable --check run from a different cwd, against a clone of this tree,
|
|
returns the same verdict as in-tree -- so the gate is a function
|
|
of the tree, not of this host's path.
|
|
|
|
The clone legs never touch the graded tree: every mutation happens in a
|
|
TemporaryDirectory copy that is thrown away.
|
|
|
|
EXIT CODES
|
|
0 every leg passed / 1 a leg failed / 2 could not grade (no git, no clone).
|
|
|
|
Usage:
|
|
python3 q87_verify.py [--repo /root/troupe-standup] [--verbose]
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
LANE = "wizard-work/forge"
|
|
GATE = "verifier_gate.py"
|
|
LEDGER = "VERIFIER_LEDGER.tsv"
|
|
WINDOWS = "VERIFIER_WINDOWS.tsv"
|
|
CHECK = "CHECK"
|
|
FILES = (GATE, LEDGER, WINDOWS, CHECK)
|
|
|
|
|
|
def run(argv, cwd=None, timeout=600):
|
|
r = subprocess.run(argv, cwd=cwd, capture_output=True, text=True,
|
|
errors="replace", timeout=timeout,
|
|
env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1"))
|
|
return r.returncode, (r.stdout or "") + (r.stderr or "")
|
|
|
|
|
|
def gate(repo, *extra, cwd=None):
|
|
return run([sys.executable, str(Path(repo) / LANE / GATE), "--repo", str(repo)]
|
|
+ list(extra), cwd=cwd)
|
|
|
|
|
|
def clone_of(repo, dest):
|
|
rc, out = run(["git", "clone", "-q", "--no-hardlinks", str(repo), str(dest)])
|
|
if rc != 0:
|
|
raise RuntimeError("git clone failed: %s" % out.strip()[:200])
|
|
for name in FILES:
|
|
src = Path(repo) / LANE / name
|
|
if src.exists():
|
|
shutil.copyfile(src, Path(dest) / LANE / name)
|
|
return Path(dest)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--repo", default=str(Path(__file__).resolve().parents[2]))
|
|
ap.add_argument("--verbose", action="store_true")
|
|
args = ap.parse_args()
|
|
repo = Path(args.repo).resolve()
|
|
if not (repo / ".git").exists():
|
|
print("q87: %s is not a git worktree -- could not grade" % repo)
|
|
return 2
|
|
|
|
legs = []
|
|
|
|
def leg(name, ok, detail):
|
|
legs.append((name, bool(ok), detail))
|
|
print(" [%s] %-9s %s" % ("ok " if ok else "FAIL", name, detail), flush=True)
|
|
|
|
# 1 -- the gate's own cells.
|
|
rc, out = gate(repo, "--selftest")
|
|
tail = [l for l in out.splitlines() if "selftest:" in l]
|
|
leg("selftest", rc == 0 and bool(tail) and "0 failed" in tail[-1]
|
|
and "cells," in tail[-1],
|
|
(tail[-1] if tail else "no selftest summary (rc %d)" % rc))
|
|
|
|
# 2 -- the shipped gate is green and says what it is green about. The count
|
|
# is read from the ledger, not hardcoded: a ledger that re-emits after a
|
|
# verifier changes would otherwise break this leg for the wrong reason.
|
|
rc, out = run(["bash", str(repo / LANE / CHECK)])
|
|
pop = [l for l in out.splitlines() if "committed verifier(s)" in l]
|
|
red = [l for l in out.splitlines() if l.strip().startswith("known-red")]
|
|
led = (repo / LANE / LEDGER).read_text(encoding="utf-8")
|
|
n_rows = sum(1 for l in led.splitlines() if l.strip() and not l.startswith("#"))
|
|
n_red = sum(1 for l in led.splitlines()
|
|
if l.strip() and not l.startswith("#") and l.split("\t")[2] != "PASS")
|
|
ok2 = (rc == 0 and bool(pop) and bool(red)
|
|
and "%d committed verifier(s), %d ledger row(s)" % (n_rows, n_rows) in pop[0]
|
|
and ("%d of %d" % (n_red, n_rows)) in red[0] and n_red >= 1)
|
|
leg("check", ok2, "%s; %s" % (pop[0].strip() if pop else "no population line",
|
|
red[0].strip() if red else "no known-red line"))
|
|
|
|
# 3 -- strict is the lane's health, and today it is not healthy.
|
|
rc, out = gate(repo, "--check", "--strict")
|
|
strict_n = [l for l in out.splitlines() if l.strip().startswith("verifier_gate:")
|
|
and "strict" in l]
|
|
n_strict = int(strict_n[-1].split()[1]) if strict_n else 0
|
|
leg("strict", rc == 1 and n_strict >= 18 and "strict: every non-PASS row" in out,
|
|
"rc %d, %d fault(s) from the non-PASS rows" % (rc, n_strict))
|
|
|
|
# 4..6 -- mutants in a throwaway clone.
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
try:
|
|
clone = clone_of(repo, Path(tmp) / "c")
|
|
except RuntimeError as exc:
|
|
print("q87: %s" % exc)
|
|
return 2
|
|
target = clone / LANE / "verify_chunker_refactor.py"
|
|
|
|
rc0, _ = gate(clone, "--check")
|
|
base_rc, base_sha = run(["git", "-C", str(clone), "rev-parse", "HEAD"])
|
|
original = target.read_text(encoding="utf-8")
|
|
target.write_text(original + "# one byte of drift\n", encoding="utf-8")
|
|
# the ledger binds to the COMMITTED blob, so the mutant has to be a
|
|
# commit -- an edit left in the worktree is the UNCOMMITTED note, not DRIFT
|
|
run(["git", "-C", str(clone), "add", str(target.relative_to(clone))])
|
|
run(["git", "-C", str(clone), "-c", "user.email=q87@example.invalid",
|
|
"-c", "user.name=q87", "commit", "-qm", "drifts one verifier"])
|
|
new_rc, new_sha = run(["git", "-C", str(clone), "rev-parse", "HEAD"])
|
|
committed = new_rc == 0 and new_sha.strip() != base_sha.strip()
|
|
rc1, out1 = gate(clone, "--check")
|
|
drift_line = [l for l in out1.splitlines()
|
|
if l.strip().startswith("DRIFT ")
|
|
and "verify_chunker_refactor.py" in l]
|
|
if committed:
|
|
run(["git", "-C", str(clone), "reset", "-q", "--hard", "HEAD~1"])
|
|
rc2, _ = gate(clone, "--check")
|
|
leg("drift", committed and rc0 == 0 and rc1 == 1 and rc2 == 0
|
|
and bool(drift_line),
|
|
"clean %d -> committed edit %d%s -> reverted %d"
|
|
% (rc0, rc1, (" (%s)" % drift_line[0].strip()[:52]) if drift_line else "",
|
|
rc2))
|
|
|
|
new = clone / LANE / "verify_added_tomorrow.py"
|
|
new.write_text("print('nothing')\n", encoding="utf-8")
|
|
run(["git", "-C", str(clone), "add", str(new.relative_to(clone))])
|
|
run(["git", "-C", str(clone), "-c", "user.email=q87@example.invalid",
|
|
"-c", "user.name=q87", "commit", "-qm", "adds a verifier"])
|
|
rc3, out3 = gate(clone, "--check")
|
|
leg("ungraded", rc3 == 1 and "UNGRADED" in out3 and "verify_added_tomorrow" in out3,
|
|
"rc %d%s" % (rc3, " naming UNGRADED verify_added_tomorrow.py"
|
|
if "UNGRADED" in out3 else " without an UNGRADED fault"))
|
|
|
|
# causeless: blank the note of the first FAIL row, in a fresh clone state
|
|
led = clone / LANE / LEDGER
|
|
lines = led.read_text(encoding="utf-8").splitlines()
|
|
hit = None
|
|
for i, line in enumerate(lines):
|
|
if line.startswith("#") or not line.strip():
|
|
continue
|
|
parts = line.split("\t")
|
|
if parts[2] == "FAIL":
|
|
parts[5] = ""
|
|
lines[i] = "\t".join(parts)
|
|
hit = parts[1]
|
|
break
|
|
led.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
rc4, out4 = gate(clone, "--check")
|
|
leg("causeless", rc4 == 1 and hit and ("CAUSELESS" in out4),
|
|
"rc %d%s" % (rc4, (" naming CAUSELESS %s" % hit) if "CAUSELESS" in out4
|
|
else " with no CAUSELESS fault"))
|
|
|
|
# 7 -- portability: same verdict from another cwd, through the clone.
|
|
rc5, out5 = gate(clone, "--check", cwd="/")
|
|
leg("portable", rc5 == 1 and "CAUSELESS" in out5,
|
|
"from cwd=/ against the clone the same fault is read (rc %d)" % rc5)
|
|
|
|
bad = [l for l in legs if not l[1]]
|
|
print("q87: %d leg(s), %d passed, %d fault(s)"
|
|
% (len(legs), len(legs) - len(bad), len(bad)))
|
|
return 0 if not bad else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|