volt: count pin -- every selftest count witnessed by a committed ledger
selftest_sweep compared a tool's printed count with a number in the same file, so a commit that deleted fixtures and decremented the docstring agreed with itself: doc_command_check.py cut 15 cells -> 12 read clean (sweep: 21 tools, 21 OK, 0 fatal, rc 0). tools/selftest_baseline.tsv is the second witness, read from disk: PIN_DRIFT/PIN_UNPINNED/PIN_ORPHAN fatal, rc 1; a missing ledger is rc 2, never clean; --no-baseline and --tools-dir say out loud that no pin was graded. +283/-4; selftest 30 -> 39 cells, 39/39.
This commit is contained in:
Volt committed
1 parent
e33f4a6a92
commit
e64a40dbec
4 files changed
+350
-15
No files matched your search
@@ -0,0 +1,32 @@
|
||||
# selftest_baseline.tsv -- the cell count each tool's selftest prints, pinned.
|
||||
# Written by: python3 tools/selftest_sweep.py --repo . --write-baseline tools/selftest_baseline.tsv
|
||||
#
|
||||
# WHY THIS FILE EXISTS: the count a tool advertises lives in its own docstring,
|
||||
# so a commit that deletes fixtures and decrements that number agrees with
|
||||
# itself and every leg stays green. This ledger is the outside witness.
|
||||
# `selftest_sweep.py` reads it on every run: a printed count that differs is
|
||||
# PIN_DRIFT and fatal; a tool with no row is PIN_UNPINNED and fatal; a row
|
||||
# whose tool is gone is PIN_ORPHAN and fatal. Re-pin on purpose, in the diff.
|
||||
#
|
||||
# tool<TAB>count<TAB>unit
|
||||
append_only_audit.py 18 cells
|
||||
append_only_owners.py 15 cells
|
||||
census_row_delta.py 33 cells
|
||||
check_lint.py 37 cells
|
||||
checkin_check.py 36 cells
|
||||
citation_hash_class.py 19 fixtures
|
||||
citation_resolve_check.py 14 fixtures
|
||||
doc_census_check.py 104 cells
|
||||
doc_command_check.py 15 cells
|
||||
gate_blame.py 21 cells
|
||||
landing_check.py 26 cells
|
||||
path_claim_check.py 27 cells
|
||||
queue_append_audit.py 25 cells
|
||||
queue_claim_check.py 24 fixtures
|
||||
queue_index.py 58 cells
|
||||
reply_check.py 49 cells
|
||||
selftest_sweep.py 39 cells
|
||||
served_drift_check.py 25 cells
|
||||
tool_map.py 20 cells
|
||||
verify.py 51 cells
|
||||
write_probe.py 15 cells
|
||||
|
+283
-4
@@ -88,6 +88,26 @@ THE MALFORMED-INPUT CLASS, ADDED 2026-10-18
|
||||
its --fail-on-stale is the gate). The axis asserts the message too, so a
|
||||
silent 0 cannot pass as a refusal.
|
||||
|
||||
THE COUNT PIN, ADDED 2026-10-22 -- the docstring was not a witness
|
||||
Every verdict above compares a tool's printed count with a number in the SAME
|
||||
file. That catches a stale number; it does not catch a shrinking corpus,
|
||||
because a commit that deletes three fixtures and decrements the docstring
|
||||
agrees with itself, and every leg stays green. Measured on a clone of this
|
||||
repository: `doc_command_check.py` cut from 15 cells to 12 at both the print
|
||||
and the usage line reads `OK 12 cells` and `sweep: 21 tools, 21 OK, 0 fatal,
|
||||
rc 0` -- a third of its fixtures gone, reported clean.
|
||||
`tools/selftest_baseline.tsv` is the second witness: one row per tool, the
|
||||
count its selftest prints, committed and read from disk. The sweep compares
|
||||
every row against it and prints four classes --
|
||||
PIN_OK / PIN_DRIFT (printed differs from pinned) / PIN_UNPINNED (a tool with
|
||||
no pinned row) / PIN_ORPHAN (a pinned row whose tool is gone). DRIFT,
|
||||
UNPINNED and ORPHAN are FATAL: a corpus may not shrink silently, and a new
|
||||
tool may not arrive unmeasured. PIN_NA (a tool with no --selftest) and
|
||||
PIN_NO_COUNT (its run already failed above) are printed and not counted
|
||||
twice. A missing baseline in repo mode is rc 2 -- could not grade -- never a
|
||||
clean run; `--no-baseline` is the explicit way to sweep without one, and
|
||||
`--write-baseline PATH` re-pins after a deliberate change.
|
||||
|
||||
WHAT COUNTS AS A CLAIM, PRECISELY
|
||||
Only the module docstring (read through `ast.get_docstring`, so a fixture
|
||||
string in the BODY is not a claim -- `doc_census_check.py` contains the literal
|
||||
@@ -111,14 +131,17 @@ NO_PROBE is not: an ungraded tool cannot be reported clean (see the read axis).
|
||||
|
||||
USAGE:
|
||||
python3 selftest_sweep.py --repo /root/troupe-standup
|
||||
python3 selftest_sweep.py --repo /root/troupe-standup --write-baseline tools/selftest_baseline.tsv
|
||||
python3 selftest_sweep.py --tools-dir /tmp/fake --json /tmp/sweep.json
|
||||
python3 selftest_sweep.py --selftest # 30 cells, parser, sweep and both input axes
|
||||
python3 selftest_sweep.py --selftest # 39 cells, parser, sweep, pin and both input axes
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import contextlib
|
||||
import glob
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -149,6 +172,10 @@ FATAL = (
|
||||
"UNREADABLE",
|
||||
)
|
||||
|
||||
# the count pin (2026-10-22): the second witness, kept out of the tool it grades
|
||||
BASELINE_NAME = "selftest_baseline.tsv"
|
||||
PIN_FAULT = ("PIN_DRIFT", "PIN_UNPINNED", "PIN_ORPHAN")
|
||||
|
||||
# --------------------------------------------------------------- read axis
|
||||
# THE SECOND GATE, ADDED 2026-10-17: the exit code a tool gives an input it
|
||||
# cannot read. The repo-wide contract is three codes and no others --
|
||||
@@ -490,6 +517,137 @@ def report(results, source, untracked, quiet=False):
|
||||
return 1 if fatal else 0, ok, len(fatal)
|
||||
|
||||
|
||||
# --------------------------------------------------------------- count pin
|
||||
# WHY THIS IS A FILE. Every verdict in `report()` compares the tool's printed
|
||||
# count with a number inside the tool. That is a real check on a stale number
|
||||
# and no check at all on a shrinking corpus: delete three fixtures, decrement
|
||||
# the docstring, and the file agrees with itself. Measured on a clone of this
|
||||
# repository at 2026-10-22 -- `doc_command_check.py` cut 15 cells -> 12 on both
|
||||
# the print and the usage line still reads `OK 12 cells`, `sweep: 21 tools, 21
|
||||
# OK, 0 fatal`, rc 0. The ledger is the witness the tool cannot edit in the
|
||||
# same commit without the change being visible in the diff.
|
||||
|
||||
def default_baseline(repo: str) -> str:
|
||||
return os.path.join(os.path.abspath(repo), "tools", BASELINE_NAME)
|
||||
|
||||
|
||||
def load_baseline(path: str):
|
||||
"""({tool: (count, unit)}, None) or (None, reason). Blank and # lines skipped."""
|
||||
rows = {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as fh:
|
||||
lines = fh.read().splitlines()
|
||||
except OSError as exc:
|
||||
return None, f"cannot read baseline {path}: {exc}"
|
||||
for n, line in enumerate(lines, 1):
|
||||
if not line.strip() or line.lstrip().startswith("#"):
|
||||
continue
|
||||
parts = line.split("\t")
|
||||
if len(parts) != 3:
|
||||
return None, (f"{path}:{n}: want 3 tab-separated columns "
|
||||
f"(tool, count, unit), got {len(parts)}")
|
||||
name, count_s, unit = (p.strip() for p in parts)
|
||||
if unit not in UNITS:
|
||||
return None, f"{path}:{n}: unit {unit!r} is not one of {UNITS}"
|
||||
try:
|
||||
count = _int(count_s)
|
||||
except ValueError:
|
||||
return None, f"{path}:{n}: count {count_s!r} is not an integer"
|
||||
if name in rows:
|
||||
return None, f"{path}:{n}: duplicate row for {name}"
|
||||
rows[name] = (count, unit)
|
||||
if not rows:
|
||||
return None, f"{path}: no pinned rows"
|
||||
return rows, None
|
||||
|
||||
|
||||
def render_baseline(results) -> str:
|
||||
"""The ledger text for these results. Rows the selftest never printed are
|
||||
skipped here and named by grade_pin() instead -- a pin cannot invent a
|
||||
number the tool did not print."""
|
||||
head = [
|
||||
"# selftest_baseline.tsv -- the cell count each tool's selftest prints, pinned.",
|
||||
"# Written by: python3 tools/selftest_sweep.py --repo . --write-baseline tools/selftest_baseline.tsv",
|
||||
"#",
|
||||
"# WHY THIS FILE EXISTS: the count a tool advertises lives in its own docstring,",
|
||||
"# so a commit that deletes fixtures and decrements that number agrees with",
|
||||
"# itself and every leg stays green. This ledger is the outside witness.",
|
||||
"# `selftest_sweep.py` reads it on every run: a printed count that differs is",
|
||||
"# PIN_DRIFT and fatal; a tool with no row is PIN_UNPINNED and fatal; a row",
|
||||
"# whose tool is gone is PIN_ORPHAN and fatal. Re-pin on purpose, in the diff.",
|
||||
"#",
|
||||
"# tool<TAB>count<TAB>unit",
|
||||
]
|
||||
for r in sorted(results, key=lambda r: r["tool"]):
|
||||
p = r.get("printed")
|
||||
if not p:
|
||||
continue
|
||||
head.append(f"{r['tool']}\t{p['count']}\t{p['unit']}")
|
||||
return "\n".join(head) + "\n"
|
||||
|
||||
|
||||
def pin_faults(rows):
|
||||
return [r for r in rows if r["verdict"] in PIN_FAULT]
|
||||
|
||||
|
||||
def grade_pin(results, baseline):
|
||||
"""One row per tool plus one per orphaned pin. Pure: no I/O, no clock."""
|
||||
rows = []
|
||||
for r in sorted(results, key=lambda r: r["tool"]):
|
||||
name, printed = r["tool"], r.get("printed")
|
||||
pin = baseline.get(name)
|
||||
pinned = {"count": pin[0], "unit": pin[1]} if pin else None
|
||||
seen = ({"count": printed["count"], "unit": printed["unit"]}
|
||||
if printed else None)
|
||||
row = {"tool": name, "pinned": pinned, "printed": seen}
|
||||
if r.get("verdict") == "NO_SELFTEST_FLAG":
|
||||
row.update(verdict="PIN_NA",
|
||||
detail="no --selftest to pin; not counted")
|
||||
elif pin is None:
|
||||
row.update(verdict="PIN_UNPINNED",
|
||||
detail=(f"prints {seen['count']} {seen['unit']}; no pinned row"
|
||||
if seen else "no pinned row, and no count to pin"))
|
||||
elif seen is None:
|
||||
row.update(verdict="PIN_NO_COUNT",
|
||||
detail=(f"pinned {pin[0]} {pin[1]}; the run printed no count "
|
||||
"(graded above)"))
|
||||
elif (seen["count"], seen["unit"]) == (pin[0], pin[1]):
|
||||
row.update(verdict="PIN_OK", detail=f"{pin[0]} {pin[1]}")
|
||||
else:
|
||||
row.update(verdict="PIN_DRIFT",
|
||||
detail=(f"pinned {pin[0]} {pin[1]}; prints "
|
||||
f"{seen['count']} {seen['unit']}"))
|
||||
rows.append(row)
|
||||
seen_tools = {r["tool"] for r in results}
|
||||
for name in sorted(set(baseline) - seen_tools):
|
||||
rows.append({"tool": name, "pinned": {"count": baseline[name][0],
|
||||
"unit": baseline[name][1]},
|
||||
"printed": None, "verdict": "PIN_ORPHAN",
|
||||
"detail": (f"pinned {baseline[name][0]} {baseline[name][1]}; "
|
||||
"no tool of that name at this rev")})
|
||||
return rows
|
||||
|
||||
|
||||
def report_pin(rows, path, quiet=False):
|
||||
bad = pin_faults(rows)
|
||||
if not quiet:
|
||||
print()
|
||||
print(f"count pin: {path} -- the number each selftest prints, pinned "
|
||||
"outside the tool it grades")
|
||||
width = max([len(r["tool"]) for r in rows] + [4])
|
||||
for r in rows:
|
||||
pin, seen = r["pinned"], r["printed"]
|
||||
pin_s = f"{pin['count']} {pin['unit']}" if pin else "-"
|
||||
seen_s = f"{seen['count']} {seen['unit']}" if seen else "-"
|
||||
print(f"{r['tool']:<{width}} {r['verdict']:<13} "
|
||||
f"pinned {pin_s:>12} printed {seen_s:>12} {r['detail']}")
|
||||
agree = sum(1 for r in rows if r["verdict"] == "PIN_OK")
|
||||
pinned = sum(1 for r in rows if r["pinned"])
|
||||
print(f"pin: {pinned} pinned, {agree} agree, {len(bad)} drift"
|
||||
+ (f" ({', '.join(sorted({r['verdict'] for r in bad}))})" if bad else ""))
|
||||
return (1 if bad else 0), len(bad)
|
||||
|
||||
|
||||
# ------------------------------------------------------- read-refusal axis
|
||||
|
||||
def _first_line(text: str) -> str:
|
||||
@@ -802,7 +960,75 @@ def selftest():
|
||||
and NO_PROBE in FATAL_READ,
|
||||
str(FATAL_READ))
|
||||
|
||||
print(f"selftest: 30 cells, {30 - len(bad)} passed, {len(bad)} failed")
|
||||
# c31-c37: the count pin. The defect these grade is the one the classes
|
||||
# above cannot see: a corpus that shrinks while every number in the tool
|
||||
# agrees with every other number in the tool.
|
||||
pin_one = {"fake_good.py": (7, "cells")}
|
||||
pr = grade_pin([{"tool": "fake_good.py", "verdict": "OK",
|
||||
"printed": {"count": 7, "unit": "cells"}}], pin_one)
|
||||
cell("c31 a printed count that matches its pinned row -> PIN_OK",
|
||||
pr[0]["verdict"] == "PIN_OK" and not pin_faults(pr), pr[0]["verdict"])
|
||||
pr = grade_pin([{"tool": "fake_good.py", "verdict": "OK",
|
||||
"printed": {"count": 4, "unit": "cells"}}], pin_one)
|
||||
cell("c32 four cells against a pin of seven -> PIN_DRIFT, fatal",
|
||||
pr[0]["verdict"] == "PIN_DRIFT" and len(pin_faults(pr)) == 1,
|
||||
f"{pr[0]['verdict']} {pr[0]['detail']}")
|
||||
pr = grade_pin([{"tool": "new_tool.py", "verdict": "OK",
|
||||
"printed": {"count": 3, "unit": "cells"}}], pin_one)
|
||||
mine = [r for r in pr if r["tool"] == "new_tool.py"]
|
||||
cell("c33 a tool with no pinned row -> PIN_UNPINNED, fatal",
|
||||
mine and mine[0]["verdict"] == "PIN_UNPINNED" and mine[0] in pin_faults(pr),
|
||||
mine[0]["verdict"] if mine else "no row")
|
||||
pr = grade_pin([], pin_one)
|
||||
cell("c34 a pinned row whose tool is gone -> PIN_ORPHAN, fatal",
|
||||
pr[0]["verdict"] == "PIN_ORPHAN" and len(pin_faults(pr)) == 1,
|
||||
pr[0]["verdict"])
|
||||
pr = grade_pin([{"tool": "flagless.py", "verdict": "NO_SELFTEST_FLAG",
|
||||
"printed": None}], pin_one)
|
||||
mine = [r for r in pr if r["tool"] == "flagless.py"]
|
||||
cell("c35 a tool with no --selftest is PIN_NA, not a fault",
|
||||
mine and mine[0]["verdict"] == "PIN_NA" and mine[0] not in pin_faults(pr),
|
||||
mine[0]["verdict"] if mine else "no row")
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
bp = os.path.join(td, "pin.tsv")
|
||||
blob = render_baseline([{"tool": "b.py", "printed": {"count": 9, "unit": "fixtures"}},
|
||||
{"tool": "a.py", "printed": {"count": 2, "unit": "cells"}}])
|
||||
with open(bp, "w", encoding="utf-8") as fh:
|
||||
fh.write(blob)
|
||||
round_trip, why = load_baseline(bp)
|
||||
cell("c36 render/load round-trip keeps both rows and their units",
|
||||
round_trip == {"b.py": (9, "fixtures"), "a.py": (2, "cells")},
|
||||
f"{round_trip} {why}")
|
||||
with open(bp, "w", encoding="utf-8") as fh:
|
||||
fh.write("a.py\tnine\tcells\n")
|
||||
bad_pin, why = load_baseline(bp)
|
||||
cell("c37 a non-integer count is refused, naming the line",
|
||||
bad_pin is None and ":1:" in (why or "") and "not an integer" in (why or ""),
|
||||
str(why))
|
||||
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
for nm, body in (("fake_good.py", good), ("fake_flagless.py", flagless)):
|
||||
with open(os.path.join(td, nm), "w") as fh:
|
||||
fh.write(body)
|
||||
bp = os.path.join(td, "pin.tsv")
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
main(["--tools-dir", td, "--write-baseline", bp, "--quiet"])
|
||||
written, why = load_baseline(bp)
|
||||
cell("c38 --write-baseline pins exactly what the run printed",
|
||||
written == {"fake_good.py": (7, "cells")}, f"{written} {why}")
|
||||
with open(bp, "w", encoding="utf-8") as fh:
|
||||
fh.write("fake_good.py\t9\tcells\n")
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf):
|
||||
rc_pin = main(["--tools-dir", td, "--baseline", bp, "--quiet"])
|
||||
out = buf.getvalue()
|
||||
cell("c39 end to end: a drifted pin names PIN_DRIFT and turns the run red",
|
||||
"PIN_DRIFT" in out and "1 drift (PIN_DRIFT)" in out and rc_pin == 1,
|
||||
(out.strip().splitlines() or ["no output"])[-1])
|
||||
|
||||
print(f"selftest: 39 cells, {39 - len(bad)} passed, {len(bad)} failed")
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
@@ -816,6 +1042,13 @@ def main(argv=None):
|
||||
help="sweep this directory instead of the repo's tracked tools/")
|
||||
ap.add_argument("--timeout", type=int, default=120, metavar="S")
|
||||
ap.add_argument("--json", default=None, metavar="PATH")
|
||||
ap.add_argument("--baseline", default=None, metavar="PATH",
|
||||
help="the pinned counts to grade against "
|
||||
f"(default: <repo>/tools/{BASELINE_NAME} in repo mode)")
|
||||
ap.add_argument("--no-baseline", action="store_true",
|
||||
help="do not grade the count pin (the run says so out loud)")
|
||||
ap.add_argument("--write-baseline", default=None, metavar="PATH",
|
||||
help="write the pinned counts from this run to PATH")
|
||||
ap.add_argument("--quiet", action="store_true")
|
||||
ap.add_argument("--verbose", action="store_true")
|
||||
ap.add_argument("--selftest", action="store_true")
|
||||
@@ -840,6 +1073,48 @@ def main(argv=None):
|
||||
axis = read_axis(rows, a.timeout)
|
||||
rrc, read_fatal = report_read(axis, a.quiet)
|
||||
|
||||
# ---- the count pin: the witness outside the tool's own docstring
|
||||
if a.no_baseline:
|
||||
pin_path, pin_note = None, "pin: not graded (--no-baseline)"
|
||||
elif a.baseline:
|
||||
pin_path, pin_note = a.baseline, None
|
||||
elif a.write_baseline:
|
||||
pin_path, pin_note = a.write_baseline, None
|
||||
elif a.tools_dir:
|
||||
pin_path, pin_note = None, ("pin: not graded (--tools-dir: a synthetic "
|
||||
"directory carries no pinned counts)")
|
||||
else:
|
||||
pin_path, pin_note = default_baseline(a.repo), None
|
||||
|
||||
pin_rc, pin_fatal, pin_rows = 0, 0, None
|
||||
if pin_path is None:
|
||||
if not a.quiet and pin_note:
|
||||
print()
|
||||
print(pin_note)
|
||||
else:
|
||||
if a.write_baseline:
|
||||
text = render_baseline(results)
|
||||
try:
|
||||
with open(a.write_baseline, "w", encoding="utf-8") as fh:
|
||||
fh.write(text)
|
||||
except OSError as exc:
|
||||
print("cannot write --write-baseline " + a.write_baseline
|
||||
+ ": " + str(exc), file=sys.stderr)
|
||||
return 2
|
||||
first = [l for l in text.splitlines() if l and not l.startswith("#")]
|
||||
print(f"pin: wrote {a.write_baseline} ({len(first)} row(s))")
|
||||
pinned, why = load_baseline(pin_path)
|
||||
if pinned is None:
|
||||
# A missing or malformed ledger is not a clean run: the pin graded
|
||||
# nothing. rc 2 (could not grade), never rc 1 and never 0.
|
||||
print("pin: cannot grade -- " + str(why)
|
||||
+ " (run --write-baseline to create one, or --no-baseline to sweep without)",
|
||||
file=sys.stderr)
|
||||
pin_rc = 2
|
||||
else:
|
||||
pin_rows = grade_pin(results, pinned)
|
||||
pin_rc, pin_fatal = report_pin(pin_rows, pin_path, a.quiet)
|
||||
|
||||
if a.json:
|
||||
try:
|
||||
with open(a.json, "w") as fh:
|
||||
@@ -851,11 +1126,15 @@ def main(argv=None):
|
||||
"fatal": fatal,
|
||||
"ok": ok,
|
||||
"read_axis": axis,
|
||||
"read_fatal": read_fatal}, fh, indent=1)
|
||||
"read_fatal": read_fatal,
|
||||
"pin": {"path": pin_path, "rows": pin_rows,
|
||||
"fatal": pin_fatal}}, fh, indent=1)
|
||||
except OSError as exc:
|
||||
print("cannot write --json " + a.json + ": " + str(exc), file=sys.stderr)
|
||||
return 2
|
||||
return 1 if (rc or rrc) else 0
|
||||
if pin_rc == 2:
|
||||
return 2
|
||||
return 1 if (rc or rrc or pin_rc) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in new issue
Block a user