volt
54451e4f6d
The 2026-10-31 artifact hardcoded an absolute home path in both the scanner and the 'independent verifier', so a clone's CHECK graded the host tree (a clone whose committed TSV held one garbage line still reported on the host's 79-row TSV). The two copies also carried different rule lists -- the generator rejected true/false, the verifier did not -- and the TSV's comment line was emitted by neither script's stdout. - draft_compliance_rule.py: the bare-command rule, stated once, 11 cases - generator: paths from __file__, artifact excluded from its own population, deterministic output (comment + header + rows) - verifier: own re-derivation, both directions, grades the report's six counts, rc 0/1/2, --selftest one control + eight mutants + rule-agreement + no-host-path - CHECK: four legs (selftest, rederive, byte-identical regeneration, copy-parity) - report: measured numbers (79 drafts, 14/14/11, 68 failing, 3 non-bare)
383 lines
14 KiB
Python
383 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Independent verifier for draft-compliance-report-2026-10-31.{tsv,md}.
|
|
|
|
WHAT IT GRADES, AND AGAINST WHAT
|
|
The committed TSV is a claim about the .md drafts beside it. This verifier
|
|
re-derives every row from the files with its OWN code -- its own line counter,
|
|
its own `VERDICT:`/`CHECK:` detection and its own implementation of the bare-
|
|
command rule -- and compares in both directions: every TSV row against the
|
|
file it names, and every file against the row it should have. It also grades
|
|
the report's Evidence table against the counts it just measured, so a stale
|
|
numeral in the prose is a fault rather than a reader's problem.
|
|
|
|
Paths come from THIS FILE's location. The 2026-10-31 version hardcoded an
|
|
absolute home-directory path in both the scanner and the verifier, so a
|
|
clone's CHECK graded the host's tree and a mutated copy of the committed TSV
|
|
still read PASS; `--selftest` cell `no-host-path` greps the sources for that
|
|
class, because this file's own fix must not reintroduce it.
|
|
|
|
EXIT CODES -- the archive's convention
|
|
0 the TSV and the report re-derive from the tree
|
|
1 a fault: a row, a file-set entry or a reported count disagrees
|
|
2 could not grade: the report or TSV is missing/unreadable, or the lane
|
|
directory does not exist. Never a pass.
|
|
|
|
--selftest runs the cells in a temp tree: one green control and eight mutants,
|
|
so a green reading is never the only thing this checker can print.
|
|
|
|
NOTE ON THE SECOND IMPLEMENTATION OF `is_bare`
|
|
It reads operators on the raw string and only checks quoting for parity, so an
|
|
operator inside a quoted argument reads as not bare where the shlex-based rule
|
|
would read it as bare. That is deliberate and conservative (it over-reports
|
|
"not bare", never over-reports "bare"), and `rule-agreement` fails loudly if
|
|
the two implementations ever disagree on a command this corpus actually holds.
|
|
"""
|
|
import argparse
|
|
import os
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
sys.path.insert(0, HERE)
|
|
|
|
import draft_compliance_rule as rule # noqa: E402
|
|
|
|
# The report's Evidence table, label -> the metric it claims. The labels are
|
|
# matched verbatim; a missing label is a fault (the report must say what it
|
|
# measured).
|
|
REPORT_LABELS = (
|
|
("Total `.md` drafts scanned", "total"),
|
|
("Drafts with `VERDICT:`", "verdict"),
|
|
("Drafts with `CHECK:`", "check"),
|
|
("Drafts with a bare `CHECK:` command", "bare"),
|
|
("Drafts failing at least one column", "failing"),
|
|
("`VERDICT:` and `CHECK:` but a non-bare command", "nonbare_both"),
|
|
)
|
|
|
|
NULL_RE = re.compile(r"(?:^|[\s;&|(])(true|false)(?=$|[\s;&|)])")
|
|
|
|
|
|
def own_is_bare(cmd):
|
|
"""The same contract as `rule.is_bare`, implemented differently.
|
|
|
|
Quoted regions are removed before the operator scan, so an operator inside a
|
|
quoted argument is an argument, not an operator; parity of quotes is checked
|
|
first, because a command that does not close its quotes does not tokenize
|
|
and is therefore not bare under the contract.
|
|
"""
|
|
if not cmd:
|
|
return False
|
|
if cmd.count("'") % 2 or cmd.count('"') % 2:
|
|
return False
|
|
stripped = re.sub(r"'[^']*'|\"[^\"]*\"", "''", cmd)
|
|
if ";" in stripped or "||" in stripped:
|
|
return False
|
|
rest = stripped.replace("&&", "")
|
|
if "&" in rest or "|" in rest:
|
|
return False
|
|
return NULL_RE.search(stripped) is None
|
|
|
|
|
|
def own_line_count(text):
|
|
"""Line count without splitlines(): newlines plus a final unterminated line."""
|
|
return text.count("\n") + (1 if text and not text.endswith("\n") else 0)
|
|
|
|
|
|
def own_rows(lane, exclude=rule.ARTIFACT_MD):
|
|
"""Re-derive the table from the files, without calling the generator."""
|
|
rows = []
|
|
names = sorted(n for n in os.listdir(lane)
|
|
if n.endswith(".md") and n != exclude)
|
|
for name in names:
|
|
path = os.path.join(lane, name)
|
|
if not os.path.isfile(path):
|
|
continue
|
|
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
|
text = f.read()
|
|
has_v = has_c = False
|
|
cmd = ""
|
|
for ln in text.split("\n"):
|
|
if ln.startswith("VERDICT:"):
|
|
has_v = True
|
|
elif ln.startswith("CHECK:"):
|
|
has_c = True
|
|
rest = ln[len("CHECK:"):].strip()
|
|
if rest and not cmd:
|
|
cmd = rest
|
|
rows.append((name, str(own_line_count(text)), str(has_v), str(has_c),
|
|
str(own_is_bare(cmd)), cmd))
|
|
return rows
|
|
|
|
|
|
def read_tsv(path):
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
raw = f.read().split("\n")
|
|
comment = ""
|
|
if raw and raw[0].startswith("#"):
|
|
comment = raw[0]
|
|
raw = raw[1:]
|
|
body = [ln for ln in raw if ln.strip()]
|
|
header = body[0].split("\t") if body else []
|
|
rows = [tuple(ln.split("\t")) for ln in body[1:]]
|
|
return comment, header, rows
|
|
|
|
|
|
def read_report_counts(path):
|
|
counts = {}
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
for ln in f:
|
|
m = re.match(r"^\|\s*(.+?)\s*\|\s*(\d+)\s*\|\s*$", ln.rstrip("\n"))
|
|
if m:
|
|
counts[m.group(1)] = int(m.group(2))
|
|
return counts
|
|
|
|
|
|
def metrics(rows):
|
|
def ok(r):
|
|
return r[2] == "True" and r[3] == "True" and r[4] == "True"
|
|
|
|
return {
|
|
"total": len(rows),
|
|
"verdict": sum(1 for r in rows if r[2] == "True"),
|
|
"check": sum(1 for r in rows if r[3] == "True"),
|
|
"bare": sum(1 for r in rows if r[4] == "True"),
|
|
"failing": sum(1 for r in rows if not ok(r)),
|
|
"nonbare_both": sum(1 for r in rows
|
|
if r[2] == "True" and r[3] == "True" and r[4] != "True"),
|
|
}
|
|
|
|
|
|
def check(lane=None, report=None, tsv=None):
|
|
"""Return (rc, faults, notes). Never raises for bad input."""
|
|
lane = lane or HERE
|
|
report = report or os.path.join(lane, rule.ARTIFACT_MD)
|
|
tsv = tsv or os.path.join(lane, rule.ARTIFACT_TSV)
|
|
faults, notes = [], []
|
|
|
|
if not os.path.isdir(lane):
|
|
return 2, ["lane directory missing: %s" % lane], notes
|
|
for path, what in ((report, "report"), (tsv, "TSV")):
|
|
if not os.path.isfile(path):
|
|
return 2, ["%s missing: %s" % (what, path)], notes
|
|
|
|
try:
|
|
comment, header, claimed = read_tsv(tsv)
|
|
reported = read_report_counts(report)
|
|
except (OSError, UnicodeDecodeError) as exc:
|
|
return 2, ["cannot read the artifact: %s" % exc], notes
|
|
|
|
if tuple(header) != rule.COLUMNS:
|
|
faults.append("TSV header is %r, contract is %r"
|
|
% (tuple(header), rule.COLUMNS))
|
|
|
|
derived = own_rows(lane)
|
|
der_map = {r[0]: r for r in derived}
|
|
claim_map = {r[0]: r for r in claimed}
|
|
|
|
for name in sorted(set(claim_map) - set(der_map)):
|
|
faults.append("TSV row names a file the lane does not hold: %s" % name)
|
|
for name in sorted(set(der_map) - set(claim_map)):
|
|
faults.append("lane file has no TSV row: %s" % name)
|
|
for name in sorted(set(der_map) & set(claim_map)):
|
|
if der_map[name] != claim_map[name]:
|
|
faults.append("row mismatch for %s: claimed=%r derived=%r"
|
|
% (name, claim_map[name], der_map[name]))
|
|
|
|
expected = "# measured %d markdown files" % len(derived)
|
|
if comment != expected:
|
|
faults.append("TSV comment is %r, measured %r" % (comment, expected))
|
|
|
|
m = metrics(derived)
|
|
for label, key in REPORT_LABELS:
|
|
if label not in reported:
|
|
faults.append("report table row missing: %r" % label)
|
|
elif reported[label] != m[key]:
|
|
faults.append("report says %r = %d, measured %d"
|
|
% (label, reported[label], m[key]))
|
|
|
|
# The contract has two implementations; they must agree on every command the
|
|
# corpus carries, or one of them is grading a different rule.
|
|
for r in derived:
|
|
if rule.is_bare(r[5]) != own_is_bare(r[5]):
|
|
faults.append("rule divergence on %s: rule=%s verifier=%s cmd=%r"
|
|
% (r[0], rule.is_bare(r[5]), own_is_bare(r[5]), r[5]))
|
|
|
|
notes.append("population %d, verdict %d, check %d, bare %d, failing %d"
|
|
% (m["total"], m["verdict"], m["check"], m["bare"], m["failing"]))
|
|
return (1 if faults else 0), faults, notes
|
|
|
|
|
|
FIXTURE_FILES = {
|
|
"good.md": "# t\n\nVERDICT: ok\nCHECK: python3 x.py\n",
|
|
"plain.md": "# nothing\n",
|
|
"piped.md": "# t\n\nVERDICT: ok\nCHECK: python3 x.py | grep -q ok\n",
|
|
}
|
|
|
|
|
|
def _fixture_report(claimed):
|
|
lines = ["# fixture", "", "| Metric | Value |", "|---|---|"]
|
|
for label, _key in REPORT_LABELS:
|
|
lines.append("| %s | %d |" % (label, claimed[label]))
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def _fixture_tree(root, mutate=None):
|
|
lane = os.path.join(root, "lane")
|
|
os.makedirs(lane)
|
|
for name, text in FIXTURE_FILES.items():
|
|
with open(os.path.join(lane, name), "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
rows = own_rows(lane)
|
|
m = metrics(rows)
|
|
claimed = {label: m[key] for label, key in REPORT_LABELS}
|
|
report = os.path.join(lane, rule.ARTIFACT_MD)
|
|
with open(report, "w", encoding="utf-8") as f:
|
|
f.write(_fixture_report(claimed))
|
|
tsv = os.path.join(lane, rule.ARTIFACT_TSV)
|
|
with open(tsv, "w", encoding="utf-8") as f:
|
|
f.write("# measured %d markdown files\n" % len(rows))
|
|
f.write("\t".join(rule.COLUMNS) + "\n")
|
|
for r in rows:
|
|
f.write("\t".join(r) + "\n")
|
|
if mutate:
|
|
mutate(lane, report, tsv)
|
|
return lane, report, tsv
|
|
|
|
|
|
def selftest(verbose=True):
|
|
cells, failed = [], []
|
|
|
|
def cell(name, want, mutate=None):
|
|
tmp = tempfile.mkdtemp(prefix="q142-")
|
|
try:
|
|
lane, report, tsv = _fixture_tree(tmp, mutate)
|
|
got, faults, _notes = check(lane, report, tsv)
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
cells.append(name)
|
|
if got != want:
|
|
failed.append((name, want, got, faults[:3]))
|
|
if verbose:
|
|
print("%-22s rc=%d want=%d %s"
|
|
% (name, got, want, "ok" if got == want else "FAIL"))
|
|
|
|
def bump_line(lane, report, tsv):
|
|
with open(os.path.join(lane, "good.md"), "a", encoding="utf-8") as f:
|
|
f.write("one more line\n")
|
|
|
|
def drop_verdict(lane, report, tsv):
|
|
path = os.path.join(lane, "good.md")
|
|
with open(path, encoding="utf-8") as f:
|
|
text = f.read()
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(text.replace("VERDICT: ok\n", ""))
|
|
|
|
def drop_row(lane, report, tsv):
|
|
with open(tsv, encoding="utf-8") as f:
|
|
lines = f.read().split("\n")
|
|
with open(tsv, "w", encoding="utf-8") as f:
|
|
f.write("\n".join(ln for ln in lines if not ln.startswith("piped.md")))
|
|
|
|
def extra_file(lane, report, tsv):
|
|
with open(os.path.join(lane, "late.md"), "w", encoding="utf-8") as f:
|
|
f.write("# late\n")
|
|
|
|
def no_report(lane, report, tsv):
|
|
os.remove(report)
|
|
|
|
def no_tsv(lane, report, tsv):
|
|
os.remove(tsv)
|
|
|
|
def stale_count(lane, report, tsv):
|
|
with open(report, encoding="utf-8") as f:
|
|
text = f.read()
|
|
text = re.sub(r"(\| Total `\.md` drafts scanned \| )3( \|)",
|
|
r"\g<1>2\g<2>", text)
|
|
with open(report, "w", encoding="utf-8") as f:
|
|
f.write(text)
|
|
|
|
def check_line_edited(lane, report, tsv):
|
|
with open(os.path.join(lane, "good.md"), "w", encoding="utf-8") as f:
|
|
f.write("# t\n\nVERDICT: ok\nCHECK: python3 y.py\n")
|
|
|
|
cell("green-control", 0)
|
|
cell("line-count-moved", 1, bump_line)
|
|
cell("verdict-removed", 1, drop_verdict)
|
|
cell("row-deleted", 1, drop_row)
|
|
cell("file-without-row", 1, extra_file)
|
|
cell("report-missing", 2, no_report)
|
|
cell("tsv-missing", 2, no_tsv)
|
|
cell("report-count-stale", 1, stale_count)
|
|
cell("check-line-edited", 1, check_line_edited)
|
|
|
|
# The two implementations of the rule, over the fixture cases and this
|
|
# directory's own corpus. A divergence here is what the 2026-10-31 copies had.
|
|
divergence = [cmd for cmd, want, _why in rule.SELFTEST_CASES
|
|
if own_is_bare(cmd) != want]
|
|
for r in own_rows(HERE):
|
|
if rule.is_bare(r[5]) != own_is_bare(r[5]):
|
|
divergence.append(r[0])
|
|
cells.append("rule-agreement")
|
|
if divergence:
|
|
failed.append(("rule-agreement", 0, 1, divergence[:3]))
|
|
if verbose:
|
|
print("%-22s %s" % ("rule-agreement",
|
|
"ok" if not divergence else "FAIL %r" % divergence[:3]))
|
|
|
|
# No host path in the sources: the 2026-10-31 defect, graded at the source.
|
|
offenders = []
|
|
needle = os.sep + "root" + os.sep
|
|
for name in (os.path.basename(__file__), "draft_compliance_report.py",
|
|
"draft_compliance_rule.py"):
|
|
with open(os.path.join(HERE, name), encoding="utf-8") as f:
|
|
if needle in f.read():
|
|
offenders.append(name)
|
|
cells.append("no-host-path")
|
|
if offenders:
|
|
failed.append(("no-host-path", 0, 1, offenders))
|
|
if verbose:
|
|
print("%-22s %s" % ("no-host-path",
|
|
"ok" if not offenders else "FAIL %r" % offenders))
|
|
|
|
print("\nselftest: %d cells, %d passed, %d failed"
|
|
% (len(cells), len(cells) - len(failed), len(failed)))
|
|
for name, want, got, detail in failed:
|
|
print(" FAIL %s: rc %d wanted %d -- %r" % (name, got, want, detail))
|
|
return 1 if failed else 0
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(
|
|
description="Verify the quill draft-compliance report.")
|
|
ap.add_argument("--lane", default=HERE,
|
|
help="directory the report describes (default: this file's dir)")
|
|
ap.add_argument("--report", default=None, help="report .md (default: in --lane)")
|
|
ap.add_argument("--tsv", default=None, help="report .tsv (default: in --lane)")
|
|
ap.add_argument("--selftest", action="store_true", help="run the cells")
|
|
a = ap.parse_args(argv)
|
|
|
|
if a.selftest:
|
|
return selftest()
|
|
|
|
rc, faults, notes = check(a.lane, a.report, a.tsv)
|
|
for note in notes:
|
|
print(note)
|
|
if rc == 0:
|
|
print("VERDICT: PASS -- TSV and report re-derive from the tree (%s)"
|
|
% notes[0])
|
|
elif rc == 2:
|
|
print("VERDICT: UNGRADABLE")
|
|
for f in faults:
|
|
print(" -", f)
|
|
else:
|
|
print("VERDICT: FAIL")
|
|
for f in faults:
|
|
print(" -", f)
|
|
return rc
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|