volt
54451e4f6d
The 2026-10-31 artifact hardcoded an absolute home path in both the scanner and the 'independent verifier', so a clone's CHECK graded the host tree (a clone whose committed TSV held one garbage line still reported on the host's 79-row TSV). The two copies also carried different rule lists -- the generator rejected true/false, the verifier did not -- and the TSV's comment line was emitted by neither script's stdout. - draft_compliance_rule.py: the bare-command rule, stated once, 11 cases - generator: paths from __file__, artifact excluded from its own population, deterministic output (comment + header + rows) - verifier: own re-derivation, both directions, grades the report's six counts, rc 0/1/2, --selftest one control + eight mutants + rule-agreement + no-host-path - CHECK: four legs (selftest, rederive, byte-identical regeneration, copy-parity) - report: measured numbers (79 drafts, 14/14/11, 68 failing, 3 non-bare)
116 lines
4.5 KiB
Python
116 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""The draft-compliance contract, stated once.
|
|
|
|
WHY THIS FILE EXISTS
|
|
The 2026-10-31 report shipped two copies of the scan: the generator and the
|
|
"independent verifier" were the same code with two `bad` lists that had already
|
|
drifted -- the generator rejected the words `true`/`false` and the verifier did
|
|
not, so a CHECK line containing either read `check_clean=False` from one and
|
|
`True` from the other. Nothing measured the disagreement, because the verifier
|
|
compared the committed TSV against its own copy of the scan and never against
|
|
the generator's.
|
|
|
|
Two things are shared here and nothing else: the constants that name the
|
|
artifact and its columns, and ONE implementation of the rule `is_bare`. The
|
|
verifier re-implements `is_bare` independently (see `verify_draft_compliance.py`)
|
|
and its `--selftest` cell `rule-agreement` requires the two implementations to
|
|
return the same answer on every CHECK line in the corpus, so a future
|
|
divergence is a red cell instead of a silent contradiction.
|
|
|
|
WHAT `is_bare` MEANS -- a CHECK command must not swallow or sequence failure
|
|
A command is bare iff no token is a control operator `;` `|` `||` `&`, and no
|
|
token is the null command `true` or `false`. `&&` IS allowed: it stops at the
|
|
first failure and returns it, which is the property the rule is about. A pipe
|
|
is rejected on this archive's own doctrine -- without `pipefail` a pipeline's
|
|
rc is its LAST command's -- and a `;` sequence returns only the last rc.
|
|
A command that does not tokenize (unbalanced quotes) is not bare.
|
|
"""
|
|
import os
|
|
import re
|
|
import shlex
|
|
|
|
# The artifact this contract grades; it is not a draft and is excluded from its
|
|
# own population (the census-artifact convention: an artifact is not a row in
|
|
# the table it publishes).
|
|
ARTIFACT_MD = "draft-compliance-report-2026-10-31.md"
|
|
ARTIFACT_TSV = "draft-compliance-report-2026-10-31.tsv"
|
|
|
|
COLUMNS = ("filename", "lines", "has_verdict", "has_check", "check_clean",
|
|
"check_command")
|
|
|
|
CONTROL_TOKENS = frozenset({";", "|", "||", "&"})
|
|
NULL_COMMANDS = frozenset({"true", "false"})
|
|
|
|
|
|
def check_line(text):
|
|
"""The text after the first `^CHECK:` line that carries text.
|
|
|
|
An empty `CHECK:` marker is skipped rather than returned as the command:
|
|
`daily-draft-verification-report.md` opens a quoted template with a bare
|
|
`CHECK:` at line 102 before its real command at line 145, and the 2026-10-31
|
|
regex (which required at least one character) read the real one. Both
|
|
implementations keep that reading.
|
|
"""
|
|
m = re.search(r"^CHECK:\s*(.+)$", text, re.MULTILINE)
|
|
return m.group(1).strip() if m else ""
|
|
|
|
|
|
def is_bare(cmd):
|
|
if not cmd:
|
|
return False
|
|
try:
|
|
tokens = shlex.split(cmd, posix=True)
|
|
except ValueError:
|
|
return False
|
|
for tok in tokens:
|
|
if tok in CONTROL_TOKENS or tok in NULL_COMMANDS:
|
|
return False
|
|
return True
|
|
|
|
|
|
def has_verdict(text):
|
|
return re.search(r"^VERDICT:", text, re.MULTILINE) is not None
|
|
|
|
|
|
def has_check(text):
|
|
return re.search(r"^CHECK:", text, re.MULTILINE) is not None
|
|
|
|
|
|
def row_for(name, text):
|
|
"""One TSV row (all strings) for one draft file's text."""
|
|
cmd = check_line(text)
|
|
return (name, str(len(text.splitlines())), str(has_verdict(text)),
|
|
str(has_check(text)), str(is_bare(cmd)), cmd)
|
|
|
|
|
|
SELFTEST_CASES = [
|
|
# (command, expected is_bare, why)
|
|
("python3 tools/verify.py --fast", True, "single command"),
|
|
("bash CHECK-quill-draft-compliance", True, "single command"),
|
|
("test -f a.md && grep -q VERDICT a.md", True, "&& stops at the first failure"),
|
|
("python3 a.py | grep -q x", False, "pipe: rc is the last command's"),
|
|
("python3 a.py || true", False, "|| swallows the failure"),
|
|
("python3 a.py; true", False, "; sequences, rc is the last"),
|
|
("python3 a.py &", False, "backgrounding discards rc"),
|
|
("curl -s x && true", False, "null command as the last word"),
|
|
("python3 a.py 'unbalanced", False, "does not tokenize"),
|
|
("", False, "no CHECK line"),
|
|
("python3 check.py --allow=true", True,
|
|
"`true` as part of a word is not the null command"),
|
|
]
|
|
|
|
|
|
def _selftest():
|
|
bad = 0
|
|
for cmd, want, why in SELFTEST_CASES:
|
|
got = is_bare(cmd)
|
|
if got != want:
|
|
bad += 1
|
|
print("FAIL rule: %r -> %s, wanted %s (%s)" % (cmd, got, want, why))
|
|
print("rule: %d case(s), %d failed" % (len(SELFTEST_CASES), bad))
|
|
return bad
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(1 if _selftest() else 0)
|