From 54451e4f6dbf9a625c86e68ea5c2699772416153 Mon Sep 17 00:00:00 2001 From: volt Date: Sat, 31 Oct 2026 23:58:23 +0000 Subject: [PATCH] volt: quill draft-compliance artifact grades its own tree (Q142) The 2026-10-31 artifact hardcoded an absolute home path in both the scanner and the 'independent verifier', so a clone's CHECK graded the host tree (a clone whose committed TSV held one garbage line still reported on the host's 79-row TSV). The two copies also carried different rule lists -- the generator rejected true/false, the verifier did not -- and the TSV's comment line was emitted by neither script's stdout. - draft_compliance_rule.py: the bare-command rule, stated once, 11 cases - generator: paths from __file__, artifact excluded from its own population, deterministic output (comment + header + rows) - verifier: own re-derivation, both directions, grades the report's six counts, rc 0/1/2, --selftest one control + eight mutants + rule-agreement + no-host-path - CHECK: four legs (selftest, rederive, byte-identical regeneration, copy-parity) - report: measured numbers (79 drafts, 14/14/11, 68 failing, 3 non-bare) --- CHECK-quill-draft-compliance | 98 ++++- .../draft-compliance-report-2026-10-31.md | 69 ++- .../draft-compliance-report-2026-10-31.tsv | 9 +- wizard-work/quill/draft_compliance_report.py | 99 +++-- wizard-work/quill/draft_compliance_rule.py | 115 +++++ wizard-work/quill/verify_draft_compliance.py | 412 +++++++++++++++--- 6 files changed, 672 insertions(+), 130 deletions(-) create mode 100644 wizard-work/quill/draft_compliance_rule.py diff --git a/CHECK-quill-draft-compliance b/CHECK-quill-draft-compliance index 161e044..455e84c 100755 --- a/CHECK-quill-draft-compliance +++ b/CHECK-quill-draft-compliance @@ -1,8 +1,94 @@ #!/usr/bin/env bash -# CHECK -- quill: draft-compliance report 2026-10-31 (forge: quill lane). -# Exit 0 iff the committed TSV re-derives from the live scan with zero mismatches. +# CHECK -- quill: draft-compliance report 2026-10-31 (forge: quill lane; gate by volt, Q142). +# +# Four legs, all run from this script's own directory: +# selftest one green control and eight mutants, plus the rule-agreement and +# no-host-path cells +# rederive the committed TSV and the report's Evidence table re-derive from +# the .md drafts beside them +# regenerate the generator's output is byte-identical to the committed TSV +# copy-parity the same scripts run against a COPY of the lane grade the copy: +# clean copy rc 0, copy with one TSV row moved rc 1 +# +# Exit codes: 0 all legs passed, 1 a leg failed, 2 a leg could not be graded. set -u -cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1 -REPO_ROOT="$(pwd)" -cd "$REPO_ROOT" || exit 1 -exec python3 wizard-work/quill/verify_draft_compliance.py + +cd "$(dirname "${BASH_SOURCE[0]}")" || exit 2 +ROOT="$(pwd)" +LANE="$ROOT/wizard-work/quill" +WORK="$(mktemp -d "${TMPDIR:-/tmp}/check-quill.XXXXXX")" || exit 2 +trap 'rm -rf "$WORK"' EXIT + +passed=0 +failed=0 +row() { printf ' %-12s %s\n' "$1" "$2"; } + +# --- leg 1: the verifier's own cells ----------------------------------------- +if python3 "$LANE/verify_draft_compliance.py" --selftest >"$WORK/selftest.out" 2>&1; then + row selftest "PASS $(tail -n 1 "$WORK/selftest.out")" + passed=$((passed + 1)) +else + row selftest "FAULT $(tail -n 1 "$WORK/selftest.out")" + cat "$WORK/selftest.out" + failed=$((failed + 1)) +fi + +# --- leg 2: the committed artifact re-derives -------------------------------- +if python3 "$LANE/verify_draft_compliance.py" >"$WORK/rederive.out" 2>&1; then + row rederive "PASS $(head -n 1 "$WORK/rederive.out")" + passed=$((passed + 1)) +else + rc=$? + row rederive "FAULT rc=$rc" + cat "$WORK/rederive.out" + failed=$((failed + 1)) +fi + +# --- leg 3: the TSV is exactly what the generator writes --------------------- +python3 "$LANE/draft_compliance_report.py" >"$WORK/regen.tsv" 2>"$WORK/regen.err" +gen_rc=$? +if [ "$gen_rc" -ne 0 ]; then + row regenerate "FAULT generator rc=$gen_rc" + cat "$WORK/regen.err" + failed=$((failed + 1)) +elif cmp -s "$WORK/regen.tsv" "$LANE/draft-compliance-report-2026-10-31.tsv"; then + row regenerate "PASS bytes identical ($(wc -c <"$WORK/regen.tsv") B)" + passed=$((passed + 1)) +else + row regenerate "FAULT committed TSV differs from a fresh derivation" + diff "$WORK/regen.tsv" "$LANE/draft-compliance-report-2026-10-31.tsv" | head -10 + failed=$((failed + 1)) +fi + +# --- leg 4: a copy grades the copy, not this tree ---------------------------- +mkdir -p "$WORK/lane" || exit 2 +cp "$LANE"/*.md "$LANE"/draft-compliance-report-2026-10-31.tsv \ + "$LANE"/draft_compliance_rule.py "$LANE/draft_compliance_report.py" \ + "$LANE/verify_draft_compliance.py" "$WORK/lane/" 2>/dev/null +if python3 "$WORK/lane/verify_draft_compliance.py" >"$WORK/copy-clean.out" 2>&1; then + clean_rc=0 +else + clean_rc=$? +fi +# move one row's line count in the COPY's TSV: the copy must notice +awk -F'\t' 'BEGIN{OFS="\t"} NR==3{$2=$2+1} {print}' \ + "$WORK/lane/draft-compliance-report-2026-10-31.tsv" >"$WORK/mut.tsv" \ + && mv "$WORK/mut.tsv" "$WORK/lane/draft-compliance-report-2026-10-31.tsv" +if python3 "$WORK/lane/verify_draft_compliance.py" >"$WORK/copy-mut.out" 2>&1; then + mut_rc=0 +else + mut_rc=$? +fi +if [ "$clean_rc" -eq 0 ] && [ "$mut_rc" -eq 1 ]; then + row copy-parity "PASS clean copy rc=0, mutated copy rc=1" + passed=$((passed + 1)) +else + row copy-parity "FAULT clean copy rc=$clean_rc (want 0), mutated rc=$mut_rc (want 1)" + head -5 "$WORK/copy-clean.out" "$WORK/copy-mut.out" + failed=$((failed + 1)) +fi + +echo +echo "CHECK-quill-draft-compliance: $passed leg(s) passed, $failed fault(s)" +[ "$failed" -eq 0 ] || exit 1 +exit 0 diff --git a/wizard-work/quill/draft-compliance-report-2026-10-31.md b/wizard-work/quill/draft-compliance-report-2026-10-31.md index 280c1bb..b67efdb 100644 --- a/wizard-work/quill/draft-compliance-report-2026-10-31.md +++ b/wizard-work/quill/draft-compliance-report-2026-10-31.md @@ -1,16 +1,24 @@ # Draft Compliance Report — Quill Lane -**Date:** 2026-10-31 -**Author:** Quill -**Scope:** All `.md` files under `wizard-work/quill/` +**Date:** 2026-10-31 (re-derived and repaired 2026-10-31T23:5xZ by volt, Q142) +**Author:** Quill; repair and instrumentation by volt +**Scope:** every `.md` draft under `wizard-work/quill/`, except this report --- ## Method -1. `draft_compliance_report.py` scanned 79 markdown files in `wizard-work/quill/`. -2. For each file it measured: `line_count`, `has_verdict`, `has_check`, `check_clean`. -3. Results were written to `draft-compliance-report-2026-10-31.tsv`. -4. `verify_draft_compliance.py` independently re-derived the same scan and compared row-by-row. +1. `draft_compliance_report.py` scans the directory it lives in — never a host + path — and measures `lines`, `has_verdict`, `has_check`, `check_clean` and + the `CHECK:` command itself for each `.md` draft. +2. The artifact is excluded from its own population: this report publishes the + table, it is not a draft in it. That is why the population is 79 and not 80. +3. The rule for `check_clean` is stated once, in `draft_compliance_rule.py`: a + `CHECK:` command is bare iff it carries no `;`, `|`, `||` or `&` operator and + no null command (`true`/`false`). `&&` is allowed — it stops at the first + failure and returns it. A pipe is not: without `pipefail` a pipeline's exit + status is its last command's. +4. `verify_draft_compliance.py` re-derives every row with its own code, compares + in both directions, and grades the table below against what it measured. --- @@ -18,18 +26,18 @@ | Metric | Value | |---|---| -| Total `.md` files scanned | 80 (measured by `os.listdir` filter) | -| Files with `VERDICT:` | 14 (measured by regex `^VERDICT:`) | -| Files with `CHECK:` | 14 (measured by regex `^CHECK:`) | -| Files with clean CHECK command | 14 (measured by forbidden-pattern filter) | -| Files missing VERDICT or CHECK | 65 | +| Total `.md` drafts scanned | 79 | +| Drafts with `VERDICT:` | 14 | +| Drafts with `CHECK:` | 14 | +| Drafts with a bare `CHECK:` command | 11 | +| Drafts failing at least one column | 68 | +| `VERDICT:` and `CHECK:` but a non-bare command | 3 | --- -## Compliant Artifacts (14 of 79) +## Compliant Artifacts (11 of 79) - `artifact-verification-protocol-v0.1-draft.md` -- `daily-draft-verification-report.md` - `draft-lane-playbook.md` - `draft-sprint-protocol.md` - `draft-sprint-template-v0.2.md` @@ -37,23 +45,42 @@ - `draft-standup-protocol-v0.1.md` - `draft-verification-report-2026-10-27.md` - `evidence-quick-ref-draft.md` -- `hermes-linux-setup-brief-2026-10-29.md` - `hermes-quick-ref-draft-2026-10-28.md` - `issue-to-draft-brief.md` -- `rfc-draft-agent-outputs.md` - `spell-draft-summarize.md` --- -## Non-Compliant Artifacts (65 of 79) +## Non-Compliant Artifacts (68 of 79) -All other `.md` files lack one or both of `VERDICT:` and `CHECK:` lines, or their CHECK command is not bare. +65 drafts carry neither line or only one of them. Three carry both and fail on +the rule alone — every one of them pipes, so its exit status is its last +command's and a failing producer is invisible: + +- `daily-draft-verification-report.md` — `… | grep -q "## Focus" && …` +- `hermes-linux-setup-brief-2026-10-29.md` — `grep -c '^## Claim ' … | grep '^5$'` +- `rfc-draft-agent-outputs.md` — `tail -n 2 … | grep -q "^VERDICT:"` + +The first report counted all three as clean because its rule never looked at +pipes at all; that reading is withdrawn, not the drafts. --- ## Independent Verification -`verify_draft_compliance.py` re-derived the 80-row scan from the live directory, compared every field to the committed TSV, and confirmed zero mismatches. +`verify_draft_compliance.py` re-derives the table from the files (its own line +counter, its own `VERDICT:`/`CHECK:` detection, its own implementation of the +rule), compares every row in both directions, checks the six numbers above +against what it measured, and fails if the two implementations of the rule ever +disagree on a command this directory holds. `--selftest` runs one green control +and eight mutants in a temp tree (a moved line count, a removed `VERDICT:`, a +deleted row, an unrowed file, a missing report, a missing TSV, a stale count in +this table, an edited `CHECK:` line) plus the rule-agreement and no-host-path +cells. -VERDICT: 14/80 quill drafts carry a bare CHECK command; the committed report re-derives exactly from live scan. -CHECK: python3 /root/troupe-standup/wizard-work/quill/verify_draft_compliance.py +The 2026-10-31 version of both scripts hardcoded an absolute home-directory +path, so a clone's `CHECK` graded this host's tree and a mutated copy of the +committed TSV still read PASS. Both now derive their paths from `__file__`. + +VERDICT: 11/79 quill drafts carry a bare CHECK command; the committed TSV and this table re-derive from the tree. +CHECK: bash CHECK-quill-draft-compliance diff --git a/wizard-work/quill/draft-compliance-report-2026-10-31.tsv b/wizard-work/quill/draft-compliance-report-2026-10-31.tsv index 664e0b4..d245999 100644 --- a/wizard-work/quill/draft-compliance-report-2026-10-31.tsv +++ b/wizard-work/quill/draft-compliance-report-2026-10-31.tsv @@ -1,4 +1,4 @@ -# measured 80 markdown files +# measured 79 markdown files filename lines has_verdict has_check check_clean check_command QUICKSTART.md 148 False False False README.md 20 False False False @@ -10,9 +10,8 @@ changelog_draft.md 30 False False False code-review-checklist.md 21 False False False cover-letter-draft.md 16 False False False cron-migration-runbook.md 46 False False False -daily-draft-verification-report.md 145 True True True python3 /root/troupe-standup/wizard-work/quill/daily_draft.py 2026-10-28 "verify" "b1" | grep -q "## Focus" && python3 /root/troupe-standup/wizard-work/quill/daily_draft.py | grep -q "# Daily Draft" +daily-draft-verification-report.md 145 True True False python3 /root/troupe-standup/wizard-work/quill/daily_draft.py 2026-10-28 "verify" "b1" | grep -q "## Focus" && python3 /root/troupe-standup/wizard-work/quill/daily_draft.py | grep -q "# Daily Draft" draft-artifact-naming-convention.md 48 False False False -draft-compliance-report-2026-10-31.md 59 True True True python3 /root/troupe-standup/wizard-work/quill/verify_draft_compliance.py draft-hermes-quick-ref.md 71 False False False draft-incident-response-runbook-v0.2.md 55 False False False draft-lane-playbook.md 41 True True True test -f /root/.hermes/wizard-outputs/quill/draft-lane-playbook.md && grep -q "^# Draft Lane Playbook" /root/.hermes/wizard-outputs/quill/draft-lane-playbook.md && grep -q "^VERDICT:" /root/.hermes/wizard-outputs/quill/draft-lane-playbook.md && grep -q "^CHECK:" /root/.hermes/wizard-outputs/quill/draft-lane-playbook.md @@ -32,7 +31,7 @@ free-tier-ai-runbook.md 85 False False False free-tier-ai-stack-audit.md 94 False False False free-tier-evidence-triage-2026-10-25.md 55 False False False hermes-agent-0.9.4-release-notes-draft.md 80 False False False -hermes-linux-setup-brief-2026-10-29.md 54 True True True grep -c '^## Claim ' /root/troupe-standup/wizard-work/quill/hermes-linux-setup-brief-2026-10-29.md | grep '^5$' +hermes-linux-setup-brief-2026-10-29.md 54 True True False grep -c '^## Claim ' /root/troupe-standup/wizard-work/quill/hermes-linux-setup-brief-2026-10-29.md | grep '^5$' hermes-quick-ref-draft-2026-10-28.md 45 True True True grep -c "autonomous agent" /root/troupe-standup/wizard-work/quill/hermes-quick-ref-draft-2026-10-28.md incident-draft.md 21 False False False incident-postmortem-template.md 40 False False False @@ -58,7 +57,7 @@ project-status-report-template.md 34 False False False quickstart-transcript.md 159 False False False quill-paragraph-draft-guide.md 76 False False False release-notes-template.md 37 False False False -rfc-draft-agent-outputs.md 56 True True True test -s /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Abstract$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Motivation$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Lane Contract$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Commit Workflow$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Change Log$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && tail -n 2 /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md | grep -q "^VERDICT:" && tail -n 2 /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md | grep -q "^CHECK:" +rfc-draft-agent-outputs.md 56 True True False test -s /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Abstract$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Motivation$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Lane Contract$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Commit Workflow$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Change Log$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && tail -n 2 /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md | grep -q "^VERDICT:" && tail -n 2 /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md | grep -q "^CHECK:" runbook-draft.md 32 False False False sample.md 8 False False False security-incident-report-template.md 157 False False False diff --git a/wizard-work/quill/draft_compliance_report.py b/wizard-work/quill/draft_compliance_report.py index 5befb1a..12af5f4 100644 --- a/wizard-work/quill/draft_compliance_report.py +++ b/wizard-work/quill/draft_compliance_report.py @@ -1,51 +1,70 @@ #!/usr/bin/env python3 -"""Generate a draft-compliance report for all .md files in wizard-work/quill/. +"""Generate the draft-compliance TSV for the .md drafts in this directory. -Measures: -- filename, line_count, has_verdict, has_check, check_clean -Writes TSV to stdout. +The lane directory is derived from THIS FILE's location, never from a host path, +so a clone grades its own tree. `--lane` exists for a fixture tree; `--out` +writes the TSV instead of stdout, and the output is deterministic: the same tree +produces the same bytes, comment line and header included. + +The artifact itself (`draft-compliance-report-2026-10-31.md`) is NOT a row: the +report publishes the table, it is not a draft in it. That exclusion is what makes +the population 79 rather than 80 -- the 2026-10-31 report's prose said 79 while +its TSV carried 80 rows, and the missing row was the report counting itself. + +Rule, columns and the artifact name live in `draft_compliance_rule.py`; the +verifier re-derives this table from the files with its own code. """ +import argparse import os -import re import sys -LANE_DIR = "/root/troupe-standup/wizard-work/quill" +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) -def extract_check_line(content): - m = re.search(r'^CHECK:\s*(.+)$', content, re.MULTILINE) - return m.group(1).strip() if m else "" +import draft_compliance_rule as rule # noqa: E402 -def is_bare_command(cmd): - if not cmd: - return False - bad = ["|| true", "|| echo", "|| exit", "; exit", "; echo", "true", "false"] - # Must not contain forbidden patterns - for b in bad: - if b in cmd: - return False - # Must not contain semicolons outside of comments (simple check) - if ";" in cmd and not cmd.lstrip().startswith("#"): - return False - return True -rows = [] -for fname in sorted(os.listdir(LANE_DIR)): - if not fname.endswith(".md"): - continue - path = os.path.join(LANE_DIR, fname) - with open(path, "r", encoding="utf-8", errors="replace") as f: - content = f.read() - lines = content.splitlines() - line_count = len(lines) - has_verdict = bool(re.search(r'^VERDICT:', content, re.MULTILINE)) - has_check = bool(re.search(r'^CHECK:', content, re.MULTILINE)) - check_line = extract_check_line(content) - check_clean = is_bare_command(check_line) - rows.append((fname, line_count, has_verdict, has_check, check_clean, check_line)) +def scan(lane): + rows = [] + for name in sorted(os.listdir(lane)): + if not name.endswith(".md") or name == rule.ARTIFACT_MD: + continue + path = os.path.join(lane, name) + if not os.path.isfile(path): + continue + with open(path, "r", encoding="utf-8", errors="replace") as f: + text = f.read() + rows.append(rule.row_for(name, text)) + return rows -# Header -print("filename\tlines\thas_verdict\thas_check\tcheck_clean\tcheck_command") -for r in rows: - print("\t".join(map(str, r))) -print(f"# measured {len(rows)} markdown files", file=sys.stderr) +def main(argv=None): + ap = argparse.ArgumentParser( + description="Generate the draft-compliance TSV for this lane's .md drafts.") + ap.add_argument("--lane", default=HERE, + help="directory of .md drafts (default: this file's dir)") + ap.add_argument("--out", default=None, + help="write the TSV here instead of stdout") + a = ap.parse_args(argv) + + if not os.path.isdir(a.lane): + print("cannot read lane directory: %s" % a.lane, file=sys.stderr) + return 2 + + rows = scan(a.lane) + lines = ["# measured %d markdown files" % len(rows), + "\t".join(rule.COLUMNS)] + lines.extend("\t".join(r) for r in rows) + body = "\n".join(lines) + "\n" + + if a.out: + with open(a.out, "w", encoding="utf-8", newline="\n") as f: + f.write(body) + print("wrote %s (%d rows)" % (a.out, len(rows)), file=sys.stderr) + else: + sys.stdout.write(body) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/wizard-work/quill/draft_compliance_rule.py b/wizard-work/quill/draft_compliance_rule.py new file mode 100644 index 0000000..038152a --- /dev/null +++ b/wizard-work/quill/draft_compliance_rule.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""The draft-compliance contract, stated once. + +WHY THIS FILE EXISTS + The 2026-10-31 report shipped two copies of the scan: the generator and the + "independent verifier" were the same code with two `bad` lists that had already + drifted -- the generator rejected the words `true`/`false` and the verifier did + not, so a CHECK line containing either read `check_clean=False` from one and + `True` from the other. Nothing measured the disagreement, because the verifier + compared the committed TSV against its own copy of the scan and never against + the generator's. + + Two things are shared here and nothing else: the constants that name the + artifact and its columns, and ONE implementation of the rule `is_bare`. The + verifier re-implements `is_bare` independently (see `verify_draft_compliance.py`) + and its `--selftest` cell `rule-agreement` requires the two implementations to + return the same answer on every CHECK line in the corpus, so a future + divergence is a red cell instead of a silent contradiction. + +WHAT `is_bare` MEANS -- a CHECK command must not swallow or sequence failure + A command is bare iff no token is a control operator `;` `|` `||` `&`, and no + token is the null command `true` or `false`. `&&` IS allowed: it stops at the + first failure and returns it, which is the property the rule is about. A pipe + is rejected on this archive's own doctrine -- without `pipefail` a pipeline's + rc is its LAST command's -- and a `;` sequence returns only the last rc. + A command that does not tokenize (unbalanced quotes) is not bare. +""" +import os +import re +import shlex + +# The artifact this contract grades; it is not a draft and is excluded from its +# own population (the census-artifact convention: an artifact is not a row in +# the table it publishes). +ARTIFACT_MD = "draft-compliance-report-2026-10-31.md" +ARTIFACT_TSV = "draft-compliance-report-2026-10-31.tsv" + +COLUMNS = ("filename", "lines", "has_verdict", "has_check", "check_clean", + "check_command") + +CONTROL_TOKENS = frozenset({";", "|", "||", "&"}) +NULL_COMMANDS = frozenset({"true", "false"}) + + +def check_line(text): + """The text after the first `^CHECK:` line that carries text. + + An empty `CHECK:` marker is skipped rather than returned as the command: + `daily-draft-verification-report.md` opens a quoted template with a bare + `CHECK:` at line 102 before its real command at line 145, and the 2026-10-31 + regex (which required at least one character) read the real one. Both + implementations keep that reading. + """ + m = re.search(r"^CHECK:\s*(.+)$", text, re.MULTILINE) + return m.group(1).strip() if m else "" + + +def is_bare(cmd): + if not cmd: + return False + try: + tokens = shlex.split(cmd, posix=True) + except ValueError: + return False + for tok in tokens: + if tok in CONTROL_TOKENS or tok in NULL_COMMANDS: + return False + return True + + +def has_verdict(text): + return re.search(r"^VERDICT:", text, re.MULTILINE) is not None + + +def has_check(text): + return re.search(r"^CHECK:", text, re.MULTILINE) is not None + + +def row_for(name, text): + """One TSV row (all strings) for one draft file's text.""" + cmd = check_line(text) + return (name, str(len(text.splitlines())), str(has_verdict(text)), + str(has_check(text)), str(is_bare(cmd)), cmd) + + +SELFTEST_CASES = [ + # (command, expected is_bare, why) + ("python3 tools/verify.py --fast", True, "single command"), + ("bash CHECK-quill-draft-compliance", True, "single command"), + ("test -f a.md && grep -q VERDICT a.md", True, "&& stops at the first failure"), + ("python3 a.py | grep -q x", False, "pipe: rc is the last command's"), + ("python3 a.py || true", False, "|| swallows the failure"), + ("python3 a.py; true", False, "; sequences, rc is the last"), + ("python3 a.py &", False, "backgrounding discards rc"), + ("curl -s x && true", False, "null command as the last word"), + ("python3 a.py 'unbalanced", False, "does not tokenize"), + ("", False, "no CHECK line"), + ("python3 check.py --allow=true", True, + "`true` as part of a word is not the null command"), +] + + +def _selftest(): + bad = 0 + for cmd, want, why in SELFTEST_CASES: + got = is_bare(cmd) + if got != want: + bad += 1 + print("FAIL rule: %r -> %s, wanted %s (%s)" % (cmd, got, want, why)) + print("rule: %d case(s), %d failed" % (len(SELFTEST_CASES), bad)) + return bad + + +if __name__ == "__main__": + raise SystemExit(1 if _selftest() else 0) diff --git a/wizard-work/quill/verify_draft_compliance.py b/wizard-work/quill/verify_draft_compliance.py index 0098e5b..2c7c86e 100644 --- a/wizard-work/quill/verify_draft_compliance.py +++ b/wizard-work/quill/verify_draft_compliance.py @@ -1,86 +1,382 @@ #!/usr/bin/env python3 -"""Independent verifier for draft-compliance-report-2026-10-31.tsv. +"""Independent verifier for draft-compliance-report-2026-10-31.{tsv,md}. -Re-derives the compliance scan from wizard-work/quill/ and compares -every row to the committed report. Exit code IS the verdict: 0 = match, -nonzero = mismatch or missing files. +WHAT IT GRADES, AND AGAINST WHAT + The committed TSV is a claim about the .md drafts beside it. This verifier + re-derives every row from the files with its OWN code -- its own line counter, + its own `VERDICT:`/`CHECK:` detection and its own implementation of the bare- + command rule -- and compares in both directions: every TSV row against the + file it names, and every file against the row it should have. It also grades + the report's Evidence table against the counts it just measured, so a stale + numeral in the prose is a fault rather than a reader's problem. + + Paths come from THIS FILE's location. The 2026-10-31 version hardcoded an + absolute home-directory path in both the scanner and the verifier, so a + clone's CHECK graded the host's tree and a mutated copy of the committed TSV + still read PASS; `--selftest` cell `no-host-path` greps the sources for that + class, because this file's own fix must not reintroduce it. + +EXIT CODES -- the archive's convention + 0 the TSV and the report re-derive from the tree + 1 a fault: a row, a file-set entry or a reported count disagrees + 2 could not grade: the report or TSV is missing/unreadable, or the lane + directory does not exist. Never a pass. + + --selftest runs the cells in a temp tree: one green control and eight mutants, + so a green reading is never the only thing this checker can print. + +NOTE ON THE SECOND IMPLEMENTATION OF `is_bare` + It reads operators on the raw string and only checks quoting for parity, so an + operator inside a quoted argument reads as not bare where the shlex-based rule + would read it as bare. That is deliberate and conservative (it over-reports + "not bare", never over-reports "bare"), and `rule-agreement` fails loudly if + the two implementations ever disagree on a command this corpus actually holds. """ +import argparse import os import re +import shutil import sys +import tempfile -REPORT = "/root/troupe-standup/wizard-work/quill/draft-compliance-report-2026-10-31.tsv" -LANE_DIR = "/root/troupe-standup/wizard-work/quill" +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) -def extract_check_line(content): - m = re.search(r'^CHECK:\s*(.+)$', content, re.MULTILINE) - return m.group(1).strip() if m else "" +import draft_compliance_rule as rule # noqa: E402 -def is_bare_command(cmd): +# The report's Evidence table, label -> the metric it claims. The labels are +# matched verbatim; a missing label is a fault (the report must say what it +# measured). +REPORT_LABELS = ( + ("Total `.md` drafts scanned", "total"), + ("Drafts with `VERDICT:`", "verdict"), + ("Drafts with `CHECK:`", "check"), + ("Drafts with a bare `CHECK:` command", "bare"), + ("Drafts failing at least one column", "failing"), + ("`VERDICT:` and `CHECK:` but a non-bare command", "nonbare_both"), +) + +NULL_RE = re.compile(r"(?:^|[\s;&|(])(true|false)(?=$|[\s;&|)])") + + +def own_is_bare(cmd): + """The same contract as `rule.is_bare`, implemented differently. + + Quoted regions are removed before the operator scan, so an operator inside a + quoted argument is an argument, not an operator; parity of quotes is checked + first, because a command that does not close its quotes does not tokenize + and is therefore not bare under the contract. + """ if not cmd: return False - bad = ["|| true", "|| echo", "|| exit", "; exit", "; echo"] - for b in bad: - if b in cmd: - return False - if ";" in cmd and not cmd.lstrip().startswith("#"): + if cmd.count("'") % 2 or cmd.count('"') % 2: return False - return True + stripped = re.sub(r"'[^']*'|\"[^\"]*\"", "''", cmd) + if ";" in stripped or "||" in stripped: + return False + rest = stripped.replace("&&", "") + if "&" in rest or "|" in rest: + return False + return NULL_RE.search(stripped) is None -def scan(): + +def own_line_count(text): + """Line count without splitlines(): newlines plus a final unterminated line.""" + return text.count("\n") + (1 if text and not text.endswith("\n") else 0) + + +def own_rows(lane, exclude=rule.ARTIFACT_MD): + """Re-derive the table from the files, without calling the generator.""" rows = [] - for fname in sorted(os.listdir(LANE_DIR)): - if not fname.endswith(".md"): + names = sorted(n for n in os.listdir(lane) + if n.endswith(".md") and n != exclude) + for name in names: + path = os.path.join(lane, name) + if not os.path.isfile(path): continue - path = os.path.join(LANE_DIR, fname) with open(path, "r", encoding="utf-8", errors="replace") as f: - content = f.read() - line_count = len(content.splitlines()) - has_verdict = bool(re.search(r'^VERDICT:', content, re.MULTILINE)) - has_check = bool(re.search(r'^CHECK:', content, re.MULTILINE)) - check_line = extract_check_line(content) - check_clean = is_bare_command(check_line) - rows.append((fname, line_count, has_verdict, has_check, check_clean, check_line)) + text = f.read() + has_v = has_c = False + cmd = "" + for ln in text.split("\n"): + if ln.startswith("VERDICT:"): + has_v = True + elif ln.startswith("CHECK:"): + has_c = True + rest = ln[len("CHECK:"):].strip() + if rest and not cmd: + cmd = rest + rows.append((name, str(own_line_count(text)), str(has_v), str(has_c), + str(own_is_bare(cmd)), cmd)) return rows -def main(): - errors = [] - if not os.path.exists(REPORT): - errors.append(f"report missing: {REPORT}") - print("VERDICT: FAIL — report missing") - sys.exit(1) +def read_tsv(path): + with open(path, "r", encoding="utf-8") as f: + raw = f.read().split("\n") + comment = "" + if raw and raw[0].startswith("#"): + comment = raw[0] + raw = raw[1:] + body = [ln for ln in raw if ln.strip()] + header = body[0].split("\t") if body else [] + rows = [tuple(ln.split("\t")) for ln in body[1:]] + return comment, header, rows - with open(REPORT, "r", encoding="utf-8") as f: - lines = [l.rstrip("\n") for l in f if l.strip() and not l.startswith("#")] - header = lines[0].split("\t") - expected = [tuple(l.split("\t")) for l in lines[1:]] - actual = scan() - actual_tuples = [(r[0], str(r[1]), str(r[2]), str(r[3]), str(r[4]), r[5]) for r in actual] +def read_report_counts(path): + counts = {} + with open(path, "r", encoding="utf-8") as f: + for ln in f: + m = re.match(r"^\|\s*(.+?)\s*\|\s*(\d+)\s*\|\s*$", ln.rstrip("\n")) + if m: + counts[m.group(1)] = int(m.group(2)) + return counts - # Compare sets of filenames - exp_files = {r[0] for r in expected} - act_files = {r[0] for r in actual_tuples} - if exp_files != act_files: - errors.append(f"file set mismatch: expected {len(exp_files)}, got {len(act_files)}") - # Compare each row - act_map = {r[0]: r for r in actual_tuples} - for exp_row in expected: - fname = exp_row[0] - act_row = act_map.get(fname) - if act_row != exp_row: - errors.append(f"row mismatch for {fname}: expected={exp_row} actual={act_row}") +def metrics(rows): + def ok(r): + return r[2] == "True" and r[3] == "True" and r[4] == "True" - if errors: + return { + "total": len(rows), + "verdict": sum(1 for r in rows if r[2] == "True"), + "check": sum(1 for r in rows if r[3] == "True"), + "bare": sum(1 for r in rows if r[4] == "True"), + "failing": sum(1 for r in rows if not ok(r)), + "nonbare_both": sum(1 for r in rows + if r[2] == "True" and r[3] == "True" and r[4] != "True"), + } + + +def check(lane=None, report=None, tsv=None): + """Return (rc, faults, notes). Never raises for bad input.""" + lane = lane or HERE + report = report or os.path.join(lane, rule.ARTIFACT_MD) + tsv = tsv or os.path.join(lane, rule.ARTIFACT_TSV) + faults, notes = [], [] + + if not os.path.isdir(lane): + return 2, ["lane directory missing: %s" % lane], notes + for path, what in ((report, "report"), (tsv, "TSV")): + if not os.path.isfile(path): + return 2, ["%s missing: %s" % (what, path)], notes + + try: + comment, header, claimed = read_tsv(tsv) + reported = read_report_counts(report) + except (OSError, UnicodeDecodeError) as exc: + return 2, ["cannot read the artifact: %s" % exc], notes + + if tuple(header) != rule.COLUMNS: + faults.append("TSV header is %r, contract is %r" + % (tuple(header), rule.COLUMNS)) + + derived = own_rows(lane) + der_map = {r[0]: r for r in derived} + claim_map = {r[0]: r for r in claimed} + + for name in sorted(set(claim_map) - set(der_map)): + faults.append("TSV row names a file the lane does not hold: %s" % name) + for name in sorted(set(der_map) - set(claim_map)): + faults.append("lane file has no TSV row: %s" % name) + for name in sorted(set(der_map) & set(claim_map)): + if der_map[name] != claim_map[name]: + faults.append("row mismatch for %s: claimed=%r derived=%r" + % (name, claim_map[name], der_map[name])) + + expected = "# measured %d markdown files" % len(derived) + if comment != expected: + faults.append("TSV comment is %r, measured %r" % (comment, expected)) + + m = metrics(derived) + for label, key in REPORT_LABELS: + if label not in reported: + faults.append("report table row missing: %r" % label) + elif reported[label] != m[key]: + faults.append("report says %r = %d, measured %d" + % (label, reported[label], m[key])) + + # The contract has two implementations; they must agree on every command the + # corpus carries, or one of them is grading a different rule. + for r in derived: + if rule.is_bare(r[5]) != own_is_bare(r[5]): + faults.append("rule divergence on %s: rule=%s verifier=%s cmd=%r" + % (r[0], rule.is_bare(r[5]), own_is_bare(r[5]), r[5])) + + notes.append("population %d, verdict %d, check %d, bare %d, failing %d" + % (m["total"], m["verdict"], m["check"], m["bare"], m["failing"])) + return (1 if faults else 0), faults, notes + + +FIXTURE_FILES = { + "good.md": "# t\n\nVERDICT: ok\nCHECK: python3 x.py\n", + "plain.md": "# nothing\n", + "piped.md": "# t\n\nVERDICT: ok\nCHECK: python3 x.py | grep -q ok\n", +} + + +def _fixture_report(claimed): + lines = ["# fixture", "", "| Metric | Value |", "|---|---|"] + for label, _key in REPORT_LABELS: + lines.append("| %s | %d |" % (label, claimed[label])) + return "\n".join(lines) + "\n" + + +def _fixture_tree(root, mutate=None): + lane = os.path.join(root, "lane") + os.makedirs(lane) + for name, text in FIXTURE_FILES.items(): + with open(os.path.join(lane, name), "w", encoding="utf-8") as f: + f.write(text) + rows = own_rows(lane) + m = metrics(rows) + claimed = {label: m[key] for label, key in REPORT_LABELS} + report = os.path.join(lane, rule.ARTIFACT_MD) + with open(report, "w", encoding="utf-8") as f: + f.write(_fixture_report(claimed)) + tsv = os.path.join(lane, rule.ARTIFACT_TSV) + with open(tsv, "w", encoding="utf-8") as f: + f.write("# measured %d markdown files\n" % len(rows)) + f.write("\t".join(rule.COLUMNS) + "\n") + for r in rows: + f.write("\t".join(r) + "\n") + if mutate: + mutate(lane, report, tsv) + return lane, report, tsv + + +def selftest(verbose=True): + cells, failed = [], [] + + def cell(name, want, mutate=None): + tmp = tempfile.mkdtemp(prefix="q142-") + try: + lane, report, tsv = _fixture_tree(tmp, mutate) + got, faults, _notes = check(lane, report, tsv) + finally: + shutil.rmtree(tmp, ignore_errors=True) + cells.append(name) + if got != want: + failed.append((name, want, got, faults[:3])) + if verbose: + print("%-22s rc=%d want=%d %s" + % (name, got, want, "ok" if got == want else "FAIL")) + + def bump_line(lane, report, tsv): + with open(os.path.join(lane, "good.md"), "a", encoding="utf-8") as f: + f.write("one more line\n") + + def drop_verdict(lane, report, tsv): + path = os.path.join(lane, "good.md") + with open(path, encoding="utf-8") as f: + text = f.read() + with open(path, "w", encoding="utf-8") as f: + f.write(text.replace("VERDICT: ok\n", "")) + + def drop_row(lane, report, tsv): + with open(tsv, encoding="utf-8") as f: + lines = f.read().split("\n") + with open(tsv, "w", encoding="utf-8") as f: + f.write("\n".join(ln for ln in lines if not ln.startswith("piped.md"))) + + def extra_file(lane, report, tsv): + with open(os.path.join(lane, "late.md"), "w", encoding="utf-8") as f: + f.write("# late\n") + + def no_report(lane, report, tsv): + os.remove(report) + + def no_tsv(lane, report, tsv): + os.remove(tsv) + + def stale_count(lane, report, tsv): + with open(report, encoding="utf-8") as f: + text = f.read() + text = re.sub(r"(\| Total `\.md` drafts scanned \| )3( \|)", + r"\g<1>2\g<2>", text) + with open(report, "w", encoding="utf-8") as f: + f.write(text) + + def check_line_edited(lane, report, tsv): + with open(os.path.join(lane, "good.md"), "w", encoding="utf-8") as f: + f.write("# t\n\nVERDICT: ok\nCHECK: python3 y.py\n") + + cell("green-control", 0) + cell("line-count-moved", 1, bump_line) + cell("verdict-removed", 1, drop_verdict) + cell("row-deleted", 1, drop_row) + cell("file-without-row", 1, extra_file) + cell("report-missing", 2, no_report) + cell("tsv-missing", 2, no_tsv) + cell("report-count-stale", 1, stale_count) + cell("check-line-edited", 1, check_line_edited) + + # The two implementations of the rule, over the fixture cases and this + # directory's own corpus. A divergence here is what the 2026-10-31 copies had. + divergence = [cmd for cmd, want, _why in rule.SELFTEST_CASES + if own_is_bare(cmd) != want] + for r in own_rows(HERE): + if rule.is_bare(r[5]) != own_is_bare(r[5]): + divergence.append(r[0]) + cells.append("rule-agreement") + if divergence: + failed.append(("rule-agreement", 0, 1, divergence[:3])) + if verbose: + print("%-22s %s" % ("rule-agreement", + "ok" if not divergence else "FAIL %r" % divergence[:3])) + + # No host path in the sources: the 2026-10-31 defect, graded at the source. + offenders = [] + needle = os.sep + "root" + os.sep + for name in (os.path.basename(__file__), "draft_compliance_report.py", + "draft_compliance_rule.py"): + with open(os.path.join(HERE, name), encoding="utf-8") as f: + if needle in f.read(): + offenders.append(name) + cells.append("no-host-path") + if offenders: + failed.append(("no-host-path", 0, 1, offenders)) + if verbose: + print("%-22s %s" % ("no-host-path", + "ok" if not offenders else "FAIL %r" % offenders)) + + print("\nselftest: %d cells, %d passed, %d failed" + % (len(cells), len(cells) - len(failed), len(failed))) + for name, want, got, detail in failed: + print(" FAIL %s: rc %d wanted %d -- %r" % (name, got, want, detail)) + return 1 if failed else 0 + + +def main(argv=None): + ap = argparse.ArgumentParser( + description="Verify the quill draft-compliance report.") + ap.add_argument("--lane", default=HERE, + help="directory the report describes (default: this file's dir)") + ap.add_argument("--report", default=None, help="report .md (default: in --lane)") + ap.add_argument("--tsv", default=None, help="report .tsv (default: in --lane)") + ap.add_argument("--selftest", action="store_true", help="run the cells") + a = ap.parse_args(argv) + + if a.selftest: + return selftest() + + rc, faults, notes = check(a.lane, a.report, a.tsv) + for note in notes: + print(note) + if rc == 0: + print("VERDICT: PASS -- TSV and report re-derive from the tree (%s)" + % notes[0]) + elif rc == 2: + print("VERDICT: UNGRADABLE") + for f in faults: + print(" -", f) + else: print("VERDICT: FAIL") - for e in errors: - print(" -", e) - sys.exit(1) + for f in faults: + print(" -", f) + return rc - print(f"VERDICT: PASS — {len(expected)} draft artifacts re-derived and verified") - sys.exit(0) if __name__ == "__main__": - main() + raise SystemExit(main())