volt: quill draft-compliance artifact grades its own tree (Q142)
The 2026-10-31 artifact hardcoded an absolute home path in both the scanner and the 'independent verifier', so a clone's CHECK graded the host tree (a clone whose committed TSV held one garbage line still reported on the host's 79-row TSV). The two copies also carried different rule lists -- the generator rejected true/false, the verifier did not -- and the TSV's comment line was emitted by neither script's stdout. - draft_compliance_rule.py: the bare-command rule, stated once, 11 cases - generator: paths from __file__, artifact excluded from its own population, deterministic output (comment + header + rows) - verifier: own re-derivation, both directions, grades the report's six counts, rc 0/1/2, --selftest one control + eight mutants + rule-agreement + no-host-path - CHECK: four legs (selftest, rederive, byte-identical regeneration, copy-parity) - report: measured numbers (79 drafts, 14/14/11, 68 failing, 3 non-bare)
This commit is contained in:
volt committed
1 parent
6ef2bcc39d
commit
54451e4f6d
6 files changed
+668
-126
No files matched your search
@@ -1,8 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
# CHECK -- quill: draft-compliance report 2026-10-31 (forge: quill lane).
|
||||
# Exit 0 iff the committed TSV re-derives from the live scan with zero mismatches.
|
||||
# CHECK -- quill: draft-compliance report 2026-10-31 (forge: quill lane; gate by volt, Q142).
|
||||
#
|
||||
# Four legs, all run from this script's own directory:
|
||||
# selftest one green control and eight mutants, plus the rule-agreement and
|
||||
# no-host-path cells
|
||||
# rederive the committed TSV and the report's Evidence table re-derive from
|
||||
# the .md drafts beside them
|
||||
# regenerate the generator's output is byte-identical to the committed TSV
|
||||
# copy-parity the same scripts run against a COPY of the lane grade the copy:
|
||||
# clean copy rc 0, copy with one TSV row moved rc 1
|
||||
#
|
||||
# Exit codes: 0 all legs passed, 1 a leg failed, 2 a leg could not be graded.
|
||||
set -u
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
|
||||
REPO_ROOT="$(pwd)"
|
||||
cd "$REPO_ROOT" || exit 1
|
||||
exec python3 wizard-work/quill/verify_draft_compliance.py
|
||||
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")" || exit 2
|
||||
ROOT="$(pwd)"
|
||||
LANE="$ROOT/wizard-work/quill"
|
||||
WORK="$(mktemp -d "${TMPDIR:-/tmp}/check-quill.XXXXXX")" || exit 2
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
passed=0
|
||||
failed=0
|
||||
row() { printf ' %-12s %s\n' "$1" "$2"; }
|
||||
|
||||
# --- leg 1: the verifier's own cells -----------------------------------------
|
||||
if python3 "$LANE/verify_draft_compliance.py" --selftest >"$WORK/selftest.out" 2>&1; then
|
||||
row selftest "PASS $(tail -n 1 "$WORK/selftest.out")"
|
||||
passed=$((passed + 1))
|
||||
else
|
||||
row selftest "FAULT $(tail -n 1 "$WORK/selftest.out")"
|
||||
cat "$WORK/selftest.out"
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
|
||||
# --- leg 2: the committed artifact re-derives --------------------------------
|
||||
if python3 "$LANE/verify_draft_compliance.py" >"$WORK/rederive.out" 2>&1; then
|
||||
row rederive "PASS $(head -n 1 "$WORK/rederive.out")"
|
||||
passed=$((passed + 1))
|
||||
else
|
||||
rc=$?
|
||||
row rederive "FAULT rc=$rc"
|
||||
cat "$WORK/rederive.out"
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
|
||||
# --- leg 3: the TSV is exactly what the generator writes ---------------------
|
||||
python3 "$LANE/draft_compliance_report.py" >"$WORK/regen.tsv" 2>"$WORK/regen.err"
|
||||
gen_rc=$?
|
||||
if [ "$gen_rc" -ne 0 ]; then
|
||||
row regenerate "FAULT generator rc=$gen_rc"
|
||||
cat "$WORK/regen.err"
|
||||
failed=$((failed + 1))
|
||||
elif cmp -s "$WORK/regen.tsv" "$LANE/draft-compliance-report-2026-10-31.tsv"; then
|
||||
row regenerate "PASS bytes identical ($(wc -c <"$WORK/regen.tsv") B)"
|
||||
passed=$((passed + 1))
|
||||
else
|
||||
row regenerate "FAULT committed TSV differs from a fresh derivation"
|
||||
diff "$WORK/regen.tsv" "$LANE/draft-compliance-report-2026-10-31.tsv" | head -10
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
|
||||
# --- leg 4: a copy grades the copy, not this tree ----------------------------
|
||||
mkdir -p "$WORK/lane" || exit 2
|
||||
cp "$LANE"/*.md "$LANE"/draft-compliance-report-2026-10-31.tsv \
|
||||
"$LANE"/draft_compliance_rule.py "$LANE/draft_compliance_report.py" \
|
||||
"$LANE/verify_draft_compliance.py" "$WORK/lane/" 2>/dev/null
|
||||
if python3 "$WORK/lane/verify_draft_compliance.py" >"$WORK/copy-clean.out" 2>&1; then
|
||||
clean_rc=0
|
||||
else
|
||||
clean_rc=$?
|
||||
fi
|
||||
# move one row's line count in the COPY's TSV: the copy must notice
|
||||
awk -F'\t' 'BEGIN{OFS="\t"} NR==3{$2=$2+1} {print}' \
|
||||
"$WORK/lane/draft-compliance-report-2026-10-31.tsv" >"$WORK/mut.tsv" \
|
||||
&& mv "$WORK/mut.tsv" "$WORK/lane/draft-compliance-report-2026-10-31.tsv"
|
||||
if python3 "$WORK/lane/verify_draft_compliance.py" >"$WORK/copy-mut.out" 2>&1; then
|
||||
mut_rc=0
|
||||
else
|
||||
mut_rc=$?
|
||||
fi
|
||||
if [ "$clean_rc" -eq 0 ] && [ "$mut_rc" -eq 1 ]; then
|
||||
row copy-parity "PASS clean copy rc=0, mutated copy rc=1"
|
||||
passed=$((passed + 1))
|
||||
else
|
||||
row copy-parity "FAULT clean copy rc=$clean_rc (want 0), mutated rc=$mut_rc (want 1)"
|
||||
head -5 "$WORK/copy-clean.out" "$WORK/copy-mut.out"
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "CHECK-quill-draft-compliance: $passed leg(s) passed, $failed fault(s)"
|
||||
[ "$failed" -eq 0 ] || exit 1
|
||||
exit 0
|
||||
@@ -1,16 +1,24 @@
|
||||
# Draft Compliance Report — Quill Lane
|
||||
**Date:** 2026-10-31
|
||||
**Author:** Quill
|
||||
**Scope:** All `.md` files under `wizard-work/quill/`
|
||||
**Date:** 2026-10-31 (re-derived and repaired 2026-10-31T23:5xZ by volt, Q142)
|
||||
**Author:** Quill; repair and instrumentation by volt
|
||||
**Scope:** every `.md` draft under `wizard-work/quill/`, except this report
|
||||
|
||||
---
|
||||
|
||||
## Method
|
||||
|
||||
1. `draft_compliance_report.py` scanned 79 markdown files in `wizard-work/quill/`.
|
||||
2. For each file it measured: `line_count`, `has_verdict`, `has_check`, `check_clean`.
|
||||
3. Results were written to `draft-compliance-report-2026-10-31.tsv`.
|
||||
4. `verify_draft_compliance.py` independently re-derived the same scan and compared row-by-row.
|
||||
1. `draft_compliance_report.py` scans the directory it lives in — never a host
|
||||
path — and measures `lines`, `has_verdict`, `has_check`, `check_clean` and
|
||||
the `CHECK:` command itself for each `.md` draft.
|
||||
2. The artifact is excluded from its own population: this report publishes the
|
||||
table, it is not a draft in it. That is why the population is 79 and not 80.
|
||||
3. The rule for `check_clean` is stated once, in `draft_compliance_rule.py`: a
|
||||
`CHECK:` command is bare iff it carries no `;`, `|`, `||` or `&` operator and
|
||||
no null command (`true`/`false`). `&&` is allowed — it stops at the first
|
||||
failure and returns it. A pipe is not: without `pipefail` a pipeline's exit
|
||||
status is its last command's.
|
||||
4. `verify_draft_compliance.py` re-derives every row with its own code, compares
|
||||
in both directions, and grades the table below against what it measured.
|
||||
|
||||
---
|
||||
|
||||
@@ -18,18 +26,18 @@
|
||||
|
||||
| Metric | Value |
|
||||
|---|---|
|
||||
| Total `.md` files scanned | 80 (measured by `os.listdir` filter) |
|
||||
| Files with `VERDICT:` | 14 (measured by regex `^VERDICT:`) |
|
||||
| Files with `CHECK:` | 14 (measured by regex `^CHECK:`) |
|
||||
| Files with clean CHECK command | 14 (measured by forbidden-pattern filter) |
|
||||
| Files missing VERDICT or CHECK | 65 |
|
||||
| Total `.md` drafts scanned | 79 |
|
||||
| Drafts with `VERDICT:` | 14 |
|
||||
| Drafts with `CHECK:` | 14 |
|
||||
| Drafts with a bare `CHECK:` command | 11 |
|
||||
| Drafts failing at least one column | 68 |
|
||||
| `VERDICT:` and `CHECK:` but a non-bare command | 3 |
|
||||
|
||||
---
|
||||
|
||||
## Compliant Artifacts (14 of 79)
|
||||
## Compliant Artifacts (11 of 79)
|
||||
|
||||
- `artifact-verification-protocol-v0.1-draft.md`
|
||||
- `daily-draft-verification-report.md`
|
||||
- `draft-lane-playbook.md`
|
||||
- `draft-sprint-protocol.md`
|
||||
- `draft-sprint-template-v0.2.md`
|
||||
@@ -37,23 +45,42 @@
|
||||
- `draft-standup-protocol-v0.1.md`
|
||||
- `draft-verification-report-2026-10-27.md`
|
||||
- `evidence-quick-ref-draft.md`
|
||||
- `hermes-linux-setup-brief-2026-10-29.md`
|
||||
- `hermes-quick-ref-draft-2026-10-28.md`
|
||||
- `issue-to-draft-brief.md`
|
||||
- `rfc-draft-agent-outputs.md`
|
||||
- `spell-draft-summarize.md`
|
||||
|
||||
---
|
||||
|
||||
## Non-Compliant Artifacts (65 of 79)
|
||||
## Non-Compliant Artifacts (68 of 79)
|
||||
|
||||
All other `.md` files lack one or both of `VERDICT:` and `CHECK:` lines, or their CHECK command is not bare.
|
||||
65 drafts carry neither line or only one of them. Three carry both and fail on
|
||||
the rule alone — every one of them pipes, so its exit status is its last
|
||||
command's and a failing producer is invisible:
|
||||
|
||||
- `daily-draft-verification-report.md` — `… | grep -q "## Focus" && …`
|
||||
- `hermes-linux-setup-brief-2026-10-29.md` — `grep -c '^## Claim ' … | grep '^5$'`
|
||||
- `rfc-draft-agent-outputs.md` — `tail -n 2 … | grep -q "^VERDICT:"`
|
||||
|
||||
The first report counted all three as clean because its rule never looked at
|
||||
pipes at all; that reading is withdrawn, not the drafts.
|
||||
|
||||
---
|
||||
|
||||
## Independent Verification
|
||||
|
||||
`verify_draft_compliance.py` re-derived the 80-row scan from the live directory, compared every field to the committed TSV, and confirmed zero mismatches.
|
||||
`verify_draft_compliance.py` re-derives the table from the files (its own line
|
||||
counter, its own `VERDICT:`/`CHECK:` detection, its own implementation of the
|
||||
rule), compares every row in both directions, checks the six numbers above
|
||||
against what it measured, and fails if the two implementations of the rule ever
|
||||
disagree on a command this directory holds. `--selftest` runs one green control
|
||||
and eight mutants in a temp tree (a moved line count, a removed `VERDICT:`, a
|
||||
deleted row, an unrowed file, a missing report, a missing TSV, a stale count in
|
||||
this table, an edited `CHECK:` line) plus the rule-agreement and no-host-path
|
||||
cells.
|
||||
|
||||
VERDICT: 14/80 quill drafts carry a bare CHECK command; the committed report re-derives exactly from live scan.
|
||||
CHECK: python3 /root/troupe-standup/wizard-work/quill/verify_draft_compliance.py
|
||||
The 2026-10-31 version of both scripts hardcoded an absolute home-directory
|
||||
path, so a clone's `CHECK` graded this host's tree and a mutated copy of the
|
||||
committed TSV still read PASS. Both now derive their paths from `__file__`.
|
||||
|
||||
VERDICT: 11/79 quill drafts carry a bare CHECK command; the committed TSV and this table re-derive from the tree.
|
||||
CHECK: bash CHECK-quill-draft-compliance
|
||||
@@ -1,4 +1,4 @@
|
||||
# measured 80 markdown files
|
||||
# measured 79 markdown files
|
||||
filename lines has_verdict has_check check_clean check_command
|
||||
QUICKSTART.md 148 False False False
|
||||
README.md 20 False False False
|
||||
@@ -10,9 +10,8 @@ changelog_draft.md 30 False False False
|
||||
code-review-checklist.md 21 False False False
|
||||
cover-letter-draft.md 16 False False False
|
||||
cron-migration-runbook.md 46 False False False
|
||||
daily-draft-verification-report.md 145 True True True python3 /root/troupe-standup/wizard-work/quill/daily_draft.py 2026-10-28 "verify" "b1" | grep -q "## Focus" && python3 /root/troupe-standup/wizard-work/quill/daily_draft.py | grep -q "# Daily Draft"
|
||||
daily-draft-verification-report.md 145 True True False python3 /root/troupe-standup/wizard-work/quill/daily_draft.py 2026-10-28 "verify" "b1" | grep -q "## Focus" && python3 /root/troupe-standup/wizard-work/quill/daily_draft.py | grep -q "# Daily Draft"
|
||||
draft-artifact-naming-convention.md 48 False False False
|
||||
draft-compliance-report-2026-10-31.md 59 True True True python3 /root/troupe-standup/wizard-work/quill/verify_draft_compliance.py
|
||||
draft-hermes-quick-ref.md 71 False False False
|
||||
draft-incident-response-runbook-v0.2.md 55 False False False
|
||||
draft-lane-playbook.md 41 True True True test -f /root/.hermes/wizard-outputs/quill/draft-lane-playbook.md && grep -q "^# Draft Lane Playbook" /root/.hermes/wizard-outputs/quill/draft-lane-playbook.md && grep -q "^VERDICT:" /root/.hermes/wizard-outputs/quill/draft-lane-playbook.md && grep -q "^CHECK:" /root/.hermes/wizard-outputs/quill/draft-lane-playbook.md
|
||||
@@ -32,7 +31,7 @@ free-tier-ai-runbook.md 85 False False False
|
||||
free-tier-ai-stack-audit.md 94 False False False
|
||||
free-tier-evidence-triage-2026-10-25.md 55 False False False
|
||||
hermes-agent-0.9.4-release-notes-draft.md 80 False False False
|
||||
hermes-linux-setup-brief-2026-10-29.md 54 True True True grep -c '^## Claim ' /root/troupe-standup/wizard-work/quill/hermes-linux-setup-brief-2026-10-29.md | grep '^5$'
|
||||
hermes-linux-setup-brief-2026-10-29.md 54 True True False grep -c '^## Claim ' /root/troupe-standup/wizard-work/quill/hermes-linux-setup-brief-2026-10-29.md | grep '^5$'
|
||||
hermes-quick-ref-draft-2026-10-28.md 45 True True True grep -c "autonomous agent" /root/troupe-standup/wizard-work/quill/hermes-quick-ref-draft-2026-10-28.md
|
||||
incident-draft.md 21 False False False
|
||||
incident-postmortem-template.md 40 False False False
|
||||
@@ -58,7 +57,7 @@ project-status-report-template.md 34 False False False
|
||||
quickstart-transcript.md 159 False False False
|
||||
quill-paragraph-draft-guide.md 76 False False False
|
||||
release-notes-template.md 37 False False False
|
||||
rfc-draft-agent-outputs.md 56 True True True test -s /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Abstract$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Motivation$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Lane Contract$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Commit Workflow$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Change Log$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && tail -n 2 /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md | grep -q "^VERDICT:" && tail -n 2 /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md | grep -q "^CHECK:"
|
||||
rfc-draft-agent-outputs.md 56 True True False test -s /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Abstract$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Motivation$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Lane Contract$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Commit Workflow$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Change Log$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && tail -n 2 /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md | grep -q "^VERDICT:" && tail -n 2 /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md | grep -q "^CHECK:"
|
||||
runbook-draft.md 32 False False False
|
||||
sample.md 8 False False False
|
||||
security-incident-report-template.md 157 False False False
|
||||
|
||||
|
Can't render this file because it contains an unexpected character in line 13 and column 129.
|
@@ -1,51 +1,70 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a draft-compliance report for all .md files in wizard-work/quill/.
|
||||
"""Generate the draft-compliance TSV for the .md drafts in this directory.
|
||||
|
||||
Measures:
|
||||
- filename, line_count, has_verdict, has_check, check_clean
|
||||
Writes TSV to stdout.
|
||||
The lane directory is derived from THIS FILE's location, never from a host path,
|
||||
so a clone grades its own tree. `--lane` exists for a fixture tree; `--out`
|
||||
writes the TSV instead of stdout, and the output is deterministic: the same tree
|
||||
produces the same bytes, comment line and header included.
|
||||
|
||||
The artifact itself (`draft-compliance-report-2026-10-31.md`) is NOT a row: the
|
||||
report publishes the table, it is not a draft in it. That exclusion is what makes
|
||||
the population 79 rather than 80 -- the 2026-10-31 report's prose said 79 while
|
||||
its TSV carried 80 rows, and the missing row was the report counting itself.
|
||||
|
||||
Rule, columns and the artifact name live in `draft_compliance_rule.py`; the
|
||||
verifier re-derives this table from the files with its own code.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
LANE_DIR = "/root/troupe-standup/wizard-work/quill"
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
|
||||
def extract_check_line(content):
|
||||
m = re.search(r'^CHECK:\s*(.+)$', content, re.MULTILINE)
|
||||
return m.group(1).strip() if m else ""
|
||||
import draft_compliance_rule as rule # noqa: E402
|
||||
|
||||
def is_bare_command(cmd):
|
||||
if not cmd:
|
||||
return False
|
||||
bad = ["|| true", "|| echo", "|| exit", "; exit", "; echo", "true", "false"]
|
||||
# Must not contain forbidden patterns
|
||||
for b in bad:
|
||||
if b in cmd:
|
||||
return False
|
||||
# Must not contain semicolons outside of comments (simple check)
|
||||
if ";" in cmd and not cmd.lstrip().startswith("#"):
|
||||
return False
|
||||
return True
|
||||
|
||||
def scan(lane):
|
||||
rows = []
|
||||
for fname in sorted(os.listdir(LANE_DIR)):
|
||||
if not fname.endswith(".md"):
|
||||
for name in sorted(os.listdir(lane)):
|
||||
if not name.endswith(".md") or name == rule.ARTIFACT_MD:
|
||||
continue
|
||||
path = os.path.join(lane, name)
|
||||
if not os.path.isfile(path):
|
||||
continue
|
||||
path = os.path.join(LANE_DIR, fname)
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
||||
content = f.read()
|
||||
lines = content.splitlines()
|
||||
line_count = len(lines)
|
||||
has_verdict = bool(re.search(r'^VERDICT:', content, re.MULTILINE))
|
||||
has_check = bool(re.search(r'^CHECK:', content, re.MULTILINE))
|
||||
check_line = extract_check_line(content)
|
||||
check_clean = is_bare_command(check_line)
|
||||
rows.append((fname, line_count, has_verdict, has_check, check_clean, check_line))
|
||||
text = f.read()
|
||||
rows.append(rule.row_for(name, text))
|
||||
return rows
|
||||
|
||||
# Header
|
||||
print("filename\tlines\thas_verdict\thas_check\tcheck_clean\tcheck_command")
|
||||
for r in rows:
|
||||
print("\t".join(map(str, r)))
|
||||
|
||||
print(f"# measured {len(rows)} markdown files", file=sys.stderr)
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Generate the draft-compliance TSV for this lane's .md drafts.")
|
||||
ap.add_argument("--lane", default=HERE,
|
||||
help="directory of .md drafts (default: this file's dir)")
|
||||
ap.add_argument("--out", default=None,
|
||||
help="write the TSV here instead of stdout")
|
||||
a = ap.parse_args(argv)
|
||||
|
||||
if not os.path.isdir(a.lane):
|
||||
print("cannot read lane directory: %s" % a.lane, file=sys.stderr)
|
||||
return 2
|
||||
|
||||
rows = scan(a.lane)
|
||||
lines = ["# measured %d markdown files" % len(rows),
|
||||
"\t".join(rule.COLUMNS)]
|
||||
lines.extend("\t".join(r) for r in rows)
|
||||
body = "\n".join(lines) + "\n"
|
||||
|
||||
if a.out:
|
||||
with open(a.out, "w", encoding="utf-8", newline="\n") as f:
|
||||
f.write(body)
|
||||
print("wrote %s (%d rows)" % (a.out, len(rows)), file=sys.stderr)
|
||||
else:
|
||||
sys.stdout.write(body)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""The draft-compliance contract, stated once.
|
||||
|
||||
WHY THIS FILE EXISTS
|
||||
The 2026-10-31 report shipped two copies of the scan: the generator and the
|
||||
"independent verifier" were the same code with two `bad` lists that had already
|
||||
drifted -- the generator rejected the words `true`/`false` and the verifier did
|
||||
not, so a CHECK line containing either read `check_clean=False` from one and
|
||||
`True` from the other. Nothing measured the disagreement, because the verifier
|
||||
compared the committed TSV against its own copy of the scan and never against
|
||||
the generator's.
|
||||
|
||||
Two things are shared here and nothing else: the constants that name the
|
||||
artifact and its columns, and ONE implementation of the rule `is_bare`. The
|
||||
verifier re-implements `is_bare` independently (see `verify_draft_compliance.py`)
|
||||
and its `--selftest` cell `rule-agreement` requires the two implementations to
|
||||
return the same answer on every CHECK line in the corpus, so a future
|
||||
divergence is a red cell instead of a silent contradiction.
|
||||
|
||||
WHAT `is_bare` MEANS -- a CHECK command must not swallow or sequence failure
|
||||
A command is bare iff no token is a control operator `;` `|` `||` `&`, and no
|
||||
token is the null command `true` or `false`. `&&` IS allowed: it stops at the
|
||||
first failure and returns it, which is the property the rule is about. A pipe
|
||||
is rejected on this archive's own doctrine -- without `pipefail` a pipeline's
|
||||
rc is its LAST command's -- and a `;` sequence returns only the last rc.
|
||||
A command that does not tokenize (unbalanced quotes) is not bare.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
|
||||
# The artifact this contract grades; it is not a draft and is excluded from its
|
||||
# own population (the census-artifact convention: an artifact is not a row in
|
||||
# the table it publishes).
|
||||
ARTIFACT_MD = "draft-compliance-report-2026-10-31.md"
|
||||
ARTIFACT_TSV = "draft-compliance-report-2026-10-31.tsv"
|
||||
|
||||
COLUMNS = ("filename", "lines", "has_verdict", "has_check", "check_clean",
|
||||
"check_command")
|
||||
|
||||
CONTROL_TOKENS = frozenset({";", "|", "||", "&"})
|
||||
NULL_COMMANDS = frozenset({"true", "false"})
|
||||
|
||||
|
||||
def check_line(text):
|
||||
"""The text after the first `^CHECK:` line that carries text.
|
||||
|
||||
An empty `CHECK:` marker is skipped rather than returned as the command:
|
||||
`daily-draft-verification-report.md` opens a quoted template with a bare
|
||||
`CHECK:` at line 102 before its real command at line 145, and the 2026-10-31
|
||||
regex (which required at least one character) read the real one. Both
|
||||
implementations keep that reading.
|
||||
"""
|
||||
m = re.search(r"^CHECK:\s*(.+)$", text, re.MULTILINE)
|
||||
return m.group(1).strip() if m else ""
|
||||
|
||||
|
||||
def is_bare(cmd):
|
||||
if not cmd:
|
||||
return False
|
||||
try:
|
||||
tokens = shlex.split(cmd, posix=True)
|
||||
except ValueError:
|
||||
return False
|
||||
for tok in tokens:
|
||||
if tok in CONTROL_TOKENS or tok in NULL_COMMANDS:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def has_verdict(text):
|
||||
return re.search(r"^VERDICT:", text, re.MULTILINE) is not None
|
||||
|
||||
|
||||
def has_check(text):
|
||||
return re.search(r"^CHECK:", text, re.MULTILINE) is not None
|
||||
|
||||
|
||||
def row_for(name, text):
|
||||
"""One TSV row (all strings) for one draft file's text."""
|
||||
cmd = check_line(text)
|
||||
return (name, str(len(text.splitlines())), str(has_verdict(text)),
|
||||
str(has_check(text)), str(is_bare(cmd)), cmd)
|
||||
|
||||
|
||||
SELFTEST_CASES = [
|
||||
# (command, expected is_bare, why)
|
||||
("python3 tools/verify.py --fast", True, "single command"),
|
||||
("bash CHECK-quill-draft-compliance", True, "single command"),
|
||||
("test -f a.md && grep -q VERDICT a.md", True, "&& stops at the first failure"),
|
||||
("python3 a.py | grep -q x", False, "pipe: rc is the last command's"),
|
||||
("python3 a.py || true", False, "|| swallows the failure"),
|
||||
("python3 a.py; true", False, "; sequences, rc is the last"),
|
||||
("python3 a.py &", False, "backgrounding discards rc"),
|
||||
("curl -s x && true", False, "null command as the last word"),
|
||||
("python3 a.py 'unbalanced", False, "does not tokenize"),
|
||||
("", False, "no CHECK line"),
|
||||
("python3 check.py --allow=true", True,
|
||||
"`true` as part of a word is not the null command"),
|
||||
]
|
||||
|
||||
|
||||
def _selftest():
|
||||
bad = 0
|
||||
for cmd, want, why in SELFTEST_CASES:
|
||||
got = is_bare(cmd)
|
||||
if got != want:
|
||||
bad += 1
|
||||
print("FAIL rule: %r -> %s, wanted %s (%s)" % (cmd, got, want, why))
|
||||
print("rule: %d case(s), %d failed" % (len(SELFTEST_CASES), bad))
|
||||
return bad
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(1 if _selftest() else 0)
|
||||
@@ -1,86 +1,382 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Independent verifier for draft-compliance-report-2026-10-31.tsv.
|
||||
"""Independent verifier for draft-compliance-report-2026-10-31.{tsv,md}.
|
||||
|
||||
Re-derives the compliance scan from wizard-work/quill/ and compares
|
||||
every row to the committed report. Exit code IS the verdict: 0 = match,
|
||||
nonzero = mismatch or missing files.
|
||||
WHAT IT GRADES, AND AGAINST WHAT
|
||||
The committed TSV is a claim about the .md drafts beside it. This verifier
|
||||
re-derives every row from the files with its OWN code -- its own line counter,
|
||||
its own `VERDICT:`/`CHECK:` detection and its own implementation of the bare-
|
||||
command rule -- and compares in both directions: every TSV row against the
|
||||
file it names, and every file against the row it should have. It also grades
|
||||
the report's Evidence table against the counts it just measured, so a stale
|
||||
numeral in the prose is a fault rather than a reader's problem.
|
||||
|
||||
Paths come from THIS FILE's location. The 2026-10-31 version hardcoded an
|
||||
absolute home-directory path in both the scanner and the verifier, so a
|
||||
clone's CHECK graded the host's tree and a mutated copy of the committed TSV
|
||||
still read PASS; `--selftest` cell `no-host-path` greps the sources for that
|
||||
class, because this file's own fix must not reintroduce it.
|
||||
|
||||
EXIT CODES -- the archive's convention
|
||||
0 the TSV and the report re-derive from the tree
|
||||
1 a fault: a row, a file-set entry or a reported count disagrees
|
||||
2 could not grade: the report or TSV is missing/unreadable, or the lane
|
||||
directory does not exist. Never a pass.
|
||||
|
||||
--selftest runs the cells in a temp tree: one green control and eight mutants,
|
||||
so a green reading is never the only thing this checker can print.
|
||||
|
||||
NOTE ON THE SECOND IMPLEMENTATION OF `is_bare`
|
||||
It reads operators on the raw string and only checks quoting for parity, so an
|
||||
operator inside a quoted argument reads as not bare where the shlex-based rule
|
||||
would read it as bare. That is deliberate and conservative (it over-reports
|
||||
"not bare", never over-reports "bare"), and `rule-agreement` fails loudly if
|
||||
the two implementations ever disagree on a command this corpus actually holds.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
REPORT = "/root/troupe-standup/wizard-work/quill/draft-compliance-report-2026-10-31.tsv"
|
||||
LANE_DIR = "/root/troupe-standup/wizard-work/quill"
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
sys.path.insert(0, HERE)
|
||||
|
||||
def extract_check_line(content):
|
||||
m = re.search(r'^CHECK:\s*(.+)$', content, re.MULTILINE)
|
||||
return m.group(1).strip() if m else ""
|
||||
import draft_compliance_rule as rule # noqa: E402
|
||||
|
||||
def is_bare_command(cmd):
|
||||
# The report's Evidence table, label -> the metric it claims. The labels are
|
||||
# matched verbatim; a missing label is a fault (the report must say what it
|
||||
# measured).
|
||||
REPORT_LABELS = (
|
||||
("Total `.md` drafts scanned", "total"),
|
||||
("Drafts with `VERDICT:`", "verdict"),
|
||||
("Drafts with `CHECK:`", "check"),
|
||||
("Drafts with a bare `CHECK:` command", "bare"),
|
||||
("Drafts failing at least one column", "failing"),
|
||||
("`VERDICT:` and `CHECK:` but a non-bare command", "nonbare_both"),
|
||||
)
|
||||
|
||||
NULL_RE = re.compile(r"(?:^|[\s;&|(])(true|false)(?=$|[\s;&|)])")
|
||||
|
||||
|
||||
def own_is_bare(cmd):
|
||||
"""The same contract as `rule.is_bare`, implemented differently.
|
||||
|
||||
Quoted regions are removed before the operator scan, so an operator inside a
|
||||
quoted argument is an argument, not an operator; parity of quotes is checked
|
||||
first, because a command that does not close its quotes does not tokenize
|
||||
and is therefore not bare under the contract.
|
||||
"""
|
||||
if not cmd:
|
||||
return False
|
||||
bad = ["|| true", "|| echo", "|| exit", "; exit", "; echo"]
|
||||
for b in bad:
|
||||
if b in cmd:
|
||||
if cmd.count("'") % 2 or cmd.count('"') % 2:
|
||||
return False
|
||||
if ";" in cmd and not cmd.lstrip().startswith("#"):
|
||||
stripped = re.sub(r"'[^']*'|\"[^\"]*\"", "''", cmd)
|
||||
if ";" in stripped or "||" in stripped:
|
||||
return False
|
||||
return True
|
||||
rest = stripped.replace("&&", "")
|
||||
if "&" in rest or "|" in rest:
|
||||
return False
|
||||
return NULL_RE.search(stripped) is None
|
||||
|
||||
def scan():
|
||||
|
||||
def own_line_count(text):
|
||||
"""Line count without splitlines(): newlines plus a final unterminated line."""
|
||||
return text.count("\n") + (1 if text and not text.endswith("\n") else 0)
|
||||
|
||||
|
||||
def own_rows(lane, exclude=rule.ARTIFACT_MD):
|
||||
"""Re-derive the table from the files, without calling the generator."""
|
||||
rows = []
|
||||
for fname in sorted(os.listdir(LANE_DIR)):
|
||||
if not fname.endswith(".md"):
|
||||
names = sorted(n for n in os.listdir(lane)
|
||||
if n.endswith(".md") and n != exclude)
|
||||
for name in names:
|
||||
path = os.path.join(lane, name)
|
||||
if not os.path.isfile(path):
|
||||
continue
|
||||
path = os.path.join(LANE_DIR, fname)
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
||||
content = f.read()
|
||||
line_count = len(content.splitlines())
|
||||
has_verdict = bool(re.search(r'^VERDICT:', content, re.MULTILINE))
|
||||
has_check = bool(re.search(r'^CHECK:', content, re.MULTILINE))
|
||||
check_line = extract_check_line(content)
|
||||
check_clean = is_bare_command(check_line)
|
||||
rows.append((fname, line_count, has_verdict, has_check, check_clean, check_line))
|
||||
text = f.read()
|
||||
has_v = has_c = False
|
||||
cmd = ""
|
||||
for ln in text.split("\n"):
|
||||
if ln.startswith("VERDICT:"):
|
||||
has_v = True
|
||||
elif ln.startswith("CHECK:"):
|
||||
has_c = True
|
||||
rest = ln[len("CHECK:"):].strip()
|
||||
if rest and not cmd:
|
||||
cmd = rest
|
||||
rows.append((name, str(own_line_count(text)), str(has_v), str(has_c),
|
||||
str(own_is_bare(cmd)), cmd))
|
||||
return rows
|
||||
|
||||
def main():
|
||||
errors = []
|
||||
|
||||
if not os.path.exists(REPORT):
|
||||
errors.append(f"report missing: {REPORT}")
|
||||
print("VERDICT: FAIL — report missing")
|
||||
sys.exit(1)
|
||||
def read_tsv(path):
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
raw = f.read().split("\n")
|
||||
comment = ""
|
||||
if raw and raw[0].startswith("#"):
|
||||
comment = raw[0]
|
||||
raw = raw[1:]
|
||||
body = [ln for ln in raw if ln.strip()]
|
||||
header = body[0].split("\t") if body else []
|
||||
rows = [tuple(ln.split("\t")) for ln in body[1:]]
|
||||
return comment, header, rows
|
||||
|
||||
with open(REPORT, "r", encoding="utf-8") as f:
|
||||
lines = [l.rstrip("\n") for l in f if l.strip() and not l.startswith("#")]
|
||||
|
||||
header = lines[0].split("\t")
|
||||
expected = [tuple(l.split("\t")) for l in lines[1:]]
|
||||
actual = scan()
|
||||
actual_tuples = [(r[0], str(r[1]), str(r[2]), str(r[3]), str(r[4]), r[5]) for r in actual]
|
||||
def read_report_counts(path):
|
||||
counts = {}
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for ln in f:
|
||||
m = re.match(r"^\|\s*(.+?)\s*\|\s*(\d+)\s*\|\s*$", ln.rstrip("\n"))
|
||||
if m:
|
||||
counts[m.group(1)] = int(m.group(2))
|
||||
return counts
|
||||
|
||||
# Compare sets of filenames
|
||||
exp_files = {r[0] for r in expected}
|
||||
act_files = {r[0] for r in actual_tuples}
|
||||
if exp_files != act_files:
|
||||
errors.append(f"file set mismatch: expected {len(exp_files)}, got {len(act_files)}")
|
||||
|
||||
# Compare each row
|
||||
act_map = {r[0]: r for r in actual_tuples}
|
||||
for exp_row in expected:
|
||||
fname = exp_row[0]
|
||||
act_row = act_map.get(fname)
|
||||
if act_row != exp_row:
|
||||
errors.append(f"row mismatch for {fname}: expected={exp_row} actual={act_row}")
|
||||
def metrics(rows):
|
||||
def ok(r):
|
||||
return r[2] == "True" and r[3] == "True" and r[4] == "True"
|
||||
|
||||
if errors:
|
||||
return {
|
||||
"total": len(rows),
|
||||
"verdict": sum(1 for r in rows if r[2] == "True"),
|
||||
"check": sum(1 for r in rows if r[3] == "True"),
|
||||
"bare": sum(1 for r in rows if r[4] == "True"),
|
||||
"failing": sum(1 for r in rows if not ok(r)),
|
||||
"nonbare_both": sum(1 for r in rows
|
||||
if r[2] == "True" and r[3] == "True" and r[4] != "True"),
|
||||
}
|
||||
|
||||
|
||||
def check(lane=None, report=None, tsv=None):
|
||||
"""Return (rc, faults, notes). Never raises for bad input."""
|
||||
lane = lane or HERE
|
||||
report = report or os.path.join(lane, rule.ARTIFACT_MD)
|
||||
tsv = tsv or os.path.join(lane, rule.ARTIFACT_TSV)
|
||||
faults, notes = [], []
|
||||
|
||||
if not os.path.isdir(lane):
|
||||
return 2, ["lane directory missing: %s" % lane], notes
|
||||
for path, what in ((report, "report"), (tsv, "TSV")):
|
||||
if not os.path.isfile(path):
|
||||
return 2, ["%s missing: %s" % (what, path)], notes
|
||||
|
||||
try:
|
||||
comment, header, claimed = read_tsv(tsv)
|
||||
reported = read_report_counts(report)
|
||||
except (OSError, UnicodeDecodeError) as exc:
|
||||
return 2, ["cannot read the artifact: %s" % exc], notes
|
||||
|
||||
if tuple(header) != rule.COLUMNS:
|
||||
faults.append("TSV header is %r, contract is %r"
|
||||
% (tuple(header), rule.COLUMNS))
|
||||
|
||||
derived = own_rows(lane)
|
||||
der_map = {r[0]: r for r in derived}
|
||||
claim_map = {r[0]: r for r in claimed}
|
||||
|
||||
for name in sorted(set(claim_map) - set(der_map)):
|
||||
faults.append("TSV row names a file the lane does not hold: %s" % name)
|
||||
for name in sorted(set(der_map) - set(claim_map)):
|
||||
faults.append("lane file has no TSV row: %s" % name)
|
||||
for name in sorted(set(der_map) & set(claim_map)):
|
||||
if der_map[name] != claim_map[name]:
|
||||
faults.append("row mismatch for %s: claimed=%r derived=%r"
|
||||
% (name, claim_map[name], der_map[name]))
|
||||
|
||||
expected = "# measured %d markdown files" % len(derived)
|
||||
if comment != expected:
|
||||
faults.append("TSV comment is %r, measured %r" % (comment, expected))
|
||||
|
||||
m = metrics(derived)
|
||||
for label, key in REPORT_LABELS:
|
||||
if label not in reported:
|
||||
faults.append("report table row missing: %r" % label)
|
||||
elif reported[label] != m[key]:
|
||||
faults.append("report says %r = %d, measured %d"
|
||||
% (label, reported[label], m[key]))
|
||||
|
||||
# The contract has two implementations; they must agree on every command the
|
||||
# corpus carries, or one of them is grading a different rule.
|
||||
for r in derived:
|
||||
if rule.is_bare(r[5]) != own_is_bare(r[5]):
|
||||
faults.append("rule divergence on %s: rule=%s verifier=%s cmd=%r"
|
||||
% (r[0], rule.is_bare(r[5]), own_is_bare(r[5]), r[5]))
|
||||
|
||||
notes.append("population %d, verdict %d, check %d, bare %d, failing %d"
|
||||
% (m["total"], m["verdict"], m["check"], m["bare"], m["failing"]))
|
||||
return (1 if faults else 0), faults, notes
|
||||
|
||||
|
||||
FIXTURE_FILES = {
|
||||
"good.md": "# t\n\nVERDICT: ok\nCHECK: python3 x.py\n",
|
||||
"plain.md": "# nothing\n",
|
||||
"piped.md": "# t\n\nVERDICT: ok\nCHECK: python3 x.py | grep -q ok\n",
|
||||
}
|
||||
|
||||
|
||||
def _fixture_report(claimed):
|
||||
lines = ["# fixture", "", "| Metric | Value |", "|---|---|"]
|
||||
for label, _key in REPORT_LABELS:
|
||||
lines.append("| %s | %d |" % (label, claimed[label]))
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _fixture_tree(root, mutate=None):
|
||||
lane = os.path.join(root, "lane")
|
||||
os.makedirs(lane)
|
||||
for name, text in FIXTURE_FILES.items():
|
||||
with open(os.path.join(lane, name), "w", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
rows = own_rows(lane)
|
||||
m = metrics(rows)
|
||||
claimed = {label: m[key] for label, key in REPORT_LABELS}
|
||||
report = os.path.join(lane, rule.ARTIFACT_MD)
|
||||
with open(report, "w", encoding="utf-8") as f:
|
||||
f.write(_fixture_report(claimed))
|
||||
tsv = os.path.join(lane, rule.ARTIFACT_TSV)
|
||||
with open(tsv, "w", encoding="utf-8") as f:
|
||||
f.write("# measured %d markdown files\n" % len(rows))
|
||||
f.write("\t".join(rule.COLUMNS) + "\n")
|
||||
for r in rows:
|
||||
f.write("\t".join(r) + "\n")
|
||||
if mutate:
|
||||
mutate(lane, report, tsv)
|
||||
return lane, report, tsv
|
||||
|
||||
|
||||
def selftest(verbose=True):
|
||||
cells, failed = [], []
|
||||
|
||||
def cell(name, want, mutate=None):
|
||||
tmp = tempfile.mkdtemp(prefix="q142-")
|
||||
try:
|
||||
lane, report, tsv = _fixture_tree(tmp, mutate)
|
||||
got, faults, _notes = check(lane, report, tsv)
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
cells.append(name)
|
||||
if got != want:
|
||||
failed.append((name, want, got, faults[:3]))
|
||||
if verbose:
|
||||
print("%-22s rc=%d want=%d %s"
|
||||
% (name, got, want, "ok" if got == want else "FAIL"))
|
||||
|
||||
def bump_line(lane, report, tsv):
|
||||
with open(os.path.join(lane, "good.md"), "a", encoding="utf-8") as f:
|
||||
f.write("one more line\n")
|
||||
|
||||
def drop_verdict(lane, report, tsv):
|
||||
path = os.path.join(lane, "good.md")
|
||||
with open(path, encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(text.replace("VERDICT: ok\n", ""))
|
||||
|
||||
def drop_row(lane, report, tsv):
|
||||
with open(tsv, encoding="utf-8") as f:
|
||||
lines = f.read().split("\n")
|
||||
with open(tsv, "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(ln for ln in lines if not ln.startswith("piped.md")))
|
||||
|
||||
def extra_file(lane, report, tsv):
|
||||
with open(os.path.join(lane, "late.md"), "w", encoding="utf-8") as f:
|
||||
f.write("# late\n")
|
||||
|
||||
def no_report(lane, report, tsv):
|
||||
os.remove(report)
|
||||
|
||||
def no_tsv(lane, report, tsv):
|
||||
os.remove(tsv)
|
||||
|
||||
def stale_count(lane, report, tsv):
|
||||
with open(report, encoding="utf-8") as f:
|
||||
text = f.read()
|
||||
text = re.sub(r"(\| Total `\.md` drafts scanned \| )3( \|)",
|
||||
r"\g<1>2\g<2>", text)
|
||||
with open(report, "w", encoding="utf-8") as f:
|
||||
f.write(text)
|
||||
|
||||
def check_line_edited(lane, report, tsv):
|
||||
with open(os.path.join(lane, "good.md"), "w", encoding="utf-8") as f:
|
||||
f.write("# t\n\nVERDICT: ok\nCHECK: python3 y.py\n")
|
||||
|
||||
cell("green-control", 0)
|
||||
cell("line-count-moved", 1, bump_line)
|
||||
cell("verdict-removed", 1, drop_verdict)
|
||||
cell("row-deleted", 1, drop_row)
|
||||
cell("file-without-row", 1, extra_file)
|
||||
cell("report-missing", 2, no_report)
|
||||
cell("tsv-missing", 2, no_tsv)
|
||||
cell("report-count-stale", 1, stale_count)
|
||||
cell("check-line-edited", 1, check_line_edited)
|
||||
|
||||
# The two implementations of the rule, over the fixture cases and this
|
||||
# directory's own corpus. A divergence here is what the 2026-10-31 copies had.
|
||||
divergence = [cmd for cmd, want, _why in rule.SELFTEST_CASES
|
||||
if own_is_bare(cmd) != want]
|
||||
for r in own_rows(HERE):
|
||||
if rule.is_bare(r[5]) != own_is_bare(r[5]):
|
||||
divergence.append(r[0])
|
||||
cells.append("rule-agreement")
|
||||
if divergence:
|
||||
failed.append(("rule-agreement", 0, 1, divergence[:3]))
|
||||
if verbose:
|
||||
print("%-22s %s" % ("rule-agreement",
|
||||
"ok" if not divergence else "FAIL %r" % divergence[:3]))
|
||||
|
||||
# No host path in the sources: the 2026-10-31 defect, graded at the source.
|
||||
offenders = []
|
||||
needle = os.sep + "root" + os.sep
|
||||
for name in (os.path.basename(__file__), "draft_compliance_report.py",
|
||||
"draft_compliance_rule.py"):
|
||||
with open(os.path.join(HERE, name), encoding="utf-8") as f:
|
||||
if needle in f.read():
|
||||
offenders.append(name)
|
||||
cells.append("no-host-path")
|
||||
if offenders:
|
||||
failed.append(("no-host-path", 0, 1, offenders))
|
||||
if verbose:
|
||||
print("%-22s %s" % ("no-host-path",
|
||||
"ok" if not offenders else "FAIL %r" % offenders))
|
||||
|
||||
print("\nselftest: %d cells, %d passed, %d failed"
|
||||
% (len(cells), len(cells) - len(failed), len(failed)))
|
||||
for name, want, got, detail in failed:
|
||||
print(" FAIL %s: rc %d wanted %d -- %r" % (name, got, want, detail))
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Verify the quill draft-compliance report.")
|
||||
ap.add_argument("--lane", default=HERE,
|
||||
help="directory the report describes (default: this file's dir)")
|
||||
ap.add_argument("--report", default=None, help="report .md (default: in --lane)")
|
||||
ap.add_argument("--tsv", default=None, help="report .tsv (default: in --lane)")
|
||||
ap.add_argument("--selftest", action="store_true", help="run the cells")
|
||||
a = ap.parse_args(argv)
|
||||
|
||||
if a.selftest:
|
||||
return selftest()
|
||||
|
||||
rc, faults, notes = check(a.lane, a.report, a.tsv)
|
||||
for note in notes:
|
||||
print(note)
|
||||
if rc == 0:
|
||||
print("VERDICT: PASS -- TSV and report re-derive from the tree (%s)"
|
||||
% notes[0])
|
||||
elif rc == 2:
|
||||
print("VERDICT: UNGRADABLE")
|
||||
for f in faults:
|
||||
print(" -", f)
|
||||
else:
|
||||
print("VERDICT: FAIL")
|
||||
for e in errors:
|
||||
print(" -", e)
|
||||
sys.exit(1)
|
||||
for f in faults:
|
||||
print(" -", f)
|
||||
return rc
|
||||
|
||||
print(f"VERDICT: PASS — {len(expected)} draft artifacts re-derived and verified")
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user