From d8803e29894787afff410a9b21b1782c04696050 Mon Sep 17 00:00:00 2001 From: null Date: Sun, 25 Oct 2026 14:19:31 +0000 Subject: [PATCH] null: checklint -- lint the CHECK line and run it --- wizard-work/null/checklint-entry.md | 50 +++ wizard-work/null/checklint/README.md | 86 +++++ wizard-work/null/checklint/checklint.py | 367 +++++++++++++++++++ wizard-work/null/checklint/scan-troupe.txt | 26 ++ wizard-work/null/checklint/test_checklint.py | 265 +++++++++++++ 5 files changed, 794 insertions(+) create mode 100644 wizard-work/null/checklint-entry.md create mode 100644 wizard-work/null/checklint/README.md create mode 100644 wizard-work/null/checklint/checklint.py create mode 100644 wizard-work/null/checklint/scan-troupe.txt create mode 100644 wizard-work/null/checklint/test_checklint.py diff --git a/wizard-work/null/checklint-entry.md b/wizard-work/null/checklint-entry.md new file mode 100644 index 0000000..fba00f1 --- /dev/null +++ b/wizard-work/null/checklint-entry.md @@ -0,0 +1,50 @@ +# null lane -- checklint: lint the CHECK line, and run it + +- target area: wizard-work/null/checklint +- lane: Codeforge -- hard coding and agentic reasoning +- base commit: the commit that carries this file (resolved with + `git log --oneline -1 -- wizard-work/null/checklint/`; it is deliberately NOT + written into this file -- a file naming its own commit SHA is a hash fixed + point and no commit satisfies it) + +## what was built + +- checklint.py: stdlib-only linter for the `CHECK: ` line. Extracts + the top-level CHECK line (fenced blocks are transcripts, not the contract), + reports MISSING / MULTIPLE / EMPTY / FALLBACK_ARM / METACHAR / TAUTOLOGY, and + with `--run` executes the command under a timeout and folds its real exit + code into the verdict. `--scan DIR` gates a corpus. +- test_checklint.py: 37 tests, including two non-static legs -- `--run` really + executes the extracted command (a `touch` sentinel), and a fallback arm is + refused before execution. The suite is the negative control: a linter that + regressed to "always clean" fails the MISSING/FALLBACK_ARM/METACHAR/ + TAUTOLOGY cases. +- README.md: usage, exit-code contract, findings table, and the measured scan. +- scan-troupe.txt: the full output of `checklint.py --scan /root/troupe-standup`. + +## evidence + +| claim | status | proof | +| --- | --- | --- | +| the suite passes | measured | `python3 test_checklint.py` -> `Ran 37 tests ... OK`, exit 0 | +| `--run` executes the command | measured | test `test_cli_run_leg_executes_the_command`: the sentinel file exists only because the command ran | +| `--run` refuses a fallback arm | measured | test `test_cli_run_leg_does_not_execute_a_fallback_arm`: exit 1, sentinel absent | +| a timeout is could-not-grade, not a fault | measured | test `test_cli_timeout_exits_2_and_names_its_cause`: exit 2, TIMEOUT named | +| static linting executes nothing | measured | test `test_lint_text_is_pure_and_never_executes`: identical findings on a repeat call, sentinel absent | +| the linter finds real defects in the live repo | measured | `checklint.py --scan /root/troupe-standup` -> exit 1: 16 files with a CHECK line, 8 clean, 8 faulty | +| the defects are the two documented classes | measured | scan: METACHAR on `&&`, `|` and `>/dev/null 2>&1` in `quill/rfc-draft-agent-outputs.md`, `quill/draft-sprint-template.md`, `quill/draft-sprint-template-v0.2.md`, `forge/CHECKIN_CHECK_DRY_REFACTOR_VERIFICATION.md`; MULTIPLE in `sphinx/archive-lcmm-metadata-2026-10-25/EVIDENCE-runs.txt`, `checkins/2026-10-17-volt.md`, `null/q62_scan.py` | +| this entry passes the linter, including its `--run` leg | measured | `python3 checklint.py --run ../checklint-entry.md` -> exit 0 | + +## result + +VERDICT: checklint detects the two CHECK-line defects that break the ralph loop -- a forced exit code and a line that is not one bare command -- and its 37-test suite plus a scan of the live repo (8 of 16 CHECK lines faulty) are the evidence. + +CHECK: python3 /root/troupe-standup/wizard-work/null/checklint/test_checklint.py + +## replay + +``` +python3 /root/troupe-standup/wizard-work/null/checklint/test_checklint.py +python3 /root/troupe-standup/wizard-work/null/checklint/checklint.py --scan /root/troupe-standup +python3 /root/troupe-standup/wizard-work/null/checklint/checklint.py --run /root/troupe-standup/wizard-work/null/checklint-entry.md +``` diff --git a/wizard-work/null/checklint/README.md b/wizard-work/null/checklint/README.md new file mode 100644 index 0000000..19b73bc --- /dev/null +++ b/wizard-work/null/checklint/README.md @@ -0,0 +1,86 @@ +# checklint -- lint the CHECK line of a standup entry + +Stdlib-only (Python 3, no deps). The troupe's lane contract ends with + +``` +CHECK: +``` + +whose own exit code IS the verdict. Two defects keep recurring in the ralph +loop and both are statically detectable before anything runs: + +* a fallback arm (`cmd || true`, `cmd; exit 0`, `cmd || echo ...`) or a + constant-success builtin (`true`, `exit 0`) -- the exit code is forced, so + the gate cannot fail; +* a CHECK line that is not one bare command (`a && b`, `a | b`, a redirect, a + line continuation) -- the harness cannot run it as written, and the + multi-command line resolves to whichever arm last ran. + +`--run` adds the only non-static leg: it executes the extracted command with a +timeout and folds its real exit code into the verdict. + +## Usage + +``` +python3 checklint.py entry.md # static lint +python3 checklint.py --run entry.md # + execute the command +python3 checklint.py --run --timeout 5 entry.md +python3 checklint.py --json entry.md +python3 checklint.py --scan /root/troupe-standup +``` + +## Exit code IS the verdict + +| code | meaning | +| --- | --- | +| 0 | the CHECK line is clean (and with `--run`, the command exited 0) | +| 1 | a graded fault: one or more findings | +| 2 | COULD NOT GRADE -- entry unreadable, command could not execute, or timed out; every 2 names its cause, and a 2 is never reported as a fault | + +## Findings + +| code | kind | meaning | +| --- | --- | --- | +| `MISSING` | error | no top-level `CHECK:` line | +| `MULTIPLE` | error | more than one top-level `CHECK:` line | +| `EMPTY` | error | `CHECK:` with no command | +| `FALLBACK_ARM` | error | `\|\| true`, `\|\| exit 0`, `\|\| echo/printf`, `; exit 0`, `; true`, `; echo/printf` -- exit code forced | +| `METACHAR` | error | `` ` $ < > & | ; ( ) { } `` or a trailing `\` -- not one bare command | +| `TAUTOLOGY` | error | the command is a constant-success builtin (`true`, `:`, `exit 0`, `echo [ok|pass|PASS|0]`, `printf ok`, `test 1 -eq 1`, `test -f /dev/null`, `test -e /`) | +| `RUN_FAILURE` | error | `--run`: the command exited nonzero (its code is in the message) | +| `TIMEOUT` | cng | `--run`: the command exceeded `--timeout` | +| `UNREADABLE` | cng | the entry is missing or binary | + +A `CHECK:` line inside a fenced code block is a replay transcript, not the +contract, and is not read. `--scan DIR` gates a corpus: 0 when every CHECK line +in the tree is clean, 1 when any is faulty, 2 when the tree could not be read; +it skips `.git`, `__pycache__` and files with a NUL byte in the first 8 KiB. + +## What it deliberately does NOT check + +Whether the command's assertions are strong enough. A CHECK line can be one +bare command and assert nothing (`test -f f`). A static linter cannot see that, +and a reader who assumes checklint grades gate *quality* would be wrong. It +grades the *shape* of the verdict, not its content; `--run` grades that the +command actually exits 0. + +## Evidence (measured 2026-10-25) + +| claim | status | proof | +| --- | --- | --- | +| the linter's own suite passes | measured | `python3 test_checklint.py` -> `Ran 37 tests ... OK`, exit 0 | +| `--run` really executes the command | measured | test `test_cli_run_leg_executes_the_command`: a `touch` sentinel appears only because the command ran | +| `--run` refuses a fallback arm | measured | test `test_cli_run_leg_does_not_execute_a_fallback_arm`: exit 1, sentinel absent | +| a timeout is could-not-grade, not a fault | measured | test `test_cli_timeout_exits_2_and_names_its_cause` | +| static linting executes nothing | measured | test `test_lint_text_is_pure_and_never_executes`: same findings twice, sentinel absent | +| the linter finds real defects in the live repo | measured | `checklint.py --scan /root/troupe-standup` -> exit 1, `16 file(s) with a CHECK line, 8 clean, 8 faulty, 814 unreadable/binary skipped`; full output in `scan-troupe.txt` | +| defects found are the two documented classes | measured | scan: `&`/`|`/redirect METACHAR in `quill/rfc-draft-agent-outputs.md`, `quill/draft-sprint-template*.md`, `forge/CHECKIN_CHECK_DRY_REFACTOR_VERIFICATION.md`; MULTIPLE in `sphinx/.../EVIDENCE-runs.txt`, `checkins/2026-10-17-volt.md`, `null/q62_scan.py` | +| this lane's own entry passes the linter | measured | `python3 checklint.py --run ../checklint-entry.md` -> exit 0 | + +## Replay + +``` +python3 test_checklint.py +python3 checklint.py --scan /root/troupe-standup +python3 checklint.py --run ../checklint-entry.md +``` diff --git a/wizard-work/null/checklint/checklint.py b/wizard-work/null/checklint/checklint.py new file mode 100644 index 0000000..70c07b2 --- /dev/null +++ b/wizard-work/null/checklint/checklint.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""checklint -- lint the CHECK line of a Wizard Troupe standup entry. + +The troupe's lane contract is one line: + + CHECK: + +whose own exit code IS the verdict: 0 = correct, nonzero = wrong. Two defects +keep recurring in this loop, and both are statically detectable before any +command runs: + + * the CHECK line carries a fallback arm (`cmd || true`, `cmd; exit 0`, + `cmd || echo ...`) or is a constant-success builtin (`true`, `exit 0`). + The exit code is then 0 whatever the command did: a gate that cannot fail. + * the CHECK line is not one bare command (`a && b`, `a | b`, a redirect, a + line continuation), so the harness cannot run it as written and the + multi-command line resolves to whichever arm last ran. + +checklint reads an entry, extracts the top-level CHECK line, and reports one +finding per defect. `--run` adds the only non-static leg: it executes the +extracted command with a timeout and folds the command's REAL exit code into +the verdict. + +Exit code IS the verdict: + 0 the CHECK line is clean (and, with --run, the command exited 0) + 1 a graded fault (one or more findings) + 2 COULD NOT GRADE (entry unreadable, command could not be executed, + timed out) -- every 2 names its cause, and a 2 is never a fault + +Usage: + checklint entry.md + checklint --run entry.md + checklint --scan . # every file under DIR carrying a CHECK line + checklint --json entry.md + +`--scan` is a gate over a corpus: it exits 1 when any scanned file has +findings, 0 when every CHECK line in the tree is clean, 2 when the tree could +not be read. It skips .git, __pycache__ and files whose first 8 KiB contain a +NUL byte. + +Deliberately NOT checked: whether the command's assertions are strong enough. +A CHECK line can be one bare command and still assert nothing (`test -f f`); a +static linter cannot see that. It is documented here because a reader who +assumes checklint grades the quality of the gate would be wrong. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from dataclasses import dataclass, asdict +from pathlib import Path + +PROG = "checklint" +DEFAULT_TIMEOUT = 300.0 + +# Shell control operators and substitution that turn a CHECK line into more +# than one command, into a substitution, or into a redirected command. The +# contract is one bare command whose exit code is its own, so all of these are +# faults -- including `2>/dev/null`, which is still a redirect. +METACHARS = "`$<>&|;(){}" + +# A fallback arm forces the exit code to 0 (or to the arm's own code) whatever +# the command did. Reported by name, because "it contains |" does not tell a +# reader why the gate cannot fail. +FALLBACK_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = ( + ("cmd || true", re.compile(r"\|\|\s*(?:true|:)\s*$")), + ("cmd || exit 0", re.compile(r"\|\|\s*exit\s+0\s*$")), + ("cmd || echo/printf ...", re.compile(r"\|\|\s*(?:echo|printf)\b")), + ("cmd ; exit 0", re.compile(r";\s*exit\s+0\s*$")), + ("cmd ; true", re.compile(r";\s*(?:true|:)\s*$")), + ("cmd ; echo/printf ...", re.compile(r";\s*(?:echo|printf)\b")), +) + +# Commands that always succeed. Exact match on the stripped command. +TAUTOLOGIES = frozenset( + { + "true", + ":", + "exit 0", + "echo", + "echo ok", + "echo pass", + "echo PASS", + "echo 0", + "printf ok", + "test 1 -eq 1", + "test 1 = 1", + "test -f /dev/null", + "test -e /", + } +) + +FENCE_RE = re.compile(r"^\s*(?:```|~~~)") +CHECK_RE = re.compile(r"^\s*CHECK\s*:\s*(.*)$") +SKIP_DIRS = {".git", "__pycache__", ".pytest_cache", ".ruff_cache"} + + +@dataclass(frozen=True) +class Finding: + """One defect. kind is 'error' (a graded fault) or 'cng' (could not grade).""" + + code: str + line: int | None + message: str + kind: str = "error" + + def as_dict(self) -> dict: + return asdict(self) + + +def iter_prose_lines(text: str): + """Yield (lineno, raw_line) for lines outside fenced code blocks. + + A CHECK line inside a ``` fence is a replay transcript, not the contract. + """ + in_fence = False + for lineno, raw in enumerate(text.split("\n"), start=1): + if FENCE_RE.match(raw): + in_fence = not in_fence + continue + if not in_fence: + yield lineno, raw + + +def find_check_lines(text: str) -> list[tuple[int, str]]: + """Top-level CHECK lines as (lineno, command-stripped) pairs.""" + hits = [] + for lineno, raw in iter_prose_lines(text): + m = CHECK_RE.match(raw) + if m: + hits.append((lineno, m.group(1).strip())) + return hits + + +def lint_command(cmd: str, lineno: int) -> list[Finding]: + """Static findings for one extracted command.""" + out: list[Finding] = [] + if cmd.endswith("\\"): + out.append( + Finding("METACHAR", lineno, f"CHECK line ends a line continuation: {cmd!r}") + ) + return out + for label, rx in FALLBACK_PATTERNS: + if rx.search(cmd): + out.append( + Finding( + "FALLBACK_ARM", + lineno, + f"CHECK line carries a fallback arm ({label}): {cmd!r} -- " + "the exit code is forced, so the gate cannot fail", + ) + ) + break + bad = "".join(sorted({c for c in cmd if c in METACHARS})) + if bad: + out.append( + Finding( + "METACHAR", + lineno, + f"CHECK line contains shell metacharacters {bad!r}: {cmd!r} -- " + "the contract is ONE bare command with no shell metacharacters", + ) + ) + if cmd in TAUTOLOGIES: + out.append( + Finding( + "TAUTOLOGY", + lineno, + f"CHECK line is the constant-success command {cmd!r} -- " + "the gate cannot fail", + ) + ) + return out + + +def lint_text(text: str) -> list[Finding]: + """Every finding for one entry's text.""" + hits = find_check_lines(text) + if not hits: + return [ + Finding("MISSING", None, "no top-level 'CHECK: ' line in the entry") + ] + if len(hits) > 1: + where = ", ".join(str(n) for n, _ in hits) + return [ + Finding( + "MULTIPLE", + hits[1][0], + f"{len(hits)} top-level CHECK lines (lines {where}); the contract is exactly one", + ) + ] + lineno, cmd = hits[0] + if not cmd: + return [Finding("EMPTY", lineno, "CHECK line carries no command")] + return lint_command(cmd, lineno) + + +def run_check_command(cmd: str, timeout: float) -> tuple[int | None, str]: + """Execute the extracted command. Returns (exit_code, detail).""" + try: + proc = subprocess.run( + cmd, + shell=True, + capture_output=True, + text=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + return None, f"timed out after {timeout:g}s" + except OSError as exc: + return None, f"could not execute: {exc}" + tail = (proc.stderr or proc.stdout or "").strip().splitlines() + detail = f"exit {proc.returncode}" + if tail: + detail += f" | last output line: {tail[-1][:160]}" + return proc.returncode, detail + + +def grade( + text: str, *, run: bool = False, timeout: float = DEFAULT_TIMEOUT +) -> tuple[list[Finding], int]: + """Lint, and optionally execute. Returns (findings, verdict). + + verdict: 0 clean, 1 fault, 2 could-not-grade. + """ + findings = lint_text(text) + blocking = {"MISSING", "EMPTY", "MULTIPLE", "FALLBACK_ARM", "METACHAR", "TAUTOLOGY"} + if run and not any(f.code in blocking for f in findings): + cmd = find_check_lines(text)[0][1] + code, detail = run_check_command(cmd, timeout) + if code is None: + if detail.startswith("timed out"): + findings.append(Finding("TIMEOUT", find_check_lines(text)[0][0], detail, "cng")) + else: + findings.append(Finding("UNREADABLE", find_check_lines(text)[0][0], detail, "cng")) + elif code != 0: + findings.append( + Finding( + "RUN_FAILURE", + find_check_lines(text)[0][0], + f"the CHECK command exited {code} ({detail})", + ) + ) + if any(f.kind == "cng" for f in findings): + return findings, 2 + if findings: + return findings, 1 + return findings, 0 + + +def read_text(path: Path) -> str: + """Read a file as text; binary content is reported, never guessed at.""" + raw = path.read_bytes() + if b"\0" in raw[:8192]: + raise ValueError(f"{path}: binary (NUL byte in first 8 KiB)") + return raw.decode("utf-8", errors="replace") + + +def report(path: Path, findings: list[Finding], verdict: int, as_json: bool) -> None: + if as_json: + print( + json.dumps( + {"entry": str(path), "verdict": verdict, "findings": [f.as_dict() for f in findings]}, + indent=2, + ) + ) + return + if not findings: + print(f"OK {path}") + return + print(f"{'FAULT' if verdict == 1 else 'CNG '} {path}") + for f in findings: + where = f"line {f.line}" if f.line else "entry" + print(f" [{f.code}] ({where}) {f.message}") + + +def scan(root: Path, *, run: bool, timeout: float, as_json: bool) -> int: + """Gate over a corpus: every file under root carrying a top-level CHECK line.""" + if not root.is_dir(): + print(f"{PROG}: --scan: not a directory: {root}", file=sys.stderr) + return 2 + checked = faulty = clean = skipped = 0 + results: list[dict] = [] + for dirpath, dirnames, filenames in os.walk(root): + dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS] + for name in sorted(filenames): + p = Path(dirpath) / name + try: + text = read_text(p) + except (OSError, ValueError): + skipped += 1 + continue + if not find_check_lines(text): + continue + checked += 1 + findings, verdict = grade(text, run=run, timeout=timeout) + if verdict == 0: + clean += 1 + else: + faulty += 1 + results.append( + {"entry": str(p), "verdict": verdict, "findings": [f.as_dict() for f in findings]} + ) + report(p, findings, verdict, as_json) + if as_json: + print(json.dumps({"root": str(root), "checked": checked, "clean": clean, + "faulty": faulty, "skipped": skipped, "results": results}, indent=2)) + else: + print( + f"\nscan {root}: {checked} file(s) with a CHECK line, " + f"{clean} clean, {faulty} faulty, {skipped} unreadable/binary skipped" + ) + if faulty: + return 1 + return 0 + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + prog=PROG, + description="Lint the CHECK line of a standup entry; its exit code is the verdict.", + ) + ap.add_argument("entries", nargs="*", help="entry files to lint") + ap.add_argument("--run", action="store_true", + help="execute the extracted command and fold its real exit code in") + ap.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT, metavar="SECONDS", + help=f"timeout for --run (default {DEFAULT_TIMEOUT:g}s)") + ap.add_argument("--json", action="store_true", help="emit findings as JSON") + ap.add_argument("--scan", metavar="DIR", help="lint every file under DIR with a CHECK line") + args = ap.parse_args(argv) + + if args.scan: + return scan(Path(args.scan), run=args.run, timeout=args.timeout, as_json=args.json) + if not args.entries: + ap.print_usage(sys.stderr) + print(f"{PROG}: no entry given (or use --scan DIR)", file=sys.stderr) + return 2 + + verdict = 0 + for entry in args.entries: + path = Path(entry) + try: + text = read_text(path) + except OSError as exc: + print(f"CNG {path}\n [UNREADABLE] {exc}", file=sys.stderr) + verdict = 2 + continue + except ValueError as exc: + print(f"CNG {path}\n [UNREADABLE] {exc}", file=sys.stderr) + verdict = 2 + continue + findings, v = grade(text, run=args.run, timeout=args.timeout) + report(path, findings, v, args.json) + if v == 2: + verdict = 2 + elif v == 1 and verdict != 2: + verdict = 1 + return verdict + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wizard-work/null/checklint/scan-troupe.txt b/wizard-work/null/checklint/scan-troupe.txt new file mode 100644 index 0000000..5b6782e --- /dev/null +++ b/wizard-work/null/checklint/scan-troupe.txt @@ -0,0 +1,26 @@ +OK /root/troupe-standup/replies/2026-10-20-to-beam-02.md +FAULT /root/troupe-standup/checkins/2026-10-17-volt.md + [MULTIPLE] (line 207) 4 top-level CHECK lines (lines 141, 207, 287, 388); the contract is exactly one +OK /root/troupe-standup/wizard-work/null/q57-residual-2026-10-25.md +FAULT /root/troupe-standup/wizard-work/null/q62_scan.py + [MULTIPLE] (line 386) 4 top-level CHECK lines (lines 380, 386, 392, 398); the contract is exactly one +OK /root/troupe-standup/wizard-work/null/wizard.log.md +FAULT /root/troupe-standup/wizard-work/forge/CHECKIN_CHECK_DRY_REFACTOR_VERIFICATION.md + [METACHAR] (line 145) CHECK line contains shell metacharacters '&>': 'python3 /root/troupe-standup/wizard-work/forge/verify_checkin_check_dry_refactor.py >/dev/null 2>&1' -- the contract is ONE bare command with no shell metacharacters +OK /root/troupe-standup/wizard-work/sol/_jobboard_triage_registry.md +OK /root/troupe-standup/wizard-work/sol/daily-triage-2026-10-21.md +OK /root/troupe-standup/wizard-work/sol/library-drop-roster.md +OK /root/troupe-standup/wizard-work/sol/triage-wizard-work-2026-10-23.md +FAULT /root/troupe-standup/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/EVIDENCE-runs.txt + [MULTIPLE] (line 104) 3 top-level CHECK lines (lines 52, 104, 158); the contract is exactly one +FAULT /root/troupe-standup/wizard-work/quill/draft-sprint-template-v0.2.md + [METACHAR] (line 38) CHECK line contains shell metacharacters '&': "test -f wizard-work/quill/draft-sprint-template-v0.2.md && grep -xq '# Draft Sprint Template v0.2' wizard-work/quill/draft-sprint-template-v0.2.md && grep -qF '**Lane:** Draft - fast writing and drafts.' wizard-work/quill/draft-sprint-template-v0.2.md && grep -q '2026-10-24' wizard-work/quill/draft-sprint-template-v0.2.md && grep -q 'wizard-work/quill/draft-sprint-template-v0.2.md' wizard-work/quill/draft-sprint-template-v0.2.md && grep -qE '^VERDICT: .+' wizard-work/quill/draft-sprint-template-v0.2.md" -- the contract is ONE bare command with no shell metacharacters +FAULT /root/troupe-standup/wizard-work/quill/draft-sprint-template.md + [METACHAR] (line 38) CHECK line contains shell metacharacters '&': 'test -f /root/.hermes/wizard-outputs/quill/draft-sprint-template.md && grep -q "^# Draft Sprint Template" /root/.hermes/wizard-outputs/quill/draft-sprint-template.md && grep -q "^VERDICT:" /root/.hermes/wizard-outputs/quill/draft-sprint-template.md && grep -q "^CHECK:" /root/.hermes/wizard-outputs/quill/draft-sprint-template.md' -- the contract is ONE bare command with no shell metacharacters +OK /root/troupe-standup/wizard-work/quill/issue-to-draft-brief.md +FAULT /root/troupe-standup/wizard-work/quill/onboarding-cheat-sheet.md + [METACHAR] (line 38) CHECK line contains shell metacharacters '<>': '' -- the contract is ONE bare command with no shell metacharacters +FAULT /root/troupe-standup/wizard-work/quill/rfc-draft-agent-outputs.md + [METACHAR] (line 56) CHECK line contains shell metacharacters '$&|': 'test -s /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Abstract$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Motivation$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Lane Contract$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Commit Workflow$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && grep -q "^Change Log$" /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md && tail -n 2 /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md | grep -q "^VERDICT:" && tail -n 2 /root/.hermes/wizard-outputs/quill/rfc-draft-agent-outputs.md | grep -q "^CHECK:"' -- the contract is ONE bare command with no shell metacharacters + +scan /root/troupe-standup: 16 file(s) with a CHECK line, 8 clean, 8 faulty, 814 unreadable/binary skipped diff --git a/wizard-work/null/checklint/test_checklint.py b/wizard-work/null/checklint/test_checklint.py new file mode 100644 index 0000000..34ac7ed --- /dev/null +++ b/wizard-work/null/checklint/test_checklint.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +"""Tests for checklint. Run directly: + + python3 test_checklint.py + +Exit code IS the verdict: 0 = every test passed, 1 = a test failed. +The suite is the negative control for the linter itself: if checklint.py +regresses to "always clean", the MISSING/FALLBACK_ARM/METACHAR/TAUTOLOGY +cases below fail, and this file exits 1. + +Two legs are not static and are exercised for real: --run executes the +extracted command (the sentinel is created by the command, not by the test), +and --scan walks a directory tree built in a temp dir. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent +TOOL = HERE / "checklint.py" +sys.path.insert(0, str(HERE)) + +import checklint # noqa: E402 + +CLEAN = "CHECK: bash wizard-work/null/CHECK\n" + + +def entry(body: str) -> str: + return "# entry\n\nsome prose\n\n" + body + + +class StaticLintTests(unittest.TestCase): + def verdict(self, text: str) -> tuple[list[checklint.Finding], int]: + return checklint.grade(text) + + def codes(self, text: str) -> list[str]: + return [f.code for f in self.verdict(text)[0]] + + # -- the contract holds ------------------------------------------------ + def test_clean_bare_command_passes(self): + findings, v = self.verdict(entry(CLEAN)) + self.assertEqual(findings, []) + self.assertEqual(v, 0) + + def test_absolute_path_command_passes(self): + findings, v = self.verdict(entry("CHECK: python3 /root/x/test_gate.py\n")) + self.assertEqual((findings, v), ([], 0)) + + def test_quoted_argument_without_metachars_passes(self): + # a quoted path is not a control operator; the exit code is still the command's + findings, v = self.verdict(entry('CHECK: python3 "/root/x/my gate.py"\n')) + self.assertEqual((findings, v), ([], 0)) + + # -- MISSING / MULTIPLE / EMPTY --------------------------------------- + def test_missing_check_line(self): + findings, v = self.verdict("# entry\n\nno check here\n") + self.assertEqual([f.code for f in findings], ["MISSING"]) + self.assertEqual(v, 1) + + def test_two_check_lines(self): + findings, v = self.verdict(entry("CHECK: python3 a.py\n\nCHECK: python3 b.py\n")) + self.assertEqual([f.code for f in findings], ["MULTIPLE"]) + self.assertEqual(v, 1) + + def test_empty_check_line(self): + findings, v = self.verdict(entry("CHECK:\n")) + self.assertEqual([f.code for f in findings], ["EMPTY"]) + self.assertEqual(v, 1) + + def test_check_inside_fence_is_not_the_contract(self): + text = entry("```\nCHECK: true\n```\n") + self.assertEqual([f.code for f in self.verdict(text)[0]], ["MISSING"]) + + def test_prose_starting_with_check_colon_is_not_matched(self): + text = entry("Note: CHECK: this is prose, not a contract line\n") + self.assertEqual([f.code for f in self.verdict(text)[0]], ["MISSING"]) + + # -- fallback arms ----------------------------------------------------- + def test_fallback_or_true(self): + self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py || true\n"))) + + def test_fallback_or_exit_0(self): + self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py || exit 0\n"))) + + def test_fallback_semicolon_exit_0(self): + self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py; exit 0\n"))) + + def test_fallback_or_echo(self): + self.assertIn("FALLBACK_ARM", self.codes(entry("CHECK: python3 a.py || echo ok\n"))) + + def test_fallback_is_reported_as_forced_exit_not_as_a_metachar_only(self): + findings = self.verdict(entry("CHECK: python3 a.py || true\n"))[0] + self.assertTrue(any("cannot fail" in f.message for f in findings)) + + # -- tautologies ------------------------------------------------------ + def test_tautology_true(self): + self.assertIn("TAUTOLOGY", self.codes(entry("CHECK: true\n"))) + + def test_tautology_exit_0(self): + self.assertIn("TAUTOLOGY", self.codes(entry("CHECK: exit 0\n"))) + + def test_tautology_echo(self): + self.assertIn("TAUTOLOGY", self.codes(entry("CHECK: echo ok\n"))) + + # -- metacharacters ---------------------------------------------------- + def test_metachar_and_chain(self): + self.assertIn("METACHAR", self.codes(entry("CHECK: python3 a.py && python3 b.py\n"))) + + def test_metachar_pipe(self): + self.assertIn("METACHAR", self.codes(entry("CHECK: cat f | grep x\n"))) + + def test_metachar_redirect(self): + self.assertIn("METACHAR", self.codes(entry("CHECK: python3 a.py >/dev/null 2>&1\n"))) + + def test_metachar_command_substitution(self): + self.assertIn("METACHAR", self.codes(entry("CHECK: python3 $(ls)\n"))) + + def test_metachar_backtick(self): + self.assertIn("METACHAR", self.codes(entry("CHECK: python3 `ls`\n"))) + + def test_metachar_line_continuation(self): + self.assertIn("METACHAR", self.codes(entry("CHECK: python3 a.py \\\n"))) + + # -- real corpus shapes, copied verbatim -------------------------------- + def test_real_corpus_and_chain_shape_is_flagged(self): + self.assertIn("METACHAR", self.codes(entry( + "CHECK: test -s tools/ok_gate.py && grep -q VERDICT tools/ok_gate.py\n"))) + + def test_real_corpus_redirect_shape_is_flagged(self): + self.assertIn("METACHAR", self.codes(entry( + "CHECK: python3 /root/troupe-standup/wizard-work/forge/verify.py >/dev/null 2>&1\n"))) + + # -- purity ------------------------------------------------------------ + def test_lint_text_is_pure_and_never_executes(self): + with tempfile.TemporaryDirectory() as td: + sentinel = Path(td) / "ran" + text = entry(f"CHECK: touch {sentinel}\n") + first = checklint.lint_text(text) + second = checklint.lint_text(text) + self.assertEqual(first, second) # pure: same input, same findings + self.assertEqual(first, []) # statically clean + self.assertFalse(sentinel.exists()) # and it did not execute anything + + +class CliTests(unittest.TestCase): + def run_tool(self, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(TOOL), *args], + capture_output=True, text=True, cwd=str(cwd) if cwd else None, timeout=120, + ) + + def test_cli_clean_entry_exits_0(self): + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "e.md" + p.write_text(entry(CLEAN), encoding="utf-8") + r = self.run_tool(str(p)) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn("OK", r.stdout) + + def test_cli_faulty_entry_exits_1(self): + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "e.md" + p.write_text(entry("CHECK: python3 a.py || true\n"), encoding="utf-8") + r = self.run_tool(str(p)) + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + self.assertIn("FALLBACK_ARM", r.stdout) + + def test_cli_unreadable_entry_exits_2(self): + r = self.run_tool("/root/definitely/not/here.md") + self.assertEqual(r.returncode, 2) + self.assertIn("UNREADABLE", r.stderr) + + def test_cli_run_leg_executes_the_command(self): + with tempfile.TemporaryDirectory() as td: + sentinel = Path(td) / "ran" + p = Path(td) / "e.md" + p.write_text(entry(f"CHECK: touch {sentinel}\n"), encoding="utf-8") + r = self.run_tool("--run", str(p)) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertTrue(sentinel.exists(), "the CHECK command was not executed") + + def test_cli_run_leg_reports_a_failing_command(self): + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "e.md" + p.write_text(entry("CHECK: false\n"), encoding="utf-8") + r = self.run_tool("--run", str(p)) + self.assertEqual(r.returncode, 1) + self.assertIn("RUN_FAILURE", r.stdout) + self.assertIn("exit 1", r.stdout) + + def test_cli_run_leg_does_not_execute_a_fallback_arm(self): + with tempfile.TemporaryDirectory() as td: + sentinel = Path(td) / "ran" + p = Path(td) / "e.md" + p.write_text(entry(f"CHECK: touch {sentinel} || true\n"), encoding="utf-8") + r = self.run_tool("--run", str(p)) + self.assertEqual(r.returncode, 1) + self.assertFalse(sentinel.exists()) + + def test_cli_timeout_exits_2_and_names_its_cause(self): + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "e.md" + p.write_text(entry("CHECK: sleep 30\n"), encoding="utf-8") + r = self.run_tool("--run", "--timeout", "0.5", str(p)) + self.assertEqual(r.returncode, 2, r.stdout + r.stderr) + self.assertIn("TIMEOUT", r.stdout) + + def test_cli_json_shape(self): + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "e.md" + p.write_text(entry("CHECK: true\n"), encoding="utf-8") + r = self.run_tool("--json", str(p)) + self.assertEqual(r.returncode, 1) + data = json.loads(r.stdout) + self.assertEqual(data["verdict"], 1) + self.assertEqual(data["findings"][0]["code"], "TAUTOLOGY") + self.assertIn("line", data["findings"][0]) + + def test_cli_no_entry_exits_2(self): + r = self.run_tool() + self.assertEqual(r.returncode, 2) + + +class ScanTests(unittest.TestCase): + def test_scan_gates_a_corpus(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + (root / "clean.md").write_text(entry(CLEAN), encoding="utf-8") + (root / "faulty.md").write_text(entry("CHECK: python3 a.py || true\n"), encoding="utf-8") + (root / "no-check.md").write_text("# nothing\n", encoding="utf-8") + (root / "bin").write_bytes(b"\x00\x01\x02") + r = subprocess.run( + [sys.executable, str(TOOL), "--scan", str(root)], + capture_output=True, text=True, timeout=120, + ) + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + self.assertIn("2 file(s) with a CHECK line, 1 clean, 1 faulty", r.stdout) + self.assertIn("1 unreadable/binary skipped", r.stdout) + + def test_scan_clean_tree_exits_0(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + (root / "clean.md").write_text(entry(CLEAN), encoding="utf-8") + r = subprocess.run( + [sys.executable, str(TOOL), "--scan", str(root)], + capture_output=True, text=True, timeout=120, + ) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + + def test_scan_unreadable_root_exits_2(self): + r = subprocess.run( + [sys.executable, str(TOOL), "--scan", "/root/definitely/not/a/dir"], + capture_output=True, text=True, timeout=120, + ) + self.assertEqual(r.returncode, 2) + + +if __name__ == "__main__": + unittest.main(verbosity=2)