#!/usr/bin/env python3 """q110_check.py -- grade wizard-work/quill/QUICKSTART.md against its execution ledger. WHY THIS EXISTS Q110 filed the document as walking a new wizard into a dead clone and then a command that does not exist. The repair is not the wording: it is that every command the document prints has been run once, in order, with its exit code and output recorded beside it. A prose document cannot hold that property on its own -- the next edit can invent a step or launder a failure into prose -- so the property lives in a pairing this checker grades: QUICKSTART.md cites steps as `[S]`; quickstart-transcript.md holds one block per step with `cmd:`, `rc:`, `status:`. TESTED means rc 0. EXPECTED-FAIL means the step is supposed to fail and the recorded rc is nonzero (a document that warns about a command has to have run it). UNTESTED means it could not be run here and says so. A green ledger for a step that actually fails, or a red one for a step that passed, is a fault. WHAT IT CHECKS 1. pairing every [S] in the doc has a block; every block is cited 2. block shape exactly one cmd/rc/status per block; status in the known set 3. status vs rc TESTED -> rc 0; EXPECTED-FAIL -> rc != 0; UNTESTED -> says so 4. dead URL no fenced command in the doc uses github.com/stackchain/library 5. live probes `hermes --help` advertises `-z PROMPT`; `hermes profile create --help` carries `--no-alias`; `hermes run ` still exits 2 with "'run' is not a" -- i.e. the document's two warnings are still true of the installed CLI EXIT CODES (the tools/ contract) 0 every leg passed 1 at least one fault 2 could not grade: a file is missing or a probe could not be run USAGE python3 wizard-work/volt/q110_check.py [--doc PATH] [--transcript PATH] [--selftest] [--json OUT] Paths default to the two files beside this checker's repository (resolved from __file__), so it runs from any cwd and in a clone. """ import argparse import json import os import re import subprocess import sys import tempfile STEP_RE = re.compile(r"\[S(\d+)\]") HEAD_RE = re.compile(r"^### S(\d+) — ") STATUSES = {"TESTED", "EXPECTED-FAIL", "UNTESTED"} FENCE_RE = re.compile(r"```[a-zA-Z0-9]*\n(.*?)```", re.S) DEAD_URL = "github.com/stackchain/library" LIVE_URL = "forge.alexanderwhitestone.com/git/stackchain/troupe-standup" def parse_transcript(text): """Return {step_id: {cmd, rc, status}}, plus a list of shape faults.""" blocks = {} faults = [] cur = None for raw in text.splitlines(): line = raw.rstrip() m = HEAD_RE.match(line) if m: cur = m.group(1) if cur in blocks: faults.append("duplicate block for S%s" % cur) blocks[cur] = {"cmd": None, "rc": None, "status": None, "line": 0} continue if cur is None: continue if line.startswith("cmd: "): if blocks[cur]["cmd"] is not None: faults.append("S%s has two cmd: lines" % cur) blocks[cur]["cmd"] = line[5:].strip() elif line.startswith("rc: "): if blocks[cur]["rc"] is not None: faults.append("S%s has two rc: lines" % cur) try: blocks[cur]["rc"] = int(line[4:].strip()) except ValueError: faults.append("S%s rc is not an integer: %r" % (cur, line[4:].strip())) elif line.startswith("status: "): st = line[8:].strip().upper() if blocks[cur]["status"] is not None: faults.append("S%s has two status: lines" % cur) blocks[cur]["status"] = st for sid, b in blocks.items(): for field in ("cmd", "rc", "status"): if b[field] is None: faults.append("S%s is missing %s:" % (sid, field)) if b["status"] is not None and b["status"] not in STATUSES: faults.append("S%s status %r is not one of %s" % (sid, b["status"], sorted(STATUSES))) return blocks, faults def parse_status_vs_rc(blocks): faults = [] for sid, b in sorted(blocks.items(), key=lambda kv: int(kv[0])): if b["status"] is None or b["rc"] is None: continue if b["status"] == "TESTED" and b["rc"] != 0: faults.append("S%s is TESTED but records rc %d" % (sid, b["rc"])) if b["status"] == "EXPECTED-FAIL" and b["rc"] == 0: faults.append("S%s is EXPECTED-FAIL but records rc 0" % sid) return faults def fenced_lines(doc): out = [] for block in FENCE_RE.findall(doc): for line in block.splitlines(): if line.strip(): out.append(line.strip()) return out def check(doc, transcript_text, probes=True, runner=subprocess.run): legs = [] def leg(name, faults): legs.append((name, faults)) return faults blocks, shape_faults = parse_transcript(transcript_text) leg("transcript-shape", shape_faults) cited = {m for m in STEP_RE.findall(doc)} pairs = [] for sid in sorted(cited, key=int): if sid not in blocks: pairs.append("doc cites [S%s]; transcript has no such block" % sid) for sid in sorted(blocks, key=int): if sid not in cited: pairs.append("transcript block S%s is never cited by the doc" % sid) leg("pairing", pairs) leg("status-vs-rc", parse_status_vs_rc(blocks)) dead = [ln for ln in fenced_lines(doc) if DEAD_URL in ln] leg("dead-url", ["a fenced command still uses the dead URL: %s" % ln for ln in dead]) if not probes: return legs probe_faults = [] def run(argv, timeout=60): try: return runner(argv, capture_output=True, text=True, timeout=timeout) except (OSError, subprocess.SubprocessError) as exc: # noqa: BLE001 probe_faults.append("could not run %r: %s" % (argv, exc)) return None p = run(["hermes", "--help"]) if p is not None: if p.returncode != 0: probe_faults.append("`hermes --help` exited %d" % p.returncode) if "-z PROMPT" not in (p.stdout + p.stderr): probe_faults.append("`hermes --help` no longer advertises the one-shot `-z PROMPT`") p = run(["hermes", "profile", "create", "--help"]) if p is not None: if p.returncode != 0: probe_faults.append("`hermes profile create --help` exited %d" % p.returncode) if "--no-alias" not in (p.stdout + p.stderr): probe_faults.append("`hermes profile create --help` no longer lists --no-alias") p = run(["hermes", "run", "q110-probe"]) if p is not None: blob = p.stdout + p.stderr if p.returncode != 2: probe_faults.append("`hermes run ` exited %d, doc warns it exits 2" % p.returncode) if "'run' is not a" not in blob: probe_faults.append("`hermes run ` no longer refuses with \"'run' is not a ...\"") leg("live-probes", probe_faults) if LIVE_URL not in doc: legs.append(("clone-url", ["the doc no longer names the forge clone URL"])) else: legs.append(("clone-url", [])) return legs def load(path): with open(path, encoding="utf-8") as fh: return fh.read() def selftest(doc, transcript_text): """Control + mutants over in-memory copies. No repository file is touched.""" cells = [] def cell(name, d, t, expect_fault): legs = check(d, t, probes=False) faults = [f for _, fs in legs for f in fs] ok = bool(faults) == expect_fault cells.append((name, ok, faults)) cell("control-clean", doc, transcript_text, False) cell("mutant-missing-block", doc, transcript_text.replace("### S5 —", "### X5 —"), True) cell("mutant-uncited-block", doc, transcript_text + "\n### S99 — orphan\ncmd: true\nrc: 0\nstatus: TESTED\n", True) cell("mutant-tested-with-rc1", doc, transcript_text.replace( "cmd: hermes -p q110demo3 config get model.default\nrc: 0", "cmd: hermes -p q110demo3 config get model.default\nrc: 1"), True) cell("mutant-expected-fail-green", doc, transcript_text.replace( "cmd: hermes run \"echo hello world\"\nrc: 2", "cmd: hermes run \"echo hello world\"\nrc: 0"), True) cell("mutant-bad-status", doc, transcript_text.replace("status: TESTED", "status: PROBABLY", 1), True) cell("mutant-dead-url-in-fence", doc.replace( "git clone https://forge.alexanderwhitestone.com/git/stackchain/troupe-standup.git", "git clone https://github.com/stackchain/library.git"), transcript_text, True) failed = [c for c in cells if not c[1]] for name, ok, faults in cells: print(" selftest %-32s %s%s" % (name, "PASS" if ok else "FAIL", "" if ok else " -> " + "; ".join(faults[:2]))) print("q110 selftest: %d cells, %d passed, %d failed" % (len(cells), len(cells) - len(failed), len(failed))) return 0 if not failed else 1 def main(argv=None): ap = argparse.ArgumentParser(description=(__doc__ or "q110 gate").splitlines()[0]) here = os.path.dirname(os.path.realpath(__file__)) repo = os.path.dirname(os.path.dirname(here)) ap.add_argument("--doc", default=os.path.join(repo, "wizard-work", "quill", "QUICKSTART.md")) ap.add_argument("--transcript", default=os.path.join(repo, "wizard-work", "quill", "quickstart-transcript.md")) ap.add_argument("--selftest", action="store_true") ap.add_argument("--json", default=None) args = ap.parse_args(argv) try: doc = load(args.doc) transcript_text = load(args.transcript) except OSError as exc: print("q110: rc 2 -- could not read input: %s" % exc) return 2 if args.selftest: return selftest(doc, transcript_text) legs = check(doc, transcript_text, probes=True) total = 0 for name, faults in legs: total += len(faults) print("%-18s %s" % (name, "PASS" if not faults else "FAIL (%d)" % len(faults))) for f in faults: print(" - %s" % f) rc = 0 if total == 0 else 1 print("q110: %d legs, %d fault(s) -- %s" % (len(legs), total, "OK" if rc == 0 else "FAULT")) if args.json: with open(args.json, "w", encoding="utf-8") as fh: json.dump({"legs": [{"leg": n, "faults": f} for n, f in legs], "faults": total, "rc": rc}, fh, indent=2) return rc if __name__ == "__main__": sys.exit(main())