263 lines
10 KiB
Python
263 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""q110_check.py -- grade wizard-work/quill/QUICKSTART.md against its execution ledger.
|
|
|
|
WHY THIS EXISTS
|
|
Q110 filed the document as walking a new wizard into a dead clone and then a
|
|
command that does not exist. The repair is not the wording: it is that every
|
|
command the document prints has been run once, in order, with its exit code
|
|
and output recorded beside it. A prose document cannot hold that property on
|
|
its own -- the next edit can invent a step or launder a failure into prose --
|
|
so the property lives in a pairing this checker grades:
|
|
|
|
QUICKSTART.md cites steps as `[S<n>]`; quickstart-transcript.md holds one
|
|
block per step with `cmd:`, `rc:`, `status:`.
|
|
|
|
TESTED means rc 0. EXPECTED-FAIL means the step is supposed to fail and the
|
|
recorded rc is nonzero (a document that warns about a command has to have run
|
|
it). UNTESTED means it could not be run here and says so. A green ledger for a
|
|
step that actually fails, or a red one for a step that passed, is a fault.
|
|
|
|
WHAT IT CHECKS
|
|
1. pairing every [S<n>] in the doc has a block; every block is cited
|
|
2. block shape exactly one cmd/rc/status per block; status in the known set
|
|
3. status vs rc TESTED -> rc 0; EXPECTED-FAIL -> rc != 0; UNTESTED -> says so
|
|
4. dead URL no fenced command in the doc uses github.com/stackchain/library
|
|
5. live probes `hermes --help` advertises `-z PROMPT`; `hermes profile
|
|
create --help` carries `--no-alias`; `hermes run <x>` still
|
|
exits 2 with "'run' is not a" -- i.e. the document's two
|
|
warnings are still true of the installed CLI
|
|
|
|
EXIT CODES (the tools/ contract)
|
|
0 every leg passed
|
|
1 at least one fault
|
|
2 could not grade: a file is missing or a probe could not be run
|
|
|
|
USAGE
|
|
python3 wizard-work/volt/q110_check.py [--doc PATH] [--transcript PATH]
|
|
[--selftest] [--json OUT]
|
|
|
|
Paths default to the two files beside this checker's repository (resolved from
|
|
__file__), so it runs from any cwd and in a clone.
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
STEP_RE = re.compile(r"\[S(\d+)\]")
|
|
HEAD_RE = re.compile(r"^### S(\d+) — ")
|
|
STATUSES = {"TESTED", "EXPECTED-FAIL", "UNTESTED"}
|
|
FENCE_RE = re.compile(r"```[a-zA-Z0-9]*\n(.*?)```", re.S)
|
|
DEAD_URL = "github.com/stackchain/library"
|
|
LIVE_URL = "forge.alexanderwhitestone.com/git/stackchain/troupe-standup"
|
|
|
|
|
|
def parse_transcript(text):
|
|
"""Return {step_id: {cmd, rc, status}}, plus a list of shape faults."""
|
|
blocks = {}
|
|
faults = []
|
|
cur = None
|
|
for raw in text.splitlines():
|
|
line = raw.rstrip()
|
|
m = HEAD_RE.match(line)
|
|
if m:
|
|
cur = m.group(1)
|
|
if cur in blocks:
|
|
faults.append("duplicate block for S%s" % cur)
|
|
blocks[cur] = {"cmd": None, "rc": None, "status": None, "line": 0}
|
|
continue
|
|
if cur is None:
|
|
continue
|
|
if line.startswith("cmd: "):
|
|
if blocks[cur]["cmd"] is not None:
|
|
faults.append("S%s has two cmd: lines" % cur)
|
|
blocks[cur]["cmd"] = line[5:].strip()
|
|
elif line.startswith("rc: "):
|
|
if blocks[cur]["rc"] is not None:
|
|
faults.append("S%s has two rc: lines" % cur)
|
|
try:
|
|
blocks[cur]["rc"] = int(line[4:].strip())
|
|
except ValueError:
|
|
faults.append("S%s rc is not an integer: %r" % (cur, line[4:].strip()))
|
|
elif line.startswith("status: "):
|
|
st = line[8:].strip().upper()
|
|
if blocks[cur]["status"] is not None:
|
|
faults.append("S%s has two status: lines" % cur)
|
|
blocks[cur]["status"] = st
|
|
for sid, b in blocks.items():
|
|
for field in ("cmd", "rc", "status"):
|
|
if b[field] is None:
|
|
faults.append("S%s is missing %s:" % (sid, field))
|
|
if b["status"] is not None and b["status"] not in STATUSES:
|
|
faults.append("S%s status %r is not one of %s" % (sid, b["status"], sorted(STATUSES)))
|
|
return blocks, faults
|
|
|
|
|
|
def parse_status_vs_rc(blocks):
|
|
faults = []
|
|
for sid, b in sorted(blocks.items(), key=lambda kv: int(kv[0])):
|
|
if b["status"] is None or b["rc"] is None:
|
|
continue
|
|
if b["status"] == "TESTED" and b["rc"] != 0:
|
|
faults.append("S%s is TESTED but records rc %d" % (sid, b["rc"]))
|
|
if b["status"] == "EXPECTED-FAIL" and b["rc"] == 0:
|
|
faults.append("S%s is EXPECTED-FAIL but records rc 0" % sid)
|
|
return faults
|
|
|
|
|
|
def fenced_lines(doc):
|
|
out = []
|
|
for block in FENCE_RE.findall(doc):
|
|
for line in block.splitlines():
|
|
if line.strip():
|
|
out.append(line.strip())
|
|
return out
|
|
|
|
|
|
def check(doc, transcript_text, probes=True, runner=subprocess.run):
|
|
legs = []
|
|
|
|
def leg(name, faults):
|
|
legs.append((name, faults))
|
|
return faults
|
|
|
|
blocks, shape_faults = parse_transcript(transcript_text)
|
|
leg("transcript-shape", shape_faults)
|
|
|
|
cited = {m for m in STEP_RE.findall(doc)}
|
|
pairs = []
|
|
for sid in sorted(cited, key=int):
|
|
if sid not in blocks:
|
|
pairs.append("doc cites [S%s]; transcript has no such block" % sid)
|
|
for sid in sorted(blocks, key=int):
|
|
if sid not in cited:
|
|
pairs.append("transcript block S%s is never cited by the doc" % sid)
|
|
leg("pairing", pairs)
|
|
|
|
leg("status-vs-rc", parse_status_vs_rc(blocks))
|
|
|
|
dead = [ln for ln in fenced_lines(doc) if DEAD_URL in ln]
|
|
leg("dead-url", ["a fenced command still uses the dead URL: %s" % ln for ln in dead])
|
|
|
|
if not probes:
|
|
return legs
|
|
probe_faults = []
|
|
|
|
def run(argv, timeout=60):
|
|
try:
|
|
return runner(argv, capture_output=True, text=True, timeout=timeout)
|
|
except (OSError, subprocess.SubprocessError) as exc: # noqa: BLE001
|
|
probe_faults.append("could not run %r: %s" % (argv, exc))
|
|
return None
|
|
|
|
p = run(["hermes", "--help"])
|
|
if p is not None:
|
|
if p.returncode != 0:
|
|
probe_faults.append("`hermes --help` exited %d" % p.returncode)
|
|
if "-z PROMPT" not in (p.stdout + p.stderr):
|
|
probe_faults.append("`hermes --help` no longer advertises the one-shot `-z PROMPT`")
|
|
|
|
p = run(["hermes", "profile", "create", "--help"])
|
|
if p is not None:
|
|
if p.returncode != 0:
|
|
probe_faults.append("`hermes profile create --help` exited %d" % p.returncode)
|
|
if "--no-alias" not in (p.stdout + p.stderr):
|
|
probe_faults.append("`hermes profile create --help` no longer lists --no-alias")
|
|
|
|
p = run(["hermes", "run", "q110-probe"])
|
|
if p is not None:
|
|
blob = p.stdout + p.stderr
|
|
if p.returncode != 2:
|
|
probe_faults.append("`hermes run <x>` exited %d, doc warns it exits 2" % p.returncode)
|
|
if "'run' is not a" not in blob:
|
|
probe_faults.append("`hermes run <x>` no longer refuses with \"'run' is not a ...\"")
|
|
leg("live-probes", probe_faults)
|
|
|
|
if LIVE_URL not in doc:
|
|
legs.append(("clone-url", ["the doc no longer names the forge clone URL"]))
|
|
else:
|
|
legs.append(("clone-url", []))
|
|
return legs
|
|
|
|
|
|
def load(path):
|
|
with open(path, encoding="utf-8") as fh:
|
|
return fh.read()
|
|
|
|
|
|
def selftest(doc, transcript_text):
|
|
"""Control + mutants over in-memory copies. No repository file is touched."""
|
|
cells = []
|
|
|
|
def cell(name, d, t, expect_fault):
|
|
legs = check(d, t, probes=False)
|
|
faults = [f for _, fs in legs for f in fs]
|
|
ok = bool(faults) == expect_fault
|
|
cells.append((name, ok, faults))
|
|
|
|
cell("control-clean", doc, transcript_text, False)
|
|
cell("mutant-missing-block", doc, transcript_text.replace("### S5 —", "### X5 —"), True)
|
|
cell("mutant-uncited-block", doc, transcript_text + "\n### S99 — orphan\ncmd: true\nrc: 0\nstatus: TESTED\n", True)
|
|
cell("mutant-tested-with-rc1", doc, transcript_text.replace(
|
|
"cmd: hermes -p q110demo3 config get model.default\nrc: 0",
|
|
"cmd: hermes -p q110demo3 config get model.default\nrc: 1"), True)
|
|
cell("mutant-expected-fail-green", doc, transcript_text.replace(
|
|
"cmd: hermes run \"echo hello world\"\nrc: 2",
|
|
"cmd: hermes run \"echo hello world\"\nrc: 0"), True)
|
|
cell("mutant-bad-status", doc, transcript_text.replace("status: TESTED", "status: PROBABLY", 1), True)
|
|
cell("mutant-dead-url-in-fence", doc.replace(
|
|
"git clone https://forge.alexanderwhitestone.com/git/stackchain/troupe-standup.git",
|
|
"git clone https://github.com/stackchain/library.git"), transcript_text, True)
|
|
|
|
failed = [c for c in cells if not c[1]]
|
|
for name, ok, faults in cells:
|
|
print(" selftest %-32s %s%s" % (name, "PASS" if ok else "FAIL",
|
|
"" if ok else " -> " + "; ".join(faults[:2])))
|
|
print("q110 selftest: %d cells, %d passed, %d failed" %
|
|
(len(cells), len(cells) - len(failed), len(failed)))
|
|
return 0 if not failed else 1
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=(__doc__ or "q110 gate").splitlines()[0])
|
|
here = os.path.dirname(os.path.realpath(__file__))
|
|
repo = os.path.dirname(os.path.dirname(here))
|
|
ap.add_argument("--doc", default=os.path.join(repo, "wizard-work", "quill", "QUICKSTART.md"))
|
|
ap.add_argument("--transcript",
|
|
default=os.path.join(repo, "wizard-work", "quill", "quickstart-transcript.md"))
|
|
ap.add_argument("--selftest", action="store_true")
|
|
ap.add_argument("--json", default=None)
|
|
args = ap.parse_args(argv)
|
|
|
|
try:
|
|
doc = load(args.doc)
|
|
transcript_text = load(args.transcript)
|
|
except OSError as exc:
|
|
print("q110: rc 2 -- could not read input: %s" % exc)
|
|
return 2
|
|
|
|
if args.selftest:
|
|
return selftest(doc, transcript_text)
|
|
|
|
legs = check(doc, transcript_text, probes=True)
|
|
total = 0
|
|
for name, faults in legs:
|
|
total += len(faults)
|
|
print("%-18s %s" % (name, "PASS" if not faults else "FAIL (%d)" % len(faults)))
|
|
for f in faults:
|
|
print(" - %s" % f)
|
|
rc = 0 if total == 0 else 1
|
|
print("q110: %d legs, %d fault(s) -- %s" % (len(legs), total, "OK" if rc == 0 else "FAULT"))
|
|
if args.json:
|
|
with open(args.json, "w", encoding="utf-8") as fh:
|
|
json.dump({"legs": [{"leg": n, "faults": f} for n, f in legs],
|
|
"faults": total, "rc": rc}, fh, indent=2)
|
|
return rc
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|