646 lines
29 KiB
Python
646 lines
29 KiB
Python
#!/usr/bin/env python3
|
|
"""check_lint.py -- grade a Ralph-loop `CHECK:` line before it is run.
|
|
|
|
WHY THIS EXISTS
|
|
STANDING-AGREEMENT.md §9 and docs/ralph-loop.md define the delivery standard:
|
|
every deliverable ends with one `CHECK: <shell command>` that exits 0 when the
|
|
work is correct, and "a `CHECK:` that is a tautology is a fault, not a
|
|
verification". The runner (/root/.hermes/scripts/ralph-loop.sh) extracts that
|
|
line, runs it under `bash -c`, and prints GREEN or RED.
|
|
|
|
Two things the runner does not do. Both were measured, not reasoned, on the
|
|
loop's first fleet night -- 2026-10-16 16:57Z, eight wizards, logs under
|
|
/root/.hermes/wizard-outputs/ralph:
|
|
|
|
1. IT DOES NOT READ THE LINE IT EXTRACTS. `... && echo PASS || echo FAIL`
|
|
exits 0 whether the thing under test passed or failed. The runner has no
|
|
defence against that: on that night two of the three GREENs were checks
|
|
that could not fail. A CHECK that cannot fail is the fault §9 names.
|
|
|
|
2. IT DOES NOT STRIP WHAT FOLLOWS THE COMMAND. An em-dash explanation on the
|
|
same line, or a carriage return the wizard's output carried, is handed to
|
|
`bash -c` verbatim -- the runner's `sed` strips whitespace *after the
|
|
colon*, never the `\\r` before the newline. Both are in that night's logs:
|
|
one RED was a bash syntax error on prose, another was a file that did not
|
|
exist under the name `...verify_chunker.py\\r`.
|
|
|
|
This tool grades the line so a fault is visible before it costs an iteration.
|
|
|
|
WHAT IT MEASURES -- EIGHT CLASSES, EACH WITH ITS RULE PRINTED
|
|
MISSING no line matches `^\\s*CHECK:`; the runner calls this RED at once.
|
|
EMPTY a CHECK line with nothing after the colon.
|
|
TAUTOLOGY the whole command is `true`, `:`, `exit 0`, `echo <literal>`,
|
|
`printf <literal>` or `sleep <n>`: nothing is read or tested.
|
|
CATCH_ALL an unconditional zero exit anywhere in the chain -- a `||` arm of
|
|
true, `:`, `exit 0`, `echo`, `printf`, or a terminal statement
|
|
`; true` / `; :` / `; exit 0` / `; echo ...`. Every path returns
|
|
0, so the runner's GREEN carries no information.
|
|
LABEL_PREFIX the command opens with a label, `word:` -- the line was written
|
|
`CHECK: selftest: bash CHECK-x` or `CHECK: line: bash CHECK-x`.
|
|
bash takes the whole first word as the command name, so the line
|
|
dies with `word:: command not found` (rc 127) before anything is
|
|
tested. Added 2026-11-02 after measuring the class on a live run:
|
|
the same line shape RED'd one of this lane's own iterations.
|
|
TRAILING_CR a literal carriage return survives at end of line; it reaches
|
|
bash as part of the last word.
|
|
PROSE_TAIL prose follows the command: an em dash, ` -- `, a trailing
|
|
`(...)` aside, or a sentence break. CANDIDATE -- the boundary
|
|
between a command and its gloss is not decidable by syntax, so
|
|
the cut point is printed for a human to confirm.
|
|
SOUND none of the above. NOT a promise. This tool never runs the
|
|
command, so SOUND means only "no zero-exit construct and no
|
|
trailing prose this rubric can name". Whether the command tests
|
|
the right thing is not a syntactic question and is not graded.
|
|
|
|
THE CROSS-CHECK, AND WHY IT IS THE POINT
|
|
`--ralph-dir` reads the runner's own logs: it re-extracts each wizard's CHECK
|
|
with the same algorithm the runner uses (last `^\\s*CHECK:` line wins, prefix
|
|
stripped, nothing else removed) and joins it to the verdict the runner
|
|
reported. A run the runner called GREEN whose line is TAUTOLOGY or CATCH_ALL
|
|
is reported GREEN_BY_CONSTRUCTION: it passed because it could not fail.
|
|
|
|
WHAT IT DOES NOT DO
|
|
It never executes an extracted command. Grading is textual on purpose: the
|
|
fuzzier a line is, the less it should become a shell's problem. Running the
|
|
CHECK stays the runner's job.
|
|
|
|
USAGE
|
|
python3 check_lint.py --check "$CHECK"
|
|
python3 check_lint.py --ralph-dir /root/.hermes/wizard-outputs/ralph
|
|
python3 check_lint.py --selftest # 51 cells, one per class and edge
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as _dt
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from collections import Counter
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Extraction -- mirrors ralph-loop.sh line 24:
|
|
# grep -iE "^\s*CHECK:" "$LOG" | tail -1 | sed -E 's/^\s*CHECK:[[:space:]]*//i'
|
|
# The mirror is deliberate: the runner's own behaviour is the subject, so a
|
|
# "cleaner" extraction here would stop reproducing what actually happened.
|
|
# --------------------------------------------------------------------------
|
|
CHECK_LINE_RE = re.compile(r"^\s*CHECK:[ \t]*", re.IGNORECASE)
|
|
|
|
|
|
def extract_check(text: str) -> str | None:
|
|
"""Last `^\\s*CHECK:` line, prefix stripped, trailing CR KEPT (the runner keeps it)."""
|
|
found = None
|
|
for line in text.split("\n"):
|
|
m = CHECK_LINE_RE.match(line)
|
|
if m:
|
|
found = line[m.end():]
|
|
return found
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The rubric
|
|
# --------------------------------------------------------------------------
|
|
CONSTANT_RE = re.compile(
|
|
r"^(?:true|:|exit\s+0|sleep\s+\d+(?:\.\d+)?"
|
|
r"|echo(?:\s+[^;|&<>$`]*)?"
|
|
r"|printf(?:\s+[^;|&<>$`]*)?)$"
|
|
)
|
|
|
|
# `|| true` / `|| echo X` arms, and terminal `; true` / `; echo X` statements.
|
|
# A `&& echo X` arm is deliberately NOT matched: it runs only if the left side
|
|
# succeeded, so it preserves the failure signal. A `&&` is excluded from the
|
|
# `[;&]` alternative by the lookbehinds, which stop the second `&`/`|` of a
|
|
# two-character operator from being read as a single one.
|
|
CATCH_ALL_RE = re.compile(
|
|
r"(?:\|\|\s*(?:true|:|exit\s+0|echo\b|printf\b))"
|
|
r"|(?:(?<!&)(?<!\|)[;&]\s*(?:true|:|exit\s+0|echo\b|printf\b)(?:\s|$))"
|
|
)
|
|
|
|
# A leading label: `selftest: python3 v.py`, `line: python3 v.py`, `Step-2: ...`.
|
|
# bash takes the WHOLE first word as the command name, so a label makes the line
|
|
# exit 127 with `word:: command not found` before a single test runs. Measured
|
|
# 2026-11-02 rather than reasoned: a ralph iteration died on
|
|
# `bash: line 1: selftest:: command not found` while this rubric read the line
|
|
# SOUND, and the runner's own 2026-10-27 forge log carries the sibling shape
|
|
# (`line: python3 ...`) one iteration before that CHECK was rewritten without it.
|
|
#
|
|
# The rule is anchored to the FIRST word and needs no space after the colon
|
|
# (`ok:true` is the same defect), and it deliberately does not match:
|
|
# `PATH=/a:/b python3 x.py` -- the `=` breaks the word class before the colon
|
|
# `python3 x.py: --selftest` -- the colon is not in the first word
|
|
# `'note: x' cmd` -- a quoted first token is not a bare word
|
|
# The last two are pinned as SOUND cells so the boundary is measured, not assumed.
|
|
LABEL_RE = re.compile(r"^(?P<label>[A-Za-z0-9][A-Za-z0-9_.+-]*):")
|
|
|
|
PROSE_MARKERS = [
|
|
(re.compile(r"\s+[\u2014\u2013]\s+"), "em dash"),
|
|
(re.compile(r"\s--\s+"), "spaced --"),
|
|
# The parenthetical may carry sentence punctuation: `bash CHECK-x (the
|
|
# artifact gate).` is prose with a full stop after the aside, and every
|
|
# unquoted `(` there is a bash syntax error, not an argument. Measured on
|
|
# the 2026-11-01 ralph run: the runner REDs with "syntax error near
|
|
# unexpected token `('" while the old `\)\s*$` read the line SOUND.
|
|
(re.compile(r"\s\([^()]*\)[.!?,;:]?\s*$"), "trailing parenthetical"),
|
|
(re.compile(r"\.\s+[A-Z]"), "sentence break"),
|
|
]
|
|
|
|
# Severity order: the first fault present names the class. EMPTY and the
|
|
# zero-exit constructs outrank the mechanical ones because they are the faults
|
|
# that make a GREEN meaningless, while TRAILING_CR/PROSE_TAIL only make one
|
|
# noisy (they break the run, loudly, and cost an iteration instead of lying).
|
|
# LABEL_PREFIX sits with the zero-exit group in rank, not in kind: it cannot
|
|
# return 0 either, but it fails the other way round -- the run REDs for a reason
|
|
# that has nothing to do with what was tested, so it is the first thing a reader
|
|
# should be told when a line carries both a label and a prose tail.
|
|
SEVERITY = ["EMPTY", "TAUTOLOGY", "CATCH_ALL", "LABEL_PREFIX", "TRAILING_CR",
|
|
"PROSE_TAIL"]
|
|
|
|
|
|
def prose_tail(core: str):
|
|
"""Earliest prose marker after at least one non-space char. Returns (name, prefix) or None."""
|
|
best = None
|
|
for rx, name in PROSE_MARKERS:
|
|
m = rx.search(core)
|
|
if m and m.start() > 0:
|
|
if best is None or m.start() < best[2]:
|
|
best = (name, core[: m.start()].rstrip(), m.start())
|
|
if best is None:
|
|
return None
|
|
return best[0], best[1]
|
|
|
|
|
|
def classify(cmd: str | None):
|
|
"""-> (primary_class, faults, detail dict). Pure text; never executes."""
|
|
detail: dict = {}
|
|
if cmd is None:
|
|
return "MISSING", ["MISSING"], detail
|
|
|
|
faults = []
|
|
if cmd.endswith("\r"):
|
|
faults.append("TRAILING_CR")
|
|
detail["cr_len"] = len(cmd) - len(cmd.rstrip("\r"))
|
|
|
|
core = cmd.rstrip("\r")
|
|
if core.strip() == "":
|
|
faults.append("EMPTY")
|
|
return "EMPTY", faults, detail
|
|
|
|
m_catch = CATCH_ALL_RE.search(core)
|
|
if CONSTANT_RE.match(core.strip()):
|
|
faults.append("TAUTOLOGY")
|
|
elif m_catch:
|
|
faults.append("CATCH_ALL")
|
|
detail["catch_all_match"] = m_catch.group(0)
|
|
|
|
m_label = LABEL_RE.match(core.lstrip())
|
|
if m_label:
|
|
faults.append("LABEL_PREFIX")
|
|
detail["label"] = m_label.group("label")
|
|
# The cut point, printed the way PROSE_TAIL prints its own: what the
|
|
# line would say with the label removed, which is what bash needs.
|
|
detail["command_after_label"] = core.lstrip()[m_label.end():].strip()
|
|
|
|
pt = prose_tail(core)
|
|
if pt:
|
|
faults.append("PROSE_TAIL")
|
|
detail["prose_marker"], detail["command_prefix"] = pt
|
|
|
|
primary = next((c for c in SEVERITY if c in faults), "SOUND")
|
|
return primary, faults, detail
|
|
|
|
|
|
def _write_json_file(path: str, obj) -> int:
|
|
"""Write obj as indented JSON to path. Returns 0 on success, 2 on OSError.
|
|
|
|
Extracted from the two sites that previously hand-duplicated the
|
|
open/dump/write/newline/except block: report() and the --check/--json
|
|
branch in main(). Both used indent=2 with default sort_keys=False.
|
|
"""
|
|
try:
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
json.dump(obj, fh, indent=2, sort_keys=False)
|
|
fh.write("\n")
|
|
except OSError as exc:
|
|
print("cannot write --json " + path + ": " + str(exc), file=sys.stderr)
|
|
return 2
|
|
return 0
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# The runner's own logs
|
|
# --------------------------------------------------------------------------
|
|
ITER_LOG_RE = re.compile(r"^(?P<wizard>.+?)-(?P<stamp>\d{4}-\d{2}-\d{2}-\d{6})-iter(?P<n>\d+)\.txt$")
|
|
|
|
|
|
def parse_runner_log(path: str) -> list[dict]:
|
|
"""Per-iteration verdicts from the runner's stdout log."""
|
|
with open(path, encoding="utf-8", errors="replace", newline="") as fh:
|
|
text = fh.read()
|
|
out = []
|
|
cur = None
|
|
for raw in text.split("\n"):
|
|
line = raw.rstrip("\r")
|
|
m = re.match(r"=== RALPH .* iter (\d+)/", line)
|
|
if m:
|
|
cur = {"iter": int(m.group(1)), "verdict": None, "reported_check": None,
|
|
"detail": ""}
|
|
out.append(cur)
|
|
continue
|
|
if cur is None:
|
|
continue
|
|
if line.startswith(" running CHECK: "):
|
|
cur["reported_check"] = line[len(" running CHECK: "):]
|
|
elif "no CHECK line emitted" in line:
|
|
cur["verdict"] = "NO_CHECK"
|
|
elif "✅ GREEN after" in line:
|
|
cur["verdict"] = "GREEN"
|
|
elif "❌ RED" in line:
|
|
cur["verdict"] = "RED"
|
|
cur["detail"] = line.split("❌ RED —", 1)[-1].strip()[:200]
|
|
return out
|
|
|
|
|
|
def read_ralph_dir(path: str, include_unlogged: bool = False):
|
|
"""-> list of runs, newest stamp first.
|
|
|
|
A run is one (stamp, wizard). The `.log` companion holds the runner's own
|
|
verdicts, but the fleet driver names it `<wizard>-<YYYY-MM-DD-HHMM>.log`
|
|
(minute resolution) while ralph-loop.sh names its iteration logs
|
|
`<wizard>-<stamp-to-the-second>-iterN.txt`. The join is therefore by prefix.
|
|
Runs with no located runner log are reported only under --all: without a
|
|
verdict there is nothing to cross-check, and the newest iter log may belong
|
|
to a run that is still in flight.
|
|
"""
|
|
if not os.path.isdir(path):
|
|
raise SystemExit(f"no such ralph dir: {path}")
|
|
log_stamps: dict[str, list[str]] = {}
|
|
for name in os.listdir(path):
|
|
m = re.match(r"^(?P<wizard>.+?)-(?P<lstr>\d{4}-\d{2}-\d{2}-\d{4})\.log$", name)
|
|
if m:
|
|
log_stamps.setdefault(m.group("wizard"), []).append(m.group("lstr"))
|
|
|
|
runs: dict[tuple, dict] = {}
|
|
for name in sorted(os.listdir(path)):
|
|
m = ITER_LOG_RE.match(name)
|
|
if not m:
|
|
continue
|
|
key = (m.group("stamp"), m.group("wizard"))
|
|
run = runs.setdefault(key, {"stamp": key[0], "wizard": key[1], "iters": {}})
|
|
with open(os.path.join(path, name), encoding="utf-8",
|
|
errors="replace", newline="") as fh:
|
|
run["iters"][int(m.group("n"))] = extract_check(fh.read())
|
|
|
|
selected = []
|
|
unlogged = 0
|
|
for (stamp, wizard), run in runs.items():
|
|
cand = [s for s in log_stamps.get(wizard, []) if stamp.startswith(s)]
|
|
run["runner_log"] = (os.path.join(path, f"{wizard}-{max(cand)}.log")
|
|
if cand else None)
|
|
if run["runner_log"] is None:
|
|
unlogged += 1
|
|
if not include_unlogged:
|
|
continue
|
|
selected.append(run)
|
|
selected.sort(key=lambda r: (r["stamp"], r["wizard"]), reverse=True)
|
|
return selected, unlogged
|
|
|
|
|
|
def grade_run(run: dict) -> dict:
|
|
iters = run["iters"]
|
|
last_n = max(iters) if iters else 0
|
|
extracted = iters.get(last_n)
|
|
primary, faults, detail = classify(extracted)
|
|
|
|
verdicts = parse_runner_log(run["runner_log"]) if run["runner_log"] else []
|
|
by_iter = {v["iter"]: v for v in verdicts}
|
|
reported = by_iter.get(last_n, {}).get("reported_check")
|
|
final = "UNKNOWN"
|
|
if verdicts:
|
|
last = by_iter[max(by_iter)]
|
|
final = last["verdict"] or "UNKNOWN"
|
|
if final is None:
|
|
final = "UNKNOWN"
|
|
# the runner's RALPH RESULT line is authoritative for the run's outcome
|
|
if run["runner_log"]:
|
|
with open(run["runner_log"], encoding="utf-8", errors="replace",
|
|
newline="") as fh:
|
|
for line in fh.read().split("\n"):
|
|
if "RALPH RESULT: GREEN" in line:
|
|
final = "GREEN"
|
|
elif "RALPH RESULT: EXHAUSTED" in line:
|
|
final = "EXHAUSTED"
|
|
|
|
cross = "none"
|
|
if final == "GREEN" and primary in ("TAUTOLOGY", "CATCH_ALL"):
|
|
cross = "GREEN_BY_CONSTRUCTION"
|
|
elif final == "GREEN" and primary in ("PROSE_TAIL", "TRAILING_CR"):
|
|
cross = "GREEN_DESPITE_FORMATTING"
|
|
# LABEL_PREFIX is deliberately in neither arm: a labelled line exits 127 at
|
|
# the shell, so a runner cannot have reported it GREEN. If one ever did, the
|
|
# run is reported with cross="none" and the class printed, rather than being
|
|
# folded into a bucket whose meaning it would dilute.
|
|
|
|
per_iteration = []
|
|
for n in sorted(iters):
|
|
p_cls, p_faults, p_det = classify(iters[n])
|
|
v = by_iter.get(n, {})
|
|
per_iteration.append({
|
|
"iter": n, "class": p_cls, "faults": p_faults, "detail": p_det,
|
|
"runner_verdict": v.get("verdict"),
|
|
"runner_detail": v.get("detail", ""),
|
|
"check": iters[n],
|
|
})
|
|
|
|
return {
|
|
"stamp": run["stamp"],
|
|
"wizard": run["wizard"],
|
|
"iterations": len(iters),
|
|
"last_iter": last_n,
|
|
"class": primary,
|
|
"faults": faults,
|
|
"detail": detail,
|
|
"check": extracted,
|
|
"per_iteration": per_iteration,
|
|
"runner_final": final,
|
|
"runner_reported_check": reported,
|
|
"extraction_matches_runner": (
|
|
None if reported is None or extracted is None
|
|
else reported.rstrip("\r") == extracted.rstrip("\r")
|
|
),
|
|
"cross_check": cross,
|
|
"runner_log": run["runner_log"],
|
|
}
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Selftest
|
|
# --------------------------------------------------------------------------
|
|
FIXTURES = [
|
|
# (what the wizard emitted, expected primary class)
|
|
("no check here at all\n", "MISSING"),
|
|
("CHECK:\n", "EMPTY"),
|
|
("CHECK: \n", "EMPTY"),
|
|
("CHECK: true\n", "TAUTOLOGY"),
|
|
("CHECK: :\n", "TAUTOLOGY"),
|
|
("CHECK: exit 0\n", "TAUTOLOGY"),
|
|
("CHECK: echo ok\n", "TAUTOLOGY"),
|
|
("CHECK: echo \"all good\"\n", "TAUTOLOGY"),
|
|
("CHECK: sleep 5\n", "TAUTOLOGY"),
|
|
("check: true\n", "TAUTOLOGY"), # runner's grep is -i
|
|
("CHECK: python3 x.py && echo OK\n", "SOUND"),
|
|
("CHECK: python3 x.py && echo PASS || echo FAIL\n", "CATCH_ALL"),
|
|
("CHECK: python3 x.py || true\n", "CATCH_ALL"),
|
|
("CHECK: python3 x.py || :\n", "CATCH_ALL"),
|
|
("CHECK: python3 x.py || exit 0\n", "CATCH_ALL"),
|
|
("CHECK: python3 x.py; echo done\n", "CATCH_ALL"),
|
|
("CHECK: python3 x.py; true\n", "CATCH_ALL"),
|
|
("CHECK: test -s f && grep -c '###' f | grep -qx '8' && echo PASS || echo FAIL\n",
|
|
"CATCH_ALL"),
|
|
("CHECK: test \"$(python3 wc.py <<< \"a b c\")\" = \"3\" && echo PASSED || echo FAILED\n",
|
|
"CATCH_ALL"),
|
|
("CHECK: python3 /tmp/verify_dl.py\r\n", "TRAILING_CR"),
|
|
("CHECK: cd /a && PYTHONPATH=/a python /a/verify_chunker.py\r\n", "TRAILING_CR"),
|
|
("CHECK: python3 /tmp/v.py \u2014 re-reads the JSON, asserts counts (Ran it: PASS, exit 0.)\n",
|
|
"PROSE_TAIL"),
|
|
("CHECK: python3 v.py -- re-reads the JSON\n", "PROSE_TAIL"),
|
|
("CHECK: cd /x && python _verify.py && python test_a.py && python test_b.py\n", "SOUND"),
|
|
("CHECK: grep -q ralph-green /root/ralph-smoke.txt\n", "SOUND"),
|
|
("CHECK: sha256sum a.tar.gz | grep -q 4844c5fc\n", "SOUND"),
|
|
("CHECK: python3 tools/x.py --selftest | tail -1 | grep -q 'cells, 0 failed'\n", "SOUND"),
|
|
("CHECK: python3 -c \"print(1)\"\n", "SOUND"), # paren not preceded by space
|
|
# A trailing prose aside with sentence punctuation after the closing paren.
|
|
# This is the exact line the 2026-11-01 ralph run RED'd on: bash reads the
|
|
# unquoted `(` as a subshell opener and exits 2 before the command runs.
|
|
("CHECK: bash CHECK-quill-draft-compliance (the artifact gate).\n", "PROSE_TAIL"),
|
|
("CHECK: python3 v.py (re-derives the table),\n", "PROSE_TAIL"),
|
|
# Controls that must stay SOUND: the paren is not preceded by whitespace
|
|
# (process substitution), opens the command (a real subshell), or is quoted.
|
|
("CHECK: diff -u <(a) <(b)\n", "SOUND"),
|
|
("CHECK: (cd sub && python3 t.py)\n", "SOUND"),
|
|
("CHECK: python3 x.py \"note (a).\"\n", "SOUND"),
|
|
("CHECK: python3 x.py --json /tmp/o.json\n", "SOUND"), # -- flag, not " -- "
|
|
("CHECK: python3 v.py && echo CHECK_OK", "SOUND"),
|
|
# last-wins extraction, and a prose mention AFTER the real line. The second
|
|
# fixture is sol's real shape (2026-10-16 16:57Z log, line 1335): a line that
|
|
# begins with an indentation then CHECK: matches the runner's grep, so prose
|
|
# written after the command becomes the graded line. Prose that does not
|
|
# itself start with CHECK: (no leading-whitespace CHECK:) is not matched and
|
|
# the earlier real line survives -- both directions are pinned here.
|
|
("CHECK: true\nCHECK: python3 x.py\n", "SOUND"),
|
|
("CHECK: python3 x.py\n\nA CHECK: line is required.\n", "SOUND"),
|
|
("CHECK: python3 x.py\n\n CHECK: line \u2014 one shell command that verifies "
|
|
"your work and exits 0\n", "PROSE_TAIL"),
|
|
# LABEL_PREFIX cells (2026-11-02, volt). The first two are the measured
|
|
# shapes: cell 1 is the line this lane's own ralph iteration died on
|
|
# (`bash: line 1: selftest:: command not found`), cell 2 is the one the
|
|
# runner's 2026-10-27 forge log carries, one iteration before that CHECK was
|
|
# rewritten without the label. The four controls pin the boundary of the
|
|
# rule: a colon that is not in the FIRST word, a colon inside an assignment,
|
|
# and a quoted first token are all SOUND.
|
|
("CHECK: selftest: python3 tools/check_lint.py --selftest\n", "LABEL_PREFIX"),
|
|
("CHECK: line: python3 tools/check_lint.py --selftest\n", "LABEL_PREFIX"),
|
|
("CHECK: Verdict: bash CHECK-x\n", "LABEL_PREFIX"), # capitalised
|
|
("CHECK: step-2: python3 x.py\n", "LABEL_PREFIX"), # hyphenated
|
|
("CHECK: 2026-11-02: python3 x.py\n", "LABEL_PREFIX"), # date label
|
|
("CHECK: ok:true\n", "LABEL_PREFIX"), # no space after
|
|
("CHECK: python3 x.py: --selftest\n", "SOUND"), # colon, word 2
|
|
("CHECK: PATH=/a:/b python3 x.py\n", "SOUND"), # assignment
|
|
("CHECK: 'note: x' cmd\n", "SOUND"), # quoted token
|
|
]
|
|
|
|
EXTRACT_FIXTURES = [
|
|
(" CHECK: cmd here\n", "cmd here"),
|
|
("CHECK: cmd2\r\n", "cmd2\r"),
|
|
("no check\n", None),
|
|
("check: lower\n", "lower"),
|
|
]
|
|
|
|
|
|
def selftest(verbose: bool) -> int:
|
|
cells = 0
|
|
failed = 0
|
|
for text, want in FIXTURES:
|
|
cells += 1
|
|
got, faults, _ = classify(extract_check(text))
|
|
if got != want:
|
|
failed += 1
|
|
print(f" cell {cells}: want {want} got {got} <- {text!r}")
|
|
elif verbose:
|
|
print(f" cell {cells}: {got:11s} {text.strip()!r}")
|
|
for text, want in EXTRACT_FIXTURES:
|
|
cells += 1
|
|
got = extract_check(text)
|
|
if got != want:
|
|
failed += 1
|
|
print(f" cell {cells}: extract want {want!r} got {got!r} <- {text!r}")
|
|
elif verbose:
|
|
print(f" cell {cells}: extract {got!r}")
|
|
print(f"selftest: {cells} cells, {cells - failed} passed, {failed} failed")
|
|
return 1 if failed else 0
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Report
|
|
# --------------------------------------------------------------------------
|
|
def print_rules():
|
|
print("RULES (textual; the command is never executed by this tool)")
|
|
print(" MISSING no line matches ^\\s*CHECK:")
|
|
print(" EMPTY a CHECK line with nothing after the colon")
|
|
print(" TAUTOLOGY whole command is true/: /exit 0/echo/printf/sleep")
|
|
print(" CATCH_ALL a || arm or terminal ; statement that exits 0 regardless")
|
|
print(" LABEL_PREFIX a leading `word:` label -- bash exits 127 on it")
|
|
print(" TRAILING_CR a literal carriage return survives at end of line")
|
|
print(" PROSE_TAIL prose follows the command (CANDIDATE, cut point printed)")
|
|
print(" SOUND none of the above -- not a promise about its meaning")
|
|
|
|
|
|
def report(runs: list[dict], as_json: str | None, quiet: bool,
|
|
unlogged: int = 0) -> int:
|
|
graded = [grade_run(r) for r in runs]
|
|
graded.sort(key=lambda r: (r["stamp"], r["wizard"]), reverse=True)
|
|
|
|
summary = {"runs_on_wizards": len(graded), "green": 0, "exhausted": 0,
|
|
"unknown": 0, "no_check_ever": 0, "classes": {},
|
|
"green_by_construction": 0, "green_despite_formatting": 0,
|
|
"extraction_mismatches": 0, "runs_without_runner_log": unlogged}
|
|
for g in graded:
|
|
if g["runner_final"] == "GREEN":
|
|
summary["green"] += 1
|
|
elif g["runner_final"] == "EXHAUSTED":
|
|
summary["exhausted"] += 1
|
|
else:
|
|
summary["unknown"] += 1
|
|
if g["class"] == "MISSING":
|
|
summary["no_check_ever"] += 1
|
|
if g["cross_check"] == "GREEN_BY_CONSTRUCTION":
|
|
summary["green_by_construction"] += 1
|
|
elif g["cross_check"] == "GREEN_DESPITE_FORMATTING":
|
|
summary["green_despite_formatting"] += 1
|
|
if g["extraction_matches_runner"] is False:
|
|
summary["extraction_mismatches"] += 1
|
|
# `collections.Counter` replaces the hand-rolled tally loop
|
|
# (`summary["classes"][g["class"]] = summary["classes"].get(g["class"], 0) + 1`):
|
|
# Counter reads a missing key as 0, so `dict(Counter(...))` yields the same
|
|
# {class: n} mapping in first-appearance order. Re-assigning the existing
|
|
# key keeps its position in `summary`, so the SUMMARY repr below and the
|
|
# json.dump payload are both byte-identical to the pre-refactor output.
|
|
summary["classes"] = dict(Counter(g["class"] for g in graded))
|
|
|
|
if not quiet:
|
|
print_rules()
|
|
stamp = None
|
|
for g in graded:
|
|
if g["stamp"] != stamp:
|
|
stamp = g["stamp"]
|
|
print(f"\nRUN {stamp}")
|
|
print(f" {'wizard':8s} {'it':>3s} {'class':12s} {'runner':10s} "
|
|
f"{'cross-check':24s} check")
|
|
chk = g["check"]
|
|
shown = "(none)" if chk is None else chk.rstrip("\r")
|
|
shown = shown if len(shown) <= 68 else shown[:65] + "..."
|
|
print(f" {g['wizard']:8s} {g['iterations']:3d} {g['class']:12s} "
|
|
f"{g['runner_final']:10s} {g['cross_check']:24s} {shown}")
|
|
if len(g["faults"]) > 1:
|
|
print(f" also: {', '.join(f for f in g['faults'] if f != g['class'])}")
|
|
if g["extraction_matches_runner"] is False:
|
|
print(" NOTE: re-extraction differs from what the runner echoed")
|
|
|
|
print("\nSUMMARY")
|
|
for k, v in summary.items():
|
|
print(f" {k}: {v}")
|
|
|
|
if as_json:
|
|
payload = {"tool": "check_lint.py",
|
|
"generated_utc": _dt.datetime.now(_dt.timezone.utc)
|
|
.strftime("%Y-%m-%dT%H:%M:%SZ"),
|
|
"ralph_dir": os.path.dirname(graded[0]["runner_log"])
|
|
if graded and graded[0]["runner_log"] else None,
|
|
"runs": graded, "summary": summary}
|
|
rc = _write_json_file(as_json, payload)
|
|
if rc:
|
|
return rc
|
|
if not quiet:
|
|
print(f"\njson -> {as_json}")
|
|
return 0
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
ap = argparse.ArgumentParser(
|
|
description="Grade Ralph-loop CHECK lines. Read-only: never executes a CHECK.")
|
|
g = ap.add_mutually_exclusive_group()
|
|
g.add_argument("--selftest", action="store_true",
|
|
help="grade the built-in fixtures")
|
|
g.add_argument("--ralph-dir", metavar="DIR", default=None,
|
|
help="grade every run in a ralph-loop output directory")
|
|
g.add_argument("--log", metavar="FILE", default=None,
|
|
help="extract and grade the CHECK in one log file")
|
|
ap.add_argument("--check", metavar="CMD", default=None,
|
|
help="grade this command string directly")
|
|
ap.add_argument("--json", metavar="FILE", default=None)
|
|
ap.add_argument("--quiet", action="store_true")
|
|
ap.add_argument("--verbose", action="store_true")
|
|
ap.add_argument("--all", action="store_true",
|
|
help="include runs that have no runner .log (no verdict to check)")
|
|
ap.add_argument("--fail-on-fault", action="store_true",
|
|
help="exit 1 if any graded CHECK is not SOUND")
|
|
a = ap.parse_args(argv)
|
|
|
|
if a.selftest:
|
|
return selftest(a.verbose)
|
|
|
|
if a.check is not None:
|
|
primary, faults, detail = classify(a.check)
|
|
if not a.quiet:
|
|
print(f"{primary}: {a.check!r}")
|
|
if faults:
|
|
print(f" faults: {', '.join(faults)}")
|
|
if detail:
|
|
print(f" detail: {json.dumps(detail)}")
|
|
if a.json:
|
|
rc = _write_json_file(a.json, {"check": a.check, "class": primary,
|
|
"faults": faults, "detail": detail})
|
|
if rc:
|
|
return rc
|
|
return 1 if (a.fail_on_fault and primary != "SOUND") else 0
|
|
|
|
if a.log:
|
|
try:
|
|
with open(a.log, encoding="utf-8", errors="replace", newline="") as fh:
|
|
text = fh.read()
|
|
except OSError as exc:
|
|
# A log that is not there is a PRECONDITION failure, not a grading
|
|
# result. It used to escape as a FileNotFoundError traceback with
|
|
# rc 1, which is the code this tool uses for "the check has a
|
|
# fault"; since 2026-10-17 the repo-wide contract is rc 2 for
|
|
# "could not grade" (see tools/selftest_sweep.py's read-refusal
|
|
# axis, which now measures this row on every reader).
|
|
print(f"cannot read log: {exc}", file=sys.stderr)
|
|
return 2
|
|
cmd = extract_check(text)
|
|
primary, faults, detail = classify(cmd)
|
|
if not a.quiet:
|
|
print(f"{a.log}: {primary}")
|
|
print(f" extracted: {cmd!r}")
|
|
if faults:
|
|
print(f" faults: {', '.join(faults)}")
|
|
if detail:
|
|
print(f" detail: {json.dumps(detail)}")
|
|
return 1 if (a.fail_on_fault and primary != "SOUND") else 0
|
|
|
|
ralph_dir = a.ralph_dir or "/root/.hermes/wizard-outputs/ralph"
|
|
runs, unlogged = read_ralph_dir(ralph_dir, include_unlogged=a.all)
|
|
if not runs:
|
|
print(f"no iter logs in {ralph_dir}")
|
|
return 0 if not a.fail_on_fault else 1
|
|
rc = report(runs, a.json, a.quiet, unlogged)
|
|
if a.fail_on_fault:
|
|
graded = [grade_run(r) for r in runs]
|
|
return 1 if any(g["class"] != "SOUND" for g in graded) else 0
|
|
return rc
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|