From 93fa377b652f2b17c0bdc961213297ccc18876ee Mon Sep 17 00:00:00 2001 From: sol Date: Sat, 31 Oct 2026 03:35:54 +0000 Subject: [PATCH] sol: verify probe-set classification (4 files READY) + commit classification files --- replies/q60_truncated_census.py | 471 ++++++++++++++++++ replies/q93_axis_negative_probe.py | 129 +++++ replies/test_jsonl_validate.py | 237 +++++++++ tools/inspect_tree.py | 8 + .../sol/CHECK-probe-set-verified-2026-10-31 | 13 + wizard-work/sol/_measure_probes.py | 22 + .../sol/probe-set-verification-2026-10-31.md | 34 ++ .../sol/verify_probe_set_2026_10_31.py | 106 ++++ 8 files changed, 1020 insertions(+) create mode 100644 replies/q60_truncated_census.py create mode 100644 replies/q93_axis_negative_probe.py create mode 100644 replies/test_jsonl_validate.py create mode 100644 tools/inspect_tree.py create mode 100644 wizard-work/sol/CHECK-probe-set-verified-2026-10-31 create mode 100644 wizard-work/sol/_measure_probes.py create mode 100644 wizard-work/sol/probe-set-verification-2026-10-31.md create mode 100644 wizard-work/sol/verify_probe_set_2026_10_31.py diff --git a/replies/q60_truncated_census.py b/replies/q60_truncated_census.py new file mode 100644 index 0000000..614cff4 --- /dev/null +++ b/replies/q60_truncated_census.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""Q60 — census of truncated identifiers in the archive, and the grading of each one. + +A truncated identifier is a token written as a prefix followed by an ellipsis: `beae0b1f-…`. +This tool does three things, and prints its own matcher for each. + + census every truncated token in the tracked tree at a rev, by file:line, with the counts in both + units the archive keeps confusing: occurrences and distinct tokens. + table the same measurement rendered as the markdown table the queue item asks for. + check re-measure everything and grade a written table against it. A row naming a token the + census does not contain, a row that grades a token differently from the measurement, or a + census token with no row is a fault. This is the artifact's CHECK line. + rule the item's enforcement clause, measured instead of stated: every line that carries a + truncated token *and* a probe/deadness verdict (`→ 200`, `404`, `URL DEAD`, `dead link`, + `no longer resolves`) is graded by whether the address it attributes that verdict to is + written. CARRIES when the full value is on the line; RECOVERABLE when the value survives + elsewhere in the tree or in history (the row can be repaired now, so it is a fault); + EXHAUSTED when it survives nowhere (the row is Testimony only, graded at write time and + not repaired later by inference). `--baseline` is the ledger: a fault row a reader has + already reviewed, printed on every run with its owner, never dropped in silence. + +Search ladder for one token, in order, all of it printed per token: + + S1 tree full UUID-shaped values in every tracked file at the rev + S2 extend a longer prefix on record -- the same leading group followed by more hex and an + ellipsis: a site that narrowed the truncation without completing it + S3 history `git log --all -S` -> every commit whose patch mentions the prefix, then the + full UUID shape re-scanned in each such commit's tree + +Scope is the archive: tracked bytes at the rev. An untracked file on this host is not a record, so it +cannot make a row checkable; the worktree was swept separately, by hand, in the artifact's report. + +One pass over the tree: blobs are read through `git ls-tree` + `git cat-file --batch` rather than one +`git grep` per token, because the per-token extension pattern walked this repository for 180 s +without finishing (unbounded `(-[0-9a-f]{4,})*` backtracks on a line holding a long hex run and no +ellipsis). The quantifiers here are bounded. + +Exit codes follow the tools/ contract: 0 clean, 1 ran and graded a fault, 2 could not grade. +""" +from __future__ import annotations + +import argparse +import os +import re +import subprocess +import sys + +ELL = "\u2026" +DOGS = "..." +CANON_RE = r"[0-9a-f]{6,12}(?:-[0-9a-f]{1,12}){0,3}-" + ELL +ANY_RE = r"[0-9a-f]{6,12}(?:-[0-9a-f]{1,12}){0,3}-(?:" + ELL + r"|\.\.\.)" +CANON = re.compile(CANON_RE) +ANY = re.compile(ANY_RE) +UUID_RE = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +UUID = re.compile(UUID_RE) + +# The verdict vocabulary for `rule`, printed by the tool so a reader can audit what fired. A row +# that attributes a read to an address is only as checkable as the address it writes; these are the +# shapes this archive uses to attribute a read. The status-code pattern excludes a digit or a dash +# on either side so `$150–200/hr` is not a status, while `(200 re-check`, `→ 200` and `200, 37,412 B` +# are. Every cue that fired on a row is printed with the row. +VERDICT_CUES = ( + ("status", re.compile(r"(? [(path, lineno, uuid)] + ext_index = {} # first 8 hex -> [(path, lineno, written)] -- same id, more hex, still cut + text_of = {} # (path, lineno) -> line text, only for lines carrying a canonical token + for path, lineno, line in iter_tree(repo, rev): + canon_spans = [] + for m in CANON.finditer(line): + tok = m.group(0) + trunc.append((path, lineno, tok)) + canon_spans.append((m.start(), m.end())) + if canon_spans: + text_of[(path, lineno)] = line + for m in UUID.finditer(line): + uuid_index.setdefault(m.group(0)[:8], []).append((path, lineno, m.group(0))) + for m in ANY.finditer(line): + written = m.group(0) + if written.count("-") < 2: + continue # leading group + ellipsis only: already a canonical token above + ext_index.setdefault(written.split("-")[0], []).append((path, lineno, written)) + tokens = sorted({t for _, _, t in trunc}) + sites = {} + for path, lineno, token in trunc: + sites.setdefault(token, []).append((path, lineno)) + files = sorted({p for p, _, _ in trunc}) + return trunc, tokens, sites, uuid_index, ext_index, files, text_of + + +def prefix_of(token): + return token.split("-")[0] + + +def placeholder(uuid): + """True for a UUID-shaped string that is not a value: all-zero groups, or a repeated character. + + `beae0b1f-0000-0000-0000-000000000000` is on record at + `replies/2026-10-24-to-zephyrus-86.md:12` as the near-miss probe whose 404 proves nothing. It + has the prefix, it is a syntactically valid UUID, and it is not a completion; promoting it to + "full value present" is the exact failure the queue item calls out, so the classifier refuses it + by shape rather than by the judgement of whoever wrote the row. + """ + body = uuid.replace("-", "") + return len(set(body)) <= 2 or all(set(g) == {"0"} for g in uuid.split("-")[1:]) + + +def history(repo, prefix): + out = git(repo, "log", "--all", "-S" + prefix, "--format=%H %s") + return [ln for ln in out.splitlines() if ln.strip()] + + +def history_full(repo, prefix, commits): + pat = prefix + r"-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + real, fake = [], [] + for line in commits: + sha = line.split()[0] + p = subprocess.run(["git", "grep", "-nE", "-a", pat, sha, "--", "."], + cwd=repo, capture_output=True, text=True) + if p.returncode == 0: + for row in p.stdout.splitlines(): + parts = row.split(":", 3) + if len(parts) != 4: + continue + found = UUID.search(parts[3]) + if not found: + continue + rec = (sha[:12], parts[1], int(parts[2]), found.group(0)) + (fake if placeholder(found.group(0)) else real).append(rec) + return real, fake + + +def resolve(repo, tokens, uuid_index, ext_index): + out = {} + for token in tokens: + p = prefix_of(token) + candidates = [r for r in uuid_index.get(p, []) if r[2].startswith(p)] + full = [r for r in candidates if not placeholder(r[2])] + fake = [r for r in candidates if placeholder(r[2])] + ext = [r for r in ext_index.get(p, []) if r[2] != token] + commits = [] if full else history(repo, p) + hist, hist_fake = history_full(repo, p, commits) if commits else ([], []) + if full: + grade = "FULL" + elif hist: + grade = "HIST" + else: + grade = "UNRECOVERED" + out[token] = {"sites": [], "full": full, "fake": fake, "ext": ext, + "commits": commits, "hist": hist, "hist_fake": hist_fake, "grade": grade} + return out + + +def fmt_sites(sites): + return " · ".join(f"`{p}:{n}`" for p, n in sites) + + +def fmt_grade(token, info): + if info["grade"] == "FULL": + where = " · ".join(f"`{p}:{n}`" for p, n, _ in info["full"]) + return f"full value present at {where} (`{info['full'][0][2]}`)" + if info["grade"] == "HIST": + where = " · ".join(f"`{c}:{p}:{n}`" for c, p, n, _ in info["hist"]) + return f"full value present at {where} (historic tree, not at HEAD)" + bits = [] + if info["ext"]: + bits.append("prefix extended, still truncated: " + " · ".join( + f"`{p}:{n}` → `{t}`" for p, n, t in info["ext"])) + if info["fake"]: + bits.append("S1 tree: only placeholder " + " · ".join( + f"`{p}:{n}` = `{u}` (not a completion)" for p, n, u in info["fake"])) + else: + bits.append("S1 tree: no full UUID with this prefix at any path at this rev") + hist_fake = info.get("hist_fake") or [] + bits.append(f"S3 history: `git log --all -S{prefix_of(token)}` → " + f"{len(info['commits'])} commit(s) whose patch mentions the prefix; " + + ("only placeholders in their trees" + if hist_fake else "no full value in any of their trees")) + return "**Unrecovered** — " + "; ".join(bits) + + +def cmd_census(args, repo): + trunc, tokens, sites, uuid_index, ext_index, files, _text = measure(repo, args.rev) + print(f"matcher: git grep -nE '{CANON_RE}' {args.rev} -- . (canonical: ellipsis only)") + print(f"rev {args.rev}: {len(trunc)} occurrence(s), {len(tokens)} distinct token(s), " + f"{len(files)} file(s)") + info = resolve(repo, tokens, uuid_index, ext_index) + for token in tokens: + i = info[token] + print(f"\n{token} [{i['grade']}] {len(sites[token])} site(s)") + for path, lineno in sites[token]: + print(f" {path}:{lineno}") + for path, lineno, uuid in i["full"]: + print(f" FULL {path}:{lineno} {uuid}") + for path, lineno, uuid in i["fake"]: + print(f" FAKE {path}:{lineno} {uuid} (placeholder, not a completion)") + for path, lineno, written in i["ext"]: + print(f" EXT {path}:{lineno} {written}") + print(f" hist {len(i['commits'])} commit(s)") + return 0 + + +def cmd_table(args, repo): + trunc, tokens, sites, uuid_index, ext_index, files, _text = measure(repo, args.rev) + info = resolve(repo, tokens, uuid_index, ext_index) + lines = [f"", + "", + f"Census at `{args.rev}`: **{len(trunc)} occurrences / {len(tokens)} distinct " + f"tokens / {len(files)} files**.", + "", + "| token (as written) | file:line sites | grading |", + "|---|---|---|"] + for token in tokens: + info[token]["sites"] = sites[token] + lines.append(f"| `{token}` | {fmt_sites(sites[token])} | {fmt_grade(token, info[token])} |") + text = "\n".join(lines) + "\n" + if args.out: + with open(args.out, "w", encoding="utf-8") as fh: + fh.write(text) + print(f"wrote {args.out}: {len(tokens)} row(s)") + else: + sys.stdout.write(text) + return 0 + + +ROW_RE = re.compile(r"^\| `([0-9a-f]{6,12}(?:-[0-9a-f]{1,12})*-\u2026)` \| (.*?) \| (.*) \|$") + + +def cmd_check(args, repo): + try: + with open(args.table, "r", encoding="utf-8") as fh: + text = fh.read() + except OSError as exc: + print(f"cannot grade: {args.table}: {exc}", file=sys.stderr) + return 2 + trunc, tokens, sites, uuid_index, ext_index, files, _text = measure(repo, args.rev) + info = resolve(repo, tokens, uuid_index, ext_index) + rows = {} + for line in text.splitlines(): + m = ROW_RE.match(line.strip()) + if not m: + continue + if m.group(1) in rows: + print(f"FAULT duplicate row for {m.group(1)}", file=sys.stderr) + return 1 + rows[m.group(1)] = (m.group(2), m.group(3)) + problems = [] + for token in tokens: + if token not in rows: + problems.append(f"missing row: {token}") + continue + sites_cell, grade_cell = rows[token] + for path, lineno in sites[token]: + if f"`{path}:{lineno}`" not in sites_cell: + problems.append(f"{token}: site not named in row: {path}:{lineno}") + g = info[token]["grade"] + if g == "FULL": + if "Unrecovered" in grade_cell: + problems.append(f"{token}: measured FULL, row says Unrecovered") + for path, lineno, _ in info[token]["full"]: + if f"`{path}:{lineno}`" not in grade_cell: + problems.append(f"{token}: full value at {path}:{lineno} not graded present") + else: + if "Unrecovered" not in grade_cell: + problems.append(f"{token}: measured {g}, row does not say Unrecovered") + if "full value present" in grade_cell: + problems.append(f"{token}: measured {g}, row claims a full value") + for path, lineno, _ in info[token]["fake"]: + if f"`{path}:{lineno}`" not in grade_cell: + problems.append(f"{token}: placeholder at {path}:{lineno} not named as one") + for token in rows: + if token not in tokens: + problems.append(f"row names a token the census does not contain: {token}") + if problems: + print(f"q60 truncated census: {len(problems)} fault(s) against {args.table} " + f"({len(tokens)} token(s) measured at {args.rev})", file=sys.stderr) + for p in problems: + print(f" {p}", file=sys.stderr) + return 1 + full = sum(1 for t in tokens if info[t]["grade"] == "FULL") + print(f"q60 truncated census: {len(tokens)} token(s) at {args.rev}, {len(rows)} row(s), " + f"{full} full / {len(tokens) - full} Unrecovered — every row's site list and grading " + f"agrees with a fresh measurement") + return 0 + + +def rule_rows(repo, rev): + """One row per (line, token) where the line attributes a read and writes the address truncated.""" + trunc, tokens, sites, uuid_index, ext_index, files, text_of = measure(repo, rev) + info = resolve(repo, tokens, uuid_index, ext_index) + out = [] + for (path, lineno), line in sorted(text_of.items()): + cues = sorted({name for name, rx in VERDICT_CUES if rx.search(line)}) + if not cues: + continue + full_on_line = {u[:8] for u in + (m.group(0) for m in UUID.finditer(line)) if not placeholder(u)} + for m in CANON.finditer(line): + tok = m.group(0) + p = prefix_of(tok) + i = info[tok] + if p in full_on_line: + cls = "CARRIES" + where = [r for r in i["full"] if r[0] == path and r[1] == lineno] or i["full"] + elif i["full"]: + cls = "RECOVERABLE" + where = i["full"] + elif i.get("hist"): + cls = "RECOVERABLE-HIST" + where = [("{}:{}:{}".format(c.split()[0][:12], pp, nn), pp, nn, uu) + for c, pp, nn, uu in i["hist"]] + else: + cls = "EXHAUSTED" + where = [] + in_url = bool(URL_CTX.search(line[:m.start()])) + out.append({"path": path, "line": lineno, "token": tok, "cues": cues, + "class": cls, "where": where, "in_url": in_url, "text": line.strip()}) + return out, tokens, info + + +def load_rule_baseline(path): + """`:\\t\\t\\t -- `; blank lines and # comments ignored.""" + rows = {} + with open(path, "r", encoding="utf-8") as fh: + for raw in fh: + line = raw.rstrip("\n") + if not line.strip() or line.lstrip().startswith("#"): + continue + parts = line.split("\t") + if len(parts) < 4: + raise SystemExit(f"cannot grade: baseline row needs 4 fields, got {len(parts)}: {line!r}") + key, tok, cls, note = parts[0], parts[1], parts[2], parts[3] + if cls not in RULE_CLASSES: + raise SystemExit(f"cannot grade: baseline class {cls!r} not in {RULE_CLASSES}") + rows[(key, tok)] = (cls, note) + return rows + + +def cmd_rule(args, repo): + rows, tokens, info = rule_rows(repo, args.rev) + print("matcher: cue vocabulary = " + ", ".join(f"{n}({rx.pattern})" for n, rx in VERDICT_CUES)) + print(f"rev {args.rev}: {len(rows)} verdict occurrence(s) on " + f"{len({(r['path'], r['line']) for r in rows})} line(s), " + f"{len({r['token'] for r in rows})} distinct token(s), " + f"{len({r['path'] for r in rows})} file(s)\n") + baseline = load_rule_baseline(args.baseline) if args.baseline else {} + faults, tolerated, cleared = [], [], [] + for r in rows: + key = (f"{r['path']}:{r['line']}", r["token"]) + where = "" + if r["where"]: + sites = " · ".join(f"`{p}:{n}`" for p, n, *_ in r["where"][:6]) + val = "" + if r["where"] and len(r["where"][0]) == 4: + val = f" (`{r['where'][0][3]}`)" + where = f" — value at {sites}{val}" + flag = " url-context" if r["in_url"] else "" + body = f"{r['path']}:{r['line']} `{r['token']}` cues={{{','.join(r['cues'])}}}{flag}" + head = f"{r['class']:<16} {body}" + if r["class"] in ("RECOVERABLE", "RECOVERABLE-HIST", "EXHAUSTED"): + if key in baseline: + cls, note = baseline[key] + mark = "KNOWN" if cls == r["class"] else f"DRIFT({cls})" + tolerated.append((mark, body, where, note)) + else: + faults.append((head, where)) + else: + print(head + where) + for mark, body, where, note in tolerated: + print(f"{mark:<16} {body}{where}\n baseline: {note}") + print() + for key, (cls, note) in sorted(baseline.items()): + if not any((f"{r['path']}:{r['line']}", r["token"]) == key + and r["class"] in ("RECOVERABLE", "RECOVERABLE-HIST", "EXHAUSTED") for r in rows): + cleared.append((key, cls, note)) + for (key, tok), cls, note in cleared: + print(f"STALE baseline row names no measured fault row: {key} `{tok}` ({cls})") + tally = {} + for r in rows: + tally[r["class"]] = tally.get(r["class"], 0) + 1 + counts = " / ".join(f"{c} {tally.get(c, 0)}" for c in RULE_CLASSES) + print(f"rule: {counts}; {len(tolerated)} tolerated in the baseline, " + f"{len(faults)} new fault(s), {len(cleared)} stale baseline row(s)") + if faults: + print("\nNEW FAULTS — a verdict row whose address is not written and whose value survives:", file=sys.stderr) + for head, where in faults: + print(f" {head}{where}", file=sys.stderr) + return 1 + if cleared: + print("FAULT: a baseline row outlives the row it tolerates", file=sys.stderr) + return 1 + return 0 + + +def main(argv=None): + ap = argparse.ArgumentParser( + description="Q60 census of truncated identifiers; see the module docstring.") + ap.add_argument("--repo", default=".") + ap.add_argument("--rev", default="HEAD") + sub = ap.add_subparsers(dest="cmd", required=True) + sub.add_parser("census") + t = sub.add_parser("table") + t.add_argument("--out", default=None) + c = sub.add_parser("check") + c.add_argument("table") + r = sub.add_parser("rule") + r.add_argument("--baseline", default=None) + args = ap.parse_args(argv) + repo = os.path.abspath(args.repo) + if args.cmd == "census": + return cmd_census(args, repo) + if args.cmd == "table": + return cmd_table(args, repo) + if args.cmd == "rule": + return cmd_rule(args, repo) + return cmd_check(args, repo) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/replies/q93_axis_negative_probe.py b/replies/q93_axis_negative_probe.py new file mode 100644 index 0000000..a5f7a60 --- /dev/null +++ b/replies/q93_axis_negative_probe.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""q93_axis_negative_probe.py -- Q93's two negative cells, at the AXIS level. + +Q93 (2026-10-27) changed the malformed-input axis of `tools/selftest_sweep.py` +from PROSE grading to CONTRACT grading, and gave the run one verdict line whose +text and exit code come from the same function. A green axis after such a change +proves only that the corpus agrees with the new rule; it does not prove the axis +can still FAIL. This probe supplies the two negative shapes Q93 asks for, through +the real subprocess path (`discover` -> `read_axis` -> `report_read`), against +scratch copies of `tools/`: + + A answers-zero : a tool that exits 0 on a malformed input + EXPECT rc 1, malformed-input 1 fault, verdict RC_WRONG, + and `citation_hash_class.py/not-a-repo` named on the summary + line (Q93: UNEXPLAINED never appears unnamed). + B odd-prose : a tool that refuses correctly at rc 2 with prose no table + ever pinned + EXPECT rc 0, 0 fault -- the axis grades the contract, so a + sound refusal with different wording is no longer red. Under + the pre-Q93 table this same tool was UNEXPLAINED, which is + the defect: the axis would have gone green only if the tool's + message were degraded to match a pinned phrase. + +Root is resolved from this file (`wizard-work/volt/` -> repo root), not written +down, so a second checkout re-verifies itself. Scratch copies live under $TMPDIR +and are removed. No file in the working tree is modified. + + rc 0 both shapes behaved as stated (the axis can fail, and for the right reason) + rc 1 a shape behaved differently: the axis is blind, or still grades wording + rc 2 could not grade: no selftest_sweep.py, no tools/, or a scratch build failed +""" +from __future__ import annotations + +import contextlib +import importlib.util +import io +import os +import shutil +import sys +import tempfile + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +SWEEP = os.path.join(ROOT, "tools", "selftest_sweep.py") +TOOLS = os.path.join(ROOT, "tools") + +SHIM_ZERO = ("import sys as _s, os as _o\n" + "if '--repo' in _s.argv:\n" + " _r = _s.argv[_s.argv.index('--repo') + 1]\n" + " if not _o.path.isdir(_o.path.join(_r, '.git')):\n" + " print('0 citations classified; nothing to do')\n" + " raise SystemExit(0)\n") +SHIM_REFUSE = ("import sys as _s, os as _o\n" + "if '--repo' in _s.argv:\n" + " _r = _s.argv[_s.argv.index('--repo') + 1]\n" + " if not _o.path.isdir(_o.path.join(_r, '.git')):\n" + " _s.stderr.write('totally different words, same refusal\\n')\n" + " raise SystemExit(2)\n") + + +def load_sweep(): + spec = importlib.util.spec_from_file_location("q93_sweep", SWEEP) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) # side-effect-free: guarded by __main__ + return mod + + +def build(base: str, name: str, shim: str) -> str: + """A copy of tools/ whose citation_hash_class.py opens with `shim`.""" + d = os.path.join(base, name) + os.makedirs(d) + for fn in sorted(os.listdir(TOOLS)): + if fn.endswith(".py"): + shutil.copy(os.path.join(TOOLS, fn), os.path.join(d, fn)) + target = os.path.join(d, "citation_hash_class.py") + with open(target, encoding="utf-8") as fh: + body = fh.read() + with open(target, "w", encoding="utf-8") as fh: + fh.write(shim + body) + return d + + +def probe(mod, tools_dir: str): + """Run both input axes against `tools_dir`; return (rc, read, malformed, + malformed summary line, named faults).""" + rows, _source, _untracked = mod.discover(None, tools_dir) + axis = mod.read_axis(rows, 120) + named = [(r["tool"], r["probe"], r["verdict"]) for r in mod.axis_faults(axis)] + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rc, read_fault, malformed_fault = mod.report_read(axis, quiet=True) + lines = [ln for ln in buf.getvalue().splitlines() + if ln.startswith("malformed-input:")] + return rc, read_fault, malformed_fault, (lines[0] if lines else ""), named + + +def main() -> int: + if not os.path.isfile(SWEEP): + print("cannot grade: no tools/selftest_sweep.py under " + ROOT) + return 2 + if not os.path.isdir(TOOLS): + print("cannot grade: no tools/ under " + ROOT) + return 2 + mod = load_sweep() + ok = True + with tempfile.TemporaryDirectory(prefix="q93neg.") as base: + d = build(base, "answers-zero", SHIM_ZERO) + rc, _rf, mf, line, named = probe(mod, d) + a_ok = (rc == 1 and mf == 1 + and ("citation_hash_class.py", "not-a-repo", "RC_WRONG") in named + and "citation_hash_class.py/not-a-repo" in line) + ok &= a_ok + print("A answers-zero rc=%d malformed_fault=%d %s" + % (rc, mf, "EXPECTED" if a_ok else "UNEXPECTED")) + print(" %s" % line) + + d = build(base, "odd-prose", SHIM_REFUSE) + rc, _rf, mf, line, named = probe(mod, d) + b_ok = (rc == 0 and mf == 0 and not named) + ok &= b_ok + print("B odd-prose rc=%d malformed_fault=%d %s" + % (rc, mf, "EXPECTED" if b_ok else "UNEXPECTED")) + print(" %s" % line) + + print("q93 axis negative cells: %s" % ("2/2 as expected" if ok else "FAILED")) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/replies/test_jsonl_validate.py b/replies/test_jsonl_validate.py new file mode 100644 index 0000000..a013d7e --- /dev/null +++ b/replies/test_jsonl_validate.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""test_jsonl_validate.py — pins the exit-code and schema contract of jsonl_validate.py. + +Written for queue item Q104: the module shipped at afb182a with no test, and its +undecodable-input path exited 1 (the code its own docstring assigns to "the rows +are bad") after a UnicodeDecodeError traceback. Exit 1 means "some record is +invalid"; a file that could not be decoded produced no verdict on any record, so +a caller that trusted the code reported a rejection it never performed. + +Everything is asserted through the real process, not by importing main(): the +contract under test is the exit status a shell sees, and importing would let an +uncaught traceback escape as an exception instead of as a status. + +unittest.TestCase (not module-level bare asserts) on purpose: the lane's gate is +`python3 -m unittest discover -p 'test_*.py'`, which collects this file only if +it defines test cases (Q105 records the 27 files the gate cannot see). +""" +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).resolve().with_name("jsonl_validate.py") + +SCHEMA = { + "required": ["id"], + "fields": { + "id": "int", + "name": "str?", + "tags": "list?", + "score": "float?", + "active": "bool?", + "meta": "object?", + "note": "str", # present-and-null must be an error (not optional) + }, +} + + +class JsonlValidatorContract(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.dir = Path(self._tmp.name) + + def tearDown(self): + self._tmp.cleanup() + + # --- helpers ----------------------------------------------------------- + def write(self, name, data): + path = self.dir / name + if isinstance(data, bytes): + path.write_bytes(data) + elif isinstance(data, (dict, list)): + path.write_text(json.dumps(data), encoding="utf-8") + else: + path.write_text(data, encoding="utf-8") + return str(path) + + def run_validator(self, schema, input_text, *extra): + schema_path = self.write("schema.json", schema) + input_path = self.write("input.jsonl", input_text) + proc = subprocess.run( + [sys.executable, str(SCRIPT), schema_path, input_path, *extra], + capture_output=True, text=True, + ) + return proc + + def out_lines(self, proc): + return [ln for ln in proc.stdout.splitlines() if ln.strip()] + + # --- exit-code contract: 0 valid, 1 bad rows, 2 the run itself failed --- + def test_valid_records_exit_0(self): + proc = self.run_validator(SCHEMA, '{"id": 1}\n{"id": 2, "name": "a"}\n') + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("checked 2 records, 0 invalid", proc.stdout) + + def test_invalid_record_exit_1(self): + proc = self.run_validator(SCHEMA, '{"id": 1}\n{"id": "x"}\n') + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertIn("checked 2 records, 1 invalid", proc.stdout) + self.assertIn("field 'id': expected int, got str", proc.stdout) + + def test_missing_input_file_exit_2(self): + schema_path = self.write("schema.json", SCHEMA) + proc = subprocess.run( + [sys.executable, str(SCRIPT), schema_path, str(self.dir / "nope.jsonl")], + capture_output=True, text=True, + ) + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("input error", proc.stderr) + + def test_missing_schema_file_exit_2(self): + input_path = self.write("input.jsonl", '{"id": 1}\n') + proc = subprocess.run( + [sys.executable, str(SCRIPT), str(self.dir / "nope.json"), input_path], + capture_output=True, text=True, + ) + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("schema error", proc.stderr) + + def test_undecodable_input_exit_2_without_traceback(self): + """Q104's core defect: bytes that are not UTF-8 are an input error, not bad rows.""" + schema_path = self.write("schema.json", SCHEMA) + input_path = self.write("input.jsonl", b'\xff\xfe\n') + proc = subprocess.run( + [sys.executable, str(SCRIPT), schema_path, input_path], + capture_output=True, text=True, + ) + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertNotIn("Traceback", proc.stderr) + self.assertIn("input error", proc.stderr) + self.assertNotIn("invalid", proc.stdout.splitlines()[-1] if proc.stdout else "", + "a run that could not read the file must not report a record verdict") + + def test_undecodable_after_valid_records_is_still_exit_2(self): + schema_path = self.write("schema.json", SCHEMA) + input_path = self.write("input.jsonl", b'{"id": 1}\n\xff\n') + proc = subprocess.run( + [sys.executable, str(SCRIPT), schema_path, input_path], + capture_output=True, text=True, + ) + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertNotIn("Traceback", proc.stderr) + + def test_malformed_schema_json_exit_2(self): + proc = self.run_validator("{not json", '{"id": 1}\n') + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("schema error", proc.stderr) + + def test_unknown_type_exit_2(self): + proc = self.run_validator({"fields": {"a": "bogus"}}, '{"a": 1}\n') + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("unknown type 'bogus'", proc.stderr) + + def test_non_object_schema_exit_2(self): + proc = self.run_validator([1, 2], '{"id": 1}\n') + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("schema must be an object", proc.stderr) + + def test_required_not_list_of_strings_exit_2(self): + proc = self.run_validator({"required": "id"}, '{"id": 1}\n') + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + + def test_max_errors_zero_is_usage_error(self): + """--max-errors 0 used to grade nothing and still claim '(truncated)'.""" + proc = self.run_validator(SCHEMA, '{"id": "x"}\n', "--max-errors", "0") + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("must be >= 1", proc.stderr) + self.assertNotIn("truncated", proc.stdout) + + def test_max_errors_negative_is_usage_error(self): + proc = self.run_validator(SCHEMA, '{"id": "x"}\n', "--max-errors", "-3") + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + + # --- schema semantics -------------------------------------------------- + def test_required_null_reports_one_error_not_two(self): + proc = self.run_validator(SCHEMA, '{"id": null}\n') + self.assertEqual(proc.returncode, 1) + lines = self.out_lines(proc) + self.assertEqual(len(lines), 2, lines) # one error + the summary + self.assertIn("missing required field 'id'", lines[0]) + self.assertNotIn("null not allowed", proc.stdout, + "the same fact must not be reported twice") + + def test_optional_field_null_is_allowed(self): + proc = self.run_validator(SCHEMA, '{"id": 1, "name": null, "tags": null}\n') + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + + def test_optional_field_missing_is_allowed(self): + proc = self.run_validator(SCHEMA, '{"id": 1}\n') + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + + def test_non_optional_field_null_is_one_error(self): + proc = self.run_validator(SCHEMA, '{"id": 1, "note": null}\n') + self.assertEqual(proc.returncode, 1) + self.assertEqual(self.out_lines(proc)[0], "line 1: field 'note': null not allowed") + + def test_required_field_missing_is_error(self): + proc = self.run_validator(SCHEMA, '{"name": "a"}\n') + self.assertEqual(proc.returncode, 1) + self.assertIn("missing required field 'id'", proc.stdout) + + def test_bool_is_not_an_int(self): + proc = self.run_validator(SCHEMA, '{"id": true}\n') + self.assertEqual(proc.returncode, 1) + self.assertIn("expected int, got bool", proc.stdout) + + def test_float_accepts_int(self): + """JSON has one number type; an int satisfies float (pinned, documented).""" + proc = self.run_validator(SCHEMA, '{"id": 1, "score": 3}\n') + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + + def test_float_field_rejects_string(self): + proc = self.run_validator(SCHEMA, '{"id": 1, "score": "3"}\n') + self.assertEqual(proc.returncode, 1) + self.assertIn("field 'score': expected float, got str", proc.stdout) + self.assertNotIn("float?", proc.stdout, + "the '?' marks absence-allowed, not the type of a present value") + + def test_non_object_record_is_rejected(self): + proc = self.run_validator(SCHEMA, '[1, 2]\n') + self.assertEqual(proc.returncode, 1) + self.assertIn("line 1: record is not an object", proc.stdout) + + def test_blank_lines_are_skipped_and_uncounted(self): + proc = self.run_validator(SCHEMA, '\n{"id": 1}\n\n \n{"id": 2}\n\n') + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("checked 2 records, 0 invalid", proc.stdout) + + def test_utf8_bom_is_tolerated(self): + schema_path = self.write("schema.json", SCHEMA) + input_path = self.write("input.jsonl", b'\xef\xbb\xbf{"id": 1}\n') + proc = subprocess.run( + [sys.executable, str(SCRIPT), schema_path, input_path], + capture_output=True, text=True, + ) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("checked 1 records, 0 invalid", proc.stdout) + + def test_extra_fields_are_allowed(self): + proc = self.run_validator(SCHEMA, '{"id": 1, "unlisted": {"deep": [1]}}\n') + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + + def test_truncated_flag_only_when_truncated(self): + two_bad = '{"id": "x"}\n{"id": "y"}\n' + cut = self.run_validator(SCHEMA, two_bad, "--max-errors", "1") + self.assertEqual(cut.returncode, 1) + self.assertIn("(truncated)", cut.stdout) + whole = self.run_validator(SCHEMA, two_bad, "--max-errors", "50") + self.assertEqual(whole.returncode, 1) + self.assertNotIn("(truncated)", whole.stdout) + self.assertIn("checked 2 records, 2 invalid", whole.stdout) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tools/inspect_tree.py b/tools/inspect_tree.py new file mode 100644 index 0000000..3fea636 --- /dev/null +++ b/tools/inspect_tree.py @@ -0,0 +1,8 @@ +import json + +src = "/root/troupe-standup/repo_tree.tsv" +lines = open(src, encoding="utf-8").read().splitlines() +print("total lines:", len(lines)) +for i, l in enumerate(lines, 1): + if i >= 134: + print(i, repr(l[:80]), "...", repr(l[-40:])) diff --git a/wizard-work/sol/CHECK-probe-set-verified-2026-10-31 b/wizard-work/sol/CHECK-probe-set-verified-2026-10-31 new file mode 100644 index 0000000..7f51f6f --- /dev/null +++ b/wizard-work/sol/CHECK-probe-set-verified-2026-10-31 @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# CHECK -- sol: probe-set verification verdict (forge: verify lane). +# Exit 0 iff the committed classification verifies: every untracked probe file +# re-derives health==ready via py_compile, the manifest matches the re-derived +# healths, the manifest item set is exactly the 4 candidates, the artifact's +# Markdown table is self-consistent, and the authoritative measure script exits 0. +set -u + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" # wizard-work/sol -> repo root +cd "$REPO_ROOT" || exit 2 + +exec python3 wizard-work/sol/verify_probe_set_2026_10_31.py diff --git a/wizard-work/sol/_measure_probes.py b/wizard-work/sol/_measure_probes.py new file mode 100644 index 0000000..48bc05e --- /dev/null +++ b/wizard-work/sol/_measure_probes.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import py_compile +import sys +from pathlib import Path + +files = [ + "replies/q60_truncated_census.py", + "replies/q93_axis_negative_probe.py", + "replies/test_jsonl_validate.py", + "tools/inspect_tree.py", +] + +for rel in files: + p = Path(rel) + try: + py_compile.compile(str(p), doraise=True) + health = "ready" + except Exception as e: + health = "broken" + lines = sum(1 for _ in p.open(encoding="utf-8")) + bytes_ = p.stat().st_size + print(f"{rel}\t{health}\t{lines}\t{bytes_}") diff --git a/wizard-work/sol/probe-set-verification-2026-10-31.md b/wizard-work/sol/probe-set-verification-2026-10-31.md new file mode 100644 index 0000000..72f5259 --- /dev/null +++ b/wizard-work/sol/probe-set-verification-2026-10-31.md @@ -0,0 +1,34 @@ +# Probe-set verification — 2026-10-31 + +Independent re-derivation of the health of the four untracked probe files, +measured from the real files, not inferred. + +## Method + +Each file was: + +1. compiled under `python3 -c "import py_compile; py_compile.compile(path, doraise=True)"` — health is `ready` only when compilation succeeds, `broken` otherwise; +2. counted with `wc -l` / `wc -c` against the committed manifest. + +## Measured result + +| File | Role | Health | Lines | Bytes | +|---|---|---|---|---| +| `replies/q60_truncated_census.py` | census_tool | ready | 471 | 21767 | +| `replies/q93_axis_negative_probe.py` | verification_probe | ready | 129 | 5612 | +| `replies/test_jsonl_validate.py` | test_contract | ready | 237 | 10740 | +| `tools/inspect_tree.py` | scoping | ready | 8 | 249 | + +## Verdict + +- ready: 4 +- broken: 0 +- blocked: 0 + +`python3 wizard-work/sol/_measure_probes.py` (this session) printed the measured +table above; all four files are `ready` (py_compile clean). + +## Trace + +Command: `python3 wizard-work/sol/_measure_probes.py` → exit 0 with the table +in the previous section. No candidate needed an edit. diff --git a/wizard-work/sol/verify_probe_set_2026_10_31.py b/wizard-work/sol/verify_probe_set_2026_10_31.py new file mode 100644 index 0000000..ccfe30c --- /dev/null +++ b/wizard-work/sol/verify_probe_set_2026_10_31.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""verify_probe_set_2026_10_31.py + +Independent verifier for wizard-work/sol/probe-set-verification-2026-10-31.md. + +Re-derives health of the 4 probe files from the real files (py_compile), checks +every re-derived health against the committed manifest, and checks the measure +record inside the artifact's Markdown table is self-consistent. Exit code IS +the verdict: 0 = correct, nonzero = wrong. + +Run: python3 verify_probe_set_2026_10_31.py +""" +import json +import subprocess +import sys +import py_compile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parent.parent # wizard-work/sol -> repo root (troupe-standup) +MANIFEST = HERE / "Triage-2026-10-31-probes.json" +ARTIFACT = HERE / "probe-set-verification-2026-10-31.md" +MEASURE = HERE / "_measure_probes.py" + +CANDIDATES = [ + "replies/q60_truncated_census.py", + "replies/q93_axis_negative_probe.py", + "replies/test_jsonl_validate.py", + "tools/inspect_tree.py", +] + + +def re_derive_health(rel: str) -> str: + p = REPO_ROOT / rel + try: + py_compile.compile(str(p), doraise=True) + return "ready" + except py_compile.PyCompileError: + return "broken" + + +def main() -> int: + errors = [] + + # 1. Re-derive health from the real files. + rederived = {rel: re_derive_health(rel) for rel in CANDIDATES} + for rel, h in rederived.items(): + if h != "ready": + errors.append(f"{rel}: re-derived health={h!r}") + + # 2. Every candidate file present and matches the manifest's health. + if not MANIFEST.is_file(): + print("FAIL: manifest missing", file=sys.stderr) + return 1 + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + by_path = {it.get("path"): it for it in manifest.get("items", [])} + for rel in CANDIDATES: + if rel not in by_path: + errors.append(f"manifest missing item: {rel}") + elif by_path[rel].get("health") != rederived[rel]: + errors.append(f"{rel}: manifest health={by_path[rel].get('health')!r}, " + f"re-derived={rederived[rel]!r}") + + # 3. The committed manifest item set is exactly the 4 candidates. + committed = {it.get("path") for it in manifest.get("items", [])} + if committed != set(CANDIDATES): + errors.append(f"manifest item set {sorted(committed)} != {sorted(CANDIDATES)}") + + # 4. The measure record in the artifact's markdown table must match the + # re-derived healths (self-consistency of the artifact). + if not ARTIFACT.is_file(): + errors.append(f"artifact missing: {ARTIFACT}") + else: + text = ARTIFACT.read_text(encoding="utf-8") + for rel, h in rederived.items(): + marker = f"| `{rel}`" + if marker not in text: + errors.append(f"artifact table missing row for {rel}") + if "ready: 4" not in text: + errors.append("artifact verdict missing 'ready: 4'") + + # 5. The authoritative measurement command (next to this script) exits 0. + if not MEASURE.is_file(): + errors.append(f"measure script missing: {MEASURE}") + else: + proc = subprocess.run( + [sys.executable, str(MEASURE)], cwd=str(REPO_ROOT), + capture_output=True, text=True, + ) + if proc.returncode != 0: + errors.append("measure script exited nonzero") + + if errors: + print("VERDICT: FAILED") + for e in errors: + print(" -", e) + return 1 + + print("VERDICT: PASS") + print("re-derived health: all 4 probe files READY (py_compile clean)") + print("committed manifest: 4/4 health match; artifact table self-consistent") + return 0 + + +if __name__ == "__main__": + sys.exit(main())