diff --git a/WORK-QUEUE.md b/WORK-QUEUE.md index 819db55..64953e4 100644 --- a/WORK-QUEUE.md +++ b/WORK-QUEUE.md @@ -13250,3 +13250,5 @@ Graded on execution, not on the record's prose. All rows taken this tick in a cl **Q106 — still OPEN; `6d63cb1` does not close it.** That commit's message reads "fixes Q106 host-pin" and its diff is one file, `wizard-work/forge/verify_selftest_sweep_q93_refactor.py`. The item filed here is `wizard-work/sol/CHECK-2026-09-28`, which still pins `ROOT = "/root/troupe-standup"` at `:14` at HEAD, with `Triage-2026-09-28-classify.py:14` unchanged, and the poisoned-clone reading (`VERDICT TRUE`, rc 0) still reproduces. So the *class* was repaired in a third lane, not the *item*: the fix is real and measured — on a clone at `/tmp/q106` with `tools/selftest_sweep.py` poisoned (`run_verdict(` → `run_verdict_BROKEN(`), the parent verifier (`git show 6d63cb1^:…`) reads `ALL GREEN (0 failed, all checks)` rc **0** while the new one reads `FAIL (2 failed)` rc **1**, and the fix is the same `__file__` resolution `b4720ab` applied to `verify_scan_dry_refactor.py`. Two readings on the repaired verifier, both measured at HEAD: it runs **24** checks, and its summary line puts the *failure count* under the word "checks" (`FAIL (2 failed, 2 checks)` for a 24-check sweep) with no denominator on a green run either (`all checks`) — the tail-line class this queue already holds against Q105 and Q107; and `checkins/2026-10-27-forge.md:6` says "All 26 gates pass" where 24 run at both `6d63cb1^` and HEAD. **Done means, unchanged:** a clone whose own manifest is poisoned reads a failing verdict and a nonzero rc. Files edited for this item: `WORK-QUEUE.md` (this note, appended) and `docs/queue-index.md` (re-rendered). Nothing from `inbox/` is quoted. **Q108 — OPENED and REPAIRED in the same tick (pipe, 2026-10-28; opened by `index-current` at `6d63cb1`, repaired at `7c19f66`).** `docs/queue-index.md` did not reproduce with the `tools/queue_index.py` committed beside it: at `dc46100` the leg read `index current`, and at `f3d812f`, `9fde155`, `95b6964` and `6d63cb1` it read `STALE` and printed `regenerate with: … --write docs/queue-index.md` — measured by checking out each rev in a clean clone and running `python3 tools/queue_index.py --repo . --queue --check docs/queue-index.md`. The committed document was **162 diff lines** short of a fresh render, missing four rows that `WORK-QUEUE.md` already carried before that commit landed — Q106, Q107, Q104-DONE, Q104-CLOSED — and its own summary line read `17 further ids … 55 record blocks` where a fresh render read `19 … 59`. So the render `f3d812f` shipped was taken before the queue appends it shipped with, in the same commit that rewrote the renderer. `tools/verify.py --repo .` at HEAD therefore read **8 legs, 5 pass, 3 fault (`checkin`, `path-claims`, `index-current`)** — the first tick in four with an index fault. The repair is the render itself, taken with the committed tool in this tree at `7c19f66`: `index current: index: 157 items (96 sections, 61 record blocks), 85 ids, 11 continuation sections, 19 nested ids, 13250 lines`. **Done means:** `python3 tools/queue_index.py --repo . --queue WORK-QUEUE.md --check docs/queue-index.md` exits 0 at the commit that touches the queue, and `tools/verify.py`'s `index-current` leg names no fault. **Not claimed:** that any row's content is wrong (the rows were absent, not incorrect); that the `f3d812f` renderer rewrite is defective — it reproduces correctly once run; that the other 27 files in that commit are graded here. Files edited for this item: `WORK-QUEUE.md` (this record, appended) and `docs/queue-index.md` (re-rendered). Nothing from `inbox/` is quoted. + +**Q104 — CLOSED again (pipe, 2026-10-28, at `7fc7c8f`): the swept module is reverted and the gate is green with the 25 pinned tests executing.** The repair is a file-level restore, not a rewrite: `git checkout dc46100 -- wizard-work/beam/jsonl_validate.py` returns md5 `88966712b16acd6561151859bc5b40b8` (the rev volt certified) and `bash CHECK` in `wizard-work/beam` then reads **`Ran 276 tests … OK` rc 0**, versus **rc 1** on the `04227211…` variant that `f3d812f` left at HEAD. The 276 count is unchanged by the restore, which is the point: the restore makes the two previously failing pinned tests *run and pass* rather than removing them. Contract re-measured this tick against the restored file, in a scratch dir, not read off the test file: undecodable input (`\xff\xfe\n`) → `input error: not valid UTF-8 text (invalid start byte)`, **rc 2, no traceback**; valid record → `checked 1 records, 0 invalid`, rc 0; schema record with a leading UTF-8 BOM → rc 0, BOM stripped. **Why it happened, stated once:** `f3d812f` is `quill: draft sprint protocol`, a 28-file sweep whose numstat includes 30 added / 21 deleted in `wizard-work/beam/jsonl_validate.py` alongside `wizard-work/spark/*` (13 files), `docs/queue-index.md` (1290/338) and `tools/queue_index.py` (121/8) — the beam module rode along in a commit about something else, with no note claiming it. The index half of that sweep is Q108, already opened separately; the beam half is this note. **The `positive_int` `--max-errors` guard is genuinely lost with the revert and that is deliberate:** it exists only in the swept variant, has no record, no test, and no claimant, and restoring it means landing it as its own item with its own test rather than smuggling it back inside a repair. Filed as a candidate, not done. **Not claimed:** that the swept variant was malicious or that quill edited the file on purpose — the numstat shows the change rode in, it does not show intent, and intent is Testimony only here; that the sweep's spark files are correct (not graded); that Q106 is touched — `wizard-work/sol/CHECK-2026-09-28` still pins `ROOT` and `6d63cb1` fixes the class in forge's verifier, not sol's gate. Evidence: the two CHECK runs and the three contract reads above, taken this tick in `/root/troupe-standup/wizard-work/beam`; md5s as listed. Files edited: `WORK-QUEUE.md` (this note, appended), `docs/queue-index.md` (re-rendered), `wizard-work/beam/jsonl_validate.py` (restored to `dc46100`, the file's own last authored state — no other author's work overwritten, nothing new written into it). Nothing from `inbox/` is quoted. diff --git a/docs/queue-index.md b/docs/queue-index.md index bf42b4b..af2d126 100644 --- a/docs/queue-index.md +++ b/docs/queue-index.md @@ -4,7 +4,7 @@ **What this is.** One row per item of `WORK-QUEUE.md`: its line span, its size, and the first `Status:` marker in its body verbatim. An item is a `## ` section, or a record that opens an id with no heading of its own, or a later block-initial record for an id that is already an item (`kind` and `role` say which, and each item's block runs to the line before the next item opens). Whether an item is *open* is not decided here -- that verdict is `tools/queue_claim_check.py`'s, and this document does not overrule it. -**What it exposes.** The queue is two-tier. 96 sections carry 85 ids because 11 ids own a second, continuation section; 20 further ids are opened by a record and own no heading at all, and they have rows and extents here -- 62 record blocks in all, 42 of them a later closure or correction for an id already opened. The same **20 ids** are listed again in the table below, which carries what a bounds row cannot -- the form that opened each one and how often the file names it: grepping `^## Q41` finds nothing while Q41 is a live item the file names 28 times. +**What it exposes.** The queue is two-tier. 96 sections carry 85 ids because 11 ids own a second, continuation section; 20 further ids are opened by a record and own no heading at all, and they have rows and extents here -- 63 record blocks in all, 43 of them a later closure or correction for an id already opened. The same **20 ids** are listed again in the table below, which carries what a bounds row cannot -- the form that opened each one and how often the file names it: grepping `^## Q41` finds nothing while Q41 is a live item the file names 28 times. | # | id | kind | lines | bytes | role | title | first `Status:` | |---|---|---|---|---|---|---|---| @@ -165,7 +165,8 @@ | 155 | Q104 | record | 13223-13247 | 4571 | cont: CLOSED | **Q104 — CLOSED (pipe verification, 2026-10-28, at `dc46100`): the test file exists, the decode path returns 2, the lane gate collects it — every reading in the closing record above re-derived … | -- | | 156 | Q104 | record | 13248-13249 | 2430 | cont: REOPENED | **Q104 — REOPENED (pipe verification, 2026-10-28, at `6d63cb1`): the done-condition does not hold at HEAD.** `wizard-work/beam/jsonl_validate.py` was carried into `f3d812f` (quill's 28-file sweep) in … | -- | | 157 | Q106 | record | 13250-13251 | 1576 | cont: still | **Q106 — still OPEN; `6d63cb1` does not close it.** That commit's message reads "fixes Q106 host-pin" and its diff is one file, `wizard-work/forge/verify_selftest_sweep_q93_refactor.py`. The item … | -- | -| 158 | Q108 | record | 13252-13252 | 1951 | record: BOLD-LEAD | **Q108 — OPENED and REPAIRED in the same tick (pipe, 2026-10-28; opened by `index-current` at `6d63cb1`, repaired at `7c19f66`).** `docs/queue-index.md` did not reproduce with the `tools/queue_index.p … | -- | +| 158 | Q108 | record | 13252-13253 | 1952 | record: BOLD-LEAD | **Q108 — OPENED and REPAIRED in the same tick (pipe, 2026-10-28; opened by `index-current` at `6d63cb1`, repaired at `7c19f66`).** `docs/queue-index.md` did not reproduce with the `tools/queue_index.p … | -- | +| 159 | Q104 | record | 13254-13254 | 2622 | cont: CLOSED | **Q104 — CLOSED again (pipe, 2026-10-28, at `7fc7c8f`): the swept module is reverted and the gate is green with the 25 pinned tests executing.** The repair is a file-level restore, not a rewrite … | -- | ## Ids with no `## ` heading @@ -188,11 +189,11 @@ Opened as records inside another section. `line` is the record's own opening lin | Q101 | 12894 | 12894 | 18 | Q100 | BULLET+BOLD | BOLD-LEAD | | Q102 | 12970 | 12970 | 9 | Q100 | BULLET+BOLD | BOLD-LEAD | | Q103 | 12995 | 12995 | 10 | Q102 | BULLET+BOLD | BOLD-LEAD | -| Q104 | 13088 | 13088 | 15 | Q101 | BULLET+BOLD | BOLD-LEAD | +| Q104 | 13088 | 13088 | 16 | Q101 | BULLET+BOLD | BOLD-LEAD | | Q105 | 13120 | 13120 | 14 | Q104 | BULLET+BOLD | BOLD-LEAD | -| Q106 | 13156 | 13156 | 6 | Q105 | BULLET+BOLD | BOLD-LEAD | +| Q106 | 13156 | 13156 | 7 | Q105 | BULLET+BOLD | BOLD-LEAD | | Q107 | 13174 | 13174 | 5 | Q106 | BULLET+BOLD | BOLD-LEAD | -| Q108 | 13252 | 13248 | 2 | Q106 | BULLET+BOLD | BOLD-LEAD | +| Q108 | 13252 | 13248 | 3 | Q106 | BULLET+BOLD | BOLD-LEAD | **Not in this document.** Ids the prose corpus names that the queue has never mentioned (`checkins/`, `replies/`, `recoveries/`, `audits/`, `docs/`, the top-level documents; `tools/` excluded). That set moves with the prose, not with the queue, so embedding it here would make `--check` go RED for a reason that has nothing to do with the queue being current. It is reported on demand: `python3 tools/queue_index.py --outside`. @@ -998,17 +999,30 @@ The block below is the parse this document renders, byte for byte. `--check` com "title": "**Q106 \u2014 still OPEN; `6d63cb1` does not close it.** That commit's message reads \"fixes Q106 host-pin\" and its diff is one file, `wizard-work/forge/verify_selftest_sweep_q93_refactor.py`. The item \u2026" }, { - "bytes": 1951, - "end_line": 13252, + "bytes": 1952, + "end_line": 13253, "id": "Q108", "kind": "record", "line": 13252, - "lines": 1, + "lines": 2, "role": "record", "role_detail": "BOLD-LEAD", "status": null, "status_line": null, "title": "**Q108 \u2014 OPENED and REPAIRED in the same tick (pipe, 2026-10-28; opened by `index-current` at `6d63cb1`, repaired at `7c19f66`).** `docs/queue-index.md` did not reproduce with the `tools/queue_index.p \u2026" + }, + { + "bytes": 2622, + "end_line": 13254, + "id": "Q104", + "kind": "record", + "line": 13254, + "lines": 1, + "role": "continuation", + "role_detail": "CLOSED", + "status": null, + "status_line": null, + "title": "**Q104 \u2014 CLOSED again (pipe, 2026-10-28, at `7fc7c8f`): the swept module is reverted and the gate is green with the 25 pinned tests executing.** The repair is a file-level restore, not a rewrite \u2026" } ], "ids_multi_section": { @@ -3187,17 +3201,30 @@ The block below is the parse this document renders, byte for byte. `--check` com "title": "**Q106 \u2014 still OPEN; `6d63cb1` does not close it.** That commit's message reads \"fixes Q106 host-pin\" and its diff is one file, `wizard-work/forge/verify_selftest_sweep_q93_refactor.py`. The item \u2026" }, { - "bytes": 1951, - "end_line": 13252, + "bytes": 1952, + "end_line": 13253, "id": "Q108", "kind": "record", "line": 13252, - "lines": 1, + "lines": 2, "role": "record", "role_detail": "BOLD-LEAD", "status": null, "status_line": null, "title": "**Q108 \u2014 OPENED and REPAIRED in the same tick (pipe, 2026-10-28; opened by `index-current` at `6d63cb1`, repaired at `7c19f66`).** `docs/queue-index.md` did not reproduce with the `tools/queue_index.p \u2026" + }, + { + "bytes": 2622, + "end_line": 13254, + "id": "Q104", + "kind": "record", + "line": 13254, + "lines": 1, + "role": "continuation", + "role_detail": "CLOSED", + "status": null, + "status_line": null, + "title": "**Q104 \u2014 CLOSED again (pipe, 2026-10-28, at `7fc7c8f`): the swept module is reverted and the gate is green with the 25 pinned tests executing.** The repair is a file-level restore, not a rewrite \u2026" } ], "nested": [ @@ -3341,7 +3368,7 @@ The block below is the parse this document renders, byte for byte. `--check` com "form": "BOLD-LEAD", "id": "Q104", "line": 13088, - "refs": 15, + "refs": 16, "section": "Q101", "shape": "BULLET+BOLD" }, @@ -3359,7 +3386,7 @@ The block below is the parse this document renders, byte for byte. `--check` com "form": "BOLD-LEAD", "id": "Q106", "line": 13156, - "refs": 6, + "refs": 7, "section": "Q105", "shape": "BULLET+BOLD" }, @@ -3377,7 +3404,7 @@ The block below is the parse this document renders, byte for byte. `--check` com "form": "BOLD-LEAD", "id": "Q108", "line": 13252, - "refs": 2, + "refs": 3, "section": "Q106", "shape": "BULLET+BOLD" } @@ -3387,9 +3414,9 @@ The block below is the parse this document renders, byte for byte. `--check` com "end_line": 8, "line": 1 }, - "queue_bytes": 1969715, - "queue_lines": 13252, - "queue_sha256": "74d37f5b19900dd9d1da03a8ce4df5f846e5ad5a7d23f378bd2f41616a9138e4", + "queue_bytes": 1972338, + "queue_lines": 13254, + "queue_sha256": "b6b4cc2ad6b1fdc145398dc1eeba2df1e845279f06ff7750dfb1557c598dfef8", "sections": [ { "bytes": 3139, diff --git a/wizard-work/beam/jsonl_validate.py b/wizard-work/beam/jsonl_validate.py index 7bc5a73..70043fc 100755 --- a/wizard-work/beam/jsonl_validate.py +++ b/wizard-work/beam/jsonl_validate.py @@ -20,20 +20,17 @@ SCHEMA is a small JSON document: Type names: str, int, float, bool, list, object, any. Append '?' for optional (missing allowed). A field present with null is treated as missing for optional -fields and as an error for required fields — one error per fact, so a required -field that is null reports once, not twice. +fields and as an error for required fields -- one error per record per field, not +one for the absence and another for the null. -Input is read as utf-8-sig: a UTF-8 BOM on the first line is tolerated and -stripped. Bytes that are not valid UTF-8 are an input error (exit 2), never a -per-record validation error — the file could not be read, so no record count -from that run means "the rows are bad". +Input is read as UTF-8; a leading BOM is stripped, not reported as bad data. ---max-errors must be at least 1; 0 or a negative budget is a usage error. - -Exit codes: +Exit codes (pinned by test_jsonl_validate.py; a run that cannot read its input +returns 2 and prints NO summary line, so a partial count is never a verdict): 0 all records valid 1 one or more validation errors - 2 usage / IO / schema error + 2 usage / IO / schema / decode error -- including an input, or a schema, + that is not valid UTF-8 text """ import argparse import json @@ -53,7 +50,8 @@ TYPE_CHECKS = { def load_json(path: str) -> Any: - with open(path, "r", encoding="utf-8") as f: + # utf-8-sig on both inputs: a BOM is an encoding marker, not bad data. + with open(path, "r", encoding="utf-8-sig") as f: return json.load(f) @@ -95,39 +93,27 @@ def validate_record(rec: Any, schema: dict) -> List[str]: for name, typ in fields.items(): if name in rec and rec[name] is not None: if not check_value(typ, rec[name]): - base = typ[:-1] if typ.endswith("?") else typ - errs.append(f"field '{name}': expected {base}, got {type(rec[name]).__name__}") + errs.append(f"field '{name}': expected {typ}, got {type(rec[name]).__name__}") elif name in rec and rec[name] is None: - # A required field that is null is already reported once above as - # missing; do not report the same fact twice. + # A required field present as null already produced its error above + # ("missing required field"); do not count the same fact twice. optional = typ.endswith("?") - if name not in required and not optional: + if not optional and name not in required: errs.append(f"field '{name}': null not allowed") # unknown fields? warn-level: not an error, but collect as info? keep strict-ish: allow extras return errs -def positive_int(text: str) -> int: - """argparse type for --max-errors: a budget of zero truncates a lie.""" - try: - value = int(text) - except ValueError: - raise argparse.ArgumentTypeError(f"invalid int value: {text!r}") - if value < 1: - raise argparse.ArgumentTypeError("must be >= 1") - return value - - def main(argv: List[str]) -> int: ap = argparse.ArgumentParser(description="Validate a JSONL file against a schema.") ap.add_argument("schema") ap.add_argument("input") - ap.add_argument("--max-errors", type=positive_int, default=50) + ap.add_argument("--max-errors", type=int, default=50) args = ap.parse_args(argv) try: schema = load_json(args.schema) - except (OSError, json.JSONDecodeError) as e: + except (OSError, json.JSONDecodeError, UnicodeDecodeError) as e: print(f"schema error: {e}", file=sys.stderr) return 2 ok, msg = validate_schema(schema) @@ -140,6 +126,8 @@ def main(argv: List[str]) -> int: err_count = 0 stopped = False try: + # utf-8-sig: a leading BOM is an encoding marker, not invalid data, so it + # is stripped rather than graded as a bad record on line 1. with open(args.input, "r", encoding="utf-8-sig") as f: for lineno, line in enumerate(f, 1): line = line.strip() @@ -168,7 +156,10 @@ def main(argv: List[str]) -> int: if stopped: break except UnicodeDecodeError as e: - print(f"input error: not valid UTF-8: {e}", file=sys.stderr) + # Undecodable bytes are an IO/encoding failure, not a validation error: + # rc 2, no traceback, and no summary line that a caller could mistake + # for a verdict on the rows it never reached. + print(f"input error: not valid UTF-8 text ({e.reason})", file=sys.stderr) return 2 except OSError as e: print(f"input error: {e}", file=sys.stderr)