diff --git a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/README.md b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/README.md
index 9a37c37..32975b4 100644
--- a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/README.md
+++ b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/README.md
@@ -1,20 +1,24 @@
# archive-lcmm-token-ledger-2026-10-26
-A machine-verified ledger of long-context multimodal papers: for each of 14 papers, the context
+A machine-verified ledger of long-context multimodal papers: for each of 15 papers, the context
length the paper itself states in its own abstract, copied verbatim, with two pinned and
sha256-hashed arXiv sources per entry.
-- `ledger.json` -- the ledger (14 entries, each with verbatim claim sentence, figure, token count,
+- `ledger.json` -- the ledger (15 entries, each with verbatim claim sentence, figure, token count,
role, metadata, source hashes)
+- `roles-expected.json` -- the expected `claim_role` per entry, with the verbatim phrase from that
+ entry's own sentence that justifies it; `verify_ledger.py` grades the ledger against it
- `REPORT.md` -- the digest; `scan-candidates.json` -- the full >=32K scan with exclusions
- `source/atom-lcmm.xml`, `source/abs/*.html`, `source/scan-probe-2026.json` -- pinned bytes
-- `verify_ledger.py` (offline), `mutation_battery.py` (13 mutations), `run_verdict.py` (live)
+- `verify_ledger.py` (offline, 183 checks), `mutation_battery.py` (mutation battery),
+ `run_verdict.py` (live)
- `manifest.json` -- sha256 of every file
Reproduce:
```
-python3 verify_ledger.py # exit 0 = ledger holds
-python3 mutation_battery.py # exit 0 = 13/13 mutations caught
-python3 run_verdict.py # exit 0 = live arXiv matches pinned bytes (needs network)
+python3 verify_ledger.py # exit 0 = ledger holds, 1 = a check failed
+python3 mutation_battery.py # exit 0 = every mutation caught (and the control held)
+python3 run_verdict.py # exit 0 = live arXiv matches pinned bytes (needs network),
+ # exit 1 = bytes differ, exit 2 = could not grade (endpoint down)
```
diff --git a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/manifest.json b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/manifest.json
index de2f62a..3f51a03 100644
--- a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/manifest.json
+++ b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/manifest.json
@@ -1,13 +1,14 @@
{
"EVIDENCE-runs.txt": "0174c4f5f3e718a371ddf33388eb579c0c1321e9f953cc86941fa5ac20ebfd15",
- "README.md": "ff76c1a71ab1363c4868aacea1e75a4a9bbd96db7f8ab2df0a9e93ce9f057ab4",
+ "README.md": "6b85987907facf24dfdfdb61d474fa39ef30251b43fae59acc82fae370ad340c",
"REPORT.md": "643e5b8b3b59ed06d8bb4a63be25f057ebcb2452984a4af852de09fbb1cc6bf3",
"build_ledger.py": "3dc27281779f69e13813087f3996861f2bb7e41a75decce62038e1a720c9e4f5",
"build_report.py": "5617f4e5fb0691452016e8e6d9b083546caa871043148556bd446055f4ab12a7",
"finalize_manifest.py": "393262c20d639ed921953bc3ee30c809df881b3ef275626085130394970139dc",
"ledger.json": "8fdac60921a221ccc07441150f372fa614bbed3bbd547f43bd4d71328fd54634",
- "mutation_battery.py": "e653666ecfc54ea5bdc0d6a6431709ba16cebaa00700c684c1649824af0a225a",
- "run_verdict.py": "a425b3b5b532e82d78847126b401edd2ef3cbc61edfc43ea260f749ac9bf61e3",
+ "mutation_battery.py": "aadd5f49a8f5ff96756c5aa8336b21b937f8b45c7b1c31482ec9aea5653ce2ca",
+ "roles-expected.json": "892231f358cf67757e87cad484bbbbd15182a756c8ac9d6cd21d9cada829e49c",
+ "run_verdict.py": "d667fccbc1ddc01789e00901aae3f6f46873a59c55023c7997b2ebaa9c0b8283",
"scan-candidates.json": "93b6d61bf2880cb8ffa13a278df24e1ca53ee717b3b74ba59cacb28e8cf4ca2b",
"source/abs/2410.07073v2.html": "2320392f1558da6947c2c614e96373eb00e8adefff9a70386f6d989e2d43c290",
"source/abs/2412.09616v2.html": "e37052780bba835e97c451bddf1a6979724bcd211ce30e69e43e4cb762ca14d2",
@@ -26,5 +27,5 @@
"source/abs/2610.15542v1.html": "ac722cac8c75f77867b01f3f2f4ffe0f4e8d85401f0111f1db01377a79817c57",
"source/atom-lcmm.xml": "7e2add7e0d3eb44c19b17cc4ff36225c0e26643cf9185ba2a421e08f1a6b9942",
"source/scan-probe-2026.json": "de026197689120e8d68fb7203ba78df3e5c59b314bbc5d5af14f08e75f7c0fd2",
- "verify_ledger.py": "3a24e3f171989d6dc3e747b05c3600e5d3bf99573e066e82620a190122a5387e"
+ "verify_ledger.py": "ba0e05efb8483e53f32162e432a2e5551cd252b787ffba81742c5f1a1c71ee5f"
}
diff --git a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/mutation_battery.py b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/mutation_battery.py
index 39c3d53..27d4543 100644
--- a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/mutation_battery.py
+++ b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/mutation_battery.py
@@ -4,8 +4,20 @@
Copies the ledger to a scratch dir, applies each mutation, runs verify_ledger.py
as a subprocess and asserts a non-zero exit. Also asserts the pristine copy
passes. Exit 0 = every mutation was caught (and the control held).
+
+Two things changed after queue item Q72:
+ * the scratch tree is taken from --scratch, else $TMPDIR, else the system
+ temp dir, and is removed on the way out (--keep leaves it). The first
+ version hard-coded an absolute path under the author's own home, so on a
+ host without that directory mkdtemp raised FileNotFoundError before grading
+ anything, and it left its trees behind where it did run;
+ * a mutation can name the check that must catch it. Without that, a
+ manifest-hash failure (any edit to ledger.json invalidates manifest.json)
+ masks whether the leg under test is awake at all -- every mutation would
+ look caught for the wrong reason.
"""
import json
+import os
import shutil
import subprocess
import sys
@@ -28,20 +40,25 @@ def run_verify(root):
p = subprocess.run([sys.executable, str(ROOT / "verify_ledger.py"), str(root)],
capture_output=True, text=True)
verdict = [l for l in p.stdout.splitlines() if l.startswith("VERDICT:")]
- return p.returncode, (verdict[-1] if verdict else "?")
+ failed = [l.strip()[7:].split(" -- ")[0] for l in p.stdout.splitlines() if l.strip().startswith("[FAIL]")]
+ return p.returncode, (verdict[-1] if verdict else "?"), failed
+
+
+def save_roles(root, R):
+ (root / "roles-expected.json").write_text(json.dumps(R, indent=1, ensure_ascii=False) + "\n")
MUTATIONS = []
-def mutation(name):
+def mutation(name, expect=None):
def deco(fn):
- MUTATIONS.append((name, fn))
+ MUTATIONS.append((name, fn, expect))
return fn
return deco
-@mutation("M1 context_tokens inflated by 1")
+@mutation("M1 context_tokens inflated by 1", "E:")
def m1(root):
L = load(root)
e = next(x for x in L["entries"] if x["claim_role"] == "model_context_window")
@@ -49,14 +66,15 @@ def m1(root):
save(root, L)
-@mutation("M2 title word altered")
+@mutation("M2 title word altered", "F:")
def m2(root):
L = load(root)
- L["entries"][0]["title"] = L["entries"][0]["title"].replace(" ", " ", 1)
+ t = L["entries"][0]["title"]
+ L["entries"][0]["title"] = t.replace(t.split(" ")[0], t.split(" ")[0] + "x", 1)
save(root, L)
-@mutation("M3 claim sentence edited (token -> token!)")
+@mutation("M3 claim sentence edited (token -> token!)", "D:")
def m3(root):
L = load(root)
e = L["entries"][0]
@@ -64,7 +82,7 @@ def m3(root):
save(root, L)
-@mutation("M4 pinned source sha256 flipped")
+@mutation("M4 pinned source sha256 flipped", "C:")
def m4(root):
L = load(root)
h = L["entries"][0]["sources"][0]["sha256"]
@@ -72,21 +90,21 @@ def m4(root):
save(root, L)
-@mutation("M5 published date shifted")
+@mutation("M5 published date shifted", "F:")
def m5(root):
L = load(root)
L["entries"][0]["published"] = "1999-01-01T00:00:00Z"
save(root, L)
-@mutation("M6 primary_category swapped")
+@mutation("M6 primary_category swapped", "F:")
def m6(root):
L = load(root)
L["entries"][0]["primary_category"] = "cs.XX"
save(root, L)
-@mutation("M7 entry dropped (n_entries kept in sync)")
+@mutation("M7 entry dropped (n_entries kept in sync)", "P6")
def m7(root):
L = load(root)
L["entries"] = L["entries"][:-1]
@@ -94,14 +112,14 @@ def m7(root):
save(root, L)
-@mutation("M8 bogus claim_role")
+@mutation("M8 bogus claim_role", "B3")
def m8(root):
L = load(root)
L["entries"][0]["claim_role"] = "magic_number"
save(root, L)
-@mutation("M9 REPORT.md citation removed")
+@mutation("M9 REPORT.md citation removed", "H1")
def m9(root):
rep = root / "REPORT.md"
L = load(root)
@@ -109,7 +127,7 @@ def m9(root):
rep.write_text(rep.read_text().replace(aid, "0000.00000"))
-@mutation("M10 pinned abs page bytes tampered")
+@mutation("M10 pinned abs page bytes tampered", "C:")
def m10(root):
L = load(root)
p = root / L["entries"][0]["sources"][1]["path"]
@@ -118,7 +136,7 @@ def m10(root):
p.write_bytes(bytes(b))
-@mutation("M11 manifest entry deleted")
+@mutation("M11 manifest entry deleted", "I1")
def m11(root):
m = json.loads((root / "manifest.json").read_text())
k = next(iter(m))
@@ -126,7 +144,7 @@ def m11(root):
(root / "manifest.json").write_text(json.dumps(m, indent=1) + "\n")
-@mutation("M12 manifest hash stale")
+@mutation("M12 manifest hash stale", "I2")
def m12(root):
m = json.loads((root / "manifest.json").read_text())
k = next(iter(m))
@@ -134,13 +152,77 @@ def m12(root):
(root / "manifest.json").write_text(json.dumps(m, indent=1) + "\n")
-@mutation("M13 claim_note emptied (honesty field)")
+@mutation("M13 claim_note emptied (honesty field)", "B6")
def m13(root):
L = load(root)
L["entries"][0]["claim_note"] = ""
save(root, L)
+@mutation("M14 three entries dropped, n_entries + manifest recomputed (the Q72 mutation)", "P6")
+def m14(root):
+ drop = {"2608.08630v1", "2610.15542v1", "2607.14548v1"}
+ L = load(root)
+ L["entries"] = [e for e in L["entries"] if e["arxiv_id"] not in drop]
+ L["n_entries"] = len(L["entries"])
+ L["api_query"] = "https://export.arxiv.org/api/query?id_list=" + ",".join(e["arxiv_id"] for e in L["entries"])
+ save(root, L)
+ rep = root / "REPORT.md"
+ rep.write_text("\n".join(l for l in rep.read_text().splitlines() if not any(d in l for d in drop)) + "\n")
+ m = json.loads((root / "manifest.json").read_text())
+ m["ledger.json"] = __import__("hashlib").sha256((root / "ledger.json").read_bytes()).hexdigest()
+ m["REPORT.md"] = __import__("hashlib").sha256((root / "REPORT.md").read_bytes()).hexdigest()
+ (root / "manifest.json").write_text(json.dumps(m, indent=1) + "\n")
+
+
+@mutation("M15 claim_role relabelled to model_context_window on 2507.02001v1 (the Q72 mutation)", "B8")
+def m15(root):
+ L = load(root)
+ e = next(x for x in L["entries"] if x["arxiv_id"] == "2507.02001v1")
+ e["claim_role"] = "model_context_window"
+ e["claim_note"] = "Model context window is 32K tokens."
+ save(root, L)
+
+
+@mutation("M16 excluded candidate's reason blanked", "P7")
+def m16(root):
+ s = json.loads((root / "scan-candidates.json").read_text())
+ ex = next(c for c in s["candidates"] if c["decision"] == "excluded")
+ ex["reason"] = " "
+ (root / "scan-candidates.json").write_text(json.dumps(s, indent=1, ensure_ascii=False) + "\n")
+
+
+@mutation("M17 README paper count one lower", "J1")
+def m17(root):
+ p = root / "README.md"
+ p.write_text(p.read_text().replace("for each of 15 papers", "for each of 14 papers"))
+
+
+@mutation("M18 ledger entry invented outside the scan", "P5")
+def m18(root):
+ L = load(root)
+ e = json.loads(json.dumps(L["entries"][0]))
+ e["arxiv_id"] = "2699.99999v1"
+ L["entries"].append(e)
+ L["n_entries"] = len(L["entries"])
+ save(root, L)
+
+
+@mutation("M19 expected-role table disagrees with the ledger", "B8")
+def m19(root):
+ R = json.loads((root / "roles-expected.json").read_text())
+ R["roles"]["2410.07073v2"]["claim_role"] = "benchmark_input_lengths"
+ save_roles(root, R)
+
+
+@mutation("M20 candidate dropped from the scan (population shrink)", "P4")
+def m20(root):
+ s = json.loads((root / "scan-candidates.json").read_text())
+ s["candidates"] = s["candidates"][:-1]
+ s["n_candidates"] = len(s["candidates"])
+ (root / "scan-candidates.json").write_text(json.dumps(s, indent=1, ensure_ascii=False) + "\n")
+
+
def fresh_copy(dest):
if dest.exists():
shutil.rmtree(dest)
@@ -149,28 +231,40 @@ def fresh_copy(dest):
def main():
- tmp = Path(tempfile.mkdtemp(prefix="lcmm-mut-", dir="/root/.hermes/profiles/sphinx/cache/scratch"))
- print(f"scratch: {tmp}")
- control = fresh_copy(tmp / "control")
- rc, verdict = run_verify(control)
- print(f" [{'PASS' if rc == 0 else 'FAIL'}] control (pristine copy must verify) -> rc={rc} {verdict}")
- if rc != 0:
- print("control FAILED; aborting battery")
- return 1
+ keep = "--keep" in sys.argv
+ scratch = os.environ.get("LCMM_SCRATCH") or tempfile.gettempdir()
+ for a in sys.argv[1:]:
+ if a.startswith("--scratch="):
+ scratch = a.split("=", 1)[1]
+ Path(scratch).mkdir(parents=True, exist_ok=True)
+ tmp = Path(tempfile.mkdtemp(prefix="lcmm-mut-", dir=scratch))
+ print(f"scratch: {tmp} (removed on exit unless --keep)")
+ try:
+ control = fresh_copy(tmp / "control")
+ rc, verdict, failed = run_verify(control)
+ print(f" [{'PASS' if rc == 0 else 'FAIL'}] control (pristine copy must verify) -> rc={rc} {verdict}")
+ if rc != 0:
+ print("control FAILED; aborting battery")
+ return 1
- caught = 0
- for name, fn in MUTATIONS:
- d = fresh_copy(tmp / name.replace(" ", "_"))
- fn(d)
- rc, verdict = run_verify(d)
- ok = rc != 0
- caught += ok
- print(f" [{'PASS' if ok else 'FAIL'}] {name} -> rc={rc} {verdict}")
+ caught = 0
+ for name, fn, expect in MUTATIONS:
+ d = fresh_copy(tmp / name.split(" ")[0])
+ fn(d)
+ rc, verdict, failed = run_verify(d)
+ hit = [f for f in failed if expect and f.startswith(expect)]
+ ok = rc != 0 and (not expect or bool(hit))
+ caught += ok
+ why = "" if ok else f" <- wanted a {expect} failure, saw {failed[:4]}"
+ print(f" [{'PASS' if ok else 'FAIL'}] {name} -> rc={rc} {verdict}{why}")
- total = len(MUTATIONS)
- print(f"\n{caught}/{total} mutations caught")
- print("VERDICT:", "FALSIFIABLE" if caught == total else "NOT FALSIFIABLE")
- return 0 if caught == total else 1
+ total = len(MUTATIONS)
+ print(f"\n{caught}/{total} mutations caught")
+ print("VERDICT:", "FALSIFIABLE" if caught == total else "NOT FALSIFIABLE")
+ return 0 if caught == total else 1
+ finally:
+ if not keep:
+ shutil.rmtree(tmp, ignore_errors=True)
if __name__ == "__main__":
diff --git a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/roles-expected.json b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/roles-expected.json
new file mode 100644
index 0000000..6bc1214
--- /dev/null
+++ b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/roles-expected.json
@@ -0,0 +1,81 @@
+{
+ "schema": "sphinx.lcmm-roles/1",
+ "note": "Expected claim_role per ledger entry, with the verbatim phrase from that entry's own claim sentence that justifies it. verify_ledger.py grades ledger.json against this table AND grades this table's evidence phrase against the sentence, so a relabel has to edit two files and cannot drop the quotation. The order of fields is fixed so the diff is readable.",
+ "roles": {
+ "2410.07073v2": {
+ "claim_role": "model_context_window",
+ "evidence_phrase": "long context window of 128K tokens",
+ "why": "the introduced model (Pixtral-12B) processes images in its own window"
+ },
+ "2412.09616v2": {
+ "claim_role": "scaling_claim",
+ "evidence_phrase": "increased to 256K tokens, the model is capable of processing multimodal sequences up to 1M tokens",
+ "why": "a train-length -> capability claim: 256K trained, 1M reported as reach"
+ },
+ "2502.05177v3": {
+ "claim_role": "model_context_window",
+ "evidence_phrase": "over 4K frames or 1M tokens",
+ "why": "the introduced model's own processing span (Long-VITA)"
+ },
+ "2504.16083v2": {
+ "claim_role": "evaluated_input_length",
+ "evidence_phrase": "at 1M tokens while maintaining accuracy",
+ "why": "the largest input the authors measure acceleration at, not a model window"
+ },
+ "2505.10610v3": {
+ "claim_role": "benchmark_input_lengths",
+ "evidence_phrase": "five standardized input lengths (8K-128K tokens)",
+ "why": "top of a benchmark's standardized input ladder (MMLongBench)"
+ },
+ "2507.01006v6": {
+ "claim_role": "model_context_window",
+ "evidence_phrase": "a 128K context window",
+ "why": "the introduced GLM-4.6V series' own window"
+ },
+ "2507.02001v1": {
+ "claim_role": "method_context_window",
+ "evidence_phrase": "our approach using a context window of 32K",
+ "why": "the authors' method uses 32K; the 700K in the same sentence is the baseline VLM's window"
+ },
+ "2510.13276v2": {
+ "claim_role": "benchmark_input_lengths",
+ "evidence_phrase": "context lengths scaled from 16K to 128K tokens",
+ "why": "the benchmark's context-length span (MMLongCite)"
+ },
+ "2511.21631v2": {
+ "claim_role": "model_context_window",
+ "evidence_phrase": "contexts of up to 256K tokens",
+ "why": "the introduced model family's native window (Qwen3-VL)"
+ },
+ "2602.00746v1": {
+ "claim_role": "evaluated_input_length",
+ "evidence_phrase": "at 1M tokens",
+ "why": "the input length at which compression latency is measured"
+ },
+ "2604.08644v1": {
+ "claim_role": "model_context_window",
+ "evidence_phrase": "extends context length up to 256K tokens",
+ "why": "the introduced model's (EXAONE 4.5) own context length"
+ },
+ "2605.14906v1": {
+ "claim_role": "benchmark_input_lengths",
+ "evidence_phrase": "four standard context lengths (32K-256K tokens)",
+ "why": "top of the benchmark's context-length ladder (MemLens)"
+ },
+ "2608.08630v1": {
+ "claim_role": "training_length",
+ "evidence_phrase": "training up to 120K tokens",
+ "why": "largest training length; the same sentence reports inference beyond 280K"
+ },
+ "2610.15542v1": {
+ "claim_role": "model_context_window",
+ "evidence_phrase": "extends the context window fourfold to 262,144 tokens",
+ "why": "the introduced models' (Apertus 1.5) own window"
+ },
+ "2607.14548v1": {
+ "claim_role": "model_context_window",
+ "evidence_phrase": "a 32K context window",
+ "why": "the base model's (Hy3.0-VL-A3B) own window"
+ }
+ }
+}
diff --git a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/run_verdict.py b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/run_verdict.py
index 8cd137b..340a4d2 100644
--- a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/run_verdict.py
+++ b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/run_verdict.py
@@ -3,13 +3,23 @@
Re-fetches the Atom feed for the ledger's ids and every abstract page, compares
sha256 with the pinned bytes, and re-checks every claim sentence against the
-live abstract. Needs network. Exit 0 = live arXiv == pinned ledger.
+live abstract. Needs network.
+
+Usage: python3 run_verdict.py [ledger_dir] [--api URL] [--abs URL] [--timeout SECS]
+
+Exit 0 = live arXiv == pinned ledger.
+Exit 1 = live bytes differ from the ledger (a real drift, with the entries named).
+Exit 2 = could not grade: the endpoint refused, timed out, or served a body that
+ is not an Atom feed. An outage must not read as a byte mismatch, which
+ is what this script did before queue item Q72 -- every failure, network
+ or content, came back as rc 1.
"""
import hashlib
import json
import re
import sys
import time
+import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
@@ -20,15 +30,62 @@ API = "https://export.arxiv.org/api/query"
ABS = "https://export.arxiv.org/abs/"
UA = "sphinx-archive-lane/1.0 (live verdict)"
-ROOT = Path(sys.argv[1] if len(sys.argv) > 1 else Path(__file__).resolve().parent).resolve()
+_args = sys.argv[1:]
+_timeout = 90
+for _a in _args:
+ if _a.startswith("--api="):
+ API = _a.split("=", 1)[1]
+ elif _a.startswith("--abs="):
+ ABS = _a.split("=", 1)[1]
+ elif _a.startswith("--timeout="):
+ _timeout = float(_a.split("=", 1)[1])
+_pos = [a for a in _args if not a.startswith("--")]
+ROOT = Path(_pos[0] if _pos else Path(__file__).resolve().parent).resolve()
L = json.loads((ROOT / "ledger.json").read_text())
entries = L["entries"]
+class Ungradable(Exception):
+ """The endpoint could not be read; nothing about the ledger was learned."""
+
+
+# arXiv stamps every Atom response with the time of the request:
+# 2026-10-26T00:27:06Z (feed level, first )
+# Two consecutive identical requests an hour apart differed in exactly that
+# element and in nothing else (same 70,326 B, all 15 entries field-equal,
+# first byte difference at offset 545 inside the element). A byte pin on the
+# raw feed therefore fails on the second request by construction, so the live
+# leg masks that one element in BOTH bodies and then demands byte equality of
+# everything else -- entry order, ids, titles, summaries, dates, categories.
+FEED_UPDATED = re.compile(rb"()[^<]*()")
+
+
+def mask_feed_updated(b):
+ return FEED_UPDATED.sub(rb"\1\2", b, count=1)
+
+
def fetch(url):
req = urllib.request.Request(url, headers={"User-Agent": UA})
- with urllib.request.urlopen(req, timeout=90) as r:
- return r.read()
+ try:
+ with urllib.request.urlopen(req, timeout=_timeout) as r:
+ body = r.read()
+ except urllib.error.HTTPError as ex:
+ raise Ungradable(f"HTTP {ex.code} {ex.reason} from {url}")
+ except urllib.error.URLError as ex:
+ raise Ungradable(f"unreachable {url}: {ex.reason}")
+ except Exception as ex: # timeout, reset, TLS
+ raise Ungradable(f"{type(ex).__name__} reading {url}: {ex}")
+ if not body:
+ raise Ungradable(f"empty body from {url}")
+ return body
+
+
+def atom_root(body, url):
+ try:
+ root = ET.fromstring(body)
+ except ET.ParseError as ex:
+ raise Ungradable(f"body from {url} is not XML ({ex}); {len(body)} B, starts {body[:40]!r}")
+ return root
def norm(s):
@@ -36,20 +93,42 @@ def norm(s):
def main():
- fails = []
+ fails = [] # rc 1: the live bytes disagree with the ledger
+ ungraded = [] # rc 2: we could not see the live bytes at all
ids = ",".join(e["arxiv_id"] for e in entries)
- live_atom = fetch(API + "?" + urllib.parse.urlencode({"id_list": ids, "start": 0, "max_results": len(entries) + 5}))
- live_sha = hashlib.sha256(live_atom).hexdigest()
- pinned_sha = next(s["sha256"] for s in entries[0]["sources"] if s["kind"] == "arxiv_api_atom")
- ok = live_sha == pinned_sha
- print(f" [{'PASS' if ok else 'FAIL'}] live Atom bytes identical to pinned ({live_sha[:12]} vs {pinned_sha[:12]})")
+ try:
+ live_atom = fetch(API + "?" + urllib.parse.urlencode({"id_list": ids, "start": 0, "max_results": len(entries) + 5}))
+ root = atom_root(live_atom, API)
+ except Ungradable as ex:
+ print(f" [UNGRADED] {ex}")
+ print("VERDICT: UNGRADED -- live arXiv could not be read; nothing claimed about the ledger")
+ return 2
+ pinned_src = next(s for s in entries[0]["sources"] if s["kind"] == "arxiv_api_atom")
+ try:
+ pinned_atom = (ROOT / pinned_src["path"]).read_bytes()
+ except OSError as ex:
+ print(f" [UNGRADED] pinned Atom unreadable: {ex}")
+ print("VERDICT: UNGRADED -- pinned bytes missing; nothing to compare against")
+ return 2
+ raw_live, raw_pin = hashlib.sha256(live_atom).hexdigest(), hashlib.sha256(pinned_atom).hexdigest()
+ ok = hashlib.sha256(mask_feed_updated(live_atom)).hexdigest() == hashlib.sha256(mask_feed_updated(pinned_atom)).hexdigest()
+ print(f" [{'PASS' if ok else 'FAIL'}] live Atom content identical to pinned, feed masked "
+ f"({raw_live[:12]} vs {raw_pin[:12]})")
+ if raw_live != raw_pin and ok:
+ print(" [note] raw bytes differ only in the feed-level stamp arXiv sets per request")
if not ok:
- fails.append("atom bytes differ")
+ fails.append("atom content differs")
+ a, b = mask_feed_updated(live_atom), mask_feed_updated(pinned_atom)
+ i = next((k for k in range(min(len(a), len(b))) if a[k] != b[k]), min(len(a), len(b)))
+ print(f" [note] first masked difference at offset {i}: live={a[i:i + 60]!r} pinned={b[i:i + 60]!r}")
- root = ET.fromstring(live_atom)
live = {}
for el in root.findall("a:entry", NS):
live[el.find("a:id", NS).text.strip().rsplit("/", 1)[-1]] = el
+ if not live:
+ print(f" [UNGRADED] live feed parsed but carries 0 entries for {len(entries)} ids")
+ print("VERDICT: UNGRADED -- empty live feed")
+ return 2
n_claim = n_abs = n_ver = 0
for e in entries:
@@ -64,7 +143,12 @@ def main():
else:
print(f" [FAIL] {aid} claim sentence not in live abstract")
fails.append(f"{aid} claim")
- b = fetch(ABS + aid)
+ try:
+ b = fetch(ABS + aid)
+ except Ungradable as ex:
+ print(f" [UNGRADED] {aid} abs page: {ex}")
+ ungraded.append(f"{aid} abs unreadable")
+ continue
if hashlib.sha256(b).hexdigest() == next(s["sha256"] for s in e["sources"] if s["kind"] == "arxiv_abs_page"):
n_abs += 1
else:
@@ -73,23 +157,35 @@ def main():
# newest version of the unversioned id
base = aid.split("v")[0]
try:
- u = ET.fromstring(fetch(API + "?" + urllib.parse.urlencode({"id_list": base, "start": 0, "max_results": 1})))
- newest = u.find("a:entry", NS).find("a:id", NS).text.strip().rsplit("/", 1)[-1]
- if newest != aid:
- print(f" [note] {aid} is not the newest version; arXiv now has {newest}")
- except Exception as ex:
+ u = atom_root(fetch(API + "?" + urllib.parse.urlencode({"id_list": base, "start": 0, "max_results": 1})), API)
+ got = u.findall("a:entry", NS)
+ if len(got) != 1:
+ print(f" [note] version probe for {base} returned {len(got)} entries; not graded")
+ else:
+ gid = got[0].find("a:id", NS)
+ newest = gid.text.strip().rsplit("/", 1)[-1] if gid is not None and gid.text else ""
+ if newest and newest != aid:
+ print(f" [note] {aid} is not the newest version; arXiv now has {newest}")
+ except (Ungradable, AttributeError) as ex:
print(f" [note] version probe failed for {base}: {ex}")
time.sleep(1)
n = len(entries)
print(f" claims present live: {n_claim}/{n}")
print(f" abs pages identical live: {n_abs}/{n}")
+ print(f" abs pages unreadable: {len(ungraded)}")
print(f" [{'PASS' if n_claim == n else 'FAIL'}] all claims present in live abstracts")
- print(f" [{'PASS' if n_abs == n else 'FAIL'}] all abs pages byte-identical live")
- if n_claim != n or n_abs != n:
- fails.append("live content mismatch")
- print("VERDICT:", "LIVE ARXIV MATCHES LEDGER" if not fails else f"LIVE DRIFT: {fails}")
- return 0 if not fails else 1
+ print(f" [{'PASS' if n_abs == n and not ungraded else 'FAIL'}] all abs pages byte-identical live")
+ if n_claim != n:
+ fails.append("claims missing live")
+ if fails:
+ print("VERDICT:", f"LIVE DRIFT: {fails}")
+ return 1
+ if ungraded:
+ print("VERDICT:", f"UNGRADED: {ungraded} -- drift not observed, so not claimed")
+ return 2
+ print("VERDICT:", "LIVE ARXIV MATCHES LEDGER")
+ return 0
if __name__ == "__main__":
diff --git a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/verify_ledger.py b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/verify_ledger.py
index c56c7da..0cfb299 100644
--- a/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/verify_ledger.py
+++ b/wizard-work/sphinx/archive-lcmm-token-ledger-2026-10-26/verify_ledger.py
@@ -4,6 +4,24 @@
Re-derives every claim from the bytes pinned next to this script. No network.
Usage: python3 verify_ledger.py [ledger_dir] (default: this file's directory)
Exit 0 = every check holds; exit 1 = at least one check failed.
+
+Three legs were added after the first release (queue item Q72), because the
+verifier as shipped graded each entry but not the ledger's population or the
+reading of its role field:
+
+ P* the chain 327 scanned -> 18 candidates -> 15 kept is re-derived from the
+ pinned probe; an entry dropped from ledger.json, invented outside the
+ candidate set, or an excluded candidate stripped of its reason is a fault.
+ B7-B10 claim_role is graded against a committed per-id table
+ (roles-expected.json) whose evidence phrase must appear verbatim in the
+ entry's own claim sentence, plus a role->marker rule that reads the
+ sentence; a relabel has to edit two files and still keep the quotation.
+ J1 the README's own "N papers" / "N entries" claims are graded against the
+ ledger, so the document cannot drift from the artifact it describes.
+
+Every check prints PASS/FAIL; the verdict is FAIL if any check fails. A leg
+that cannot read its input fails rather than being skipped, so a deleted file
+is never silent.
"""
import hashlib
import html as htmlmod
@@ -22,6 +40,23 @@ ROLES = {
"method_context_window",
"scaling_claim",
}
+# What each role's own claim sentence must say, and what a model-window entry
+# must not say. Marker rules are graded against the quoted sentence, not
+# against the role name, so a relabel has to survive contact with the text.
+ROLE_MARKERS = {
+ "model_context_window": r"(context window|context length|contexts? of up to|process\w+[^.]{0,160}tokens)",
+ "method_context_window": r"(our (method|approach|framework|system)|we propose)",
+ "benchmark_input_lengths": r"(benchmark|standard(?:ized)?\s+(?:input|context)\s+lengths?|context lengths?\s+(?:scaled|of)|examples across|questions)",
+ "evaluated_input_length": r"(experiment|evaluat|measur|latency|compar|accelerat)",
+ "training_length": r"\btrain(?:ing|ed)?\b",
+ "scaling_claim": r"(increas|scal|generaliz|capable of|potential)",
+}
+# A sentence that says how a *method* or a *benchmark* was run must not be
+# filed as the model's own window: this is the distinction REPORT.md section 1
+# promises ("a benchmark's longest input is never reported as a model's
+# context window"), and it is the one a relabel of 2507.02001v1 breaks.
+MODEL_WINDOW_MUST_NOT = r"(benchmark|standard(?:ized)?\s+(?:input|context)\s+lengths?|context lengths?\s+scaled|our (method|approach|framework)|we propose|dataset is increased)"
+SCAN_THRESHOLD = 32000
ID_RX = re.compile(r"^\d{4}\.\d{4,5}v\d+$")
ROOT = Path(sys.argv[1] if len(sys.argv) > 1 else Path(__file__).resolve().parent).resolve()
@@ -49,6 +84,24 @@ def text_of_html(b):
return norm(htmlmod.unescape(s))
+def loads(path):
+ """Parse JSON, or return None. A missing/broken file is a graded fault."""
+ try:
+ return json.loads(Path(path).read_text())
+ except Exception:
+ return None
+
+
+def tokens_in(fig):
+ """The last length figure in a string, in tokens; -1 if there is none."""
+ ms = list(re.finditer(r"(\d[\d,]*(?:\.\d+)?)\s*([kKmM])\b", fig))
+ if ms:
+ m = ms[-1]
+ return int(float(m.group(1).replace(",", "")) * (1e3 if m.group(2).lower() == "k" else 1e6))
+ mb = list(re.finditer(r"(\d[\d,]*(?:\.\d+)?)\s*tokens?\b", fig, re.I))
+ return int(float(mb[-1].group(1).replace(",", ""))) if mb else -1
+
+
def main():
print(f"verifying ledger at {ROOT}")
ledger_p = ROOT / "ledger.json"
@@ -65,6 +118,47 @@ def main():
check("A6 REPORT.md present", (ROOT / "REPORT.md").exists())
ids = [e["arxiv_id"] for e in entries]
+ id_set = set(ids)
+
+ # P: the population, re-derived. 327 scanned -> 18 candidates -> 15 kept.
+ scan = loads(ROOT / "scan-candidates.json")
+ check("P1 scan-candidates.json present and parses", scan is not None)
+ cands = (scan or {}).get("candidates", [])
+ check("P2 n_candidates matches candidate list",
+ (scan or {}).get("n_candidates") == len(cands) and bool(str((scan or {}).get("rule", "")).strip()),
+ f"{len(cands)} rows, rule={str((scan or {}).get('rule', ''))[:40]!r}")
+ probe = loads(ROOT / "source" / "scan-probe-2026.json")
+ check("P3 the pinned scan probe parses", isinstance(probe, dict) and bool(probe),
+ f"{len(probe) if isinstance(probe, dict) else 0} records")
+ derived = {}
+ for k, v in (probe or {}).items():
+ if isinstance(v, dict) and v.get("ctx"):
+ n = tokens_in(v["ctx"])
+ if n >= SCAN_THRESHOLD:
+ derived[k] = n
+ cand_ids = {c.get("arxiv_id") for c in cands}
+ check("P4 candidates re-derive from the probe at >= 32K tokens",
+ bool(probe) and set(derived) == cand_ids,
+ f"{len(probe or {})} scanned, {len(derived)} >= {SCAN_THRESHOLD}, {len(cand_ids)} candidates; "
+ f"derived-only={sorted(set(derived) - cand_ids)[:4]} candidates-only={sorted(cand_ids - set(derived))[:4]}")
+ check("P5 every ledger id is an included candidate (nothing invented)",
+ id_set <= cand_ids, f"outside the scan: {sorted(id_set - cand_ids)}")
+ included = {c.get("arxiv_id") for c in cands if c.get("decision") == "included"}
+ check("P6 every included candidate is in the ledger (nothing dropped)",
+ included <= id_set, f"missing from ledger: {sorted(included - id_set)}")
+ excluded = [c for c in cands if c.get("decision") == "excluded"]
+ check("P7 every excluded candidate carries a non-empty reason",
+ all(str(c.get("reason", "")).strip() for c in excluded),
+ f"{sum(1 for c in excluded if not str(c.get('reason', '')).strip())} of {len(excluded)} blank")
+ check("P8 excluded candidates are absent from the ledger",
+ not (id_set & {c.get("arxiv_id") for c in excluded}),
+ f"in both: {sorted(id_set & {c.get('arxiv_id') for c in excluded})}")
+ api_ids = set()
+ if "id_list=" in str(L.get("api_query", "")):
+ api_ids = {i.strip() for i in L["api_query"].split("id_list=", 1)[1].split("&")[0].split(",") if i.strip()}
+ check("P9 api_query id_list matches the ledger ids", api_ids == id_set,
+ f"{len(api_ids)} queried vs {len(id_set)} kept; only-query={sorted(api_ids - id_set)} only-ledger={sorted(id_set - api_ids)}")
+
check("B1 ids unique", len(ids) == len(set(ids)))
check("B2 ids canonical (NNNN.NNNNNvN)", all(ID_RX.match(i) for i in ids))
check("B3 roles from allowed set", all(e["claim_role"] in ROLES for e in entries))
@@ -73,6 +167,34 @@ def main():
check("B5 >=5 model_context_window entries", sum(1 for e in entries if e["claim_role"] == "model_context_window") >= 5)
check("B6 claim_note non-empty everywhere", all(str(e.get("claim_note", "")).strip() for e in entries))
+ # B7-B10: claim_role graded against a second, committed file and the sentence.
+ roles = loads(ROOT / "roles-expected.json")
+ table = (roles or {}).get("roles", {})
+ check("B7 roles-expected.json present and covers exactly the ledger ids",
+ roles is not None and set(table) == id_set,
+ f"unexpected={sorted(set(table) - id_set)} missing={sorted(id_set - set(table))}")
+ bad_role = [i for i in ids if (table.get(i) or {}).get("claim_role") != next(e["claim_role"] for e in entries if e["arxiv_id"] == i)]
+ check("B8 claim_role matches the expected-role table", not bad_role,
+ "; ".join(f"{i}: ledger={next(e['claim_role'] for e in entries if e['arxiv_id'] == i)} table={(table.get(i) or {}).get('claim_role')}" for i in bad_role))
+ bad_ev = [i for i in ids
+ if norm((table.get(i) or {}).get("evidence_phrase", "")) and
+ norm((table.get(i) or {}).get("evidence_phrase", "")) not in
+ norm(next(e["claim_sentence"] for e in entries if e["arxiv_id"] == i))]
+ check("B9 table evidence phrase is verbatim in the claim sentence", not bad_ev, f"not quoted: {bad_ev}")
+ marker_bad = []
+ for e in entries:
+ role, sent = e["claim_role"], norm(e["claim_sentence"])
+ why = []
+ if role not in ROLE_MARKERS:
+ why.append("no marker rule for role")
+ elif not re.search(ROLE_MARKERS[role], sent, re.I):
+ why.append(f"sentence has no {role} marker")
+ if role == "model_context_window" and re.search(MODEL_WINDOW_MUST_NOT, sent, re.I):
+ why.append("sentence is about a method/benchmark, not the model's own window")
+ if why:
+ marker_bad.append(f"{e['arxiv_id']}[{'/'.join(why)}]")
+ check("B10 role agrees with the sentence-level marker rule", not marker_bad, "; ".join(marker_bad))
+
atom_p = ROOT / "source" / "atom-lcmm.xml"
check("C1 pinned Atom XML present", atom_p.exists())
if not atom_p.exists():
@@ -108,14 +230,8 @@ def main():
fig = e["context_figure"]
ok_fig = fig in e["claim_sentence"]
check(f"E:{aid} figure inside claim sentence", ok_fig)
- ms = list(re.finditer(r"(\d[\d,]*(?:\.\d+)?)\s*([kKmM])\b", fig))
- if ms:
- m = ms[-1]
- v = float(m.group(1).replace(",", "")) * (1e3 if m.group(2).lower() == "k" else 1e6)
- else:
- mb = list(re.finditer(r"(\d[\d,]*(?:\.\d+)?)\s*tokens?\b", fig, re.I))
- v = float(mb[-1].group(1).replace(",", "")) if mb else -1
- check(f"E:{aid} parsed figure == context_tokens", int(v) == e["context_tokens"], f"{int(v)} vs {e['context_tokens']}")
+ v = tokens_in(fig)
+ check(f"E:{aid} parsed figure == context_tokens", v == e["context_tokens"], f"{v} vs {e['context_tokens']}")
check(f"E:{aid} context >= 32000 tokens (32K)", e["context_tokens"] >= 32000, str(e["context_tokens"]))
# F: metadata re-derived from the same pinned bytes
check(f"F:{aid} title matches Atom", norm(el.find("a:title", NS).text) == norm(e["title"]))
@@ -158,6 +274,12 @@ def main():
bad = [f for f, h in manifest.items() if sha(ROOT / f) != h]
check("I2 manifest hashes match bytes", not bad, f"mismatch {bad}")
+ # J: the README describes this ledger, not an earlier one
+ readme_p = ROOT / "README.md"
+ counts = sorted({int(m.group(1)) for m in re.finditer(r"(\d+)\s+(?:papers|entries)\b", readme_p.read_text())}) if readme_p.exists() else []
+ check("J1 README paper/entry counts match the ledger", counts == [len(entries)],
+ f"README says {counts or 'nothing'}, ledger has {len(entries)}")
+
return finish()