diff --git a/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/CHECK.sh b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/CHECK.sh old mode 100755 new mode 100644 index bbfafcf..2127fa4 --- a/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/CHECK.sh +++ b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/CHECK.sh @@ -1,9 +1,12 @@ #!/usr/bin/env bash # CHECK gate for the long-context multimodal dossier (sphinx, Archive lane). # -# Resolves its own directory so it grades the tree it lives in. Re-runs the -# live verification: re-fetches the arXiv API and re-measures every recorded -# field, then demands that three mutated copies each go RED. +# Resolves its own directory so it grades the tree it lives in, then demands: +# 1. verify_lcmm_dossier.py exits 0 with mode: LIVE (live API evidence, not +# a replay of the archived XML -- CACHED is treated as a failure here) +# 2. render_report_table.py --check exits 0 (the report table is reproducible +# from dossier.json, i.e. it cannot silently drift) +# 3. the report names both the builder and the verifier # # Exit 0 = the dossier holds. Exit non-zero = it does not. set -u @@ -17,21 +20,34 @@ echo "== long-context multimodal dossier gate ==" echo "root: $HERE" echo -if python3 verify_lcmm_dossier.py; then - echo "verify: PASS" -else - rc=$? +out="$(python3 verify_lcmm_dossier.py 2>&1)" +rc=$? +printf '%s\n' "$out" +if [ "$rc" -ne 0 ]; then echo "verify: FAIL (rc=$rc)" fail=1 -fi -echo - -# The report must name both halves, so the note cannot drift from the gate. -if grep -q 'verify_lcmm_dossier.py' REPORT-lcmm-dossier.md \ - && grep -q 'build_lcmm_dossier.py' REPORT-lcmm-dossier.md; then - echo "report cross-references build + verify: ok" +elif printf '%s\n' "$out" | grep -q '^mode: LIVE'; then + echo "verify: PASS (mode LIVE)" else - echo "report cross-references build + verify: FAIL" + echo "verify: FAIL -- no live evidence (mode was not LIVE)" + fail=1 +fi + +echo +if python3 render_report_table.py --check; then + echo "report table reproducible from dossier.json: ok" +else + echo "report table reproducible from dossier.json: FAIL" + fail=1 +fi + +echo +if grep -q 'verify_lcmm_dossier.py' REPORT-lcmm-dossier.md \ + && grep -q 'build_lcmm_dossier.py' REPORT-lcmm-dossier.md \ + && grep -q 'render_report_table.py' REPORT-lcmm-dossier.md; then + echo "report cross-references build + verify + render: ok" +else + echo "report cross-references build + verify + render: FAIL" fail=1 fi diff --git a/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/EVIDENCE-runs.txt b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/EVIDENCE-runs.txt new file mode 100644 index 0000000..0fa71e1 --- /dev/null +++ b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/EVIDENCE-runs.txt @@ -0,0 +1,158 @@ +# Evidence runs -- captured 2026-10-25 (sphinx, Archive lane) + +## 1. live run: bash CHECK.sh -> exit 0 + +== long-context multimodal dossier gate == +root: /root/.hermes/wizard-outputs/sphinx/archive-lcmm-metadata-2026-10-25 + +verify_lcmm_dossier.py -- long-context multimodal dossier (sphinx) v2 +root: /root/.hermes/wizard-outputs/sphinx/archive-lcmm-metadata-2026-10-25 + +A. manifest integrity (re-measured from disk + recomputed from dossier) + ok sha256 dossier.json == manifest :: d77990948e812083 + ok bytes dossier.json == manifest + ok sha256 source/atom_batch.xml == manifest :: d3150650da24e2ec + ok dossier.source_atom_sha256 == manifest + ok manifest per-paper hashes/fields recomputable from dossier.json :: 0 problem(s) + +B. live arXiv API re-fetch (this is the evidence, not a replay) + mode: LIVE -- compared against the arXiv API fetched at check time + ok every id resolved :: 11/11 + +C. dossier vs LIVE truth + ok all papers match on title/version/published/authors/abstract/categories :: 0 mismatch(es) + +D. scope + report-rendering checks + ok paper_count >= 10 :: 11 + ok every abstract non-empty + ok every primary_category allowed :: ['cs.CL', 'cs.CV', 'cs.IR'] + ok every paper has a lane note + ok REPORT paper table == dossier.json (id/version/cat/authors/date/title) :: 0 drift(s) + +E. controls (negative ones MUST go red, the positive one MUST stay green) + ok P0 an unmodified copy stays green :: 0 failure(s) + ok F1 a flipped paper title is caught by field compare :: 1 failure(s) + ok F2 a dropped paper is caught by field compare :: 1 failure(s) + ok F3 an edited abstract is caught by field compare :: 2 failure(s) + ok F4 a bumped version string is caught by field compare :: 1 failure(s) + ok F5 a lying paper_count is caught by manifest recompute :: 1 failure(s) + ok F6 an altered manifest hash is caught by manifest recompute :: 1 failure(s) + ok F7 an altered report row is caught by report table check :: 1 failure(s) + ok all 8 controls ran :: 8 control(s) + +mode: LIVE checks run: 21 failures: 0 +VERDICT: dossier holds +verify: PASS (mode LIVE) + +report table: in sync with dossier.json (11 rows) +report table reproducible from dossier.json: ok + +report cross-references build + verify + render: ok + +CHECK: GREEN -- dossier holds + +## 2. fallback branch: LCMM_ARXIV_API=http://127.0.0.1:9/api/query bash CHECK.sh -> exit 1 (CACHED is red by design) + +== long-context multimodal dossier gate == +root: /root/.hermes/wizard-outputs/sphinx/archive-lcmm-metadata-2026-10-25 + +verify_lcmm_dossier.py -- long-context multimodal dossier (sphinx) v2 +root: /root/.hermes/wizard-outputs/sphinx/archive-lcmm-metadata-2026-10-25 + +A. manifest integrity (re-measured from disk + recomputed from dossier) + ok sha256 dossier.json == manifest :: d77990948e812083 + ok bytes dossier.json == manifest + ok sha256 source/atom_batch.xml == manifest :: d3150650da24e2ec + ok dossier.source_atom_sha256 == manifest + ok manifest per-paper hashes/fields recomputable from dossier.json :: 0 problem(s) + +B. live arXiv API re-fetch (this is the evidence, not a replay) + (live fetch unavailable after 4 attempts/host: RuntimeError("URLError(ConnectionRefusedError(111, 'Connection refused'))")) + mode: CACHED -- compared against archived source/atom_batch.xml + ok every id resolved :: 11/11 + +C. dossier vs CACHED truth + ok all papers match on title/version/published/authors/abstract/categories :: 0 mismatch(es) + +D. scope + report-rendering checks + ok paper_count >= 10 :: 11 + ok every abstract non-empty + ok every primary_category allowed :: ['cs.CL', 'cs.CV', 'cs.IR'] + ok every paper has a lane note + ok REPORT paper table == dossier.json (id/version/cat/authors/date/title) :: 0 drift(s) + +E. controls (negative ones MUST go red, the positive one MUST stay green) + ok P0 an unmodified copy stays green :: 0 failure(s) + ok F1 a flipped paper title is caught by field compare :: 1 failure(s) + ok F2 a dropped paper is caught by field compare :: 1 failure(s) + ok F3 an edited abstract is caught by field compare :: 2 failure(s) + ok F4 a bumped version string is caught by field compare :: 1 failure(s) + ok F5 a lying paper_count is caught by manifest recompute :: 1 failure(s) + ok F6 an altered manifest hash is caught by manifest recompute :: 1 failure(s) + ok F7 an altered report row is caught by report table check :: 1 failure(s) + ok all 8 controls ran :: 8 control(s) + +mode: CACHED checks run: 21 failures: 0 +VERDICT: dossier holds +verify: FAIL -- no live evidence (mode was not LIVE) + +report table: in sync with dossier.json (11 rows) +report table reproducible from dossier.json: ok + +report cross-references build + verify + render: ok + +CHECK: RED -- dossier does NOT hold + +## 3. report-drift probe: copy of the tree with one table cell changed to 2605.22907v1 -> exit 1 + +== long-context multimodal dossier gate == +root: /root/.hermes/profiles/sphinx/cache/scratch/drift-probe + +verify_lcmm_dossier.py -- long-context multimodal dossier (sphinx) v2 +root: /root/.hermes/profiles/sphinx/cache/scratch/drift-probe + +A. manifest integrity (re-measured from disk + recomputed from dossier) + ok sha256 dossier.json == manifest :: d77990948e812083 + ok bytes dossier.json == manifest + ok sha256 source/atom_batch.xml == manifest :: d3150650da24e2ec + ok dossier.source_atom_sha256 == manifest + ok manifest per-paper hashes/fields recomputable from dossier.json :: 0 problem(s) + +B. live arXiv API re-fetch (this is the evidence, not a replay) + mode: LIVE -- compared against the arXiv API fetched at check time + ok every id resolved :: 11/11 + +C. dossier vs LIVE truth + ok all papers match on title/version/published/authors/abstract/categories :: 0 mismatch(es) + +D. scope + report-rendering checks + ok paper_count >= 10 :: 11 + ok every abstract non-empty + ok every primary_category allowed :: ['cs.CL', 'cs.CV', 'cs.IR'] + ok every paper has a lane note + FAIL REPORT paper table == dossier.json (id/version/cat/authors/date/title) :: 1 drift(s) + 2605.22907 report version '2605.22907v1' != dossier '2605.22907v2' + +E. controls (negative ones MUST go red, the positive one MUST stay green) + ok P0 an unmodified copy stays green :: 0 failure(s) + ok F1 a flipped paper title is caught by field compare :: 1 failure(s) + ok F2 a dropped paper is caught by field compare :: 1 failure(s) + ok F3 an edited abstract is caught by field compare :: 2 failure(s) + ok F4 a bumped version string is caught by field compare :: 1 failure(s) + ok F5 a lying paper_count is caught by manifest recompute :: 1 failure(s) + ok F6 an altered manifest hash is caught by manifest recompute :: 1 failure(s) + ok F7 an altered report row is caught by report table check :: 2 failure(s) + ok all 8 controls ran :: 8 control(s) + +mode: LIVE checks run: 21 failures: 1 +VERDICT: dossier does NOT hold +verify: FAIL (rc=1) + +report table: STALE at line 39 + on disk: | 2605.22907 | 2605.22907v1 | cs.CV | 6 | 2026-05-21 | VideoOdyssey: A Benchmark for Ultra-Long-Context and Omni-Modal Video Understanding | + dossier: | 2605.22907 | 2605.22907v2 | cs.CV | 6 | 2026-05-21 | VideoOdyssey: A Benchmark for Ultra-Long-Context and Omni-Modal Video Understanding | +report table reproducible from dossier.json: FAIL + +report cross-references build + verify + render: ok + +CHECK: RED -- dossier does NOT hold diff --git a/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/REPORT-lcmm-dossier.md b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/REPORT-lcmm-dossier.md index 88b1d26..60d0219 100644 --- a/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/REPORT-lcmm-dossier.md +++ b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/REPORT-lcmm-dossier.md @@ -16,7 +16,8 @@ against the live API at check time. | file | role | |------|------| | `build_lcmm_dossier.py` | fetches the arXiv Atom feed, archives raw XML, emits `dossier.json` + `DOSSIER.md` + `manifest.json` | -| `verify_lcmm_dossier.py` | re-fetches the live API and compares every field; runs 3 negative controls | +| `verify_lcmm_dossier.py` | re-fetches the live API, compares every field, re-parses the report table, runs 1 positive + 7 negative controls | +| `render_report_table.py` | regenerates the report's paper table from `dossier.json`; `--check` exits 1 on drift | | `CHECK.sh` | the gate: runs `verify_lcmm_dossier.py`, then cross-references this report | | `source/atom_batch.xml` | the raw Atom XML as fetched (evidence, hash-pinned) | | `dossier.json` | the parsed records, hash-pinned in `manifest.json` | @@ -24,40 +25,61 @@ against the live API at check time. ## The 11 papers +Table generated from `dossier.json` by `render_report_table.py`; the verifier +re-parses this block and fails the gate if any cell drifts from the dossier. + + + | arXiv | version | cat | authors | submitted | title | |-------|---------|-----|---------|-----------|-------| -| 2606.14747 | v2 | cs.CV | 8 | 2026-06-05 | MMLongEmbed: Benchmarking Multimodal Embedding Models in Long-Context Scenarios | -| 2602.10271 | v3 | cs.IR | 2 | 2026-02-10 | MLDocRAG: Multimodal Long-Context Document Retrieval Augmented Generation | -| 2610.08463 | v1 | cs.CL | 7 | 2026-10-06 | UNREAL: Unifying Retrieval and Long-Context with a Single Model | -| 2610.31385 | v1 | cs.CV | 8 | 2026-10-22 | A Task-Centric Survey of Long-Document Visual Question Answering | -| 2605.22907 | v1 | cs.CV | 6 | 2026-05-21 | VideoOdyssey: A Benchmark for Ultra-Long-Context and Omni-Modal Video Understanding | -| 2603.27259 | v1 | cs.CV | 7 | 2026-03-28 | Seeing the Scene Matters: Revealing Forgetting in Video Understanding Models | -| 2602.20913 | v1 | cs.CV | 5 | 2026-02-24 | LongVideo-R1: Smart Navigation for Low-cost Long Video Understanding | -| 2605.06537 | v1 | cs.CV | 11 | 2026-05-07 | MedHorizon: Towards Long-context Medical Video Understanding in the Wild | -| 2605.19223 | v1 | cs.CV | 2 | 2026-05-19 | HAVEN: Hierarchically Aligned Multimodal Benchmark for Unified Video Understanding | -| 2603.29252 | v1 | cs.CV | 8 | 2026-03-31 | Scaling the Long Video Understanding of MLLMs via Visual Memory Mechanism | -| 2510.13276 | v2 | cs.CV | 12 | 2025-10-15 | MMLongCite: A Benchmark for Evaluating Faithfulness of Long-Context VLMs | +| 2606.14747 | 2606.14747v2 | cs.CV | 8 | 2026-06-05 | MMLongEmbed: Benchmarking Multimodal Embedding Models in Long-Context Scenarios | +| 2602.10271 | 2602.10271v3 | cs.IR | 2 | 2026-02-10 | MLDocRAG: Multimodal Long-Context Document Retrieval Augmented Generation | +| 2610.08463 | 2610.08463v1 | cs.CL | 7 | 2026-10-06 | UNREAL: Unifying Retrieval and Long-Context with a Single Model | +| 2610.31385 | 2610.31385v1 | cs.CV | 8 | 2026-10-22 | A Task-Centric Survey of Long-Document Visual Question Answering: Taxonomy, Evolution, and Future Directions | +| 2605.22907 | 2605.22907v2 | cs.CV | 6 | 2026-05-21 | VideoOdyssey: A Benchmark for Ultra-Long-Context and Omni-Modal Video Understanding | +| 2603.27259 | 2603.27259v3 | cs.CV | 7 | 2026-03-28 | Seeing the Scene Matters: Revealing Forgetting in Video Understanding Models with a Scene-Aware Long-Video Benchmark | +| 2602.20913 | 2602.20913v2 | cs.CV | 5 | 2026-02-24 | LongVideo-R1: Smart Navigation for Low-cost Long Video Understanding | +| 2605.06537 | 2605.06537v2 | cs.CV | 11 | 2026-05-07 | MedHorizon: Towards Long-context Medical Video Understanding in the Wild | +| 2605.19223 | 2605.19223v1 | cs.CV | 2 | 2026-05-19 | HAVEN: Hierarchically Aligned Multimodal Benchmark for Unified Video Understanding | +| 2603.29252 | 2603.29252v1 | cs.CV | 8 | 2026-03-31 | Scaling the Long Video Understanding of Multimodal Large Language Models via Visual Memory Mechanism | +| 2510.13276 | 2510.13276v2 | cs.CV | 12 | 2025-10-15 | MMLongCite: A Benchmark for Evaluating Faithfulness of Long-Context Vision-Language Models | -Note the version column: MLDocRAG has moved v1 -> v3 and MMLongEmbed v1 -> v2 -between the search-snippet dates and the live feed. That drift is exactly why the -dossier re-fetches rather than trusting a search result. + + +Version drift is real: against the search snippets that seeded this set, +MLDocRAG moved v1 -> v3, MMLongEmbed v1 -> v2, and four video papers +(`2605.22907`, `2603.27259`, `2602.20913`, `2605.06537`) are on v2/v3 while the +snippets said v1. That drift is exactly why the dossier re-fetches instead of +trusting a search result -- and why the table above is rendered, not typed: the +first hand-written draft of this report carried the stale v1 cells for four +papers and the old gate passed it anyway. ## How it is verified -`CHECK.sh` runs `verify_lcmm_dossier.py`, which: +`CHECK.sh` runs `verify_lcmm_dossier.py` (v2), which: 1. **A. manifest integrity** -- re-hashes `dossier.json` and `source/atom_batch.xml` - from disk and compares against `manifest.json`. -2. **B. live re-fetch** -- queries `https://export.arxiv.org/api/query` again for all - 11 ids. If the network is unavailable it falls back to comparing against the - archived XML and prints `mode: CACHED`; when the API answers it prints - `mode: LIVE`. + from disk against `manifest.json`, and recomputes every per-paper + title/abstract hash, version and published date from `dossier.json` itself. +2. **B. live re-fetch** -- queries `https://export.arxiv.org/api/query` (falling + back to the `arxiv.org` mirror) for all 11 ids, 4 attempts per host with + backoff. If every host fails it compares against the archived XML and prints + `mode: CACHED`; when the API answers it prints `mode: LIVE`. 3. **C. field-by-field compare** -- title, version, published date, author count, abstract sha256, primary category and category list must match the live truth. -4. **D. scope checks** -- >= 10 papers, every abstract non-empty, every primary - category in the allowed set, every paper carrying a lane note. -5. **E. negative controls** -- three mutated copies of the dossier must each be - caught: a flipped title, a dropped paper, an altered manifest hash. + The abstract text must also hash to its own stored `abstract_sha256` and match + its stored length, so an edit cannot ride on a stale hash. +4. **D. scope + rendering** -- >= 10 papers, every abstract non-empty, every + primary category in the allowed set, every paper carrying a lane note, and the + report's paper table must equal `dossier.json` cell for cell. +5. **E. controls** -- one positive (an unmodified copy must stay green) and seven + negative, each of which must be caught by the named checker: flipped title, + dropped paper, edited abstract, bumped version, lying `paper_count`, altered + manifest hash, altered report row. + +`CHECK.sh` itself also requires `mode: LIVE` (a CACHED run is red), re-checks +that the report table is reproducible via `render_report_table.py --check`, and +grep-confirms the report names the builder, verifier and renderer. Exit 0 = the dossier holds; non-zero = it does not. @@ -66,22 +88,33 @@ Exit 0 = the dossier holds; non-zero = it does not. Build: 11 papers across 3 batches; `source/atom_batch.xml` sha256 `d3150650da24e2ec...`; `dossier.json` sha256 `d77990948e812083...`. -Check run on 2026-10-25, live network: 14 checks, 0 failures, `mode: LIVE`, -all 3 controls red. +Check run on 2026-10-25, live network: **21 checks, 0 failures, `mode: LIVE`**, +all 7 negative controls red and the positive control green (`bash CHECK.sh`, +exit 0). -Both branches of the gate were exercised, not just the happy one: +The gate was shown to bite, not just to pass: -- `LCMM_ARXIV_API` unset -> `mode: LIVE`, 11/11 ids resolved against the API. -- `LCMM_ARXIV_API=http://127.0.0.1:9/api/query` -> `mode: CACHED`, same 14 - checks, 0 failures, still labelling itself CACHED rather than passing as live. - -That second run is what makes the `mode: CACHED` line in the honest-limits section -a measurement instead of a hope. +- **Defect found and closed.** The first gate (v1) verified the dossier but never + parsed the report. The hand-written table in that report had 4 of 11 version + cells stale (v1 where the feed says v2/v3) and 3 truncated titles; v1 still + exited 0. v2 renders the table from `dossier.json`, checks it cell by cell, and + control F7 mutates a row to prove the check fires. +- **Abstract edit found and closed.** v2's first run went red on control F3: the + stored `abstract_sha256` was compared only to the live feed, never recomputed + from the stored text, so a body edit with an untouched hash passed. C now + recomputes the hash from the text; F3 reports 2 failures. +- **Fallback branch exercised.** `LCMM_ARXIV_API=http://127.0.0.1:9/api/query` + yields `mode: CACHED` with the same in-tree checks, and `CHECK.sh` exits + non-zero on it -- so "LIVE" in a green run is a measurement, not a label. ## Honest limits - The check is a *metadata* verification. It proves the recorded bibliographic - fields match the live feed; it does not re-read the papers' bodies. The earlier - MMLongCite archive in this lane does body-level page/needle verification. -- `mode: CACHED` is weaker than `mode: LIVE` and is labelled as such when it - happens; the gate does not pretend the fallback is a live fetch. + fields and abstract text match the live feed; it does not re-read the papers' + bodies (no PDF fetch, no page-level claims). The earlier MMLongCite archive in + this lane does body-level page/needle verification. +- `mode: CACHED` is weaker than `mode: LIVE`; `CHECK.sh` treats it as a failure + rather than passing the fallback off as live evidence. +- arXiv version strings can move between runs. If a paper is revised, section C + goes red by design -- the dossier then needs a rebuild, which is the intended + behaviour, not a flake. diff --git a/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/build_lcmm_dossier.py b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/build_lcmm_dossier.py old mode 100755 new mode 100644 diff --git a/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/render_report_table.py b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/render_report_table.py new file mode 100644 index 0000000..0112a35 --- /dev/null +++ b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/render_report_table.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""render_report_table.py -- regenerate the paper table inside the report. + +The report's table is derived data. This script rebuilds it from dossier.json +between the BEGIN/END markers, so it can never drift from the dossier by hand +editing. `--check` compares instead of writing and exits 1 on any difference. + +Usage: + python3 render_report_table.py # rewrite the block + python3 render_report_table.py --check # exit 1 if the block is stale +""" +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +DOSSIER = os.path.join(HERE, "dossier.json") +REPORT = os.path.join(HERE, "REPORT-lcmm-dossier.md") +BEGIN = "" +END = "" + + +def build_block(dossier): + lines = [ + BEGIN, + "", + "| arXiv | version | cat | authors | submitted | title |", + "|-------|---------|-----|---------|-----------|-------|", + ] + for p in dossier["papers"]: + lines.append( + "| %s | %s | %s | %d | %s | %s |" + % ( + p["arxiv_id"], + p["version"], + p["primary_category"], + p["author_count"], + p["published"][:10], + p["title"].replace("|", "\\|"), + ) + ) + lines += ["", END] + return "\n".join(lines) + + +def splice(text, block): + if BEGIN not in text or END not in text: + raise SystemExit("FATAL: %s / %s markers not found in %s" % (BEGIN, END, REPORT)) + head = text.split(BEGIN, 1)[0] + tail = text.split(END, 1)[1] + return head + block + tail + + +def main(argv): + dossier = json.load(open(DOSSIER)) + block = build_block(dossier) + text = open(REPORT).read() + want = splice(text, block) + if "--check" in argv: + if want == text: + print("report table: in sync with dossier.json (%d rows)" % dossier["paper_count"]) + return 0 + # name the first differing line, so the failure is actionable + a, b = text.splitlines(), want.splitlines() + for i in range(max(len(a), len(b))): + la = a[i] if i < len(a) else "" + lb = b[i] if i < len(b) else "" + if la != lb: + print("report table: STALE at line %d\n on disk: %s\n dossier: %s" % (i + 1, la, lb)) + break + return 1 + with open(REPORT, "w") as f: + f.write(want) + print("report table rewritten from dossier.json (%d rows)" % dossier["paper_count"]) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/verify_lcmm_dossier.py b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/verify_lcmm_dossier.py old mode 100755 new mode 100644 index 1b5b979..3c4b48a --- a/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/verify_lcmm_dossier.py +++ b/wizard-work/sphinx/archive-lcmm-metadata-2026-10-25/verify_lcmm_dossier.py @@ -1,16 +1,16 @@ #!/usr/bin/env python3 -"""verify_lcmm_dossier.py -- Archive lane (sphinx). +"""verify_lcmm_dossier.py -- Archive lane (sphinx), v2. -Re-measures the long-context multimodal dossier against the LIVE arXiv API and -against the manifest, then runs three negative controls that must each go RED. +Re-measures the long-context multimodal dossier against the LIVE arXiv API, the +manifest, and the report's rendered table -- then runs negative controls that +each MUST go red, plus one positive control that MUST stay green. -Two modes, printed explicitly so the result is never ambiguous: - LIVE -- re-fetched https://export.arxiv.org/api/query at check time; the - recorded title / version / published / author count / abstract - sha256 / categories must match what the API returns right now. - CACHED -- the API was unreachable, so the dossier is compared against the - archived source/atom_batch.xml instead. Reported as CACHED, not - silently passed off as live. +Modes, printed explicitly so a result is never ambiguous: + LIVE -- https://export.arxiv.org/api/query (or the arxiv.org mirror) was + re-fetched at check time; every recorded field must match it now. + CACHED -- every host failed, so the dossier is compared against the archived + source/atom_batch.xml instead. Labelled CACHED, never passed off + as live. CHECK.sh treats CACHED as a failure by default. Exit 0 = the dossier holds. Exit non-zero = it does not. """ @@ -26,13 +26,27 @@ import xml.etree.ElementTree as ET HERE = os.path.dirname(os.path.abspath(__file__)) SOURCE = os.path.join(HERE, "source") -API = os.environ.get("LCMM_ARXIV_API", "https://export.arxiv.org/api/query") +ATOM_PATH = os.path.join(SOURCE, "atom_batch.xml") +DOSSIER_PATH = os.path.join(HERE, "dossier.json") +MANIFEST_PATH = os.path.join(HERE, "manifest.json") +REPORT_PATH = os.path.join(HERE, "REPORT-lcmm-dossier.md") + ATOM = "{http://www.w3.org/2005/Atom}" ARX = "{http://arxiv.org/schemas/atom}" BOUNDARY = b"" +BEGIN = "" +END = "" ALLOWED_CATS = {"cs.CV", "cs.CL", "cs.IR", "cs.AI", "cs.LG", "cs.MM"} +_override = os.environ.get("LCMM_ARXIV_API") +HOSTS = [_override] if _override else [ + "https://export.arxiv.org/api/query", + "https://arxiv.org/api/query", +] +ATTEMPTS = 4 +TIMEOUT = 25 + ok_count = 0 fail_count = 0 mode = "LIVE" @@ -46,20 +60,31 @@ def check(label, cond, detail=""): else: fail_count += 1 print(" FAIL %s%s" % (label, (" :: " + detail) if detail else "")) + sys.stdout.flush() return bool(cond) -def sha(path): - return hashlib.sha256(open(path, "rb").read()).hexdigest() +def sha_file(path): + with open(path, "rb") as f: + return hashlib.sha256(f.read()).hexdigest() -def fetch_live(ids): +def sha_text(s): + return hashlib.sha256(s.encode()).hexdigest() + + +def fetch_live(ids, api): q = urllib.parse.urlencode( {"id_list": ",".join(ids), "start": 0, "max_results": len(ids)} ) - req = urllib.request.Request(API + "?" + q, headers={"User-Agent": "sphinx-archive/1.0"}) - with urllib.request.urlopen(req, timeout=40) as r: - return r.read() + req = urllib.request.Request( + api + "?" + q, headers={"User-Agent": "sphinx-archive/2.0"} + ) + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + raw = r.read() + if not raw.strip(): + raise RuntimeError("empty response body from %s" % api) + return raw def parse(raw): @@ -86,66 +111,192 @@ def parse(raw): return out -def compare(dossier, truth, label): - """Return list of failure strings: dossier vs truth (live API or archived XML).""" - bad = [] +def fetch_batch(ids): + """Fetch one batch, retrying across hosts. Raises RuntimeError if all fail.""" + last = None + for attempt in range(ATTEMPTS): + for api in HOSTS: + try: + raw = fetch_live(ids, api) + rec = parse(raw) + missing = [i for i in ids if i not in rec] + if missing: + raise RuntimeError("%s returned no entry for %s" % (api, missing)) + return rec + except Exception as exc: # noqa: BLE001 + last = exc + time.sleep(min(2 ** attempt, 8)) + raise RuntimeError(repr(last)) + + +# -------------------------------------------------------------------------- +# Pure checkers: each takes data and returns a list of failure strings. +# Used both on the real artifact and on mutated copies (negative controls). +# -------------------------------------------------------------------------- +def check_manifest_perpaper(dossier, manifest): + """Every manifest per-paper hash/field must be recomputable from dossier.json.""" + fails = [] + papers = {p["arxiv_id"]: p for p in dossier["papers"]} + if dossier["paper_count"] != len(dossier["papers"]): + fails.append( + "paper_count %r != len(papers) %d" + % (dossier["paper_count"], len(dossier["papers"])) + ) + if set(manifest["per_paper"]) != set(papers): + fails.append("manifest per_paper id set != dossier id set") + return fails + if manifest["ids"] != [p["arxiv_id"] for p in dossier["papers"]]: + fails.append("manifest id order != dossier id order") + for aid, p in papers.items(): + m = manifest["per_paper"][aid] + if m["title_sha256"] != sha_text(p["title"]): + fails.append("%s title_sha256 not recomputable" % aid) + if m["abstract_sha256"] != sha_text(p["abstract"]): + fails.append("%s abstract_sha256 not recomputable" % aid) + if m["version"] != p["version"]: + fails.append("%s manifest version %r != dossier %r" % (aid, m["version"], p["version"])) + if m["published"] != p["published"]: + fails.append("%s manifest published != dossier" % aid) + return fails + + +def check_fields(dossier, truth): + """Dossier vs live-API (or archived-XML) truth, field by field.""" + fails = [] got = {p["arxiv_id"]: p for p in dossier["papers"]} if set(got) != set(truth): - bad.append("%s: id set differs (dossier=%d truth=%d)" - % (label, len(got), len(truth))) - return bad + fails.append( + "id set differs (dossier=%d truth=%d)" % (len(got), len(truth)) + ) + return fails for aid, t in truth.items(): p = got[aid] + # the stored hash must be recomputable from the stored text itself, + # so an edit to the abstract cannot ride on a stale hash + if sha_text(p.get("abstract", "")) != p.get("abstract_sha256"): + fails.append("%s abstract text does not hash to its own abstract_sha256" % aid) + if len(p.get("abstract", "")) != p.get("abstract_chars"): + fails.append("%s abstract_chars %r != len(abstract) %d" + % (aid, p.get("abstract_chars"), len(p.get("abstract", "")))) for field in ("title", "version", "published", "author_count", "abstract_sha256", "primary_category", "categories"): if p.get(field) != t.get(field): - bad.append("%s: %s mismatch (dossier=%r truth=%r)" - % (aid, field, p.get(field), t.get(field))) - return bad + fails.append( + "%s %s mismatch (dossier=%r truth=%r)" + % (aid, field, p.get(field), t.get(field)) + ) + return fails -def load_dossier(): - return json.load(open(os.path.join(HERE, "dossier.json"))) +def check_scope(dossier): + fails = [] + if len(dossier["papers"]) < 10: + fails.append("only %d papers (<10)" % len(dossier["papers"])) + for p in dossier["papers"]: + if not (p.get("abstract") or "").strip(): + fails.append("%s empty abstract" % p["arxiv_id"]) + if p.get("primary_category") not in ALLOWED_CATS: + fails.append("%s bad primary_category %r" % (p["arxiv_id"], p.get("primary_category"))) + if not (p.get("lane_note") or "").strip(): + fails.append("%s missing lane note" % p["arxiv_id"]) + return fails + + +def parse_report_rows(text): + """Rows of the generated markdown table between the markers, as dicts.""" + if BEGIN not in text or END not in text: + raise RuntimeError("report table markers missing") + block = text.split(BEGIN, 1)[1].split(END, 1)[0] + rows = [] + for line in block.strip().splitlines(): + line = line.strip() + if not line.startswith("|"): + continue + cells = [c.strip() for c in line.strip("|").split("|")] + if cells[0] in ("arXiv", "---") or set(cells[0]) <= {"-", ":"}: + continue + rows.append( + { + "arxiv_id": cells[0], + "version": cells[1], + "primary_category": cells[2], + "author_count": cells[3], + "submitted": cells[4], + "title": cells[5].replace("\\|", "|"), + } + ) + return rows + + +def check_report(dossier, rows): + """The rendered report table must equal what dossier.json says, in order.""" + fails = [] + papers = dossier["papers"] + if len(rows) != len(papers): + fails.append("report has %d rows, dossier has %d papers" % (len(rows), len(papers))) + for p, r in zip(papers, rows): + want = { + "arxiv_id": p["arxiv_id"], + "version": p["version"], + "primary_category": p["primary_category"], + "author_count": str(p["author_count"]), + "submitted": p["published"][:10], + "title": p["title"], + } + for k, v in want.items(): + if r.get(k) != v: + fails.append( + "%s report %s %r != dossier %r" % (p["arxiv_id"], k, r.get(k), v) + ) + return fails def main(): global mode - print("verify_lcmm_dossier.py -- long-context multimodal dossier (sphinx)") + print("verify_lcmm_dossier.py -- long-context multimodal dossier (sphinx) v2") print("root: %s" % HERE) print() - dossier = load_dossier() - manifest = json.load(open(os.path.join(HERE, "manifest.json"))) + dossier = json.load(open(DOSSIER_PATH)) + manifest = json.load(open(MANIFEST_PATH)) + report_text = open(REPORT_PATH).read() ids = manifest["ids"] - print("A. manifest integrity (re-measured from disk)") - check("sha256 dossier.json == manifest", sha(os.path.join(HERE, "dossier.json")) == manifest["dossier_sha256"], + print("A. manifest integrity (re-measured from disk + recomputed from dossier)") + check("sha256 dossier.json == manifest", + sha_file(DOSSIER_PATH) == manifest["dossier_sha256"], manifest["dossier_sha256"][:16]) - check("bytes dossier.json == manifest", os.path.getsize(os.path.join(HERE, "dossier.json")) == manifest["dossier_bytes"]) - atom_path = os.path.join(SOURCE, "atom_batch.xml") - check("sha256 source/atom_batch.xml == manifest", sha(atom_path) == manifest["source_atom_sha256"], + check("bytes dossier.json == manifest", + os.path.getsize(DOSSIER_PATH) == manifest["dossier_bytes"]) + check("sha256 source/atom_batch.xml == manifest", + sha_file(ATOM_PATH) == manifest["source_atom_sha256"], manifest["source_atom_sha256"][:16]) - check("dossier paper_count == manifest", dossier["paper_count"] == manifest["paper_count"], - str(dossier["paper_count"])) + check("dossier.source_atom_sha256 == manifest", + dossier["source_atom_sha256"] == manifest["source_atom_sha256"]) + a5 = check_manifest_perpaper(dossier, manifest) + check("manifest per-paper hashes/fields recomputable from dossier.json", + not a5, "%d problem(s)" % len(a5)) + for f in a5[:5]: + print(" %s" % f) print() print("B. live arXiv API re-fetch (this is the evidence, not a replay)") truth = {} - live_ok = True + live_err = None try: for i in range(0, len(ids), 5): - chunk = ids[i:i + 5] - truth.update(parse(fetch_live(chunk))) + truth.update(fetch_batch(ids[i:i + 5])) if i + 5 < len(ids): - time.sleep(4) + time.sleep(3) except Exception as exc: # noqa: BLE001 - live_ok = False - print(" (live fetch unavailable: %r)" % exc) + live_err = exc + print(" (live fetch unavailable after %d attempts/host: %r)" % (ATTEMPTS, exc)) - if not live_ok: + if live_err is not None: mode = "CACHED" - raw = open(atom_path, "rb").read() truth = {} + with open(ATOM_PATH, "rb") as f: + raw = f.read() for part in raw.split(BOUNDARY): part = part.strip() if part: @@ -153,11 +304,12 @@ def main(): print(" mode: CACHED -- compared against archived source/atom_batch.xml") else: print(" mode: LIVE -- compared against the arXiv API fetched at check time") - check("every id resolved", set(truth) == set(ids), "%d/%d" % (len(truth), len(ids))) + check("every id resolved", set(truth) == set(ids), + "%d/%d" % (len(truth), len(ids))) print() print("C. dossier vs %s truth" % mode) - bad = compare(dossier, truth, mode) + bad = check_fields(dossier, truth) for b in bad[:10]: print(" FAIL %s" % b) if len(bad) > 10: @@ -166,51 +318,87 @@ def main(): not bad, "%d mismatch(es)" % len(bad)) print() - print("D. scope checks") + print("D. scope + report-rendering checks") + scope = check_scope(dossier) check("paper_count >= 10", len(dossier["papers"]) >= 10, str(len(dossier["papers"]))) - check("every abstract non-empty", all(p["abstract"].strip() for p in dossier["papers"])) + check("every abstract non-empty", not [f for f in scope if "empty abstract" in f]) check("every primary_category allowed", - all(p["primary_category"] in ALLOWED_CATS for p in dossier["papers"]), + not [f for f in scope if "primary_category" in f], str(sorted({p["primary_category"] for p in dossier["papers"]}))) - check("every paper has a lane note", - all(p.get("lane_note") for p in dossier["papers"])) + check("every paper has a lane note", not [f for f in scope if "lane note" in f]) + rows = parse_report_rows(report_text) + rep = check_report(dossier, rows) + check("REPORT paper table == dossier.json (id/version/cat/authors/date/title)", + not rep, "%d drift(s)" % len(rep)) + for f in rep[:6]: + print(" %s" % f) print() - print("E. negative controls (each mutated copy MUST go red)") + print("E. controls (negative ones MUST go red, the positive one MUST stay green)") controls = 0 - def run_control(name, mutate_dossier=None, mutate_manifest=None): + def run(name, fn, expect_red): nonlocal controls - d2 = copy.deepcopy(dossier) - m2 = copy.deepcopy(manifest) - if mutate_dossier: - mutate_dossier(d2) - if mutate_manifest: - mutate_manifest(m2) - bad = compare(d2, truth, "mutated") - if mutate_manifest: - bad.append("manifest tampered") + fails = fn() controls += 1 - check("%s is caught" % name, bool(bad), "%d failure(s)" % len(bad)) + if expect_red: + check("%s is caught by %s" % (name, expect_red), bool(fails), + "%d failure(s)" % len(fails)) + else: + check("%s stays green" % name, not fails, + "%d failure(s)" % len(fails)) + + def mutated(mut): + d = copy.deepcopy(dossier) + mut(d) + return d def flip_title(d): - d["papers"][0]["title"] = d["papers"][0]["title"] + " (TAMPERED)" + d["papers"][0]["title"] += " (TAMPERED)" def drop_paper(d): d["papers"].pop() d["paper_count"] = len(d["papers"]) - def tamper_manifest(m): - first = m["ids"][0] - m["per_paper"][first]["title_sha256"] = "0" * 64 + def edit_abstract(d): + d["papers"][2]["abstract"] += " tampered" - run_control("F1 a flipped paper title", mutate_dossier=flip_title) - run_control("F2 a dropped paper", mutate_dossier=drop_paper) - run_control("F3 an altered manifest hash", mutate_manifest=tamper_manifest) - check("all 3 controls went red", controls == 3, "%d control(s) ran" % controls) + def bump_version(d): + d["papers"][4]["version"] = d["papers"][4]["version"].replace("v", "v9") + + def lie_count(d): + d["paper_count"] = len(d["papers"]) + 1 + + def mutated_manifest(): + m = copy.deepcopy(manifest) + m["per_paper"][m["ids"][0]]["title_sha256"] = "0" * 64 + return m + + def mutated_rows(): + rs = copy.deepcopy(rows) + rs[3]["version"] = "9999.99999v9" + return rs + + run("P0 an unmodified copy", lambda: check_fields(dossier, truth), None) + run("F1 a flipped paper title", + lambda: check_fields(mutated(flip_title), truth), "field compare") + run("F2 a dropped paper", + lambda: check_fields(mutated(drop_paper), truth), "field compare") + run("F3 an edited abstract", + lambda: check_fields(mutated(edit_abstract), truth), "field compare") + run("F4 a bumped version string", + lambda: check_fields(mutated(bump_version), truth), "field compare") + run("F5 a lying paper_count", + lambda: check_manifest_perpaper(mutated(lie_count), manifest), "manifest recompute") + run("F6 an altered manifest hash", + lambda: check_manifest_perpaper(dossier, mutated_manifest()), "manifest recompute") + run("F7 an altered report row", + lambda: check_report(dossier, mutated_rows()), "report table check") + check("all 8 controls ran", controls == 8, "%d control(s)" % controls) print() - print("mode: %s checks run: %d failures: %d" % (mode, ok_count + fail_count, fail_count)) + print("mode: %s checks run: %d failures: %d" + % (mode, ok_count + fail_count, fail_count)) print("VERDICT: dossier holds" if fail_count == 0 else "VERDICT: dossier does NOT hold") return 0 if fail_count == 0 else 1