sphinx: harden lcmm dossier gate -- report table rendered from dossier.json, 8 controls, live-only green
This commit is contained in:
sphinx committed
1 parent
d7e6e8e50c
commit
0430d457bd
6 files changed
+601
-127
No files matched your search
Executable → Regular
+31
-15
@@ -1,9 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
# CHECK gate for the long-context multimodal dossier (sphinx, Archive lane).
|
||||
#
|
||||
# Resolves its own directory so it grades the tree it lives in. Re-runs the
|
||||
# live verification: re-fetches the arXiv API and re-measures every recorded
|
||||
# field, then demands that three mutated copies each go RED.
|
||||
# Resolves its own directory so it grades the tree it lives in, then demands:
|
||||
# 1. verify_lcmm_dossier.py exits 0 with mode: LIVE (live API evidence, not
|
||||
# a replay of the archived XML -- CACHED is treated as a failure here)
|
||||
# 2. render_report_table.py --check exits 0 (the report table is reproducible
|
||||
# from dossier.json, i.e. it cannot silently drift)
|
||||
# 3. the report names both the builder and the verifier
|
||||
#
|
||||
# Exit 0 = the dossier holds. Exit non-zero = it does not.
|
||||
set -u
|
||||
@@ -17,21 +20,34 @@ echo "== long-context multimodal dossier gate =="
|
||||
echo "root: $HERE"
|
||||
echo
|
||||
|
||||
if python3 verify_lcmm_dossier.py; then
|
||||
echo "verify: PASS"
|
||||
else
|
||||
rc=$?
|
||||
out="$(python3 verify_lcmm_dossier.py 2>&1)"
|
||||
rc=$?
|
||||
printf '%s\n' "$out"
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "verify: FAIL (rc=$rc)"
|
||||
fail=1
|
||||
fi
|
||||
echo
|
||||
|
||||
# The report must name both halves, so the note cannot drift from the gate.
|
||||
if grep -q 'verify_lcmm_dossier.py' REPORT-lcmm-dossier.md \
|
||||
&& grep -q 'build_lcmm_dossier.py' REPORT-lcmm-dossier.md; then
|
||||
echo "report cross-references build + verify: ok"
|
||||
elif printf '%s\n' "$out" | grep -q '^mode: LIVE'; then
|
||||
echo "verify: PASS (mode LIVE)"
|
||||
else
|
||||
echo "report cross-references build + verify: FAIL"
|
||||
echo "verify: FAIL -- no live evidence (mode was not LIVE)"
|
||||
fail=1
|
||||
fi
|
||||
|
||||
echo
|
||||
if python3 render_report_table.py --check; then
|
||||
echo "report table reproducible from dossier.json: ok"
|
||||
else
|
||||
echo "report table reproducible from dossier.json: FAIL"
|
||||
fail=1
|
||||
fi
|
||||
|
||||
echo
|
||||
if grep -q 'verify_lcmm_dossier.py' REPORT-lcmm-dossier.md \
|
||||
&& grep -q 'build_lcmm_dossier.py' REPORT-lcmm-dossier.md \
|
||||
&& grep -q 'render_report_table.py' REPORT-lcmm-dossier.md; then
|
||||
echo "report cross-references build + verify + render: ok"
|
||||
else
|
||||
echo "report cross-references build + verify + render: FAIL"
|
||||
fail=1
|
||||
fi
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
# Evidence runs -- captured 2026-10-25 (sphinx, Archive lane)
|
||||
|
||||
## 1. live run: bash CHECK.sh -> exit 0
|
||||
|
||||
== long-context multimodal dossier gate ==
|
||||
root: /root/.hermes/wizard-outputs/sphinx/archive-lcmm-metadata-2026-10-25
|
||||
|
||||
verify_lcmm_dossier.py -- long-context multimodal dossier (sphinx) v2
|
||||
root: /root/.hermes/wizard-outputs/sphinx/archive-lcmm-metadata-2026-10-25
|
||||
|
||||
A. manifest integrity (re-measured from disk + recomputed from dossier)
|
||||
ok sha256 dossier.json == manifest :: d77990948e812083
|
||||
ok bytes dossier.json == manifest
|
||||
ok sha256 source/atom_batch.xml == manifest :: d3150650da24e2ec
|
||||
ok dossier.source_atom_sha256 == manifest
|
||||
ok manifest per-paper hashes/fields recomputable from dossier.json :: 0 problem(s)
|
||||
|
||||
B. live arXiv API re-fetch (this is the evidence, not a replay)
|
||||
mode: LIVE -- compared against the arXiv API fetched at check time
|
||||
ok every id resolved :: 11/11
|
||||
|
||||
C. dossier vs LIVE truth
|
||||
ok all papers match on title/version/published/authors/abstract/categories :: 0 mismatch(es)
|
||||
|
||||
D. scope + report-rendering checks
|
||||
ok paper_count >= 10 :: 11
|
||||
ok every abstract non-empty
|
||||
ok every primary_category allowed :: ['cs.CL', 'cs.CV', 'cs.IR']
|
||||
ok every paper has a lane note
|
||||
ok REPORT paper table == dossier.json (id/version/cat/authors/date/title) :: 0 drift(s)
|
||||
|
||||
E. controls (negative ones MUST go red, the positive one MUST stay green)
|
||||
ok P0 an unmodified copy stays green :: 0 failure(s)
|
||||
ok F1 a flipped paper title is caught by field compare :: 1 failure(s)
|
||||
ok F2 a dropped paper is caught by field compare :: 1 failure(s)
|
||||
ok F3 an edited abstract is caught by field compare :: 2 failure(s)
|
||||
ok F4 a bumped version string is caught by field compare :: 1 failure(s)
|
||||
ok F5 a lying paper_count is caught by manifest recompute :: 1 failure(s)
|
||||
ok F6 an altered manifest hash is caught by manifest recompute :: 1 failure(s)
|
||||
ok F7 an altered report row is caught by report table check :: 1 failure(s)
|
||||
ok all 8 controls ran :: 8 control(s)
|
||||
|
||||
mode: LIVE checks run: 21 failures: 0
|
||||
VERDICT: dossier holds
|
||||
verify: PASS (mode LIVE)
|
||||
|
||||
report table: in sync with dossier.json (11 rows)
|
||||
report table reproducible from dossier.json: ok
|
||||
|
||||
report cross-references build + verify + render: ok
|
||||
|
||||
CHECK: GREEN -- dossier holds
|
||||
|
||||
## 2. fallback branch: LCMM_ARXIV_API=http://127.0.0.1:9/api/query bash CHECK.sh -> exit 1 (CACHED is red by design)
|
||||
|
||||
== long-context multimodal dossier gate ==
|
||||
root: /root/.hermes/wizard-outputs/sphinx/archive-lcmm-metadata-2026-10-25
|
||||
|
||||
verify_lcmm_dossier.py -- long-context multimodal dossier (sphinx) v2
|
||||
root: /root/.hermes/wizard-outputs/sphinx/archive-lcmm-metadata-2026-10-25
|
||||
|
||||
A. manifest integrity (re-measured from disk + recomputed from dossier)
|
||||
ok sha256 dossier.json == manifest :: d77990948e812083
|
||||
ok bytes dossier.json == manifest
|
||||
ok sha256 source/atom_batch.xml == manifest :: d3150650da24e2ec
|
||||
ok dossier.source_atom_sha256 == manifest
|
||||
ok manifest per-paper hashes/fields recomputable from dossier.json :: 0 problem(s)
|
||||
|
||||
B. live arXiv API re-fetch (this is the evidence, not a replay)
|
||||
(live fetch unavailable after 4 attempts/host: RuntimeError("URLError(ConnectionRefusedError(111, 'Connection refused'))"))
|
||||
mode: CACHED -- compared against archived source/atom_batch.xml
|
||||
ok every id resolved :: 11/11
|
||||
|
||||
C. dossier vs CACHED truth
|
||||
ok all papers match on title/version/published/authors/abstract/categories :: 0 mismatch(es)
|
||||
|
||||
D. scope + report-rendering checks
|
||||
ok paper_count >= 10 :: 11
|
||||
ok every abstract non-empty
|
||||
ok every primary_category allowed :: ['cs.CL', 'cs.CV', 'cs.IR']
|
||||
ok every paper has a lane note
|
||||
ok REPORT paper table == dossier.json (id/version/cat/authors/date/title) :: 0 drift(s)
|
||||
|
||||
E. controls (negative ones MUST go red, the positive one MUST stay green)
|
||||
ok P0 an unmodified copy stays green :: 0 failure(s)
|
||||
ok F1 a flipped paper title is caught by field compare :: 1 failure(s)
|
||||
ok F2 a dropped paper is caught by field compare :: 1 failure(s)
|
||||
ok F3 an edited abstract is caught by field compare :: 2 failure(s)
|
||||
ok F4 a bumped version string is caught by field compare :: 1 failure(s)
|
||||
ok F5 a lying paper_count is caught by manifest recompute :: 1 failure(s)
|
||||
ok F6 an altered manifest hash is caught by manifest recompute :: 1 failure(s)
|
||||
ok F7 an altered report row is caught by report table check :: 1 failure(s)
|
||||
ok all 8 controls ran :: 8 control(s)
|
||||
|
||||
mode: CACHED checks run: 21 failures: 0
|
||||
VERDICT: dossier holds
|
||||
verify: FAIL -- no live evidence (mode was not LIVE)
|
||||
|
||||
report table: in sync with dossier.json (11 rows)
|
||||
report table reproducible from dossier.json: ok
|
||||
|
||||
report cross-references build + verify + render: ok
|
||||
|
||||
CHECK: RED -- dossier does NOT hold
|
||||
|
||||
## 3. report-drift probe: copy of the tree with one table cell changed to 2605.22907v1 -> exit 1
|
||||
|
||||
== long-context multimodal dossier gate ==
|
||||
root: /root/.hermes/profiles/sphinx/cache/scratch/drift-probe
|
||||
|
||||
verify_lcmm_dossier.py -- long-context multimodal dossier (sphinx) v2
|
||||
root: /root/.hermes/profiles/sphinx/cache/scratch/drift-probe
|
||||
|
||||
A. manifest integrity (re-measured from disk + recomputed from dossier)
|
||||
ok sha256 dossier.json == manifest :: d77990948e812083
|
||||
ok bytes dossier.json == manifest
|
||||
ok sha256 source/atom_batch.xml == manifest :: d3150650da24e2ec
|
||||
ok dossier.source_atom_sha256 == manifest
|
||||
ok manifest per-paper hashes/fields recomputable from dossier.json :: 0 problem(s)
|
||||
|
||||
B. live arXiv API re-fetch (this is the evidence, not a replay)
|
||||
mode: LIVE -- compared against the arXiv API fetched at check time
|
||||
ok every id resolved :: 11/11
|
||||
|
||||
C. dossier vs LIVE truth
|
||||
ok all papers match on title/version/published/authors/abstract/categories :: 0 mismatch(es)
|
||||
|
||||
D. scope + report-rendering checks
|
||||
ok paper_count >= 10 :: 11
|
||||
ok every abstract non-empty
|
||||
ok every primary_category allowed :: ['cs.CL', 'cs.CV', 'cs.IR']
|
||||
ok every paper has a lane note
|
||||
FAIL REPORT paper table == dossier.json (id/version/cat/authors/date/title) :: 1 drift(s)
|
||||
2605.22907 report version '2605.22907v1' != dossier '2605.22907v2'
|
||||
|
||||
E. controls (negative ones MUST go red, the positive one MUST stay green)
|
||||
ok P0 an unmodified copy stays green :: 0 failure(s)
|
||||
ok F1 a flipped paper title is caught by field compare :: 1 failure(s)
|
||||
ok F2 a dropped paper is caught by field compare :: 1 failure(s)
|
||||
ok F3 an edited abstract is caught by field compare :: 2 failure(s)
|
||||
ok F4 a bumped version string is caught by field compare :: 1 failure(s)
|
||||
ok F5 a lying paper_count is caught by manifest recompute :: 1 failure(s)
|
||||
ok F6 an altered manifest hash is caught by manifest recompute :: 1 failure(s)
|
||||
ok F7 an altered report row is caught by report table check :: 2 failure(s)
|
||||
ok all 8 controls ran :: 8 control(s)
|
||||
|
||||
mode: LIVE checks run: 21 failures: 1
|
||||
VERDICT: dossier does NOT hold
|
||||
verify: FAIL (rc=1)
|
||||
|
||||
report table: STALE at line 39
|
||||
on disk: | 2605.22907 | 2605.22907v1 | cs.CV | 6 | 2026-05-21 | VideoOdyssey: A Benchmark for Ultra-Long-Context and Omni-Modal Video Understanding |
|
||||
dossier: | 2605.22907 | 2605.22907v2 | cs.CV | 6 | 2026-05-21 | VideoOdyssey: A Benchmark for Ultra-Long-Context and Omni-Modal Video Understanding |
|
||||
report table reproducible from dossier.json: FAIL
|
||||
|
||||
report cross-references build + verify + render: ok
|
||||
|
||||
CHECK: RED -- dossier does NOT hold
|
||||
@@ -16,7 +16,8 @@ against the live API at check time.
|
||||
| file | role |
|
||||
|------|------|
|
||||
| `build_lcmm_dossier.py` | fetches the arXiv Atom feed, archives raw XML, emits `dossier.json` + `DOSSIER.md` + `manifest.json` |
|
||||
| `verify_lcmm_dossier.py` | re-fetches the live API and compares every field; runs 3 negative controls |
|
||||
| `verify_lcmm_dossier.py` | re-fetches the live API, compares every field, re-parses the report table, runs 1 positive + 7 negative controls |
|
||||
| `render_report_table.py` | regenerates the report's paper table from `dossier.json`; `--check` exits 1 on drift |
|
||||
| `CHECK.sh` | the gate: runs `verify_lcmm_dossier.py`, then cross-references this report |
|
||||
| `source/atom_batch.xml` | the raw Atom XML as fetched (evidence, hash-pinned) |
|
||||
| `dossier.json` | the parsed records, hash-pinned in `manifest.json` |
|
||||
@@ -24,40 +25,61 @@ against the live API at check time.
|
||||
|
||||
## The 11 papers
|
||||
|
||||
Table generated from `dossier.json` by `render_report_table.py`; the verifier
|
||||
re-parses this block and fails the gate if any cell drifts from the dossier.
|
||||
|
||||
<!-- BEGIN PAPER TABLE -->
|
||||
|
||||
| arXiv | version | cat | authors | submitted | title |
|
||||
|-------|---------|-----|---------|-----------|-------|
|
||||
| 2606.14747 | v2 | cs.CV | 8 | 2026-06-05 | MMLongEmbed: Benchmarking Multimodal Embedding Models in Long-Context Scenarios |
|
||||
| 2602.10271 | v3 | cs.IR | 2 | 2026-02-10 | MLDocRAG: Multimodal Long-Context Document Retrieval Augmented Generation |
|
||||
| 2610.08463 | v1 | cs.CL | 7 | 2026-10-06 | UNREAL: Unifying Retrieval and Long-Context with a Single Model |
|
||||
| 2610.31385 | v1 | cs.CV | 8 | 2026-10-22 | A Task-Centric Survey of Long-Document Visual Question Answering |
|
||||
| 2605.22907 | v1 | cs.CV | 6 | 2026-05-21 | VideoOdyssey: A Benchmark for Ultra-Long-Context and Omni-Modal Video Understanding |
|
||||
| 2603.27259 | v1 | cs.CV | 7 | 2026-03-28 | Seeing the Scene Matters: Revealing Forgetting in Video Understanding Models |
|
||||
| 2602.20913 | v1 | cs.CV | 5 | 2026-02-24 | LongVideo-R1: Smart Navigation for Low-cost Long Video Understanding |
|
||||
| 2605.06537 | v1 | cs.CV | 11 | 2026-05-07 | MedHorizon: Towards Long-context Medical Video Understanding in the Wild |
|
||||
| 2605.19223 | v1 | cs.CV | 2 | 2026-05-19 | HAVEN: Hierarchically Aligned Multimodal Benchmark for Unified Video Understanding |
|
||||
| 2603.29252 | v1 | cs.CV | 8 | 2026-03-31 | Scaling the Long Video Understanding of MLLMs via Visual Memory Mechanism |
|
||||
| 2510.13276 | v2 | cs.CV | 12 | 2025-10-15 | MMLongCite: A Benchmark for Evaluating Faithfulness of Long-Context VLMs |
|
||||
| 2606.14747 | 2606.14747v2 | cs.CV | 8 | 2026-06-05 | MMLongEmbed: Benchmarking Multimodal Embedding Models in Long-Context Scenarios |
|
||||
| 2602.10271 | 2602.10271v3 | cs.IR | 2 | 2026-02-10 | MLDocRAG: Multimodal Long-Context Document Retrieval Augmented Generation |
|
||||
| 2610.08463 | 2610.08463v1 | cs.CL | 7 | 2026-10-06 | UNREAL: Unifying Retrieval and Long-Context with a Single Model |
|
||||
| 2610.31385 | 2610.31385v1 | cs.CV | 8 | 2026-10-22 | A Task-Centric Survey of Long-Document Visual Question Answering: Taxonomy, Evolution, and Future Directions |
|
||||
| 2605.22907 | 2605.22907v2 | cs.CV | 6 | 2026-05-21 | VideoOdyssey: A Benchmark for Ultra-Long-Context and Omni-Modal Video Understanding |
|
||||
| 2603.27259 | 2603.27259v3 | cs.CV | 7 | 2026-03-28 | Seeing the Scene Matters: Revealing Forgetting in Video Understanding Models with a Scene-Aware Long-Video Benchmark |
|
||||
| 2602.20913 | 2602.20913v2 | cs.CV | 5 | 2026-02-24 | LongVideo-R1: Smart Navigation for Low-cost Long Video Understanding |
|
||||
| 2605.06537 | 2605.06537v2 | cs.CV | 11 | 2026-05-07 | MedHorizon: Towards Long-context Medical Video Understanding in the Wild |
|
||||
| 2605.19223 | 2605.19223v1 | cs.CV | 2 | 2026-05-19 | HAVEN: Hierarchically Aligned Multimodal Benchmark for Unified Video Understanding |
|
||||
| 2603.29252 | 2603.29252v1 | cs.CV | 8 | 2026-03-31 | Scaling the Long Video Understanding of Multimodal Large Language Models via Visual Memory Mechanism |
|
||||
| 2510.13276 | 2510.13276v2 | cs.CV | 12 | 2025-10-15 | MMLongCite: A Benchmark for Evaluating Faithfulness of Long-Context Vision-Language Models |
|
||||
|
||||
Note the version column: MLDocRAG has moved v1 -> v3 and MMLongEmbed v1 -> v2
|
||||
between the search-snippet dates and the live feed. That drift is exactly why the
|
||||
dossier re-fetches rather than trusting a search result.
|
||||
<!-- END PAPER TABLE -->
|
||||
|
||||
Version drift is real: against the search snippets that seeded this set,
|
||||
MLDocRAG moved v1 -> v3, MMLongEmbed v1 -> v2, and four video papers
|
||||
(`2605.22907`, `2603.27259`, `2602.20913`, `2605.06537`) are on v2/v3 while the
|
||||
snippets said v1. That drift is exactly why the dossier re-fetches instead of
|
||||
trusting a search result -- and why the table above is rendered, not typed: the
|
||||
first hand-written draft of this report carried the stale v1 cells for four
|
||||
papers and the old gate passed it anyway.
|
||||
|
||||
## How it is verified
|
||||
|
||||
`CHECK.sh` runs `verify_lcmm_dossier.py`, which:
|
||||
`CHECK.sh` runs `verify_lcmm_dossier.py` (v2), which:
|
||||
|
||||
1. **A. manifest integrity** -- re-hashes `dossier.json` and `source/atom_batch.xml`
|
||||
from disk and compares against `manifest.json`.
|
||||
2. **B. live re-fetch** -- queries `https://export.arxiv.org/api/query` again for all
|
||||
11 ids. If the network is unavailable it falls back to comparing against the
|
||||
archived XML and prints `mode: CACHED`; when the API answers it prints
|
||||
`mode: LIVE`.
|
||||
from disk against `manifest.json`, and recomputes every per-paper
|
||||
title/abstract hash, version and published date from `dossier.json` itself.
|
||||
2. **B. live re-fetch** -- queries `https://export.arxiv.org/api/query` (falling
|
||||
back to the `arxiv.org` mirror) for all 11 ids, 4 attempts per host with
|
||||
backoff. If every host fails it compares against the archived XML and prints
|
||||
`mode: CACHED`; when the API answers it prints `mode: LIVE`.
|
||||
3. **C. field-by-field compare** -- title, version, published date, author count,
|
||||
abstract sha256, primary category and category list must match the live truth.
|
||||
4. **D. scope checks** -- >= 10 papers, every abstract non-empty, every primary
|
||||
category in the allowed set, every paper carrying a lane note.
|
||||
5. **E. negative controls** -- three mutated copies of the dossier must each be
|
||||
caught: a flipped title, a dropped paper, an altered manifest hash.
|
||||
The abstract text must also hash to its own stored `abstract_sha256` and match
|
||||
its stored length, so an edit cannot ride on a stale hash.
|
||||
4. **D. scope + rendering** -- >= 10 papers, every abstract non-empty, every
|
||||
primary category in the allowed set, every paper carrying a lane note, and the
|
||||
report's paper table must equal `dossier.json` cell for cell.
|
||||
5. **E. controls** -- one positive (an unmodified copy must stay green) and seven
|
||||
negative, each of which must be caught by the named checker: flipped title,
|
||||
dropped paper, edited abstract, bumped version, lying `paper_count`, altered
|
||||
manifest hash, altered report row.
|
||||
|
||||
`CHECK.sh` itself also requires `mode: LIVE` (a CACHED run is red), re-checks
|
||||
that the report table is reproducible via `render_report_table.py --check`, and
|
||||
grep-confirms the report names the builder, verifier and renderer.
|
||||
|
||||
Exit 0 = the dossier holds; non-zero = it does not.
|
||||
|
||||
@@ -66,22 +88,33 @@ Exit 0 = the dossier holds; non-zero = it does not.
|
||||
Build: 11 papers across 3 batches; `source/atom_batch.xml` sha256
|
||||
`d3150650da24e2ec...`; `dossier.json` sha256 `d77990948e812083...`.
|
||||
|
||||
Check run on 2026-10-25, live network: 14 checks, 0 failures, `mode: LIVE`,
|
||||
all 3 controls red.
|
||||
Check run on 2026-10-25, live network: **21 checks, 0 failures, `mode: LIVE`**,
|
||||
all 7 negative controls red and the positive control green (`bash CHECK.sh`,
|
||||
exit 0).
|
||||
|
||||
Both branches of the gate were exercised, not just the happy one:
|
||||
The gate was shown to bite, not just to pass:
|
||||
|
||||
- `LCMM_ARXIV_API` unset -> `mode: LIVE`, 11/11 ids resolved against the API.
|
||||
- `LCMM_ARXIV_API=http://127.0.0.1:9/api/query` -> `mode: CACHED`, same 14
|
||||
checks, 0 failures, still labelling itself CACHED rather than passing as live.
|
||||
|
||||
That second run is what makes the `mode: CACHED` line in the honest-limits section
|
||||
a measurement instead of a hope.
|
||||
- **Defect found and closed.** The first gate (v1) verified the dossier but never
|
||||
parsed the report. The hand-written table in that report had 4 of 11 version
|
||||
cells stale (v1 where the feed says v2/v3) and 3 truncated titles; v1 still
|
||||
exited 0. v2 renders the table from `dossier.json`, checks it cell by cell, and
|
||||
control F7 mutates a row to prove the check fires.
|
||||
- **Abstract edit found and closed.** v2's first run went red on control F3: the
|
||||
stored `abstract_sha256` was compared only to the live feed, never recomputed
|
||||
from the stored text, so a body edit with an untouched hash passed. C now
|
||||
recomputes the hash from the text; F3 reports 2 failures.
|
||||
- **Fallback branch exercised.** `LCMM_ARXIV_API=http://127.0.0.1:9/api/query`
|
||||
yields `mode: CACHED` with the same in-tree checks, and `CHECK.sh` exits
|
||||
non-zero on it -- so "LIVE" in a green run is a measurement, not a label.
|
||||
|
||||
## Honest limits
|
||||
|
||||
- The check is a *metadata* verification. It proves the recorded bibliographic
|
||||
fields match the live feed; it does not re-read the papers' bodies. The earlier
|
||||
MMLongCite archive in this lane does body-level page/needle verification.
|
||||
- `mode: CACHED` is weaker than `mode: LIVE` and is labelled as such when it
|
||||
happens; the gate does not pretend the fallback is a live fetch.
|
||||
fields and abstract text match the live feed; it does not re-read the papers'
|
||||
bodies (no PDF fetch, no page-level claims). The earlier MMLongCite archive in
|
||||
this lane does body-level page/needle verification.
|
||||
- `mode: CACHED` is weaker than `mode: LIVE`; `CHECK.sh` treats it as a failure
|
||||
rather than passing the fallback off as live evidence.
|
||||
- arXiv version strings can move between runs. If a paper is revised, section C
|
||||
goes red by design -- the dossier then needs a rebuild, which is the intended
|
||||
behaviour, not a flake.
|
||||
Executable → Regular
File mode changed.
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""render_report_table.py -- regenerate the paper table inside the report.
|
||||
|
||||
The report's table is derived data. This script rebuilds it from dossier.json
|
||||
between the BEGIN/END markers, so it can never drift from the dossier by hand
|
||||
editing. `--check` compares instead of writing and exits 1 on any difference.
|
||||
|
||||
Usage:
|
||||
python3 render_report_table.py # rewrite the block
|
||||
python3 render_report_table.py --check # exit 1 if the block is stale
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DOSSIER = os.path.join(HERE, "dossier.json")
|
||||
REPORT = os.path.join(HERE, "REPORT-lcmm-dossier.md")
|
||||
BEGIN = "<!-- BEGIN PAPER TABLE -->"
|
||||
END = "<!-- END PAPER TABLE -->"
|
||||
|
||||
|
||||
def build_block(dossier):
|
||||
lines = [
|
||||
BEGIN,
|
||||
"",
|
||||
"| arXiv | version | cat | authors | submitted | title |",
|
||||
"|-------|---------|-----|---------|-----------|-------|",
|
||||
]
|
||||
for p in dossier["papers"]:
|
||||
lines.append(
|
||||
"| %s | %s | %s | %d | %s | %s |"
|
||||
% (
|
||||
p["arxiv_id"],
|
||||
p["version"],
|
||||
p["primary_category"],
|
||||
p["author_count"],
|
||||
p["published"][:10],
|
||||
p["title"].replace("|", "\\|"),
|
||||
)
|
||||
)
|
||||
lines += ["", END]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def splice(text, block):
|
||||
if BEGIN not in text or END not in text:
|
||||
raise SystemExit("FATAL: %s / %s markers not found in %s" % (BEGIN, END, REPORT))
|
||||
head = text.split(BEGIN, 1)[0]
|
||||
tail = text.split(END, 1)[1]
|
||||
return head + block + tail
|
||||
|
||||
|
||||
def main(argv):
|
||||
dossier = json.load(open(DOSSIER))
|
||||
block = build_block(dossier)
|
||||
text = open(REPORT).read()
|
||||
want = splice(text, block)
|
||||
if "--check" in argv:
|
||||
if want == text:
|
||||
print("report table: in sync with dossier.json (%d rows)" % dossier["paper_count"])
|
||||
return 0
|
||||
# name the first differing line, so the failure is actionable
|
||||
a, b = text.splitlines(), want.splitlines()
|
||||
for i in range(max(len(a), len(b))):
|
||||
la = a[i] if i < len(a) else "<missing>"
|
||||
lb = b[i] if i < len(b) else "<missing>"
|
||||
if la != lb:
|
||||
print("report table: STALE at line %d\n on disk: %s\n dossier: %s" % (i + 1, la, lb))
|
||||
break
|
||||
return 1
|
||||
with open(REPORT, "w") as f:
|
||||
f.write(want)
|
||||
print("report table rewritten from dossier.json (%d rows)" % dossier["paper_count"])
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Executable → Regular
+262
-74
@@ -1,16 +1,16 @@
|
||||
#!/usr/bin/env python3
|
||||
"""verify_lcmm_dossier.py -- Archive lane (sphinx).
|
||||
"""verify_lcmm_dossier.py -- Archive lane (sphinx), v2.
|
||||
|
||||
Re-measures the long-context multimodal dossier against the LIVE arXiv API and
|
||||
against the manifest, then runs three negative controls that must each go RED.
|
||||
Re-measures the long-context multimodal dossier against the LIVE arXiv API, the
|
||||
manifest, and the report's rendered table -- then runs negative controls that
|
||||
each MUST go red, plus one positive control that MUST stay green.
|
||||
|
||||
Two modes, printed explicitly so the result is never ambiguous:
|
||||
LIVE -- re-fetched https://export.arxiv.org/api/query at check time; the
|
||||
recorded title / version / published / author count / abstract
|
||||
sha256 / categories must match what the API returns right now.
|
||||
CACHED -- the API was unreachable, so the dossier is compared against the
|
||||
archived source/atom_batch.xml instead. Reported as CACHED, not
|
||||
silently passed off as live.
|
||||
Modes, printed explicitly so a result is never ambiguous:
|
||||
LIVE -- https://export.arxiv.org/api/query (or the arxiv.org mirror) was
|
||||
re-fetched at check time; every recorded field must match it now.
|
||||
CACHED -- every host failed, so the dossier is compared against the archived
|
||||
source/atom_batch.xml instead. Labelled CACHED, never passed off
|
||||
as live. CHECK.sh treats CACHED as a failure by default.
|
||||
|
||||
Exit 0 = the dossier holds. Exit non-zero = it does not.
|
||||
"""
|
||||
@@ -26,13 +26,27 @@ import xml.etree.ElementTree as ET
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
SOURCE = os.path.join(HERE, "source")
|
||||
API = os.environ.get("LCMM_ARXIV_API", "https://export.arxiv.org/api/query")
|
||||
ATOM_PATH = os.path.join(SOURCE, "atom_batch.xml")
|
||||
DOSSIER_PATH = os.path.join(HERE, "dossier.json")
|
||||
MANIFEST_PATH = os.path.join(HERE, "manifest.json")
|
||||
REPORT_PATH = os.path.join(HERE, "REPORT-lcmm-dossier.md")
|
||||
|
||||
ATOM = "{http://www.w3.org/2005/Atom}"
|
||||
ARX = "{http://arxiv.org/schemas/atom}"
|
||||
BOUNDARY = b"<!-- batch boundary -->"
|
||||
BEGIN = "<!-- BEGIN PAPER TABLE -->"
|
||||
END = "<!-- END PAPER TABLE -->"
|
||||
|
||||
ALLOWED_CATS = {"cs.CV", "cs.CL", "cs.IR", "cs.AI", "cs.LG", "cs.MM"}
|
||||
|
||||
_override = os.environ.get("LCMM_ARXIV_API")
|
||||
HOSTS = [_override] if _override else [
|
||||
"https://export.arxiv.org/api/query",
|
||||
"https://arxiv.org/api/query",
|
||||
]
|
||||
ATTEMPTS = 4
|
||||
TIMEOUT = 25
|
||||
|
||||
ok_count = 0
|
||||
fail_count = 0
|
||||
mode = "LIVE"
|
||||
@@ -46,20 +60,31 @@ def check(label, cond, detail=""):
|
||||
else:
|
||||
fail_count += 1
|
||||
print(" FAIL %s%s" % (label, (" :: " + detail) if detail else ""))
|
||||
sys.stdout.flush()
|
||||
return bool(cond)
|
||||
|
||||
|
||||
def sha(path):
|
||||
return hashlib.sha256(open(path, "rb").read()).hexdigest()
|
||||
def sha_file(path):
|
||||
with open(path, "rb") as f:
|
||||
return hashlib.sha256(f.read()).hexdigest()
|
||||
|
||||
|
||||
def fetch_live(ids):
|
||||
def sha_text(s):
|
||||
return hashlib.sha256(s.encode()).hexdigest()
|
||||
|
||||
|
||||
def fetch_live(ids, api):
|
||||
q = urllib.parse.urlencode(
|
||||
{"id_list": ",".join(ids), "start": 0, "max_results": len(ids)}
|
||||
)
|
||||
req = urllib.request.Request(API + "?" + q, headers={"User-Agent": "sphinx-archive/1.0"})
|
||||
with urllib.request.urlopen(req, timeout=40) as r:
|
||||
return r.read()
|
||||
req = urllib.request.Request(
|
||||
api + "?" + q, headers={"User-Agent": "sphinx-archive/2.0"}
|
||||
)
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
||||
raw = r.read()
|
||||
if not raw.strip():
|
||||
raise RuntimeError("empty response body from %s" % api)
|
||||
return raw
|
||||
|
||||
|
||||
def parse(raw):
|
||||
@@ -86,66 +111,192 @@ def parse(raw):
|
||||
return out
|
||||
|
||||
|
||||
def compare(dossier, truth, label):
|
||||
"""Return list of failure strings: dossier vs truth (live API or archived XML)."""
|
||||
bad = []
|
||||
def fetch_batch(ids):
|
||||
"""Fetch one batch, retrying across hosts. Raises RuntimeError if all fail."""
|
||||
last = None
|
||||
for attempt in range(ATTEMPTS):
|
||||
for api in HOSTS:
|
||||
try:
|
||||
raw = fetch_live(ids, api)
|
||||
rec = parse(raw)
|
||||
missing = [i for i in ids if i not in rec]
|
||||
if missing:
|
||||
raise RuntimeError("%s returned no entry for %s" % (api, missing))
|
||||
return rec
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last = exc
|
||||
time.sleep(min(2 ** attempt, 8))
|
||||
raise RuntimeError(repr(last))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Pure checkers: each takes data and returns a list of failure strings.
|
||||
# Used both on the real artifact and on mutated copies (negative controls).
|
||||
# --------------------------------------------------------------------------
|
||||
def check_manifest_perpaper(dossier, manifest):
|
||||
"""Every manifest per-paper hash/field must be recomputable from dossier.json."""
|
||||
fails = []
|
||||
papers = {p["arxiv_id"]: p for p in dossier["papers"]}
|
||||
if dossier["paper_count"] != len(dossier["papers"]):
|
||||
fails.append(
|
||||
"paper_count %r != len(papers) %d"
|
||||
% (dossier["paper_count"], len(dossier["papers"]))
|
||||
)
|
||||
if set(manifest["per_paper"]) != set(papers):
|
||||
fails.append("manifest per_paper id set != dossier id set")
|
||||
return fails
|
||||
if manifest["ids"] != [p["arxiv_id"] for p in dossier["papers"]]:
|
||||
fails.append("manifest id order != dossier id order")
|
||||
for aid, p in papers.items():
|
||||
m = manifest["per_paper"][aid]
|
||||
if m["title_sha256"] != sha_text(p["title"]):
|
||||
fails.append("%s title_sha256 not recomputable" % aid)
|
||||
if m["abstract_sha256"] != sha_text(p["abstract"]):
|
||||
fails.append("%s abstract_sha256 not recomputable" % aid)
|
||||
if m["version"] != p["version"]:
|
||||
fails.append("%s manifest version %r != dossier %r" % (aid, m["version"], p["version"]))
|
||||
if m["published"] != p["published"]:
|
||||
fails.append("%s manifest published != dossier" % aid)
|
||||
return fails
|
||||
|
||||
|
||||
def check_fields(dossier, truth):
|
||||
"""Dossier vs live-API (or archived-XML) truth, field by field."""
|
||||
fails = []
|
||||
got = {p["arxiv_id"]: p for p in dossier["papers"]}
|
||||
if set(got) != set(truth):
|
||||
bad.append("%s: id set differs (dossier=%d truth=%d)"
|
||||
% (label, len(got), len(truth)))
|
||||
return bad
|
||||
fails.append(
|
||||
"id set differs (dossier=%d truth=%d)" % (len(got), len(truth))
|
||||
)
|
||||
return fails
|
||||
for aid, t in truth.items():
|
||||
p = got[aid]
|
||||
# the stored hash must be recomputable from the stored text itself,
|
||||
# so an edit to the abstract cannot ride on a stale hash
|
||||
if sha_text(p.get("abstract", "")) != p.get("abstract_sha256"):
|
||||
fails.append("%s abstract text does not hash to its own abstract_sha256" % aid)
|
||||
if len(p.get("abstract", "")) != p.get("abstract_chars"):
|
||||
fails.append("%s abstract_chars %r != len(abstract) %d"
|
||||
% (aid, p.get("abstract_chars"), len(p.get("abstract", ""))))
|
||||
for field in ("title", "version", "published", "author_count",
|
||||
"abstract_sha256", "primary_category", "categories"):
|
||||
if p.get(field) != t.get(field):
|
||||
bad.append("%s: %s mismatch (dossier=%r truth=%r)"
|
||||
% (aid, field, p.get(field), t.get(field)))
|
||||
return bad
|
||||
fails.append(
|
||||
"%s %s mismatch (dossier=%r truth=%r)"
|
||||
% (aid, field, p.get(field), t.get(field))
|
||||
)
|
||||
return fails
|
||||
|
||||
|
||||
def load_dossier():
|
||||
return json.load(open(os.path.join(HERE, "dossier.json")))
|
||||
def check_scope(dossier):
|
||||
fails = []
|
||||
if len(dossier["papers"]) < 10:
|
||||
fails.append("only %d papers (<10)" % len(dossier["papers"]))
|
||||
for p in dossier["papers"]:
|
||||
if not (p.get("abstract") or "").strip():
|
||||
fails.append("%s empty abstract" % p["arxiv_id"])
|
||||
if p.get("primary_category") not in ALLOWED_CATS:
|
||||
fails.append("%s bad primary_category %r" % (p["arxiv_id"], p.get("primary_category")))
|
||||
if not (p.get("lane_note") or "").strip():
|
||||
fails.append("%s missing lane note" % p["arxiv_id"])
|
||||
return fails
|
||||
|
||||
|
||||
def parse_report_rows(text):
|
||||
"""Rows of the generated markdown table between the markers, as dicts."""
|
||||
if BEGIN not in text or END not in text:
|
||||
raise RuntimeError("report table markers missing")
|
||||
block = text.split(BEGIN, 1)[1].split(END, 1)[0]
|
||||
rows = []
|
||||
for line in block.strip().splitlines():
|
||||
line = line.strip()
|
||||
if not line.startswith("|"):
|
||||
continue
|
||||
cells = [c.strip() for c in line.strip("|").split("|")]
|
||||
if cells[0] in ("arXiv", "---") or set(cells[0]) <= {"-", ":"}:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"arxiv_id": cells[0],
|
||||
"version": cells[1],
|
||||
"primary_category": cells[2],
|
||||
"author_count": cells[3],
|
||||
"submitted": cells[4],
|
||||
"title": cells[5].replace("\\|", "|"),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def check_report(dossier, rows):
|
||||
"""The rendered report table must equal what dossier.json says, in order."""
|
||||
fails = []
|
||||
papers = dossier["papers"]
|
||||
if len(rows) != len(papers):
|
||||
fails.append("report has %d rows, dossier has %d papers" % (len(rows), len(papers)))
|
||||
for p, r in zip(papers, rows):
|
||||
want = {
|
||||
"arxiv_id": p["arxiv_id"],
|
||||
"version": p["version"],
|
||||
"primary_category": p["primary_category"],
|
||||
"author_count": str(p["author_count"]),
|
||||
"submitted": p["published"][:10],
|
||||
"title": p["title"],
|
||||
}
|
||||
for k, v in want.items():
|
||||
if r.get(k) != v:
|
||||
fails.append(
|
||||
"%s report %s %r != dossier %r" % (p["arxiv_id"], k, r.get(k), v)
|
||||
)
|
||||
return fails
|
||||
|
||||
|
||||
def main():
|
||||
global mode
|
||||
print("verify_lcmm_dossier.py -- long-context multimodal dossier (sphinx)")
|
||||
print("verify_lcmm_dossier.py -- long-context multimodal dossier (sphinx) v2")
|
||||
print("root: %s" % HERE)
|
||||
print()
|
||||
|
||||
dossier = load_dossier()
|
||||
manifest = json.load(open(os.path.join(HERE, "manifest.json")))
|
||||
dossier = json.load(open(DOSSIER_PATH))
|
||||
manifest = json.load(open(MANIFEST_PATH))
|
||||
report_text = open(REPORT_PATH).read()
|
||||
ids = manifest["ids"]
|
||||
|
||||
print("A. manifest integrity (re-measured from disk)")
|
||||
check("sha256 dossier.json == manifest", sha(os.path.join(HERE, "dossier.json")) == manifest["dossier_sha256"],
|
||||
print("A. manifest integrity (re-measured from disk + recomputed from dossier)")
|
||||
check("sha256 dossier.json == manifest",
|
||||
sha_file(DOSSIER_PATH) == manifest["dossier_sha256"],
|
||||
manifest["dossier_sha256"][:16])
|
||||
check("bytes dossier.json == manifest", os.path.getsize(os.path.join(HERE, "dossier.json")) == manifest["dossier_bytes"])
|
||||
atom_path = os.path.join(SOURCE, "atom_batch.xml")
|
||||
check("sha256 source/atom_batch.xml == manifest", sha(atom_path) == manifest["source_atom_sha256"],
|
||||
check("bytes dossier.json == manifest",
|
||||
os.path.getsize(DOSSIER_PATH) == manifest["dossier_bytes"])
|
||||
check("sha256 source/atom_batch.xml == manifest",
|
||||
sha_file(ATOM_PATH) == manifest["source_atom_sha256"],
|
||||
manifest["source_atom_sha256"][:16])
|
||||
check("dossier paper_count == manifest", dossier["paper_count"] == manifest["paper_count"],
|
||||
str(dossier["paper_count"]))
|
||||
check("dossier.source_atom_sha256 == manifest",
|
||||
dossier["source_atom_sha256"] == manifest["source_atom_sha256"])
|
||||
a5 = check_manifest_perpaper(dossier, manifest)
|
||||
check("manifest per-paper hashes/fields recomputable from dossier.json",
|
||||
not a5, "%d problem(s)" % len(a5))
|
||||
for f in a5[:5]:
|
||||
print(" %s" % f)
|
||||
|
||||
print()
|
||||
print("B. live arXiv API re-fetch (this is the evidence, not a replay)")
|
||||
truth = {}
|
||||
live_ok = True
|
||||
live_err = None
|
||||
try:
|
||||
for i in range(0, len(ids), 5):
|
||||
chunk = ids[i:i + 5]
|
||||
truth.update(parse(fetch_live(chunk)))
|
||||
truth.update(fetch_batch(ids[i:i + 5]))
|
||||
if i + 5 < len(ids):
|
||||
time.sleep(4)
|
||||
time.sleep(3)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
live_ok = False
|
||||
print(" (live fetch unavailable: %r)" % exc)
|
||||
live_err = exc
|
||||
print(" (live fetch unavailable after %d attempts/host: %r)" % (ATTEMPTS, exc))
|
||||
|
||||
if not live_ok:
|
||||
if live_err is not None:
|
||||
mode = "CACHED"
|
||||
raw = open(atom_path, "rb").read()
|
||||
truth = {}
|
||||
with open(ATOM_PATH, "rb") as f:
|
||||
raw = f.read()
|
||||
for part in raw.split(BOUNDARY):
|
||||
part = part.strip()
|
||||
if part:
|
||||
@@ -153,11 +304,12 @@ def main():
|
||||
print(" mode: CACHED -- compared against archived source/atom_batch.xml")
|
||||
else:
|
||||
print(" mode: LIVE -- compared against the arXiv API fetched at check time")
|
||||
check("every id resolved", set(truth) == set(ids), "%d/%d" % (len(truth), len(ids)))
|
||||
check("every id resolved", set(truth) == set(ids),
|
||||
"%d/%d" % (len(truth), len(ids)))
|
||||
|
||||
print()
|
||||
print("C. dossier vs %s truth" % mode)
|
||||
bad = compare(dossier, truth, mode)
|
||||
bad = check_fields(dossier, truth)
|
||||
for b in bad[:10]:
|
||||
print(" FAIL %s" % b)
|
||||
if len(bad) > 10:
|
||||
@@ -166,51 +318,87 @@ def main():
|
||||
not bad, "%d mismatch(es)" % len(bad))
|
||||
|
||||
print()
|
||||
print("D. scope checks")
|
||||
print("D. scope + report-rendering checks")
|
||||
scope = check_scope(dossier)
|
||||
check("paper_count >= 10", len(dossier["papers"]) >= 10, str(len(dossier["papers"])))
|
||||
check("every abstract non-empty", all(p["abstract"].strip() for p in dossier["papers"]))
|
||||
check("every abstract non-empty", not [f for f in scope if "empty abstract" in f])
|
||||
check("every primary_category allowed",
|
||||
all(p["primary_category"] in ALLOWED_CATS for p in dossier["papers"]),
|
||||
not [f for f in scope if "primary_category" in f],
|
||||
str(sorted({p["primary_category"] for p in dossier["papers"]})))
|
||||
check("every paper has a lane note",
|
||||
all(p.get("lane_note") for p in dossier["papers"]))
|
||||
check("every paper has a lane note", not [f for f in scope if "lane note" in f])
|
||||
rows = parse_report_rows(report_text)
|
||||
rep = check_report(dossier, rows)
|
||||
check("REPORT paper table == dossier.json (id/version/cat/authors/date/title)",
|
||||
not rep, "%d drift(s)" % len(rep))
|
||||
for f in rep[:6]:
|
||||
print(" %s" % f)
|
||||
|
||||
print()
|
||||
print("E. negative controls (each mutated copy MUST go red)")
|
||||
print("E. controls (negative ones MUST go red, the positive one MUST stay green)")
|
||||
controls = 0
|
||||
|
||||
def run_control(name, mutate_dossier=None, mutate_manifest=None):
|
||||
def run(name, fn, expect_red):
|
||||
nonlocal controls
|
||||
d2 = copy.deepcopy(dossier)
|
||||
m2 = copy.deepcopy(manifest)
|
||||
if mutate_dossier:
|
||||
mutate_dossier(d2)
|
||||
if mutate_manifest:
|
||||
mutate_manifest(m2)
|
||||
bad = compare(d2, truth, "mutated")
|
||||
if mutate_manifest:
|
||||
bad.append("manifest tampered")
|
||||
fails = fn()
|
||||
controls += 1
|
||||
check("%s is caught" % name, bool(bad), "%d failure(s)" % len(bad))
|
||||
if expect_red:
|
||||
check("%s is caught by %s" % (name, expect_red), bool(fails),
|
||||
"%d failure(s)" % len(fails))
|
||||
else:
|
||||
check("%s stays green" % name, not fails,
|
||||
"%d failure(s)" % len(fails))
|
||||
|
||||
def mutated(mut):
|
||||
d = copy.deepcopy(dossier)
|
||||
mut(d)
|
||||
return d
|
||||
|
||||
def flip_title(d):
|
||||
d["papers"][0]["title"] = d["papers"][0]["title"] + " (TAMPERED)"
|
||||
d["papers"][0]["title"] += " (TAMPERED)"
|
||||
|
||||
def drop_paper(d):
|
||||
d["papers"].pop()
|
||||
d["paper_count"] = len(d["papers"])
|
||||
|
||||
def tamper_manifest(m):
|
||||
first = m["ids"][0]
|
||||
m["per_paper"][first]["title_sha256"] = "0" * 64
|
||||
def edit_abstract(d):
|
||||
d["papers"][2]["abstract"] += " tampered"
|
||||
|
||||
run_control("F1 a flipped paper title", mutate_dossier=flip_title)
|
||||
run_control("F2 a dropped paper", mutate_dossier=drop_paper)
|
||||
run_control("F3 an altered manifest hash", mutate_manifest=tamper_manifest)
|
||||
check("all 3 controls went red", controls == 3, "%d control(s) ran" % controls)
|
||||
def bump_version(d):
|
||||
d["papers"][4]["version"] = d["papers"][4]["version"].replace("v", "v9")
|
||||
|
||||
def lie_count(d):
|
||||
d["paper_count"] = len(d["papers"]) + 1
|
||||
|
||||
def mutated_manifest():
|
||||
m = copy.deepcopy(manifest)
|
||||
m["per_paper"][m["ids"][0]]["title_sha256"] = "0" * 64
|
||||
return m
|
||||
|
||||
def mutated_rows():
|
||||
rs = copy.deepcopy(rows)
|
||||
rs[3]["version"] = "9999.99999v9"
|
||||
return rs
|
||||
|
||||
run("P0 an unmodified copy", lambda: check_fields(dossier, truth), None)
|
||||
run("F1 a flipped paper title",
|
||||
lambda: check_fields(mutated(flip_title), truth), "field compare")
|
||||
run("F2 a dropped paper",
|
||||
lambda: check_fields(mutated(drop_paper), truth), "field compare")
|
||||
run("F3 an edited abstract",
|
||||
lambda: check_fields(mutated(edit_abstract), truth), "field compare")
|
||||
run("F4 a bumped version string",
|
||||
lambda: check_fields(mutated(bump_version), truth), "field compare")
|
||||
run("F5 a lying paper_count",
|
||||
lambda: check_manifest_perpaper(mutated(lie_count), manifest), "manifest recompute")
|
||||
run("F6 an altered manifest hash",
|
||||
lambda: check_manifest_perpaper(dossier, mutated_manifest()), "manifest recompute")
|
||||
run("F7 an altered report row",
|
||||
lambda: check_report(dossier, mutated_rows()), "report table check")
|
||||
check("all 8 controls ran", controls == 8, "%d control(s)" % controls)
|
||||
|
||||
print()
|
||||
print("mode: %s checks run: %d failures: %d" % (mode, ok_count + fail_count, fail_count))
|
||||
print("mode: %s checks run: %d failures: %d"
|
||||
% (mode, ok_count + fail_count, fail_count))
|
||||
print("VERDICT: dossier holds" if fail_count == 0 else "VERDICT: dossier does NOT hold")
|
||||
return 0 if fail_count == 0 else 1
|
||||
|
||||
|
||||
Reference in new issue
Block a user