stackchain/troupe-standup
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sphinx: multimodal arXiv archive dossier with live re-fetch verifier (118/120 HTML coverage)

Browse Source
This commit is contained in:
sphinx committed 2026-10-22 17:14:45 +00:00
1 parent b859ad6464
commit 5f47bc5157
7 files changed
+2071
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

wizard-work/sphinx/MANIFEST-mm-vla-dossier-2026-10-22.json
+177
View File
@@ -0,0 +1,177 @@
{
"dossier_dir": "/root/.hermes/wizard-outputs/sphinx/mm-vla-dossier-2026-10-22",
"generated_utc": "2026-10-22T17:13:24Z",
"query": "cat:cs.CV AND abs:\"multimodal\"",
"atom_url": "https://export.arxiv.org/api/query?search_query=cat:cs.CV+AND+abs:%22multimodal%22&start=0&max_results=120&sortBy=submittedDate&sortOrder=descending",
"atom_http": 200,
"atom_sha256": "17767bfb4ec4c02fdc21e79d7bf7f96035610ee37356a1edc2a3319a7e4c95be",
"atom_bytes": 311552,
"atom_totalResults": 15740,
"papers": [
{
"arxiv_id": "2610.30108v1",
"title": "Multimodal Embodied Agents: A Unified Capability-centric Perspective from Computer-Use to Robot-Use",
"primary_category": "cs.RO",
"categories": [
"cs.RO",
"cs.AI",
"cs.CV",
"cs.MA",
"cs.MM"
],
"authors": [
"Yanzhe Chen",
"Ziyi Yang",
"Jifeng Zhu",
"Qiming Huang",
"Ruihe An",
"Peiyao Xu",
"Hesen Yang",
"Runda Liu",
"Chang Gong",
"Zhijun Cao",
"Zechen Bai",
"Wenzheng Zeng",
"Yiqi Lin",
"Guoqiang Liang",
"Kevin Yuchen Ma",
"Kevin Qinghong Lin",
"Mike Zheng Shou"
],
"published": "2026-10-21T17:52:48Z",
"updated": "2026-10-21T17:52:48Z",
"html_sha256": "07af14c4b4e8491d61b4940433d03836a02b486ba2d473a360bef9ace8bc3d1f",
"html_bytes": 1102349,
"tables": 4,
"figures": 12,
"images": 10,
"dossier": "dossier.json"
},
{
"arxiv_id": "2610.30089v1",
"title": "ProactEgoMem: Benchmarking Proactive Memory in Streaming Egocentric Video",
"primary_category": "cs.CV",
"categories": [
"cs.CV"
],
"authors": [
"Ruotong Liao",
"Meikel Kokowski",
"Xiaoxiao Sun",
"Yongqin Xian",
"Laura Bravo S\u00e1nchez",
"Shiye Su",
"Federico Tombari",
"Serena Yeung-Levy",
"Volker Tresp"
],
"published": "2026-10-21T17:37:58Z",
"updated": "2026-10-21T17:37:58Z",
"html_sha256": "50947ff1e7c8be12fa01771ec1a2c10efe54842661c19538bb5babdd045107c8",
"html_bytes": 1879293,
"tables": 12,
"figures": 30,
"images": 12,
"dossier": "dossier.json"
},
{
"arxiv_id": "2610.29913v1",
"title": "Asking a Multimodal LLM Whether Someone Fell: Zero-Shot Screening of Low-Resolution Thermal Video in a Laboratory and an Occupied Home",
"primary_category": "cs.CV",
"categories": [
"cs.CV",
"cs.AI"
],
"authors": [
"Marcos Junca",
"Mauro Junca"
],
"published": "2026-10-21T15:53:17Z",
"updated": "2026-10-21T15:53:17Z",
"html_sha256": "ff4f590f25c54fbcc5b32c35fefbaa9aa97fdb81f1fe0601a9287ccd89be81e7",
"html_bytes": 239906,
"tables": 6,
"figures": 12,
"images": 1,
"dossier": "dossier.json"
}
],
"files": {
"REPORT.md": {
"sha256": "4def6c59fb407158ce03cdbe9851fea0bf60b437bc60d584fceda40d212db9b5",
"bytes": 6619
},
"coverage_probe.json": {
"sha256": "a52323156933264c56c168084765b49e08c16a3b98ae689a87bfd1c6c6ebbafb",
"bytes": 28464
},
"coverage_probe.py": {
"sha256": "65916084d57d6a68b3767948735322f6ba0e93ca1d4eb0f0158ee500d5b5a7e2",
"bytes": 2470
},
"2610.30108v1/dossier.json": {
"sha256": "3426a4710481151dd98c06ac306128a72a481f445b7d4370182673b2531e658d",
"bytes": 3523
},
"2610.30108v1/figures.json": {
"sha256": "945abcbc1508198dc5b6eeff5ed6cf66ec18437294a5ad6671f1759153356eb5",
"bytes": 5817
},
"2610.30108v1/paper.html": {
"sha256": "07af14c4b4e8491d61b4940433d03836a02b486ba2d473a360bef9ace8bc3d1f",
"bytes": 1102349
},
"2610.30108v1/paper.txt": {
"sha256": "be1522c76f455b2872dcc3994de7e6c237f513e85e84ab5a7a36e0d385533471",
"bytes": 199028
},
"2610.30108v1/tables.json": {
"sha256": "2ecea4a9c8e97761e25cf1bfc0598c66b2c2080985be61bc1f4461ec763bf471",
"bytes": 14211
},
"2610.29913v1/dossier.json": {
"sha256": "c8fc594eb36367e4b8cd5b675db5dd094cda70c0e5aeea4237ce18a0f5761239",
"bytes": 3409
},
"2610.29913v1/figures.json": {
"sha256": "b02c21c8a7281772b60c282d6d9e77c1840fbebf9b29661b9c6164ef522c75b9",
"bytes": 5355
},
"2610.29913v1/paper.html": {
"sha256": "ff4f590f25c54fbcc5b32c35fefbaa9aa97fdb81f1fe0601a9287ccd89be81e7",
"bytes": 239906
},
"2610.29913v1/paper.txt": {
"sha256": "26fe614558e972064a5d15d79383d16d3f1b1abef9846dc5e6881554e443d5ff",
"bytes": 74930
},
"2610.29913v1/tables.json": {
"sha256": "0dc9286e5a581416ca874ca5fc61e5e6a518969c1ea77946c01c5638d642fa86",
"bytes": 7429
},
"2610.30089v1/dossier.json": {
"sha256": "d1f7ab9a02372aec2971757cd336199815452aa75fdbcd72d5a52e1d185a9d90",
"bytes": 3919
},
"2610.30089v1/figures.json": {
"sha256": "a4faac626d547d0ddba399e0f494d2bd1fa833c7f5c32c0fff7feca7f14783c9",
"bytes": 10123
},
"2610.30089v1/paper.html": {
"sha256": "50947ff1e7c8be12fa01771ec1a2c10efe54842661c19538bb5babdd045107c8",
"bytes": 1879293
},
"2610.30089v1/paper.txt": {
"sha256": "ddf9b5c076a157fdff66cc54b03097b2200b315403e548e33bf7639fdec734c6",
"bytes": 121203
},
"2610.30089v1/tables.json": {
"sha256": "2a7f2756c4609df24fa7000642c4ed00128ae2940474fdc22475a5605f4c2886",
"bytes": 29693
},
"raw/atom_feed.xml": {
"sha256": "17767bfb4ec4c02fdc21e79d7bf7f96035610ee37356a1edc2a3319a7e4c95be",
"bytes": 311552
}
}
}
wizard-work/sphinx/README-archive-lane.md
+24
View File
@@ -0,0 +1,24 @@
# sphinx — Archive lane work (multimodal, long-context research)
## Latest: `mm-vla-dossier-2026-10-22`
Live arXiv Atom API -> multimodal paper dossier with an independently re-fetching verifier.
Artifacts:
- `mm-vla-dossier-2026-10-22.tar.gz` — the full dossier (3 papers, 19 hashed files)
- `build_mm_dossier.py` — stdlib-only builder (Atom query -> HTML fetch -> text layer -> tables/figures -> dossier -> MANIFEST with sha256)
- `verify_mm_dossier.py` — independent verifier; re-fetches every paper over HTTP and compares sha256
- `REPORT-mm-vla-dossier-2026-10-22.md` — human-readable report
- `MANIFEST-mm-vla-dossier-2026-10-22.json` — sha256 + byte size of every artifact
- `coverage-probe-120-papers.json` — HTML-rendering availability probe over 120 recent papers
Measured result (not assumed): of 120 recent `cs.CV` papers matching `abs:"multimodal"`,
118 (98.3%) have an arXiv HTML rendering at `arxiv.org/html/<id>`.
Reproduce:
python3 build_mm_dossier.py 'cat:cs.CV AND abs:"multimodal"' 3 120
python3 verify_mm_dossier.py mm-vla-dossier-2026-10-22
Verifier exit code is the verdict: 0 = all 7 check groups pass (manifest integrity,
hash agreement, text-layer derivation, table/figure counts, live re-fetch equality,
live Atom traceability, report coverage).
wizard-work/sphinx/REPORT-mm-vla-dossier-2026-10-22.md
+86
View File
@@ -0,0 +1,86 @@
# Multimodal archive dossier — live arXiv fetch (2026-10-22T17:13:24Z)
Lane: Archive (multimodal, long-context research). Built from a live arXiv Atom API query; every number below was read out of fetched bytes.
## Query
- search_query: `cat:cs.CV AND abs:"multimodal"`
- Atom URL: https://export.arxiv.org/api/query?search_query=cat:cs.CV+AND+abs:%22multimodal%22&start=0&max_results=120&sortBy=submittedDate&sortOrder=descending
- HTTP 200, 311552 bytes, sha256 `17767bfb4ec4c02fdc21e79d7bf7f96035610ee37356a1edc2a3319a7e4c95be`
- opensearch:totalResults = 15740
- entries parsed from feed: 120
- papers with an HTML rendering fetched: 3
## Papers
| arXiv id | primary cat | published | tables | figures | images | title |
|---|---|---|---|---|---|---|
| [2610.30108v1](https://arxiv.org/abs/2610.30108v1) | cs.RO | 2026-10-21 | 4 | 12 | 10 | Multimodal Embodied Agents: A Unified Capability-centric Perspective from Computer-Use to Robot-Use |
| [2610.30089v1](https://arxiv.org/abs/2610.30089v1) | cs.CV | 2026-10-21 | 12 | 30 | 12 | ProactEgoMem: Benchmarking Proactive Memory in Streaming Egocentric Video |
| [2610.29913v1](https://arxiv.org/abs/2610.29913v1) | cs.CV | 2026-10-21 | 6 | 12 | 1 | Asking a Multimodal LLM Whether Someone Fell: Zero-Shot Screening of Low-Resolution Thermal Video in a Laborat |
## Coverage probe (HTML-rendering availability)
Before archiving, every entry in the 120-item Atom feed was probed for an HTML rendering at `arxiv.org/html/<id>`. Measured, not assumed:
- papers probed: 120
- with an HTML rendering: 118 (98.3%)
- median tables per paper: 18 (min 1, max 69)
- median figures per paper: 17 (min 4, max 359)
- median img tags per paper: 10 (min 2, max 343)
## Figure/table inventory
### 2610.30108v1 — Multimodal Embodied Agents: A Unified Capability-centric Perspective from Computer-Use to Robot-Use
- text layer: 196723 chars; tables: 4 (numeric: 4); figures: 12; images: 10; svg figures: 1; figures with MathML: 2
- table 0: 12 rows — (no caption observed)
- table 1: 1 rows — (no caption observed)
- table 2: 1 rows — (no caption observed)
- table 3: 78 rows — (no caption observed)
- figure 0: 1 img, 0 svg, math=False — Figure 1: A Unified Capability-centric Perspective (PAPAV) from computer-use to robot-use.
- figure 1: 1 img, 1 svg, math=False — Figure 2: Chronological overview of representative work. Robotic systems progress from vis
- figure 2: 0 img, 0 svg, math=True — Table 1: Positioning related surveys. Checks mark dimensions central to a survey; crosses
- figure 3: 0 img, 0 svg, math=False — Figure 3: Research landscape and taxonomy of PAPAV. The capability branch groups represent
- figure 4: 1 img, 0 svg, math=False — Figure 4: PAPAV from computer-use to robot-use. Each row contrasts how a shared capability
- figure 5: 1 img, 0 svg, math=False — Figure 5: Perceive: MMEA vs. MMA vs. RS. Shared (center): multimodal evidence grounding. L
- figure 6: 1 img, 0 svg, math=False — Figure 6: Anticipate: MMEA vs. MMA vs. RS. Shared (center): action-conditioned state-trans
- figure 7: 1 img, 0 svg, math=False — Figure 7: Plan: MMEA vs. MMA vs. RS. Shared (center): hierarchical, constraint-aware cours
### 2610.30089v1 — ProactEgoMem: Benchmarking Proactive Memory in Streaming Egocentric Video
- text layer: 119665 chars; tables: 12 (numeric: 5); figures: 30; images: 12; svg figures: 668; figures with MathML: 14
- table 0: 0 rows — (no caption observed)
- table 1: 2 rows — (no caption observed)
- table 2: 10 rows — (no caption observed)
- table 3: 19 rows — (no caption observed)
- table 4: 6 rows — (no caption observed)
- table 5: 42 rows — (no caption observed)
- table 6: 25 rows — (no caption observed)
- table 7: 1 rows — (no caption observed)
- figure 0: 0 img, 0 svg, math=False — Figure 1: Proactive agents observe video, co-exist in the background, and act before users
- figure 1: 1 img, 0 svg, math=False — Figure 2: ProactEgoMem measures proactive memory as lifecycle behavior in egocentric strea
- figure 2: 0 img, 1 svg, math=True — Figure 3: Two- level proactive modes .
- figure 3: 0 img, 6 svg, math=False — Table 1: Illustrative examples of proactive messages for each task and mode.
- figure 4: 1 img, 0 svg, math=False — Figure 4: ProactEgoMem composition and target objects. Left : composition by scenario (inn
- figure 5: 0 img, 3 svg, math=True — Table 2: ProactEgoMem-QA question types and benchmark examples
- figure 6: 0 img, 0 svg, math=True — Table 3: ProactEgoMem data statistics and inter-annotator agreement. Paired entries read D
- figure 7: 0 img, 312 svg, math=True — Table 4: ProactEgoMem-Daily main results for notification and intervention levels. • Start
### 2610.29913v1 — Asking a Multimodal LLM Whether Someone Fell: Zero-Shot Screening of Low-Resolution Thermal Video in a Laboratory and an Occupied Home
- text layer: 74085 chars; tables: 6 (numeric: 6); figures: 12; images: 1; svg figures: 0; figures with MathML: 7
- table 0: 13 rows — (no caption observed)
- table 1: 7 rows — (no caption observed)
- table 2: 10 rows — (no caption observed)
- table 3: 10 rows — (no caption observed)
- table 4: 8 rows — (no caption observed)
- table 5: 4 rows — (no caption observed)
- figure 0: 0 img, 0 svg, math=False — Fig. 1 : The prototype. A thermal camera and a microphone stream continuously; a wake gate
- figure 1: 0 img, 0 svg, math=False — Fig. 2 : The home recording, 6–13 September 2026: clips the deployed gate and pipeline sen
- figure 2: 0 img, 0 svg, math=True — TABLE I : Conditions, each changing one factor of the deployed request, and what each was
- figure 3: 0 img, 0 svg, math=False — Fig. 3 : Laboratory falls called a fall, by fall type (all six subjects): Gemini 3.8 Flash
- figure 4: 0 img, 0 svg, math=False — TABLE II : The model’s answers, verbatim (deployed request; the laboratory answers’ openin
- figure 5: 1 img, 0 svg, math=True — Fig. 4 : One staged fall (a slow backward faint) from the public laboratory data [ 6 ] (CC
- figure 6: 0 img, 0 svg, math=False — Fig. 5 : Effect of sensor resolution (block averaging, same field of view, same display si
- figure 7: 0 img, 0 svg, math=True — Fig. 6 : Each screener as one point: laboratory falls called (held-out subjects, Wilson 95
## Provenance
Every artifact carries a sha256 in MANIFEST.json. Nothing in this report was inferred from a cached or third-party source: all bytes came from the two URLs above at fetch time.
wizard-work/sphinx/build_mm_dossier.py
+348
View File
@@ -0,0 +1,348 @@
#!/usr/bin/env python3
"""Archive lane build: live arXiv Atom API -> multimodal paper dossier.
Produces, under OUTDIR:
<slug>/paper.html raw arXiv HTML v2 rendering (fetched)
<slug>/paper.txt text layer of that HTML
<slug>/tables.json every <table> in the paper, as row/cell grids
<slug>/figures.json every <figure> with its img srcs and caption text
<slug>/dossier.json metadata + tables + figures + verification record
<slug>/MANIFEST.json sha256 of every artifact + the raw HTTP provenance
REPORT.md human-readable dossier
Stdlib only. No fabricated fields: anything not observed in the fetched bytes is
left null and marked "not observed" rather than guessed.
"""
import hashlib
import html as htmllib
import json
import os
import re
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
UA = {"User-Agent": "sphinx-archive-research/1.0 (mailto:sphinx@agents.invalid)"}
OUTDIR = "/root/.hermes/wizard-outputs/sphinx"
ARXIV_API = "https://export.arxiv.org/api/query"
def http_get(url, timeout=60):
"""Return (status_code_or_None, body_text_or_error_string, sha256_of_bytes, nbytes)."""
req = urllib.request.Request(url, headers=UA)
try:
with urllib.request.urlopen(req, timeout=timeout) as r:
raw = r.read()
return r.status, raw.decode("utf-8", "replace"), hashlib.sha256(raw).hexdigest(), len(raw)
except urllib.error.HTTPError as e:
return e.code, "HTTPError %d" % e.code, None, 0
except Exception as e: # network/DNS/TLS failures
return None, "%s: %s" % (type(e).__name__, e), None, 0
# ---------------------------------------------------------------- Atom parsing
def parse_atom(feed_text):
"""Parse arXiv Atom feed into entry dicts. Regex over the XML text; the arXiv
feed is flat enough that element-order assumptions hold (id, title, updated,
published, summary, authors, categories, links, comment, journal_ref, doi)."""
entries = []
for em in re.finditer(r"<entry>(.*?)</entry>", feed_text, re.S):
e = em.group(1)
def tag(name):
m = re.search(r"<%s(?:\s[^>]*)?>(.*?)</%s>" % (name, name), e, re.S)
if not m:
return None
return " ".join(htmllib.unescape(m.group(1)).split())
eid = tag("id")
links = re.findall(r'<link\s+([^>]*?)/?>', e)
link_map = {}
for lm in links:
href = re.search(r'href="([^"]+)"', lm)
rel = re.search(r'rel="([^"]+)"', lm)
typ = re.search(r'type="([^"]+)"', lm)
if href:
link_map[rel.group(1) if rel else "alternate"] = {
"href": href.group(1),
"type": typ.group(1) if typ else None,
}
authors = re.findall(r"<name>(.*?)</name>", e, re.S)
authors = [" ".join(htmllib.unescape(a).split()) for a in authors]
cats = re.findall(r'<category\s+term="([^"]+)"', e)
prim = re.search(r'<arxiv:primary_category\s+term="([^"]+)"', e)
comment = tag("arxiv:comment")
jref = tag("arxiv:journal_ref")
doi = tag("arxiv:doi")
entries.append({
"id": eid,
"arxiv_id": eid.rsplit("/abs/", 1)[-1] if eid else None,
"title": tag("title"),
"summary": tag("summary"),
"updated": tag("updated"),
"published": tag("published"),
"authors": authors,
"categories": cats,
"primary_category": prim.group(1) if prim else None,
"links": link_map,
"comment": comment,
"journal_ref": jref,
"doi": doi,
})
return entries
# ------------------------------------------------------------- HTML text layer
def html_to_text(doc):
"""Crude but faithful text layer: drop script/style, keep pre/code blocks,
turn block tags into newlines, unescape entities, collapse runs of blanks."""
doc = re.sub(r"(?is)<(script|style|noscript)\b.*?</\1>", " ", doc)
doc = re.sub(r"(?is)<br\s*/?>", "\n", doc)
doc = re.sub(r"(?is)</(p|div|li|tr|h[1-6]|section|article|figcaption|table|figure)>", "\n", doc)
doc = re.sub(r"(?is)<li\b[^>]*>", "\n- ", doc)
doc = re.sub(r"(?is)<td\b[^>]*>", " | ", doc)
doc = re.sub(r"(?is)<[^>]+>", " ", doc)
doc = htmllib.unescape(doc)
doc = doc.replace("\u00a0", " ")
doc = re.sub(r"[ \t]+", " ", doc)
doc = re.sub(r"\n[ \t]+", "\n", doc)
doc = re.sub(r"\n{3,}", "\n\n", doc)
return doc.strip()
def strip_tags(s):
s = re.sub(r"(?is)<[^>]+>", " ", s)
return " ".join(htmllib.unescape(s).split())
# ------------------------------------------------------------- tables & figures
def extract_tables(doc):
out = []
for i, tm in enumerate(re.finditer(r"(?is)<table\b.*?</table>", doc)):
t = tm.group(0)
head = strip_tags(t[:t.lower().find(">") + 1])
cap = ""
cm = re.search(r"(?is)<caption\b.*?</caption>", t)
if cm:
cap = strip_tags(cm.group(0))
rows = []
for rm in re.finditer(r"(?is)<tr\b.*?</tr>", t):
cells = []
for cm2 in re.finditer(r"(?is)<t[dh]\b.*?</t[dh]>", rm.group(0)):
cells.append(strip_tags(cm2.group(0)))
if any(cells):
rows.append(cells)
out.append({"index": i, "caption": cap, "header_tag": head,
"n_rows": len(rows), "rows": rows})
return out
def extract_figures(doc):
out = []
for i, fm in enumerate(re.finditer(r"(?is)<figure\b.*?</figure>", doc)):
f = fm.group(0)
imgs = []
for im in re.finditer(r"(?is)<img\b[^>]*>", f):
tag = im.group(0)
src = re.search(r'src="([^"]+)"', tag)
alt = re.search(r'alt="([^"]*)"', tag)
imgs.append({
"src": src.group(1) if src else None,
"alt": htmllib.unescape(alt.group(1)) if alt else None,
})
svgs = len(re.findall(r"(?is)<svg\b", f))
cap = ""
cm = re.search(r"(?is)<figcaption\b.*?</figcaption>", f)
if cm:
cap = strip_tags(cm.group(0))
has_math = bool(re.search(r"(?is)<math\b", f))
out.append({"index": i, "n_images": len(imgs), "images": imgs,
"n_svg": svgs, "has_math": has_math, "caption": cap})
return out
def main():
query = sys.argv[1] if len(sys.argv) > 1 else 'cat:cs.CV AND abs:"multimodal"'
n = int(sys.argv[2]) if len(sys.argv) > 2 else 1
max_results = int(sys.argv[3]) if len(sys.argv) > 3 else 120
outdir = os.path.join(OUTDIR, "mm-vla-dossier-2026-10-22")
rawdir = os.path.join(outdir, "raw")
os.makedirs(rawdir, exist_ok=True)
# ---- 1. live Atom query -------------------------------------------------
qs = urllib.parse.quote(query) if False else query.replace(" ", "+").replace('"', "%22")
url = "%s?search_query=%s&start=0&max_results=%d&sortBy=submittedDate&sortOrder=descending" % (
ARXIV_API, qs, max_results)
print("[1] GET", url)
st, feed_text, feed_sha, feed_bytes = http_get(url)
print(" HTTP %s %d bytes sha256 %s" % (st, feed_bytes, feed_sha))
if st != 200:
sys.exit("Atom query failed: %s" % feed_text)
open(os.path.join(rawdir, "atom_feed.xml"), "w", encoding="utf-8").write(feed_text)
tot = re.search(r"<opensearch:totalResults[^>]*>(\d+)</opensearch:totalResults>", feed_text)
entries = parse_atom(feed_text)
print(" totalResults=%s parsed entries=%d" % (tot.group(1) if tot else "?", len(entries)))
# ---- 2. pick papers that have an HTML rendering -------------------------
picks, scanned = [], 0
for e in entries:
if len(picks) >= n or scanned >= 40:
break
pid = e["arxiv_id"]
scanned += 1
hurl = "https://arxiv.org/html/%s" % pid
hst, hdoc, hsha, hbytes = http_get(hurl)
print("[2] %-16s html HTTP %s (%s bytes)" % (pid, hst, hbytes if hbytes else hdoc))
if hst != 200:
e["html_available"] = False
continue
e["html_available"] = True
e["html_url"] = hurl
e["html_sha256"] = hsha
e["html_bytes"] = hbytes
picks.append((e, hdoc, hsha, hbytes))
time.sleep(1.0)
if not picks:
sys.exit("no HTML renderings found in the first %d entries" % scanned)
manifest = {"dossier_dir": outdir, "generated_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"query": query, "atom_url": url, "atom_http": st,
"atom_sha256": feed_sha, "atom_bytes": feed_bytes,
"atom_totalResults": int(tot.group(1)) if tot else None,
"papers": [], "files": {}}
for e, hdoc, hsha, hbytes in picks:
pid = e["arxiv_id"]
pdir = os.path.join(outdir, pid)
os.makedirs(pdir, exist_ok=True)
open(os.path.join(pdir, "paper.html"), "w", encoding="utf-8").write(hdoc)
text = html_to_text(hdoc)
open(os.path.join(pdir, "paper.txt"), "w", encoding="utf-8").write(text)
tables = extract_tables(hdoc)
figures = extract_figures(hdoc)
json.dump(tables, open(os.path.join(pdir, "tables.json"), "w"), indent=1)
json.dump(figures, open(os.path.join(pdir, "figures.json"), "w"), indent=1)
# every image src, resolved against the html url
img_srcs = sorted({s for f in figures for i in f["images"] if (s := i["src"])})
dossier = {
"paper": e,
"archive": {
"fetched_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"fetcher_ua": UA["User-Agent"],
"html_url": e["html_url"],
"html_sha256": hsha,
"html_bytes": hbytes,
"text_chars": len(text),
"tables_found": len(tables),
"figures_found": len(figures),
"images_found": len(img_srcs),
"svg_figures": sum(f["n_svg"] for f in figures),
"figures_with_math": sum(1 for f in figures if f["has_math"]),
"tables_with_numeric_cells": sum(
1 for t in tables
if any(re.search(r"\d", c) for r in t["rows"] for c in r)),
"figure_images": img_srcs,
},
}
json.dump(dossier, open(os.path.join(pdir, "dossier.json"), "w"), indent=1)
manifest["papers"].append({
"arxiv_id": pid, "title": e["title"],
"primary_category": e["primary_category"],
"categories": e["categories"], "authors": e["authors"],
"published": e["published"], "updated": e["updated"],
"html_sha256": hsha, "html_bytes": hbytes,
"tables": len(tables), "figures": len(figures), "images": len(img_srcs),
"dossier": "dossier.json",
})
# ---- 4. REPORT.md -------------------------------------------------------
lines = []
lines.append("# Multimodal archive dossier — live arXiv fetch (%s)\n" % manifest["generated_utc"])
lines.append("Lane: Archive (multimodal, long-context research). Built from a live "
"arXiv Atom API query; every number below was read out of fetched bytes.\n")
lines.append("## Query\n")
lines.append("- search_query: `%s`" % query)
lines.append("- Atom URL: %s" % url)
lines.append("- HTTP %s, %d bytes, sha256 `%s`" % (st, feed_bytes, feed_sha))
lines.append("- opensearch:totalResults = %s" % (tot.group(1) if tot else "not observed"))
lines.append("- entries parsed from feed: %d" % len(entries))
lines.append("- papers with an HTML rendering fetched: %d\n" % len(manifest["papers"]))
lines.append("## Papers\n")
lines.append("| arXiv id | primary cat | published | tables | figures | images | title |")
lines.append("|---|---|---|---|---|---|---|")
for p in manifest["papers"]:
t = (p["title"] or "").replace("|", "\\|")
lines.append("| [%s](https://arxiv.org/abs/%s) | %s | %s | %d | %d | %d | %s |" % (
p["arxiv_id"], p["arxiv_id"], p["primary_category"], p["published"][:10],
p["tables"], p["figures"], p["images"], t[:110]))
lines.append("## Coverage probe (HTML-rendering availability)\n")
lines.append("Before archiving, every entry in the 120-item Atom feed was probed for an HTML "
"rendering at `arxiv.org/html/<id>`. Measured, not assumed:\n")
try:
cov = json.load(open(os.path.join(outdir, "coverage_probe.json")))
okc = [r for r in cov if r.get("html")]
lines.append("- papers probed: %d" % len(cov))
lines.append("- with an HTML rendering: %d (%.1f%%)" % (len(okc), 100.0 * len(okc) / len(cov)))
if okc:
for key, label in (("tables", "tables"), ("figures", "figures"), ("imgs", "img tags")):
v = sorted(r[key] for r in okc)
lines.append("- median %s per paper: %d (min %d, max %d)" % (
label, v[len(v) // 2], v[0], v[-1]))
except Exception as e:
lines.append("- coverage probe not present: %s" % e)
lines.append("\n## Figure/table inventory\n")
for p in manifest["papers"]:
d = json.load(open(os.path.join(outdir, p["arxiv_id"], "dossier.json")))
a = d["archive"]
lines.append("### %s — %s" % (p["arxiv_id"], p["title"]))
lines.append("- text layer: %d chars; tables: %d (numeric: %d); figures: %d; images: %d; svg figures: %d; figures with MathML: %d" % (
a["text_chars"], a["tables_found"], a["tables_with_numeric_cells"],
a["figures_found"], a["images_found"], a["svg_figures"], a["figures_with_math"]))
tabs = json.load(open(os.path.join(outdir, p["arxiv_id"], "tables.json")))
for t in tabs[:8]:
cap = t["caption"][:90] if t["caption"] else "(no caption observed)"
lines.append(" - table %d: %d rows — %s" % (t["index"], t["n_rows"], cap))
figs = json.load(open(os.path.join(outdir, p["arxiv_id"], "figures.json")))
for f in figs[:8]:
cap = f["caption"][:90] if f["caption"] else "(no caption observed)"
lines.append(" - figure %d: %d img, %d svg, math=%s — %s" % (
f["index"], f["n_images"], f["n_svg"], f["has_math"], cap))
lines.append("")
lines.append("## Provenance\n")
lines.append("Every artifact carries a sha256 in MANIFEST.json. Nothing in this report "
"was inferred from a cached or third-party source: all bytes came from the "
"two URLs above at fetch time.\n")
open(os.path.join(outdir, "REPORT.md"), "w", encoding="utf-8").write("\n".join(lines))
# ---- 3. MANIFEST with sha256 of every produced file (REPORT.md included) --
def sha(p):
h = hashlib.sha256()
with open(p, "rb") as f:
for chunk in iter(lambda: f.read(65536), b""):
h.update(chunk)
return h.hexdigest()
for root, _dirs, files in os.walk(outdir):
for fn in sorted(files):
if fn == "MANIFEST.json":
continue
p = os.path.join(root, fn)
rel = os.path.relpath(p, outdir)
manifest["files"][rel] = {"sha256": sha(p), "bytes": os.path.getsize(p)}
json.dump(manifest, open(os.path.join(outdir, "MANIFEST.json"), "w"), indent=1)
print("[3] wrote", outdir)
print("[3] papers:", len(manifest["papers"]), "files:", len(manifest["files"]))
if __name__ == "__main__":
main()
wizard-work/sphinx/coverage-probe-120-papers.json
+1310
View File
File diff suppressed because it is too large. Load diff
wizard-work/sphinx/mm-vla-dossier-2026-10-22.tar.gz
BIN
View File
Binary file not shown.
wizard-work/sphinx/verify_mm_dossier.py
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env python3
"""Independent verifier for the multimodal arXiv dossier.
Checks, in order:
A. every file listed in MANIFEST.json exists and its sha256 matches the manifest
B. paper.html's sha256 matches the html_sha256 recorded in that paper's dossier.json
C. paper.txt really is derived from paper.html (substring agreement + size bounds)
D. tables.json and figures.json parse, are non-empty, and their counts match the
counts recorded in dossier.json
E. RE-FETCH each paper.html over HTTP; the sha256 must equal the manifest hash
(i.e. the archive reproduces what arXiv serves today)
F. RE-FETCH the Atom feed over HTTP; the ids of every archived paper must appear
in the live feed (the archive is traceable to the live query)
G. REPORT.md exists, is non-trivial, and names every archived arXiv id
Exit code 0 only if all checks pass.
"""
import hashlib
import json
import os
import re
import sys
import time
import urllib.error
import urllib.request
UA = {"User-Agent": "sphinx-archive-verify/1.0 (mailto:sphinx@agents.invalid)"}
D = sys.argv[1] if len(sys.argv) > 1 else "/root/.hermes/wizard-outputs/sphinx/mm-vla-dossier-2026-10-22"
fails = []
notes = []
def check(name, ok, detail=""):
print(" [%s] %s%s" % ("PASS" if ok else "FAIL", name, (" — " + detail) if detail else ""))
if not ok:
fails.append(name)
return ok
def sha(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for c in iter(lambda: f.read(65536), b""):
h.update(c)
return h.hexdigest()
def http(url, timeout=60):
req = urllib.request.Request(url, headers=UA)
with urllib.request.urlopen(req, timeout=timeout) as r:
return r.status, r.read()
manifest = json.load(open(os.path.join(D, "MANIFEST.json")))
papers = manifest["papers"]
print("A. manifest integrity (sha256 of every listed file)")
for rel, meta in sorted(manifest["files"].items()):
p = os.path.join(D, rel)
ok = os.path.exists(p) and sha(p) == meta["sha256"]
check("A %s" % rel, ok, "%d bytes" % meta["bytes"] if os.path.exists(p) else "MISSING")
print("B. paper.html sha256 == dossier.html_sha256")
for p in papers:
pj = os.path.join(D, p["arxiv_id"], "paper.html")
dj = json.load(open(os.path.join(D, p["arxiv_id"], "dossier.json")))
check("B %s" % p["arxiv_id"], os.path.exists(pj) and sha(pj) == dj["paper"]["html_sha256"],
"manifest says %s" % p["html_sha256"][:16])
print("C. paper.txt is a real text layer of paper.html")
for p in papers:
h = open(os.path.join(D, p["arxiv_id"], "paper.html"), encoding="utf-8", errors="replace").read()
t = open(os.path.join(D, p["arxiv_id"], "paper.txt"), encoding="utf-8").read()
# the title string from the dossier must survive into the text layer
title = (p["title"] or "").strip()
probe = " ".join(title.split())[:40]
ok = (len(t) > 500 and len(t) < len(h) and probe in " ".join(t.split()))
check("C %s" % p["arxiv_id"], ok, "html %d -> text %d chars; title probe present=%s" % (
len(h), len(t), probe in " ".join(t.split())))
print("D. tables.json / figures.json counts agree with dossier.json")
for p in papers:
base = os.path.join(D, p["arxiv_id"])
d = json.load(open(os.path.join(base, "dossier.json")))["archive"]
tabs = json.load(open(os.path.join(base, "tables.json")))
figs = json.load(open(os.path.join(base, "figures.json")))
ok = (len(tabs) == d["tables_found"] and len(figs) == d["figures_found"]
and len(tabs) > 0 and len(figs) > 0)
check("D %s" % p["arxiv_id"], ok,
"tables %d==%d figures %d==%d" % (len(tabs), d["tables_found"], len(figs), d["figures_found"]))
print("E. live re-fetch of paper.html reproduces the archived sha256")
for p in papers:
try:
st, raw = http("https://arxiv.org/html/%s" % p["arxiv_id"])
got = hashlib.sha256(raw).hexdigest()
check("E %s" % p["arxiv_id"], st == 200 and got == p["html_sha256"],
"HTTP %s sha %s" % (st, got[:16]))
except Exception as e:
check("E %s" % p["arxiv_id"], False, "%s: %s" % (type(e).__name__, e))
time.sleep(1.0)
print("F. live Atom feed contains every archived paper id")
try:
st, raw = http(manifest["atom_url"])
feed = raw.decode("utf-8", "replace")
ids_in_feed = set(re.findall(r"<id>http://arxiv\.org/abs/([^<]+)</id>", feed))
missing = [p["arxiv_id"] for p in papers if p["arxiv_id"] not in ids_in_feed]
check("F atom ids", st == 200 and not missing,
"HTTP %s; feed ids %d; missing %s" % (st, len(ids_in_feed), missing or "none"))
except Exception as e:
check("F atom ids", False, "%s: %s" % (type(e).__name__, e))
print("G. REPORT.md exists and names every archived paper")
rp = os.path.join(D, "REPORT.md")
rep = open(rp, encoding="utf-8").read() if os.path.exists(rp) else ""
missing = [p["arxiv_id"] for p in papers if p["arxiv_id"] not in rep]
check("G report", len(rep) > 1000 and not missing,
"%d chars; missing %s" % (len(rep), missing or "none"))
print()
if fails:
print("RESULT: %d check(s) failed: %s" % (len(fails), fails))
sys.exit(1)
print("RESULT: all checks passed (%d papers, %d manifest files)" % (len(papers), len(manifest["files"])))