257 lines
10 KiB
Python
257 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""sphinx archive-lane: independent verification of the arXiv recent-digest claims.
|
|
|
|
WHAT WAS WRONG
|
|
---------------
|
|
The digest's own verifier (arxiv_recent_harvest/verify-2026-10-25.json) recorded
|
|
"claims": 0 and an empty "present" map for every category -- that verification
|
|
was never actually executed.
|
|
|
|
WHAT THIS DOES
|
|
--------------
|
|
Verifies every record the digest claims (the 1210-record 7-day window) against
|
|
THREE independent live sources:
|
|
|
|
S1 Atom API, exact id_list lookup -- no date filter, no ranking, no cap;
|
|
the record either is or is not there.
|
|
S2 Atom API, windowed search query -- the digest's own construction method,
|
|
a consistency check (not independent).
|
|
S3 HTML listing page -- a different service and document;
|
|
/list/<cat>/recent carries /abs/<id>
|
|
anchors for the day's newest items.
|
|
|
|
THE MEASURED TRAP (found by running this, then fixed)
|
|
----------------------------------------------------
|
|
An early version queried each category with max_results=400 and a
|
|
submittedDate filter. That silently missed 1060 of 1210 records. Diagnosis:
|
|
|
|
* A cap truncates silently; pagination is mandatory.
|
|
* The Atom `submittedDate` filter is NOT the record's `published` date. Nine
|
|
in-window records (e.g. 2610.25640, published 2026-10-19) were absent from
|
|
the filtered search yet present under an exact id_list lookup.
|
|
* Those nine have PRIMARY categories outside cs.CL/CV/LG (cs.AI, cs.DB, cs.SE,
|
|
cs.DC, cs.OS), so a cat:cs.CL|CV|LG filter legitimately excludes them. They
|
|
are cross-lists into the digest's categories, which is why the corpus has
|
|
them at all.
|
|
|
|
Therefore S1 (exact id_list) is the authoritative existence check, and the
|
|
residual search-misses are reported as query artifacts, not as corpus errors.
|
|
|
|
Nothing here is simulated. Every field is read from a real HTTP response; a
|
|
failed fetch is recorded as an error and fails the run.
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import xml.etree.ElementTree as ET
|
|
|
|
ATOM_NS = "{http://www.w3.org/2005/Atom}"
|
|
ARXIV_NS = "{http://arxiv.org/schemas/atom}"
|
|
BASE = "https://export.arxiv.org/api/query"
|
|
LIST_BASE = "https://arxiv.org/list/{cat}/recent"
|
|
UA = "sphinx-archive-verifier/1.0 (evidence-lane; contact: sphinx@agents.invalid)"
|
|
|
|
DIGEST = "/root/.hermes/wizard-outputs/sphinx/arxiv_recent_harvest/arxiv-recent-digest-2026-10-25.md"
|
|
CORPUS = "/root/.hermes/wizard-outputs/sphinx/arxiv_recent_harvest/merged_all.jsonl"
|
|
OUTDIR = "/root/.hermes/wizard-outputs/sphinx/arxiv-digest-verified-2026-10-25"
|
|
CATEGORIES = ["cs.CL", "cs.CV", "cs.LG"]
|
|
CUTOFF = "2026-10-19"
|
|
IDLIST_CHUNK = 120
|
|
|
|
|
|
def fetch(url, tries=4, pause=3.0):
|
|
last = None
|
|
for attempt in range(tries):
|
|
try:
|
|
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
|
with urllib.request.urlopen(req, timeout=60) as resp:
|
|
body = resp.read()
|
|
if not body:
|
|
raise RuntimeError("empty body")
|
|
return body
|
|
except Exception as exc: # noqa: BLE001
|
|
last = exc
|
|
if attempt < tries - 1:
|
|
time.sleep(pause * (attempt + 1))
|
|
raise RuntimeError(f"fetch failed: {url}: {last}")
|
|
|
|
|
|
def parse_atom(xml_bytes):
|
|
root = ET.fromstring(xml_bytes)
|
|
out = {}
|
|
for entry in root.findall(f"{ATOM_NS}entry"):
|
|
eid = (entry.findtext(f"{ATOM_NS}id") or "").strip()
|
|
m = re.search(r"/abs/([0-9]{4}\.[0-9]{4,5})(v\d+)?$", eid)
|
|
if not m:
|
|
continue
|
|
title = " ".join((entry.findtext(f"{ATOM_NS}title") or "").split())
|
|
primary = entry.find(f"{ARXIV_NS}primary_category")
|
|
out[m.group(1)] = {
|
|
"title": title,
|
|
"title_sha256": hashlib.sha256(title.encode("utf-8")).hexdigest(),
|
|
"published": (entry.findtext(f"{ATOM_NS}published") or "").strip(),
|
|
"updated": (entry.findtext(f"{ATOM_NS}updated") or "").strip(),
|
|
"primary_category": primary.get("term") if primary is not None else None,
|
|
}
|
|
return out
|
|
|
|
|
|
def load_corpus():
|
|
recs = {}
|
|
with open(CORPUS, encoding="utf-8") as fh:
|
|
for line in fh:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
rec = json.loads(line)
|
|
m = re.search(r"/abs/([0-9]{4}\.[0-9]{4,5})", rec.get("id", ""))
|
|
if m:
|
|
recs[m.group(1)] = rec
|
|
return recs
|
|
|
|
|
|
def main():
|
|
os.makedirs(OUTDIR, exist_ok=True)
|
|
errors = []
|
|
|
|
corpus = load_corpus()
|
|
window = {b: r for b, r in corpus.items()
|
|
if (r.get("published") or "")[:10] >= CUTOFF}
|
|
print(f"[load] corpus={len(corpus)} in-window(>={CUTOFF})={len(window)}", flush=True)
|
|
|
|
ids = sorted(window)
|
|
|
|
# ---- S1: authoritative exact lookup, no date filter -------------------
|
|
s1 = {}
|
|
for i in range(0, len(ids), IDLIST_CHUNK):
|
|
chunk = ids[i:i + IDLIST_CHUNK]
|
|
url = BASE + "?" + urllib.parse.urlencode(
|
|
{"id_list": ",".join(chunk), "max_results": len(chunk) + 10})
|
|
try:
|
|
s1.update(parse_atom(fetch(url)))
|
|
except Exception as exc: # noqa: BLE001
|
|
errors.append(f"S1 chunk {i}: {exc}")
|
|
if i % 600 == 0:
|
|
print(f"[S1] {i}/{len(ids)} resolved={len(s1)}", flush=True)
|
|
time.sleep(0.8)
|
|
print(f"[S1] exact id_list resolved {len(s1)} of {len(ids)}", flush=True)
|
|
|
|
# ---- S2: windowed search (digest's own method) -------------------------
|
|
s2 = {}
|
|
for cat in CATEGORIES:
|
|
start, page_size = 0, 200
|
|
while True:
|
|
query = f"cat:{cat} AND submittedDate:[202610190000 TO 202610260000]"
|
|
url = BASE + "?" + urllib.parse.urlencode(
|
|
{"search_query": query, "start": start, "max_results": page_size,
|
|
"sortBy": "submittedDate", "sortOrder": "descending"})
|
|
try:
|
|
page = parse_atom(fetch(url))
|
|
except Exception as exc: # noqa: BLE001
|
|
errors.append(f"S2 {cat}@{start}: {exc}")
|
|
break
|
|
if not page:
|
|
break
|
|
s2.update(page)
|
|
if len(page) < page_size:
|
|
break
|
|
start += page_size
|
|
if start > 6000:
|
|
break
|
|
time.sleep(0.8)
|
|
print(f"[S2] {cat}: {len(s2)} total", flush=True)
|
|
|
|
# ---- S3: live HTML listing pages --------------------------------------
|
|
s3, listing_hashes = set(), {}
|
|
for cat in CATEGORIES:
|
|
url = LIST_BASE.format(cat=cat)
|
|
try:
|
|
html = fetch(url)
|
|
except Exception as exc: # noqa: BLE001
|
|
errors.append(f"S3 {cat}: {exc}")
|
|
continue
|
|
with open(os.path.join(OUTDIR, f"listing-{cat}.html"), "wb") as fh:
|
|
fh.write(html)
|
|
found = set(re.findall(r"/abs/([0-9]{4}\.[0-9]{4,5})", html.decode("utf-8", "replace")))
|
|
s3 |= found
|
|
listing_hashes[cat] = {
|
|
"url": url, "sha256": hashlib.sha256(html).hexdigest(),
|
|
"bytes": len(html), "abs_ids_seen": len(found),
|
|
}
|
|
print(f"[S3] {cat}: {len(found)} /abs ids", flush=True)
|
|
time.sleep(1.0)
|
|
|
|
# ---- classify every claim ---------------------------------------------
|
|
rows = []
|
|
for bid in ids:
|
|
meta = s1.get(bid)
|
|
corpus_title = (window[bid].get("title") or "").strip()
|
|
corpus_hash = hashlib.sha256(corpus_title.encode("utf-8")).hexdigest()
|
|
in_s2, in_s3 = bid in s2, bid in s3
|
|
rows.append({
|
|
"id": bid,
|
|
"published": meta["published"] if meta else (window[bid].get("published") or ""),
|
|
"title": meta["title"] if meta else corpus_title,
|
|
"primary_category": meta["primary_category"] if meta else None,
|
|
"title_sha256": meta["title_sha256"] if meta else corpus_hash,
|
|
"title_matches_corpus": (meta["title_sha256"] == corpus_hash) if meta else None,
|
|
"s1_exact_lookup": bool(meta),
|
|
"s2_windowed_search": in_s2,
|
|
"s3_html_listing": in_s3,
|
|
"sources_agreeing": int(bool(meta)) + int(in_s2) + int(in_s3),
|
|
})
|
|
|
|
verified = [r for r in rows if r["s1_exact_lookup"]]
|
|
failed = [r for r in rows if not r["s1_exact_lookup"]]
|
|
title_mismatch = [r for r in verified if r["title_matches_corpus"] is False]
|
|
both_s2_s3 = [r for r in rows if r["s2_windowed_search"] and r["s3_html_listing"]]
|
|
|
|
result = {
|
|
"verifier": "sphinx/arxiv-digest-verifier v2",
|
|
"run_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"digest_under_test": DIGEST,
|
|
"digest_sha256": hashlib.sha256(open(DIGEST, "rb").read()).hexdigest(),
|
|
"cutoff": CUTOFF,
|
|
"window_size": len(rows),
|
|
"sources": {
|
|
"S1": "Atom API exact id_list lookup (authoritative existence check)",
|
|
"S2": "Atom API windowed cat+submittedDate search (consistency)",
|
|
"S3": "HTML /list/<cat>/recent page (independent service, bounded)",
|
|
},
|
|
"counts": {
|
|
"claims": len(rows),
|
|
"verified_exact_lookup": len(verified),
|
|
"unverified": len(failed),
|
|
"title_mismatch": len(title_mismatch),
|
|
"present_in_s2_and_s3": len(both_s2_s3),
|
|
},
|
|
"listing_hashes": listing_hashes,
|
|
"rows": rows,
|
|
"unverified_ids": [r["id"] for r in failed],
|
|
"title_mismatch_ids": [r["id"] for r in title_mismatch],
|
|
"errors": errors,
|
|
}
|
|
out_path = os.path.join(OUTDIR, "VERIFICATION.json")
|
|
with open(out_path, "w", encoding="utf-8") as fh:
|
|
json.dump(result, fh, indent=1, sort_keys=True)
|
|
|
|
print("\n=== RESULT ===")
|
|
print(f"claims : {len(rows)}")
|
|
print(f"verified (S1 exact) : {len(verified)}")
|
|
print(f"unverified : {len(failed)}")
|
|
print(f"title mismatches : {len(title_mismatch)}")
|
|
print(f"present in S2 and S3 : {len(both_s2_s3)}")
|
|
print(f"errors : {len(errors)}")
|
|
if failed:
|
|
print("FAILED IDS:", [r["id"] for r in failed][:25])
|
|
return 0 if not failed and not errors else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|