77 lines
2.8 KiB
Python
77 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Third-source probe: do the 9 residual IDs appear in a live /list/ page?
|
|
|
|
The /recent page shows only the day's newest items, so it can never cover a
|
|
7-day window. arXiv's listing system also serves /list/<cat>/<YYMM> (a full
|
|
month) and ?skip=N&show=M (pagination). This probe searches those pages for the
|
|
residual IDs.
|
|
|
|
A hit here is a genuinely independent corroboration: a different service,
|
|
different document, no shared code path with the Atom API.
|
|
"""
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
UA = "sphinx-archive-verifier/1.0 (evidence-lane)"
|
|
OUTDIR = "/root/.hermes/wizard-outputs/sphinx/arxiv-digest-verified-2026-10-25"
|
|
RESIDUAL = ["2610.25640", "2610.27032", "2610.27234", "2610.27634",
|
|
"2610.28223", "2610.28385", "2610.28441", "2610.29820", "2610.31169"]
|
|
|
|
|
|
def fetch(url):
|
|
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
|
with urllib.request.urlopen(req, timeout=60) as resp:
|
|
return resp.read()
|
|
|
|
|
|
def main():
|
|
want = set(RESIDUAL)
|
|
hits, pages = {}, []
|
|
cats = ["cs.CL", "cs.CV", "cs.LG"]
|
|
# The /list/<cat>/<YYMM> form 404s; the live scheme is /list/<cat>/<YYYY-MM>
|
|
# (measured: 2610 -> 404, 2026-10 -> 200).
|
|
months = ["2026-10"]
|
|
|
|
for cat in cats:
|
|
for month in months:
|
|
for url in (
|
|
f"https://arxiv.org/list/{cat}/{month}",
|
|
f"https://arxiv.org/list/{cat}/{month}?skip=0&show=2000",
|
|
):
|
|
try:
|
|
html = fetch(url).decode("utf-8", "replace")
|
|
except Exception as exc: # noqa: BLE001
|
|
pages.append({"url": url, "error": str(exc)})
|
|
print(f"[list] {url} -> ERROR {exc}", flush=True)
|
|
continue
|
|
found = set(re.findall(r"/abs/([0-9]{4}\.[0-9]{4,5})", html))
|
|
overlap = want & found
|
|
for bid in overlap:
|
|
hits.setdefault(bid, []).append(url)
|
|
pages.append({"url": url, "bytes": len(html), "abs_ids": len(found),
|
|
"residual_hits": sorted(overlap)})
|
|
print(f"[list] {url} -> {len(found)} ids, residual hits: {sorted(overlap)}",
|
|
flush=True)
|
|
time.sleep(1.5)
|
|
|
|
out = {
|
|
"probe": "month-listing / pagination search for residual IDs",
|
|
"run_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"residual": RESIDUAL,
|
|
"hits": hits,
|
|
"corroborated": sorted(hits),
|
|
"uncorroborated": sorted(want - set(hits)),
|
|
"pages": pages,
|
|
}
|
|
json.dump(out, open(f"{OUTDIR}/LISTING_PROBE.json", "w", encoding="utf-8"),
|
|
indent=1, sort_keys=True)
|
|
print(f"\ncorroborated by listing pages: {len(hits)}/{len(want)}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|