#!/usr/bin/env python3
"""Independent verifier for the multimodal arXiv dossier.

Checks, in order:
  A. every file listed in MANIFEST.json exists and its sha256 matches the manifest
  B. paper.html's sha256 matches the html_sha256 recorded in that paper's dossier.json
  C. paper.txt really is derived from paper.html (substring agreement + size bounds)
  D. tables.json and figures.json parse, are non-empty, and their counts match the
     counts recorded in dossier.json
  E. RE-FETCH each paper.html over HTTP; the sha256 must equal the manifest hash
     (i.e. the archive reproduces what arXiv serves today)
  F. RE-FETCH the Atom feed over HTTP; the ids of every archived paper must appear
     in the live feed (the archive is traceable to the live query)
  G. REPORT.md exists, is non-trivial, and names every archived arXiv id

Exit code 0 only if all checks pass.
"""
import hashlib
import json
import os
import re
import sys
import time
import urllib.error
import urllib.request

UA = {"User-Agent": "sphinx-archive-verify/1.0 (mailto:sphinx@agents.invalid)"}
D = sys.argv[1] if len(sys.argv) > 1 else "/root/.hermes/wizard-outputs/sphinx/mm-vla-dossier-2026-10-22"

fails = []
notes = []


def check(name, ok, detail=""):
    print("  [%s] %s%s" % ("PASS" if ok else "FAIL", name, (" — " + detail) if detail else ""))
    if not ok:
        fails.append(name)
    return ok


def sha(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for c in iter(lambda: f.read(65536), b""):
            h.update(c)
    return h.hexdigest()


def http(url, timeout=60):
    req = urllib.request.Request(url, headers=UA)
    with urllib.request.urlopen(req, timeout=timeout) as r:
        return r.status, r.read()


manifest = json.load(open(os.path.join(D, "MANIFEST.json")))
papers = manifest["papers"]

print("A. manifest integrity (sha256 of every listed file)")
for rel, meta in sorted(manifest["files"].items()):
    p = os.path.join(D, rel)
    ok = os.path.exists(p) and sha(p) == meta["sha256"]
    check("A %s" % rel, ok, "%d bytes" % meta["bytes"] if os.path.exists(p) else "MISSING")

print("B. paper.html sha256 == dossier.html_sha256")
for p in papers:
    pj = os.path.join(D, p["arxiv_id"], "paper.html")
    dj = json.load(open(os.path.join(D, p["arxiv_id"], "dossier.json")))
    check("B %s" % p["arxiv_id"], os.path.exists(pj) and sha(pj) == dj["paper"]["html_sha256"],
          "manifest says %s" % p["html_sha256"][:16])

print("C. paper.txt is a real text layer of paper.html")
for p in papers:
    h = open(os.path.join(D, p["arxiv_id"], "paper.html"), encoding="utf-8", errors="replace").read()
    t = open(os.path.join(D, p["arxiv_id"], "paper.txt"), encoding="utf-8").read()
    # the title string from the dossier must survive into the text layer
    title = (p["title"] or "").strip()
    probe = " ".join(title.split())[:40]
    ok = (len(t) > 500 and len(t) < len(h) and probe in " ".join(t.split()))
    check("C %s" % p["arxiv_id"], ok, "html %d -> text %d chars; title probe present=%s" % (
        len(h), len(t), probe in " ".join(t.split())))

print("D. tables.json / figures.json counts agree with dossier.json")
for p in papers:
    base = os.path.join(D, p["arxiv_id"])
    d = json.load(open(os.path.join(base, "dossier.json")))["archive"]
    tabs = json.load(open(os.path.join(base, "tables.json")))
    figs = json.load(open(os.path.join(base, "figures.json")))
    ok = (len(tabs) == d["tables_found"] and len(figs) == d["figures_found"]
          and len(tabs) > 0 and len(figs) > 0)
    check("D %s" % p["arxiv_id"], ok,
          "tables %d==%d figures %d==%d" % (len(tabs), d["tables_found"], len(figs), d["figures_found"]))

print("E. live re-fetch of paper.html reproduces the archived sha256")
for p in papers:
    try:
        st, raw = http("https://arxiv.org/html/%s" % p["arxiv_id"])
        got = hashlib.sha256(raw).hexdigest()
        check("E %s" % p["arxiv_id"], st == 200 and got == p["html_sha256"],
              "HTTP %s sha %s" % (st, got[:16]))
    except Exception as e:
        check("E %s" % p["arxiv_id"], False, "%s: %s" % (type(e).__name__, e))
    time.sleep(1.0)

print("F. live Atom feed contains every archived paper id")
try:
    st, raw = http(manifest["atom_url"])
    feed = raw.decode("utf-8", "replace")
    ids_in_feed = set(re.findall(r"<id>http://arxiv\.org/abs/([^<]+)</id>", feed))
    missing = [p["arxiv_id"] for p in papers if p["arxiv_id"] not in ids_in_feed]
    check("F atom ids", st == 200 and not missing,
          "HTTP %s; feed ids %d; missing %s" % (st, len(ids_in_feed), missing or "none"))
except Exception as e:
    check("F atom ids", False, "%s: %s" % (type(e).__name__, e))

print("G. REPORT.md exists and names every archived paper")
rp = os.path.join(D, "REPORT.md")
rep = open(rp, encoding="utf-8").read() if os.path.exists(rp) else ""
missing = [p["arxiv_id"] for p in papers if p["arxiv_id"] not in rep]
check("G report", len(rep) > 1000 and not missing,
      "%d chars; missing %s" % (len(rep), missing or "none"))

print()
if fails:
    print("RESULT: %d check(s) failed: %s" % (len(fails), fails))
    sys.exit(1)
print("RESULT: all checks passed (%d papers, %d manifest files)" % (len(papers), len(manifest["files"])))