307 lines
13 KiB
Python
307 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""verify.py — audit the long-context multimodal archive against its sources.
|
|
|
|
Checks, in order:
|
|
1. structural integrity of papers.jsonl (unique ids, non-empty fields, parseable dates)
|
|
2. raw/ Atom responses: sha256 matches raw_manifest.json
|
|
3. every raw entry is present in papers.jsonl with identical title/abstract/date
|
|
4. CORPUS.txt size equals build_stats.corpus_chars
|
|
5. every claim in claims.jsonl recomputes to its stated expected value
|
|
6. ANALYSIS.md exists, is substantive, cites every claim id, and cites no
|
|
arXiv id that is absent from the archive
|
|
|
|
Exit 0 only if every check passes. Usage: python3 verify.py [--quiet]
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
|
|
OUT = os.path.dirname(os.path.abspath(__file__))
|
|
ATOM = "{http://www.w3.org/2005/Atom}"
|
|
ARX = "{http://arxiv.org/schemas/atom}"
|
|
|
|
fails = []
|
|
checks = 0
|
|
|
|
|
|
def ok(msg):
|
|
global checks
|
|
checks += 1
|
|
if "--quiet" not in sys.argv:
|
|
print("PASS %s" % msg)
|
|
|
|
|
|
def bad(msg):
|
|
global checks
|
|
checks += 1
|
|
fails.append(msg)
|
|
print("FAIL %s" % msg)
|
|
|
|
|
|
def norm(s):
|
|
return " ".join((s or "").split())
|
|
|
|
|
|
def term_re(term):
|
|
return re.compile(r"\b" + re.escape(term) + r"\b", re.IGNORECASE)
|
|
|
|
|
|
def load_jsonl(path):
|
|
with open(path) as f:
|
|
return [json.loads(l) for l in f if l.strip()]
|
|
|
|
|
|
def main():
|
|
papers_path = os.path.join(OUT, "papers.jsonl")
|
|
for req in ("papers.jsonl", "build_stats.json", "raw_manifest.json", "claims.jsonl", "CORPUS.txt", "ANALYSIS.md"):
|
|
if not os.path.isfile(os.path.join(OUT, req)):
|
|
bad("missing required file %s" % req)
|
|
if fails:
|
|
return finish()
|
|
|
|
papers = load_jsonl(papers_path)
|
|
stats = json.load(open(os.path.join(OUT, "build_stats.json")))
|
|
manifest = json.load(open(os.path.join(OUT, "raw_manifest.json")))
|
|
claims = load_jsonl(os.path.join(OUT, "claims.jsonl"))
|
|
|
|
# 1. structural
|
|
ids = [p["base_id"] for p in papers]
|
|
if len(ids) == len(set(ids)):
|
|
ok("1a unique base_id across %d records" % len(ids))
|
|
else:
|
|
bad("1a duplicate base_id in papers.jsonl")
|
|
bad_recs = [p["base_id"] for p in papers
|
|
if not p.get("title") or not p.get("abstract") or not p.get("published")]
|
|
if not bad_recs:
|
|
ok("1b all %d records have title, abstract, published" % len(papers))
|
|
else:
|
|
bad("1b empty fields in %s" % bad_recs[:5])
|
|
baddate = [p["base_id"] for p in papers
|
|
if not re.match(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}", p["published"])]
|
|
if not baddate:
|
|
ok("1c all published timestamps ISO-8601")
|
|
else:
|
|
bad("1c malformed timestamps in %s" % baddate[:5])
|
|
if len(papers) == stats["papers"]:
|
|
ok("1d papers.jsonl count %d == build_stats.papers" % len(papers))
|
|
else:
|
|
bad("1d count mismatch %d vs %s" % (len(papers), stats["papers"]))
|
|
if any(len(p["abstract"]) < 200 for p in papers):
|
|
bad("1e some abstracts suspiciously short")
|
|
else:
|
|
ok("1e every abstract >= 200 chars (min %d)" % min(len(p["abstract"]) for p in papers))
|
|
|
|
by_id = {p["base_id"]: p for p in papers}
|
|
|
|
# 2. raw sha256
|
|
raw_ok = 0
|
|
for m in manifest:
|
|
path = os.path.join(OUT, m["file"])
|
|
if not os.path.isfile(path):
|
|
bad("2 raw file missing: %s" % m["file"])
|
|
continue
|
|
blob = open(path, "rb").read()
|
|
got = hashlib.sha256(blob).hexdigest()
|
|
if got != m["sha256"]:
|
|
bad("2 sha256 mismatch %s: %s != %s" % (m["file"], got[:16], m["sha256"][:16]))
|
|
elif len(blob) != m["bytes"]:
|
|
bad("2 byte length mismatch %s" % m["file"])
|
|
else:
|
|
raw_ok += 1
|
|
if raw_ok == len(manifest) == 5:
|
|
ok("2 all %d raw Atom responses match manifest sha256 + byte length" % raw_ok)
|
|
|
|
# 3. re-parse raw and compare to papers.jsonl
|
|
raw_ids = set()
|
|
mismatches = []
|
|
entries = 0
|
|
for m in manifest:
|
|
root = ET.parse(os.path.join(OUT, m["file"])).getroot()
|
|
n = 0
|
|
for e in root.findall(ATOM + "entry"):
|
|
raw_id = (e.findtext(ATOM + "id") or "").strip()
|
|
mm = re.search(r"abs/(.+)$", raw_id)
|
|
if not mm:
|
|
continue
|
|
base = re.sub(r"v\d+$", "", mm.group(1))
|
|
raw_ids.add(base)
|
|
entries += 1
|
|
n += 1
|
|
rec = by_id.get(base)
|
|
if rec is None:
|
|
mismatches.append("%s absent from papers.jsonl" % base)
|
|
continue
|
|
if norm(e.findtext(ATOM + "title")) != norm(rec["title"]):
|
|
mismatches.append("%s title differs" % base)
|
|
if norm(e.findtext(ATOM + "summary")) != norm(rec["abstract"]):
|
|
mismatches.append("%s abstract differs" % base)
|
|
if (e.findtext(ATOM + "published") or "").strip() != rec["published"]:
|
|
mismatches.append("%s published differs" % base)
|
|
if n != m["entries"]:
|
|
mismatches.append("%s entry count %d != manifest %d" % (m["file"], n, m["entries"]))
|
|
if not mismatches:
|
|
ok("3 all %d raw entries re-parse to identical title/abstract/date in papers.jsonl" % entries)
|
|
else:
|
|
bad("3 raw/archive mismatches: %s" % mismatches[:5])
|
|
if raw_ids == set(by_id):
|
|
ok("3b raw base-id set == archive base-id set (%d ids)" % len(raw_ids))
|
|
else:
|
|
bad("3b id set mismatch: raw-only=%s archive-only=%s"
|
|
% (sorted(raw_ids - set(by_id))[:5], sorted(set(by_id) - raw_ids)[:5]))
|
|
|
|
# 4. corpus size
|
|
csize = os.path.getsize(os.path.join(OUT, "CORPUS.txt"))
|
|
if csize == stats["corpus_chars"]:
|
|
ok("4 CORPUS.txt %d chars == build_stats.corpus_chars" % csize)
|
|
else:
|
|
bad("4 CORPUS.txt %d != build_stats %d" % (csize, stats["corpus_chars"]))
|
|
corpus = open(os.path.join(OUT, "CORPUS.txt")).read()
|
|
missing_in_pack = [p["base_id"] for p in papers if ("[%s]" % p["base_id"]) not in corpus]
|
|
if not missing_in_pack:
|
|
ok("4b all %d papers appear as headers in CORPUS.txt" % len(papers))
|
|
else:
|
|
bad("4b papers missing from CORPUS.txt: %s" % missing_in_pack[:5])
|
|
|
|
# 5. claims
|
|
claim_fail = 0
|
|
for c in claims:
|
|
cid, kind, exp = c["id"], c["kind"], c.get("expected")
|
|
if kind == "stat":
|
|
got = stats.get(c["field"])
|
|
(ok if got == exp else bad)("5 %s stat %s = %r (expected %r)" % (cid, c["field"], got, exp))
|
|
claim_fail += 0 if got == exp else 1
|
|
elif kind == "term_count":
|
|
pat = term_re(c["term"])
|
|
got = sum(1 for p in papers if pat.search(p["abstract"]))
|
|
(ok if got == exp else bad)("5 %s term_count %r = %d (expected %r)" % (cid, c["term"], got, exp))
|
|
claim_fail += 0 if got == exp else 1
|
|
elif kind == "cooccur":
|
|
pats = [term_re(t) for t in c["terms"]]
|
|
got = sum(1 for p in papers if all(x.search(p["abstract"]) for x in pats))
|
|
(ok if got == exp else bad)("5 %s cooccur %s = %d (expected %r)" % (cid, "+".join(c["terms"]), got, exp))
|
|
claim_fail += 0 if got == exp else 1
|
|
elif kind == "month_count":
|
|
got = sum(1 for p in papers if p["published"][:7] == c["month"])
|
|
(ok if got == exp else bad)("5 %s month_count %s = %d (expected %r)" % (cid, c["month"], got, exp))
|
|
claim_fail += 0 if got == exp else 1
|
|
elif kind == "category_count":
|
|
got = sum(1 for p in papers if p["primary_category"] == c["category"])
|
|
(ok if got == exp else bad)("5 %s category_count %s = %d (expected %r)" % (cid, c["category"], got, exp))
|
|
claim_fail += 0 if got == exp else 1
|
|
elif kind == "quote":
|
|
rec = by_id.get(c["paper_id"])
|
|
if rec is None:
|
|
bad("5 %s quote cites unknown paper %s" % (cid, c["paper_id"]))
|
|
claim_fail += 1
|
|
continue
|
|
found = norm(c["quote"]) in norm(rec["abstract"])
|
|
(ok if found else bad)("5 %s verbatim quote present in %s abstract" % (cid, c["paper_id"]))
|
|
claim_fail += 0 if found else 1
|
|
else:
|
|
bad("5 %s unknown claim kind %r" % (cid, kind))
|
|
claim_fail += 1
|
|
if claim_fail == 0:
|
|
ok("5 all %d claims recomputed to their stated values" % len(claims))
|
|
|
|
# 6. report integrity
|
|
report = open(os.path.join(OUT, "ANALYSIS.md")).read()
|
|
if len(report) >= 4000:
|
|
ok("6a ANALYSIS.md is %d chars" % len(report))
|
|
else:
|
|
bad("6a ANALYSIS.md too short (%d chars)" % len(report))
|
|
cited = set(re.findall(r"\bC\d{2}\b", report))
|
|
uncited = sorted({c["id"] for c in claims} - cited)
|
|
if not uncited:
|
|
ok("6b ANALYSIS.md cites all %d claim ids" % len(claims))
|
|
else:
|
|
bad("6b claim ids never referenced in ANALYSIS.md: %s" % uncited)
|
|
stale = sorted(cited - {c["id"] for c in claims})
|
|
if not stale:
|
|
ok("6c ANALYSIS.md cites no claim id absent from the ledger")
|
|
else:
|
|
bad("6c phantom claim ids in ANALYSIS.md: %s" % stale)
|
|
mentioned = set(re.findall(r"\b(\d{4}\.\d{4,5})\b", report))
|
|
unknown = sorted(mentioned - set(by_id))
|
|
if not unknown:
|
|
ok("6d every arXiv id mentioned in ANALYSIS.md (%d) exists in the archive" % len(mentioned))
|
|
else:
|
|
bad("6d ANALYSIS.md mentions ids not in archive: %s" % unknown)
|
|
|
|
# 7. mutation self-test: the verifier must reject corrupted copies
|
|
if "--no-selftest" not in sys.argv and not fails:
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
|
|
tmp = tempfile.mkdtemp(prefix="verify-mutation-")
|
|
try:
|
|
work = os.path.join(tmp, "copy")
|
|
shutil.copytree(OUT, work, ignore=shutil.ignore_patterns("verify.out", "*.pyc", "__pycache__"))
|
|
clean = subprocess.run([sys.executable, os.path.join(work, "verify.py"), "--no-selftest", "--quiet"],
|
|
capture_output=True, text=True)
|
|
if clean.returncode != 0:
|
|
bad("7a clean copy of the archive failed verification: %s" % clean.stdout.strip()[-200:])
|
|
else:
|
|
ok("7a clean copy of the archive verifies (control)")
|
|
|
|
# mutation A: nudge one quantitative claim
|
|
cp = os.path.join(work, "claims.jsonl")
|
|
lines = open(cp).read().splitlines()
|
|
first = json.loads(lines[0])
|
|
first["expected"] = (first["expected"] + 1) if isinstance(first["expected"], int) else first["expected"]
|
|
lines[0] = json.dumps(first)
|
|
open(cp, "w").write("\n".join(lines) + "\n")
|
|
mut_a = subprocess.run([sys.executable, os.path.join(work, "verify.py"), "--no-selftest", "--quiet"],
|
|
capture_output=True, text=True)
|
|
if mut_a.returncode == 0:
|
|
bad("7b verifier accepted a falsified claim value (C01 expected+1)")
|
|
else:
|
|
ok("7b verifier rejects a falsified claim value (C01 expected+1)")
|
|
|
|
# mutation B: alter one character of a stored abstract
|
|
pp = os.path.join(work, "papers.jsonl")
|
|
recs = [json.loads(l) for l in open(pp).read().splitlines() if l.strip()]
|
|
recs[0]["abstract"] = recs[0]["abstract"].replace("the", "teh", 1)
|
|
with open(pp, "w") as f:
|
|
for r in recs:
|
|
f.write(json.dumps(r, sort_keys=True) + "\n")
|
|
mut_b = subprocess.run([sys.executable, os.path.join(work, "verify.py"), "--no-selftest", "--quiet"],
|
|
capture_output=True, text=True)
|
|
if mut_b.returncode == 0:
|
|
bad("7c verifier accepted a tampered abstract")
|
|
else:
|
|
ok("7c verifier rejects a tampered abstract (raw Atom mismatch)")
|
|
|
|
# mutation C: corrupt a raw response byte
|
|
raw_file = os.path.join(work, manifest[0]["file"])
|
|
blob = bytearray(open(raw_file, "rb").read())
|
|
blob[len(blob) // 2] ^= 0xFF
|
|
open(raw_file, "wb").write(bytes(blob))
|
|
mut_c = subprocess.run([sys.executable, os.path.join(work, "verify.py"), "--no-selftest", "--quiet"],
|
|
capture_output=True, text=True)
|
|
if mut_c.returncode == 0:
|
|
bad("7d verifier accepted a corrupted raw response")
|
|
else:
|
|
ok("7d verifier rejects a corrupted raw response (sha256)")
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
return finish()
|
|
|
|
|
|
def finish():
|
|
print("\n%d checks, %d failures" % (checks, len(fails)))
|
|
if fails:
|
|
for f in fails:
|
|
print(" FAILED: %s" % f)
|
|
return 1
|
|
print("ARCHIVE VERIFIED")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|