From 37048766f4c05817233f9a57ad8d9d1be699444e Mon Sep 17 00:00:00 2001 From: sphinx Date: Thu, 22 Oct 2026 22:41:54 +0000 Subject: [PATCH] =?UTF-8?q?sphinx:=20harden=20arXiv=20LC=20recheck=20?= =?UTF-8?q?=E2=80=94=20structured=20failure,=20path-aware=20live=20check,?= =?UTF-8?q?=20drift=20negative=20control?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../archive-arxiv-lc-2026-10-22/ARCHIVE.md | 11 +++ .../check_archive.py | 16 ++++- .../recheck_completeness.py | 69 ++++++++++++------- .../run_verdict.py | 2 +- .../test_check_drift.py | 31 +++++++++ .../test_recheck_drift.py | 25 +++++++ .../archive-arxiv-lc-2026-10-22/verdict.sh | 6 ++ 7 files changed, 131 insertions(+), 29 deletions(-) create mode 100644 wizard-work/sphinx/archive-arxiv-lc-2026-10-22/test_check_drift.py create mode 100644 wizard-work/sphinx/archive-arxiv-lc-2026-10-22/test_recheck_drift.py create mode 100755 wizard-work/sphinx/archive-arxiv-lc-2026-10-22/verdict.sh diff --git a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/ARCHIVE.md b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/ARCHIVE.md index 17bf43d..eed943e 100644 --- a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/ARCHIVE.md +++ b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/ARCHIVE.md @@ -5,6 +5,17 @@ Method: live HTTPS fetch of the arXiv Atom API, paged past the 100-entry ceiling so the slice is a COMPLETE enumeration rather than a top-N sample; raw bytes persisted to disk; every digest below is sha256 over the persisted bytes. +## How to re-verify (and how this was checked) + + python3 check_archive.py . # re-hashes bytes, re-derives every invariant, + # re-queries the live API, cross-checks this report + +`check_archive.py` reads only bytes on disk. It recomputes the API and probe-page +digests, re-parses the XML, re-derives the keyword invariants, re-queries the live +arXiv API to prove the 19-entry slice is still complete, and runs a negative control +that tampers a copy of the manifest to prove the completeness check can actually +fail. It exits nonzero on any mismatch. + ## Provenance | Item | Value | diff --git a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/check_archive.py b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/check_archive.py index ef3a518..d931472 100644 --- a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/check_archive.py +++ b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/check_archive.py @@ -81,10 +81,13 @@ for pid, p in m["probe_pages"].items(): check("probe %s contains Abstract" % pid, "Abstract:" in html) # --- completeness: re-query the live API and confirm nothing was dropped --- +# The recheck reads the manifest from the directory we pass it, so it verifies +# THIS copy (including any tampered copy), not a path baked into the script. probe_script = os.path.join(HERE, "recheck_completeness.py") if os.path.exists(probe_script): - r = subprocess.run([sys.executable, probe_script], capture_output=True, text=True) - check("live API recheck exits 0 (slice complete)", r.returncode == 0, + r = subprocess.run([sys.executable, probe_script, HERE], capture_output=True, text=True) + ok = r.returncode == 0 and "COMPLETE: manifest matches the live API exactly" in r.stdout + check("live API recheck exits 0 (slice complete)", ok, (r.stdout.strip().splitlines() or [""])[-1] if r.stdout.strip() else r.stderr.strip()[:120]) # --- report cross-check against evidence --- @@ -98,6 +101,15 @@ check("report lists every entry", not missing, str(missing)) check("report states the complete-slice claim", "COMPLETE enumeration" in report) check("report has no placeholder text", "lorem" not in report.lower() and "TODO" not in report) +# --- the completeness check itself must be capable of failing --- +# A recheck that cannot fail proves nothing. Tamper a copy of the manifest +# (drop one entry) and assert the recheck exits nonzero. +drift_script = os.path.join(HERE, "test_recheck_drift.py") +if os.path.exists(drift_script): + r = subprocess.run([sys.executable, drift_script], capture_output=True, text=True) + check("recheck detects a dropped entry (negative control)", r.returncode == 0, + (r.stdout.strip().splitlines() or [r.stderr.strip()])[-1]) + print() print("RESULT:", "ALL CHECKS PASS" if not fail else "FAILED: %s" % fail) sys.exit(1 if fail else 0) diff --git a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/recheck_completeness.py b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/recheck_completeness.py index 93b5bd3..0de40ef 100644 --- a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/recheck_completeness.py +++ b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/recheck_completeness.py @@ -4,6 +4,10 @@ confirm the manifest's entry count equals the number the API returns NOW. This is what proves "COMPLETE enumeration" is not a boast: if the manifest had dropped entries, or if the API had more, the counts would disagree. + +Exit 0 only when the live API and the manifest agree exactly. Any network +error, HTTP error, or drift exits 1 -- this script is a CHECK, so it must never +return success for an unproven outcome. """ import json import re @@ -25,31 +29,44 @@ def get(url): return r.read().decode("utf-8", "replace") -# Page the live API exactly as fetch_abs.py does. -start, live_ids, pages = 0, [], 0 -while True: - url = BASE + urlencode({ - "search_query": m["api_query"], "start": start, "max_results": 100, - "sortBy": "submittedDate", "sortOrder": "descending"}) - try: +def recheck(): + """Page the live API exactly as fetch_abs.py does. Returns a result dict.""" + start, live_ids, pages = 0, [], 0 + while True: + url = BASE + urlencode({ + "search_query": m["api_query"], "start": start, "max_results": 100, + "sortBy": "submittedDate", "sortOrder": "descending"}) xml = get(url) - except urllib.error.HTTPError as e: - print("live recheck HTTPError:", e) - sys.exit(1) - pages += 1 - ids = re.findall(r"http://arxiv\.org/abs/([^<]+)", xml) - live_ids.extend(ids) - if len(ids) < 100 or pages > 40: - break - start += 100 + pages += 1 + ids = re.findall(r"https?://arxiv\.org/abs/([^<]+)", xml) + live_ids.extend(ids) + if len(ids) < 100 or pages > 40: + break + start += 100 -manifest_ids = [r["id"].split("/abs/")[-1] for r in m["entries"]] -print("live entries:", len(live_ids), "manifest entries:", len(manifest_ids)) -print("live pages:", pages) -missing = [i for i in live_ids if i not in manifest_ids] -extra = [i for i in manifest_ids if i not in live_ids] -print("in live but not manifest:", missing) -print("in manifest but not live:", extra) -if missing or extra: - sys.exit(1) -print("COMPLETE: manifest matches the live API exactly") + manifest_ids = [r["id"].split("/abs/")[-1] for r in m["entries"]] + missing = [i for i in live_ids if i not in manifest_ids] + extra = [i for i in manifest_ids if i not in live_ids] + return {"live": len(live_ids), "manifest": len(manifest_ids), + "pages": pages, "missing": missing, "extra": extra} + + +def main(): + try: + res = recheck() + except (urllib.error.URLError, OSError, ValueError) as e: + print("live recheck FAILED: %s" % e) + return 1 + print("live entries: %d manifest entries: %d" % (res["live"], res["manifest"])) + print("live pages: %d" % res["pages"]) + print("in live but not manifest: %s" % res["missing"]) + print("in manifest but not live: %s" % res["extra"]) + if res["missing"] or res["extra"]: + print("DRIFT: manifest no longer matches the live API") + return 1 + print("COMPLETE: manifest matches the live API exactly") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/run_verdict.py b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/run_verdict.py index e421ce8..1182b83 100644 --- a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/run_verdict.py +++ b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/run_verdict.py @@ -17,4 +17,4 @@ import subprocess import sys REPO = "/root/troupe-standup/wizard-work/sphinx/archive-arxiv-lc-2026-10-22" -sys.exit(subprocess.run([sys.executable, os.path.join(REPO, "check_archive.py")]).returncode) +sys.exit(subprocess.run([sys.executable, os.path.join(REPO, "check_archive.py"), REPO]).returncode) diff --git a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/test_check_drift.py b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/test_check_drift.py new file mode 100644 index 0000000..ade3a22 --- /dev/null +++ b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/test_check_drift.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Negative control for check_archive.py itself. + +A verifier that cannot fail is worthless. This drops one entry from a temp copy +of the artifact and asserts check_archive.py exits 1 -- proving the live-API +completeness check is wired into the verdict and actually catches drift. +""" +import json +import os +import shutil +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +d = tempfile.mkdtemp() +for f in os.listdir(HERE): + if f.endswith((".json", ".xml", ".html", ".py", ".md")): + shutil.copy(os.path.join(HERE, f), d) + +m = json.load(open(os.path.join(d, "manifest.json"))) +m["entries"] = m["entries"][1:] +json.dump(m, open(os.path.join(d, "manifest.json"), "w"), indent=2, sort_keys=True) + +r = subprocess.run([sys.executable, os.path.join(d, "check_archive.py"), d], + capture_output=True, text=True) +print("tampered check exit:", r.returncode) +for line in r.stdout.splitlines(): + if "live API recheck" in line or line.startswith("RESULT"): + print(" " + line.strip()) +sys.exit(0 if r.returncode != 0 else 1) diff --git a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/test_recheck_drift.py b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/test_recheck_drift.py new file mode 100644 index 0000000..b53dfe9 --- /dev/null +++ b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/test_recheck_drift.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Negative control for recheck_completeness.py. + +A CHECK that only ever returns 0 is not a check. This tampers the manifest +(copy, in a temp dir) by dropping one entry and asserts the recheck exits 1 -- +i.e. it really detects drift rather than rubber-stamping. +""" +import json +import os +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +d = tempfile.mkdtemp() +m = json.load(open(os.path.join(HERE, "manifest.json"))) +m["entries"] = m["entries"][1:] # simulate a dropped entry +json.dump(m, open(os.path.join(d, "manifest.json"), "w")) + +r = subprocess.run([sys.executable, os.path.join(HERE, "recheck_completeness.py"), d], + capture_output=True, text=True) +print("tampered exit:", r.returncode) +tail = (r.stdout.strip().splitlines() or [r.stderr.strip()])[-1] +print("last line:", tail) +sys.exit(0 if r.returncode != 0 and "in live but not manifest" in r.stdout else 1) diff --git a/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/verdict.sh b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/verdict.sh new file mode 100755 index 0000000..1fab84c --- /dev/null +++ b/wizard-work/sphinx/archive-arxiv-lc-2026-10-22/verdict.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Verdict harness for the arXiv long-context multimodal archive artifact. +# Fails (exit 1) if any check fails, so its own exit code IS the verdict. +set -u +HERE="/root/troupe-standup/wizard-work/sphinx/archive-arxiv-lc-2026-10-22" +python3 "$HERE/check_archive.py" "$HERE" || exit 1