sphinx: long-context needle grounding — 7 arXiv papers, 819KB, 35 needles; verifier falsifiable on chunk, offset, and forged-report mutations

wizard-work/sphinx/longctx-needle-grounding-2026-10-22/REPORT.md

@@ -51,16 +51,22 @@ rejected by this artifact's own test.
 - `tools/run_retrieval.py` — needle ranking
 - `tools/build_negative_control.py` — distractor comparison
 - `verify.py` — recomputes hashes, spans, contiguity, needle offsets, ownership,
-  and re-ranks every needle with its own copy of the ranker. **365 checks, 0 failed.**
+  and re-ranks every needle with its own copy of the ranker. It also reads
+  REPORT.md and re-derives the published headline numbers from the corpus, so the
+  prose cannot drift from the bytes. **371 checks, 0 failed.**
 - `selftest_tamper.py` — corrupts two independent things in scratch copies
   (a chunk byte; a needle offset) and requires `verify.py` to reject both.
   **4 mutation checks, 0 missed.**
+- `check_report_forgery.py` — the third, independent mutation: it forges
+  REPORT.md to claim 35/35 top-1 while the corpus says 34/35, and requires
+  `verify.py` to reject the forgery. **9 checks, 0 failed.**
 - `check.sh` — the single verdict command: `sh check.sh` exits 0 only if
-  verification passes AND the verifier proves falsifiable.
+  verification passes AND the verifier proves falsifiable on three kinds of lie.
 
 ## Reproduce
 
     sh /root/troupe-standup/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check.sh
 
-Measured on this run: `VERIFICATION PASSED — 365 checks, 0 failed`,
-`SELFTEST PASSED — 4 mutation checks, 0 missed`, shell exit 0.
+Measured on this run: `VERIFICATION PASSED — 371 checks, 0 failed`,
+`SELFTEST PASSED — 4 mutation checks, 0 missed`,
+`REPORT FORGERY DETECTED — 9 checks, 0 failed`, shell exit 0.

wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check.sh

@@ -15,3 +15,4 @@ LOG="${TMPDIR:-/tmp}/sphinx-needle-verify.log"
 python3 "$HERE/verify.py" >"$LOG" 2>&1 || { tail -n 25 "$LOG"; exit 1; }
 tail -n 1 "$LOG"
 python3 "$HERE/selftest_tamper.py" || exit 1
+python3 "$HERE/check_report_forgery.py" || exit 1

wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check_report_forgery.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""check_report_forgery.py — a CHECK that is not a copy of check.sh.
+
+The artifact's REPORT.md asserts headline numbers:
+
+    top-1 chunk correct: 34 / 35
+    top-5 contains the owning chunk: 35 / 35
+
+verify.py already proves it can fail on a corrupted chunk byte and a shifted
+needle offset.  It does NOT prove anything about the *report text* being bound
+to the numbers it publishes: a forge that edited REPORT.md to read "35 / 35"
+while the bytes say 34 / 35 would walk straight through check.sh.
+
+This tool is that missing mutation.  It forges REPORT.md, runs the artifact's
+own verifier, and demands that the verifier REJECT the forged report.  It also
+re-derives the two headline numbers from the bytes on disk and fails if they do
+not match what the current REPORT.md claims — so the check passes only while the
+report is honest AND the verifier is capable of noticing a lie.
+
+Exit 0 iff both hold.
+"""
+import hashlib
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+failures = []
+checks = [0]
+
+
+def report(name, ok, detail=""):
+    checks[0] += 1
+    print("  %s  %s %s" % ("PASS" if ok else "FAIL", name, detail))
+    if not ok:
+        failures.append("%s %s" % (name, detail))
+
+
+def sha256(b):
+    return hashlib.sha256(b).hexdigest()
+
+
+def recompute_top1():
+    """Independent recomputation of top-1 accuracy from the bytes on disk.
+
+    Re-implements the ranker from scratch (same definition as verify.py) and
+    reads chunk files directly.  Deliberately does not import verify.py.
+    """
+    index = json.load(open(os.path.join(HERE, "index.json"), encoding="utf-8"))
+    ndoc = json.load(open(os.path.join(HERE, "needles.json"), encoding="utf-8"))
+    by_slug = {}
+    for c in index["chunks"]:
+        by_slug.setdefault(c["slug"], []).append(c)
+
+    def words(s):
+        return re.findall(r"[a-z0-9]+", s.lower())
+
+    def rank_score(needle, body):
+        nw = set(words(needle))
+        if not nw:
+            return 0.0
+        return len(nw & set(words(body))) / len(nw) - 0.0001 * len(body)
+
+    top1 = 0
+    for nd in ndoc["needles"]:
+        slug = nd["slug"]
+        rows = sorted(by_slug[slug], key=lambda c: c["chunk"])
+        scored = []
+        for c in rows:
+            body = open(os.path.join(HERE, c["path"]), encoding="utf-8").read()
+            scored.append((rank_score(nd["needle"], body), c["chunk"]))
+        scored.sort(key=lambda t: (-t[0], t[1]))
+        if scored[0][1] == nd["owning_chunk"]:
+            top1 += 1
+    return top1, len(ndoc["needles"])
+
+
+def main():
+    print("== 1. the published headline numbers match the bytes ==")
+    report_path = os.path.join(HERE, "REPORT.md")
+    if not os.path.exists(report_path):
+        report("REPORT.md exists", False, "missing")
+        return 1
+    report("REPORT.md exists", True)
+    text = open(report_path, encoding="utf-8").read()
+
+    m1 = re.search(r"top-1 chunk correct:\s*\*?\*?(\d+)\s*/\s*(\d+)", text)
+    m5 = re.search(r"top-5 contains the owning chunk:\s*\*?\*?(\d+)\s*/\s*(\d+)",
+                   text)
+    report("REPORT.md states a top-1 headline", m1 is not None)
+    report("REPORT.md states a top-5 headline", m5 is not None)
+    if not (m1 and m5):
+        return 1
+
+    rep_top1, rep_total = int(m1.group(1)), int(m1.group(2))
+    rep_top5 = int(m5.group(1))
+    real_top1, real_total = recompute_top1()
+    report("top-1 headline equals recomputed %d/%d" % (real_top1, real_total),
+           rep_top1 == real_top1 and rep_total == real_total,
+           "report says %d/%d" % (rep_top1, rep_total))
+    report("top-5 headline is not inflated above needle count",
+           rep_top5 <= real_total, "report says %d/%d" % (rep_top5, real_total))
+
+    print("\n== 2. the verifier rejects a forged headline ==")
+    dst = tempfile.mkdtemp(prefix="forgery-")
+    shutil.copytree(HERE, dst, dirs_exist_ok=True,
+                    ignore=shutil.ignore_patterns("__pycache__", "*.pyc"))
+    try:
+        forged = os.path.join(dst, "REPORT.md")
+        ftext = open(forged, encoding="utf-8").read()
+        # Forge the strongest possible lie: claim every needle's top-1 was right.
+        ftext, nsub = re.subn(r"top-1 chunk correct:\s*\*?\*?(\d+)\s*/\s*(\d+)",
+                              lambda mm: "top-1 chunk correct: **%s" % mm.group(2),
+                              ftext, count=1)
+        open(forged, "w", encoding="utf-8").write(ftext)
+        forged_sha = sha256(open(forged, "rb").read())
+        orig_sha = sha256(open(report_path, "rb").read())
+        report("forged report actually differs from the real one",
+               nsub == 1 and forged_sha != orig_sha and ftext != text)
+
+        p = subprocess.run([sys.executable, os.path.join(dst, "verify.py")],
+                           capture_output=True, text=True, timeout=1800)
+        report("verify.py exits nonzero on the forged report", p.returncode != 0,
+               "exit %d" % p.returncode)
+        report("failure output says VERIFICATION FAILED",
+               "VERIFICATION FAILED" in (p.stdout + p.stderr))
+        report("forgery was noticed (not silently accepted)",
+               p.returncode != 0)
+    finally:
+        shutil.rmtree(dst, ignore_errors=True)
+
+    print("\n%s — %d checks, %d failed" % (
+        "REPORT FORGERY DETECTED (report is bound to the bytes)" if not failures
+        else "REPORT IS NOT BOUND TO THE BYTES", checks[0], len(failures)))
+    if failures:
+        for f in failures[:20]:
+            print("   ! %s" % f)
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

wizard-work/sphinx/longctx-needle-grounding-2026-10-22/verify.py

@@ -24,6 +24,7 @@ import sys
 HERE = os.path.dirname(os.path.abspath(__file__))
 TXT = os.path.join(HERE, "text")
 CHUNKS = os.path.join(HERE, "chunks")
+REPORT = os.path.join(HERE, "REPORT.md")
 
 failures = []
 checks = [0]
@@ -209,6 +210,34 @@ def main():
         print("  NOTE  %d of %d needles tie or lose to a non-containing chunk "
               "(ranker tie-break is chunk index, not relevance)" % (ties, neg["needles"]))
 
+    print("\n== 6. the published report is bound to the bytes it publishes ==")
+    # A report is data too.  If REPORT.md can be edited to claim "35 / 35" while
+    # the corpus says 34 / 35, then check.sh passes on a lie.  So verify.py reads
+    # the report text, re-derives each headline number, and demands they agree.
+    if not os.path.exists(REPORT):
+        check("REPORT.md exists", False, "missing")
+    else:
+        rtext = open(REPORT, encoding="utf-8").read()
+        check("REPORT.md exists", True)
+        m1 = re.search(r"top-1 chunk correct:\s*\*?\*?(\d+)\s*/\s*(\d+)", rtext)
+        m5 = re.search(r"top-5 contains the owning chunk:\s*\*?\*?(\d+)\s*/\s*(\d+)",
+                       rtext)
+        check("REPORT.md states the top-1 headline", m1 is not None)
+        check("REPORT.md states the top-5 headline", m5 is not None)
+        if m1:
+            check("REPORT.md top-1 headline equals recomputed value",
+                  int(m1.group(1)) == recomputed_top1 and int(m1.group(2)) == n,
+                  "report %s/%s vs recomputed %d/%d"
+                  % (m1.group(1), m1.group(2), recomputed_top1, n))
+        if m5:
+            check("REPORT.md top-5 headline equals recomputed value",
+                  int(m5.group(1)) == recomputed_top5 and int(m5.group(2)) == n,
+                  "report %s/%s vs recomputed %d/%d"
+                  % (m5.group(1), m5.group(2), recomputed_top5, n))
+        check("REPORT.md claims nothing above 100%",
+              all(int(m.group(1)) <= int(m.group(2))
+                  for m in [m1, m5] if m))
+
     print("\n%s — %d checks, %d failed" % (
         "VERIFICATION PASSED" if not failures else "VERIFICATION FAILED",
         checks[0], len(failures)))