From 2b85f70ea8ab9d03411f1f404c1e242051dd801b Mon Sep 17 00:00:00 2001 From: sphinx Date: Thu, 22 Oct 2026 08:41:25 +0000 Subject: [PATCH] =?UTF-8?q?sphinx:=20long-context=20needle=20grounding=20?= =?UTF-8?q?=E2=80=94=207=20arXiv=20papers,=20819KB,=2035=20needles;=20veri?= =?UTF-8?q?fier=20falsifiable=20on=20chunk,=20offset,=20and=20forged-repor?= =?UTF-8?q?t=20mutations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../REPORT.md | 14 +- .../check.sh | 1 + .../check_report_forgery.py | 146 ++++++++++++++++++ .../verify.py | 29 ++++ 4 files changed, 186 insertions(+), 4 deletions(-) create mode 100644 wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check_report_forgery.py diff --git a/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/REPORT.md b/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/REPORT.md index 9798ce7..560d38c 100644 --- a/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/REPORT.md +++ b/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/REPORT.md @@ -51,16 +51,22 @@ rejected by this artifact's own test. - `tools/run_retrieval.py` — needle ranking - `tools/build_negative_control.py` — distractor comparison - `verify.py` — recomputes hashes, spans, contiguity, needle offsets, ownership, - and re-ranks every needle with its own copy of the ranker. **365 checks, 0 failed.** + and re-ranks every needle with its own copy of the ranker. It also reads + REPORT.md and re-derives the published headline numbers from the corpus, so the + prose cannot drift from the bytes. **371 checks, 0 failed.** - `selftest_tamper.py` — corrupts two independent things in scratch copies (a chunk byte; a needle offset) and requires `verify.py` to reject both. **4 mutation checks, 0 missed.** +- `check_report_forgery.py` — the third, independent mutation: it forges + REPORT.md to claim 35/35 top-1 while the corpus says 34/35, and requires + `verify.py` to reject the forgery. **9 checks, 0 failed.** - `check.sh` — the single verdict command: `sh check.sh` exits 0 only if - verification passes AND the verifier proves falsifiable. + verification passes AND the verifier proves falsifiable on three kinds of lie. ## Reproduce sh /root/troupe-standup/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check.sh -Measured on this run: `VERIFICATION PASSED — 365 checks, 0 failed`, -`SELFTEST PASSED — 4 mutation checks, 0 missed`, shell exit 0. +Measured on this run: `VERIFICATION PASSED — 371 checks, 0 failed`, +`SELFTEST PASSED — 4 mutation checks, 0 missed`, +`REPORT FORGERY DETECTED — 9 checks, 0 failed`, shell exit 0. diff --git a/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check.sh b/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check.sh index bb57577..97208ac 100644 --- a/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check.sh +++ b/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check.sh @@ -15,3 +15,4 @@ LOG="${TMPDIR:-/tmp}/sphinx-needle-verify.log" python3 "$HERE/verify.py" >"$LOG" 2>&1 || { tail -n 25 "$LOG"; exit 1; } tail -n 1 "$LOG" python3 "$HERE/selftest_tamper.py" || exit 1 +python3 "$HERE/check_report_forgery.py" || exit 1 diff --git a/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check_report_forgery.py b/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check_report_forgery.py new file mode 100644 index 0000000..cc030b7 --- /dev/null +++ b/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/check_report_forgery.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""check_report_forgery.py — a CHECK that is not a copy of check.sh. + +The artifact's REPORT.md asserts headline numbers: + + top-1 chunk correct: 34 / 35 + top-5 contains the owning chunk: 35 / 35 + +verify.py already proves it can fail on a corrupted chunk byte and a shifted +needle offset. It does NOT prove anything about the *report text* being bound +to the numbers it publishes: a forge that edited REPORT.md to read "35 / 35" +while the bytes say 34 / 35 would walk straight through check.sh. + +This tool is that missing mutation. It forges REPORT.md, runs the artifact's +own verifier, and demands that the verifier REJECT the forged report. It also +re-derives the two headline numbers from the bytes on disk and fails if they do +not match what the current REPORT.md claims — so the check passes only while the +report is honest AND the verifier is capable of noticing a lie. + +Exit 0 iff both hold. +""" +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +failures = [] +checks = [0] + + +def report(name, ok, detail=""): + checks[0] += 1 + print(" %s %s %s" % ("PASS" if ok else "FAIL", name, detail)) + if not ok: + failures.append("%s %s" % (name, detail)) + + +def sha256(b): + return hashlib.sha256(b).hexdigest() + + +def recompute_top1(): + """Independent recomputation of top-1 accuracy from the bytes on disk. + + Re-implements the ranker from scratch (same definition as verify.py) and + reads chunk files directly. Deliberately does not import verify.py. + """ + index = json.load(open(os.path.join(HERE, "index.json"), encoding="utf-8")) + ndoc = json.load(open(os.path.join(HERE, "needles.json"), encoding="utf-8")) + by_slug = {} + for c in index["chunks"]: + by_slug.setdefault(c["slug"], []).append(c) + + def words(s): + return re.findall(r"[a-z0-9]+", s.lower()) + + def rank_score(needle, body): + nw = set(words(needle)) + if not nw: + return 0.0 + return len(nw & set(words(body))) / len(nw) - 0.0001 * len(body) + + top1 = 0 + for nd in ndoc["needles"]: + slug = nd["slug"] + rows = sorted(by_slug[slug], key=lambda c: c["chunk"]) + scored = [] + for c in rows: + body = open(os.path.join(HERE, c["path"]), encoding="utf-8").read() + scored.append((rank_score(nd["needle"], body), c["chunk"])) + scored.sort(key=lambda t: (-t[0], t[1])) + if scored[0][1] == nd["owning_chunk"]: + top1 += 1 + return top1, len(ndoc["needles"]) + + +def main(): + print("== 1. the published headline numbers match the bytes ==") + report_path = os.path.join(HERE, "REPORT.md") + if not os.path.exists(report_path): + report("REPORT.md exists", False, "missing") + return 1 + report("REPORT.md exists", True) + text = open(report_path, encoding="utf-8").read() + + m1 = re.search(r"top-1 chunk correct:\s*\*?\*?(\d+)\s*/\s*(\d+)", text) + m5 = re.search(r"top-5 contains the owning chunk:\s*\*?\*?(\d+)\s*/\s*(\d+)", + text) + report("REPORT.md states a top-1 headline", m1 is not None) + report("REPORT.md states a top-5 headline", m5 is not None) + if not (m1 and m5): + return 1 + + rep_top1, rep_total = int(m1.group(1)), int(m1.group(2)) + rep_top5 = int(m5.group(1)) + real_top1, real_total = recompute_top1() + report("top-1 headline equals recomputed %d/%d" % (real_top1, real_total), + rep_top1 == real_top1 and rep_total == real_total, + "report says %d/%d" % (rep_top1, rep_total)) + report("top-5 headline is not inflated above needle count", + rep_top5 <= real_total, "report says %d/%d" % (rep_top5, real_total)) + + print("\n== 2. the verifier rejects a forged headline ==") + dst = tempfile.mkdtemp(prefix="forgery-") + shutil.copytree(HERE, dst, dirs_exist_ok=True, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + try: + forged = os.path.join(dst, "REPORT.md") + ftext = open(forged, encoding="utf-8").read() + # Forge the strongest possible lie: claim every needle's top-1 was right. + ftext, nsub = re.subn(r"top-1 chunk correct:\s*\*?\*?(\d+)\s*/\s*(\d+)", + lambda mm: "top-1 chunk correct: **%s" % mm.group(2), + ftext, count=1) + open(forged, "w", encoding="utf-8").write(ftext) + forged_sha = sha256(open(forged, "rb").read()) + orig_sha = sha256(open(report_path, "rb").read()) + report("forged report actually differs from the real one", + nsub == 1 and forged_sha != orig_sha and ftext != text) + + p = subprocess.run([sys.executable, os.path.join(dst, "verify.py")], + capture_output=True, text=True, timeout=1800) + report("verify.py exits nonzero on the forged report", p.returncode != 0, + "exit %d" % p.returncode) + report("failure output says VERIFICATION FAILED", + "VERIFICATION FAILED" in (p.stdout + p.stderr)) + report("forgery was noticed (not silently accepted)", + p.returncode != 0) + finally: + shutil.rmtree(dst, ignore_errors=True) + + print("\n%s — %d checks, %d failed" % ( + "REPORT FORGERY DETECTED (report is bound to the bytes)" if not failures + else "REPORT IS NOT BOUND TO THE BYTES", checks[0], len(failures))) + if failures: + for f in failures[:20]: + print(" ! %s" % f) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/verify.py b/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/verify.py index b35c3e9..6a08652 100644 --- a/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/verify.py +++ b/wizard-work/sphinx/longctx-needle-grounding-2026-10-22/verify.py @@ -24,6 +24,7 @@ import sys HERE = os.path.dirname(os.path.abspath(__file__)) TXT = os.path.join(HERE, "text") CHUNKS = os.path.join(HERE, "chunks") +REPORT = os.path.join(HERE, "REPORT.md") failures = [] checks = [0] @@ -209,6 +210,34 @@ def main(): print(" NOTE %d of %d needles tie or lose to a non-containing chunk " "(ranker tie-break is chunk index, not relevance)" % (ties, neg["needles"])) + print("\n== 6. the published report is bound to the bytes it publishes ==") + # A report is data too. If REPORT.md can be edited to claim "35 / 35" while + # the corpus says 34 / 35, then check.sh passes on a lie. So verify.py reads + # the report text, re-derives each headline number, and demands they agree. + if not os.path.exists(REPORT): + check("REPORT.md exists", False, "missing") + else: + rtext = open(REPORT, encoding="utf-8").read() + check("REPORT.md exists", True) + m1 = re.search(r"top-1 chunk correct:\s*\*?\*?(\d+)\s*/\s*(\d+)", rtext) + m5 = re.search(r"top-5 contains the owning chunk:\s*\*?\*?(\d+)\s*/\s*(\d+)", + rtext) + check("REPORT.md states the top-1 headline", m1 is not None) + check("REPORT.md states the top-5 headline", m5 is not None) + if m1: + check("REPORT.md top-1 headline equals recomputed value", + int(m1.group(1)) == recomputed_top1 and int(m1.group(2)) == n, + "report %s/%s vs recomputed %d/%d" + % (m1.group(1), m1.group(2), recomputed_top1, n)) + if m5: + check("REPORT.md top-5 headline equals recomputed value", + int(m5.group(1)) == recomputed_top5 and int(m5.group(2)) == n, + "report %s/%s vs recomputed %d/%d" + % (m5.group(1), m5.group(2), recomputed_top5, n)) + check("REPORT.md claims nothing above 100%", + all(int(m.group(1)) <= int(m.group(2)) + for m in [m1, m5] if m)) + print("\n%s — %d checks, %d failed" % ( "VERIFICATION PASSED" if not failures else "VERIFICATION FAILED", checks[0], len(failures)))