sphinx: Gutenberg long-context corpus (6 books, 1.13M words, SHA-256 provenance)

2026-10-22 14:23:13 +00:00
1 parent 69661276fa
commit e33f4a6a92
@@ -0,0 +1,64 @@
+# Gutenberg Long-Context Corpus — Archive Report
+
+**Artifact:** `archive-gutenberg/` (raw HTML + normalised text + manifest)
+**Built:** 2026-10-22 UTC · **Lane:** Archive — multimodal, long-context research
+**Corpus size:** 6 documents, **1,129,516 words**, 6,813,315 bytes of normalised text
+
+## What this is
+
+A fetched, hashed, structurally-verified archive of six public-domain long-form
+books from Project Gutenberg, normalised to plain UTF-8 text with per-document
+SHA-256 provenance. Built for long-context retrieval and chunking work: each
+document is large enough (29k–567k words) to stress a long context window, and
+the manifest pins exact content hashes so any chunking experiment is reproducible.
+
+## Documents
+
+| slug | title | words | txt bytes | txt SHA-256 (first 16) |
+|---|---|---|---|---|
+| war_and_peace | War and Peace | 566,794 | 3,476,843 | 6cb9dd25004552dd |
+| moby_dick | Moby-Dick; or, The Whale | 216,325 | 1,312,754 | 9f7e9224bfb6baa8 |
+| pride_and_prejudice | Pride and Prejudice | 130,933 | 771,500 | 7c8f3b9cd10c2324 |
+| sherlock_holmes | The Adventures of Sherlock Holmes | 107,654 | 619,475 | d00a2b51c3a403ed |
+| frankenstein | Frankenstein; or, The Modern Prometheus | 78,137 | 453,921 | 0d113dcfc3a1203f |
+| alice_in_wonderland | Alice's Adventures in Wonderland | 29,673 | 178,232 | 29d04d721acfd87d |
+
+Full hashes, source URLs, byte counts and fetch timestamps are in `manifest.json`.
+
+## Method
+
+1. `build_corpus.py` fetches `https://www.gutenberg.org/cache/epub/{id}/pg{id}-images.html`
+   for each document with a browser User-Agent.
+2. HTML is normalised: `<script>`/`<style>` stripped, block tags converted to
+   newlines, entities decoded, runs of whitespace collapsed. Raw HTML is kept
+   alongside the normalised text so the transform is auditable.
+3. Every file is hashed **before** the manifest is written, so the manifest
+   cannot describe bytes that were never written.
+
+## Verification (measured)
+
+- `python3 verify_archive.py .` → `PASS: 6 documents, 1129516 words, all hashes verified`, exit 0.
+  It recomputes SHA-256 from the bytes **on disk** (not from the manifest),
+  checks byte counts and word counts, confirms each book's title appears in its
+  own archived text, and enforces floors of 6 documents / 1,000,000 words so a
+  truncated or stubbed corpus cannot pass.
+- **Negative control:** appending one word to `txt/alice_in_wonderland.txt`
+  makes the verifier exit 1 with `txt hash mismatch`. The check discriminates
+  real from broken rather than always passing.
+
+## Files
+
+```
+archive-gutenberg/
+├── build_corpus.py     # fetches + normalises + hashes + writes manifest
+├── verify_archive.py   # independent verifier (recomputes hashes from disk)
+├── manifest.json       # schema sphinx-archive/1, 6 pinned documents
+├── raw/                # 6 .html files, byte-for-byte as fetched
+└── txt/                # 6 .txt files, normalised UTF-8
+```
+
+## Verdict
+
+The archive is built from real network fetches, every document is content-hashed,
+and a measured verifier confirms integrity from disk while demonstrably failing
+on tampered input.
@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+"""sphinx Archive-lane worker.
+
+Fetches a long-context corpus, normalises it to plain text, records per-document
+SHA-256 provenance, and writes a machine-verifiable manifest. No stubs: every
+document is fetched, hashed, and written to disk before the manifest is emitted.
+"""
+import hashlib
+import json
+import os
+import re
+import sys
+import time
+import urllib.request
+
+CORPUS_DIR = sys.argv[1] if len(sys.argv) > 1 else "."
+CORPUS_DIR = os.path.abspath(CORPUS_DIR)
+RAW = os.path.join(CORPUS_DIR, "raw")
+TXT = os.path.join(CORPUS_DIR, "txt")
+MANIFEST = os.path.join(CORPUS_DIR, "manifest.json")
+
+UA = ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+      "(KHTML, like Gecko) Chrome/124.0 Safari/537.36")
+
+# Long-context public-domain corpus. GutenIDs are stable; SHA-256 pins content.
+DOCS = [
+    ("moby_dick", "2701", "Moby-Dick; or, The Whale"),
+    ("pride_and_prejudice", "1342", "Pride and Prejudice"),
+    ("war_and_peace", "2600", "War and Peace"),
+    ("alice_in_wonderland", "11", "Alice's Adventures in Wonderland"),
+    ("sherlock_holmes", "1661", "The Adventures of Sherlock Holmes"),
+    ("frankenstein", "84", "Frankenstein; or, The Modern Prometheus"),
+]
+
+
+def fetch(url, timeout=60):
+    req = urllib.request.Request(url, headers={"User-Agent": UA})
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return resp.read()
+
+
+def to_text(raw_bytes):
+    raw = raw_bytes.decode("utf-8", errors="replace")
+    m = re.search(r"<body[^>]*>(.*)</body>", raw, re.S | re.I)
+    body = m.group(1) if m else raw
+    body = re.sub(r"(?is)<(script|style)[^>]*>.*?</\1>", " ", body)
+    body = re.sub(r"(?i)<br\s*/?>", "\n", body)
+    body = re.sub(r"(?i)</p>", "\n\n", body)
+    body = re.sub(r"(?s)<[^>]+>", " ", body)
+    body = body.replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">")
+    body = body.replace("&quot;", '"').replace("&#39;", "'").replace("&nbsp;", " ")
+    body = re.sub(r"[ \t]+", " ", body)
+    return body.strip()
+
+
+def main():
+    for d in (RAW, TXT):
+        os.makedirs(d, exist_ok=True)
+
+    entries = []
+    for slug, gid, title in DOCS:
+        url = f"https://www.gutenberg.org/cache/epub/{gid}/pg{gid}-images.html"
+        t0 = time.time()
+        raw = fetch(url)
+        raw_sha = hashlib.sha256(raw).hexdigest()
+        text = to_text(raw)
+        txt_bytes = text.encode("utf-8")
+        txt_sha = hashlib.sha256(txt_bytes).hexdigest()
+
+        raw_path = os.path.join(RAW, f"{slug}.html")
+        txt_path = os.path.join(TXT, f"{slug}.txt")
+        with open(raw_path, "wb") as f:
+            f.write(raw)
+        with open(txt_path, "wb") as f:
+            f.write(txt_bytes)
+
+        words = len(text.split())
+        entries.append({
+            "slug": slug,
+            "title": title,
+            "gutenberg_id": gid,
+            "source_url": url,
+            "fetched_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(t0)),
+            "raw_sha256": raw_sha,
+            "txt_sha256": txt_sha,
+            "raw_bytes": len(raw),
+            "txt_bytes": len(txt_bytes),
+            "word_count": words,
+            "txt_file": f"txt/{slug}.txt",
+            "raw_file": f"raw/{slug}.html",
+        })
+        print(f"[{slug}] raw={len(raw)}B txt={len(txt_bytes)}B words={words} "
+              f"sha={txt_sha[:16]}…")
+
+    manifest = {
+        "schema": "sphinx-archive/1",
+        "corpus": "gutenberg-longcontext",
+        "created_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "documents": sorted(entries, key=lambda e: -e["word_count"]),
+    }
+    with open(MANIFEST, "w", encoding="utf-8") as f:
+        json.dump(manifest, f, indent=2, sort_keys=True)
+    total_words = sum(e["word_count"] for e in entries)
+    print(f"\nmanifest -> {MANIFEST}")
+    print(f"documents={len(entries)} total_words={total_words}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,91 @@
+{
+  "corpus": "gutenberg-longcontext",
+  "created_utc": "2026-10-22T14:21:38Z",
+  "documents": [
+    {
+      "fetched_at_utc": "2026-10-22T14:21:35Z",
+      "gutenberg_id": "2600",
+      "raw_bytes": 3878334,
+      "raw_file": "raw/war_and_peace.html",
+      "raw_sha256": "95204a39054713cb9f3bb4468177e64e1cef19f2e60384283aef6403d23ee30e",
+      "slug": "war_and_peace",
+      "source_url": "https://www.gutenberg.org/cache/epub/2600/pg2600-images.html",
+      "title": "War and Peace",
+      "txt_bytes": 3476843,
+      "txt_file": "txt/war_and_peace.txt",
+      "txt_sha256": "6cb9dd25004552dd6362f55f8560d926c2d145d89c192b02884b590ceb8c093b",
+      "word_count": 566794
+    },
+    {
+      "fetched_at_utc": "2026-10-22T14:21:34Z",
+      "gutenberg_id": "2701",
+      "raw_bytes": 1472550,
+      "raw_file": "raw/moby_dick.html",
+      "raw_sha256": "3e6cb01b2e2a839a50934121f511a2bfffc5695489b7d61ddd7eeae252c0277e",
+      "slug": "moby_dick",
+      "source_url": "https://www.gutenberg.org/cache/epub/2701/pg2701-images.html",
+      "title": "Moby-Dick; or, The Whale",
+      "txt_bytes": 1312754,
+      "txt_file": "txt/moby_dick.txt",
+      "txt_sha256": "9f7e9224bfb6baa821e8e1ff04302e3d5d2e41e716f7f6da1a226b01eeecbd0e",
+      "word_count": 216325
+    },
+    {
+      "fetched_at_utc": "2026-10-22T14:21:35Z",
+      "gutenberg_id": "1342",
+      "raw_bytes": 852590,
+      "raw_file": "raw/pride_and_prejudice.html",
+      "raw_sha256": "11a120c25838974d55bb542d424df322ee10a9058cd884e1166208d1c0fbfeaa",
+      "slug": "pride_and_prejudice",
+      "source_url": "https://www.gutenberg.org/cache/epub/1342/pg1342-images.html",
+      "title": "Pride and Prejudice",
+      "txt_bytes": 771500,
+      "txt_file": "txt/pride_and_prejudice.txt",
+      "txt_sha256": "7c8f3b9cd10c2324d1308ec99dc5018bc73b5809bf9a1c2dd3107e09d6d5d321",
+      "word_count": 130933
+    },
+    {
+      "fetched_at_utc": "2026-10-22T14:21:37Z",
+      "gutenberg_id": "1661",
+      "raw_bytes": 640229,
+      "raw_file": "raw/sherlock_holmes.html",
+      "raw_sha256": "26adf6c8ff1384574fca7466df77e226fc048e9ef1eac5c341f0dd997bcb71a4",
+      "slug": "sherlock_holmes",
+      "source_url": "https://www.gutenberg.org/cache/epub/1661/pg1661-images.html",
+      "title": "The Adventures of Sherlock Holmes",
+      "txt_bytes": 619475,
+      "txt_file": "txt/sherlock_holmes.txt",
+      "txt_sha256": "d00a2b51c3a403edf40edc4f5be40d109206cf0b4a83f09f1bc8cebafba30594",
+      "word_count": 107654
+    },
+    {
+      "fetched_at_utc": "2026-10-22T14:21:38Z",
+      "gutenberg_id": "84",
+      "raw_bytes": 468370,
+      "raw_file": "raw/frankenstein.html",
+      "raw_sha256": "23403a6a3b1cace42a06d6c378a6f1acf8ec97c940c652d12fe3c0d30275e58f",
+      "slug": "frankenstein",
+      "source_url": "https://www.gutenberg.org/cache/epub/84/pg84-images.html",
+      "title": "Frankenstein; or, The Modern Prometheus",
+      "txt_bytes": 453921,
+      "txt_file": "txt/frankenstein.txt",
+      "txt_sha256": "0d113dcfc3a1203f1ab24840a70a804562fbc474c2bf3f5050b614f3abd48406",
+      "word_count": 78137
+    },
+    {
+      "fetched_at_utc": "2026-10-22T14:21:37Z",
+      "gutenberg_id": "11",
+      "raw_bytes": 192539,
+      "raw_file": "raw/alice_in_wonderland.html",
+      "raw_sha256": "7df9c19e9fccac5b3ae9b1f837b247db1c1da93845dfb9e1903dce85013c43d3",
+      "slug": "alice_in_wonderland",
+      "source_url": "https://www.gutenberg.org/cache/epub/11/pg11-images.html",
+      "title": "Alice's Adventures in Wonderland",
+      "txt_bytes": 178232,
+      "txt_file": "txt/alice_in_wonderland.txt",
+      "txt_sha256": "29d04d721acfd87d176b2931f87c934852268b3fe36ed4259abe9c035ed005b6",
+      "word_count": 29673
+    }
+  ],
+  "schema": "sphinx-archive/1"
+}
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""sphinx Archive-lane verifier.
+
+Reads manifest.json, recomputes SHA-256 from the bytes ACTUALLY ON DISK, and
+asserts structural properties of the archived corpus. Exit 0 only if every
+document is present, every hash matches, and the corpus is non-trivial.
+
+Usage: verify_archive.py <corpus_dir>
+"""
+import hashlib
+import json
+import os
+import sys
+
+MIN_DOCS = 6
+MIN_TOTAL_WORDS = 1_000_000
+
+
+def fail(msg):
+    print(f"FAIL: {msg}")
+    sys.exit(1)
+
+
+def main():
+    corpus = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else ".")
+    manifest_path = os.path.join(corpus, "manifest.json")
+    assert os.path.isfile(manifest_path), f"no manifest at {manifest_path}"
+
+    with open(manifest_path, encoding="utf-8") as f:
+        man = json.load(f)
+
+    if man.get("schema") != "sphinx-archive/1":
+        fail(f"unexpected schema {man.get('schema')!r}")
+
+    docs = man.get("documents", [])
+    if len(docs) < MIN_DOCS:
+        fail(f"only {len(docs)} documents, need >= {MIN_DOCS}")
+
+    total_words = 0
+    seen = set()
+    for d in docs:
+        slug = d["slug"]
+        if slug in seen:
+            fail(f"duplicate slug {slug}")
+        seen.add(slug)
+
+        txt_path = os.path.join(corpus, d["txt_file"])
+        raw_path = os.path.join(corpus, d["raw_file"])
+        if not os.path.isfile(txt_path):
+            fail(f"{slug}: missing txt {txt_path}")
+        if not os.path.isfile(raw_path):
+            fail(f"{slug}: missing raw {raw_path}")
+
+        on_disk = open(txt_path, "rb").read()
+        disk_sha = hashlib.sha256(on_disk).hexdigest()
+        if disk_sha != d["txt_sha256"]:
+            fail(f"{slug}: txt hash mismatch manifest={d['txt_sha256']} disk={disk_sha}")
+        if len(on_disk) != d["txt_bytes"]:
+            fail(f"{slug}: txt size mismatch manifest={d['txt_bytes']} disk={len(on_disk)}")
+
+        raw_on_disk = open(raw_path, "rb").read()
+        raw_disk_sha = hashlib.sha256(raw_on_disk).hexdigest()
+        if raw_disk_sha != d["raw_sha256"]:
+            fail(f"{slug}: raw hash mismatch")
+
+        # Structural: the archived text must actually contain its own title
+        # (case-insensitive) and be wordy enough to count as a book, not a stub.
+        text = on_disk.decode("utf-8", errors="replace")
+        if len(text.split()) != d["word_count"]:
+            fail(f"{slug}: word_count mismatch manifest={d['word_count']} disk={len(text.split())}")
+        if d["word_count"] < 20_000:
+            fail(f"{slug}: only {d['word_count']} words, looks like a stub")
+        if d["title"].split(";")[0].split(",")[0].strip().lower() not in text.lower():
+            fail(f"{slug}: title {d['title']!r} not found in archived text")
+        total_words += d["word_count"]
+
+    if total_words < MIN_TOTAL_WORDS:
+        fail(f"total_words={total_words} < {MIN_TOTAL_WORDS}")
+
+    print(f"PASS: {len(docs)} documents, {total_words} words, all hashes verified")
+    print(f"      manifest={manifest_path}")
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()