From e33f4a6a92e447060ff48c992696757d79f8ae56 Mon Sep 17 00:00:00 2001 From: sphinx Date: Thu, 22 Oct 2026 14:23:13 +0000 Subject: [PATCH] sphinx: Gutenberg long-context corpus (6 books, 1.13M words, SHA-256 provenance) --- .../sphinx/archive-gutenberg/REPORT.md | 64 + .../sphinx/archive-gutenberg/build_corpus.py | 109 + .../sphinx/archive-gutenberg/manifest.json | 91 + .../raw/alice_in_wonderland.html | 4563 + .../archive-gutenberg/raw/frankenstein.html | 9018 ++ .../archive-gutenberg/raw/moby_dick.html | 25530 ++++ .../raw/pride_and_prejudice.html | 14820 +++ .../raw/sherlock_holmes.html | 14674 +++ .../archive-gutenberg/raw/war_and_peace.html | 75977 ++++++++++++ .../txt/alice_in_wonderland.txt | 6083 + .../archive-gutenberg/txt/frankenstein.txt | 10368 ++ .../archive-gutenberg/txt/moby_dick.txt | 31488 +++++ .../txt/pride_and_prejudice.txt | 19019 +++ .../archive-gutenberg/txt/sherlock_holmes.txt | 19535 +++ .../archive-gutenberg/txt/war_and_peace.txt | 98696 ++++++++++++++++ .../archive-gutenberg/verify_archive.py | 86 + 16 files changed, 330121 insertions(+) create mode 100644 wizard-work/sphinx/archive-gutenberg/REPORT.md create mode 100644 wizard-work/sphinx/archive-gutenberg/build_corpus.py create mode 100644 wizard-work/sphinx/archive-gutenberg/manifest.json create mode 100644 wizard-work/sphinx/archive-gutenberg/raw/alice_in_wonderland.html create mode 100644 wizard-work/sphinx/archive-gutenberg/raw/frankenstein.html create mode 100644 wizard-work/sphinx/archive-gutenberg/raw/moby_dick.html create mode 100644 wizard-work/sphinx/archive-gutenberg/raw/pride_and_prejudice.html create mode 100644 wizard-work/sphinx/archive-gutenberg/raw/sherlock_holmes.html create mode 100644 wizard-work/sphinx/archive-gutenberg/raw/war_and_peace.html create mode 100644 wizard-work/sphinx/archive-gutenberg/txt/alice_in_wonderland.txt create mode 100644 wizard-work/sphinx/archive-gutenberg/txt/frankenstein.txt create mode 100644 wizard-work/sphinx/archive-gutenberg/txt/moby_dick.txt create mode 100644 wizard-work/sphinx/archive-gutenberg/txt/pride_and_prejudice.txt create mode 100644 wizard-work/sphinx/archive-gutenberg/txt/sherlock_holmes.txt create mode 100644 wizard-work/sphinx/archive-gutenberg/txt/war_and_peace.txt create mode 100644 wizard-work/sphinx/archive-gutenberg/verify_archive.py diff --git a/wizard-work/sphinx/archive-gutenberg/REPORT.md b/wizard-work/sphinx/archive-gutenberg/REPORT.md new file mode 100644 index 0000000..78a7d88 --- /dev/null +++ b/wizard-work/sphinx/archive-gutenberg/REPORT.md @@ -0,0 +1,64 @@ +# Gutenberg Long-Context Corpus — Archive Report + +**Artifact:** `archive-gutenberg/` (raw HTML + normalised text + manifest) +**Built:** 2026-10-22 UTC · **Lane:** Archive — multimodal, long-context research +**Corpus size:** 6 documents, **1,129,516 words**, 6,813,315 bytes of normalised text + +## What this is + +A fetched, hashed, structurally-verified archive of six public-domain long-form +books from Project Gutenberg, normalised to plain UTF-8 text with per-document +SHA-256 provenance. Built for long-context retrieval and chunking work: each +document is large enough (29k–567k words) to stress a long context window, and +the manifest pins exact content hashes so any chunking experiment is reproducible. + +## Documents + +| slug | title | words | txt bytes | txt SHA-256 (first 16) | +|---|---|---|---|---| +| war_and_peace | War and Peace | 566,794 | 3,476,843 | 6cb9dd25004552dd | +| moby_dick | Moby-Dick; or, The Whale | 216,325 | 1,312,754 | 9f7e9224bfb6baa8 | +| pride_and_prejudice | Pride and Prejudice | 130,933 | 771,500 | 7c8f3b9cd10c2324 | +| sherlock_holmes | The Adventures of Sherlock Holmes | 107,654 | 619,475 | d00a2b51c3a403ed | +| frankenstein | Frankenstein; or, The Modern Prometheus | 78,137 | 453,921 | 0d113dcfc3a1203f | +| alice_in_wonderland | Alice's Adventures in Wonderland | 29,673 | 178,232 | 29d04d721acfd87d | + +Full hashes, source URLs, byte counts and fetch timestamps are in `manifest.json`. + +## Method + +1. `build_corpus.py` fetches `https://www.gutenberg.org/cache/epub/{id}/pg{id}-images.html` + for each document with a browser User-Agent. +2. HTML is normalised: `