87 lines
3.0 KiB
Python
87 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""sphinx Archive-lane verifier.
|
|
|
|
Reads manifest.json, recomputes SHA-256 from the bytes ACTUALLY ON DISK, and
|
|
asserts structural properties of the archived corpus. Exit 0 only if every
|
|
document is present, every hash matches, and the corpus is non-trivial.
|
|
|
|
Usage: verify_archive.py <corpus_dir>
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
MIN_DOCS = 6
|
|
MIN_TOTAL_WORDS = 1_000_000
|
|
|
|
|
|
def fail(msg):
|
|
print(f"FAIL: {msg}")
|
|
sys.exit(1)
|
|
|
|
|
|
def main():
|
|
corpus = os.path.abspath(sys.argv[1] if len(sys.argv) > 1 else ".")
|
|
manifest_path = os.path.join(corpus, "manifest.json")
|
|
assert os.path.isfile(manifest_path), f"no manifest at {manifest_path}"
|
|
|
|
with open(manifest_path, encoding="utf-8") as f:
|
|
man = json.load(f)
|
|
|
|
if man.get("schema") != "sphinx-archive/1":
|
|
fail(f"unexpected schema {man.get('schema')!r}")
|
|
|
|
docs = man.get("documents", [])
|
|
if len(docs) < MIN_DOCS:
|
|
fail(f"only {len(docs)} documents, need >= {MIN_DOCS}")
|
|
|
|
total_words = 0
|
|
seen = set()
|
|
for d in docs:
|
|
slug = d["slug"]
|
|
if slug in seen:
|
|
fail(f"duplicate slug {slug}")
|
|
seen.add(slug)
|
|
|
|
txt_path = os.path.join(corpus, d["txt_file"])
|
|
raw_path = os.path.join(corpus, d["raw_file"])
|
|
if not os.path.isfile(txt_path):
|
|
fail(f"{slug}: missing txt {txt_path}")
|
|
if not os.path.isfile(raw_path):
|
|
fail(f"{slug}: missing raw {raw_path}")
|
|
|
|
on_disk = open(txt_path, "rb").read()
|
|
disk_sha = hashlib.sha256(on_disk).hexdigest()
|
|
if disk_sha != d["txt_sha256"]:
|
|
fail(f"{slug}: txt hash mismatch manifest={d['txt_sha256']} disk={disk_sha}")
|
|
if len(on_disk) != d["txt_bytes"]:
|
|
fail(f"{slug}: txt size mismatch manifest={d['txt_bytes']} disk={len(on_disk)}")
|
|
|
|
raw_on_disk = open(raw_path, "rb").read()
|
|
raw_disk_sha = hashlib.sha256(raw_on_disk).hexdigest()
|
|
if raw_disk_sha != d["raw_sha256"]:
|
|
fail(f"{slug}: raw hash mismatch")
|
|
|
|
# Structural: the archived text must actually contain its own title
|
|
# (case-insensitive) and be wordy enough to count as a book, not a stub.
|
|
text = on_disk.decode("utf-8", errors="replace")
|
|
if len(text.split()) != d["word_count"]:
|
|
fail(f"{slug}: word_count mismatch manifest={d['word_count']} disk={len(text.split())}")
|
|
if d["word_count"] < 20_000:
|
|
fail(f"{slug}: only {d['word_count']} words, looks like a stub")
|
|
if d["title"].split(";")[0].split(",")[0].strip().lower() not in text.lower():
|
|
fail(f"{slug}: title {d['title']!r} not found in archived text")
|
|
total_words += d["word_count"]
|
|
|
|
if total_words < MIN_TOTAL_WORDS:
|
|
fail(f"total_words={total_words} < {MIN_TOTAL_WORDS}")
|
|
|
|
print(f"PASS: {len(docs)} documents, {total_words} words, all hashes verified")
|
|
print(f" manifest={manifest_path}")
|
|
sys.exit(0)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|