152 lines
6.2 KiB
Python
152 lines
6.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Independent verification of the chunker refactor (forge-lane Refactor build).
|
|
|
|
The module docstring of ``chunker.py`` claims two things the refactor achieved:
|
|
|
|
1. The *fixed* ``reconstruct`` is **lossless** on all inputs (including the
|
|
highly-repetitive / sentence-grouped cases the original broke on).
|
|
2. The **pre-refactor** ``reconstruct`` stripped the carried-overlap tail from
|
|
the previous chunk's content *unconditionally*, which is only correct for
|
|
genuine hard-split overlaps and is **lossy** for sentence-grouped chunks
|
|
(where the carried tail is pure context, not duplicated into the chunk's
|
|
own content).
|
|
|
|
This harness proves BOTH claims with real execution against the genuine
|
|
``chunker`` module under test:
|
|
|
|
* ``reconstruct`` (fixed) -- imported from the real module.
|
|
* ``reconstruct_naive`` -- a faithful reproduction of the pre-fix algorithm
|
|
(identical to the fixed code except the ``rest.startswith(head)``
|
|
structural discriminator is dropped, exactly as the docstring describes the
|
|
original "strip unconditionally" behaviour).
|
|
|
|
Exit code:
|
|
0 fixed is lossless on every corpus case AND naive is lossy on >=1 case
|
|
1 fixed is lossy on some case (the refactor is broken)
|
|
2 naive is lossless everywhere (cannot demonstrate the fix was necessary)
|
|
|
|
Stdlib only; no external data files.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import pathlib
|
|
import random
|
|
import sys
|
|
|
|
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent))
|
|
|
|
from chunker import ( # the module under test
|
|
_OVERLAP_SENTINEL,
|
|
chunk_text,
|
|
reconstruct,
|
|
)
|
|
|
|
|
|
def reconstruct_naive(chunks: list, overlap_tokens: int = 32) -> str:
|
|
"""Pre-refactor ``reconstruct``: strip the carried tail unconditionally.
|
|
|
|
This is the fixed algorithm verbatim EXCEPT the ``rest.startswith(head)``
|
|
branch discriminator is removed -- i.e. whenever the carried ``head`` is a
|
|
suffix of the accumulator, it is deleted regardless of whether ``rest``
|
|
itself begins with ``head``. That is precisely the lossy behaviour
|
|
described in chunker.py's module docstring.
|
|
"""
|
|
if not chunks:
|
|
return ""
|
|
parts: list[str] = []
|
|
for i, c in enumerate(chunks):
|
|
if i == 0:
|
|
# First chunk never carries an overlap; peel a stray sentinel off
|
|
# defensively in case callers pass pre-tagged content.
|
|
parts.append(c.partition(_OVERLAP_SENTINEL)[2] if _OVERLAP_SENTINEL in c else c)
|
|
continue
|
|
prev = parts[-1]
|
|
if _OVERLAP_SENTINEL in c:
|
|
head, _, rest = c.partition(_OVERLAP_SENTINEL)
|
|
head_len = len(head)
|
|
# PRE-FIX: no `rest.startswith(head)` check -> always strip.
|
|
if head_len and head_len <= len(prev) and prev.endswith(head):
|
|
parts[-1] = prev[: len(prev) - head_len]
|
|
else:
|
|
parts.append(" " + rest if rest else "")
|
|
continue
|
|
parts.append(rest)
|
|
else:
|
|
parts.append(" " + c)
|
|
return "".join(parts).strip()
|
|
|
|
|
|
def corpus() -> list[tuple[str, str, int, int]]:
|
|
"""(label, text, max_tokens, overlap_tokens) cases to round-trip."""
|
|
items = [
|
|
# Adversarial repetitive sentences (the exact class the docstring cites).
|
|
("repetitive", ("Sentence one. Sentence two. Sentence three. ") * 40, 24, 4),
|
|
# Varied distinct sentences.
|
|
("varied", ("The quick brown fox jumps over the lazy dog. Pack my box with five "
|
|
"dozen liquor jugs. How vexingly quick daft zebras jump. ") * 20, 64, 8),
|
|
# Single-character hard-split path (genuine overlap; both must be lossless).
|
|
("monorep", "a" * 5000, 32, 4),
|
|
# Zero overlap (no sentinels inserted; both must be lossless).
|
|
("zero-overlap", "One. Two. Three. Four. Five. Six. Seven. Eight. " * 15, 16, 0),
|
|
# Single chunk.
|
|
("single", "Hello world. This is a test. A test of chunker. ", 100, 4),
|
|
# Tiny max_tokens to force many sentence-grouped chunks.
|
|
("tiny", ("The quick brown fox. " * 40), 12, 4),
|
|
]
|
|
rng = random.Random(2026)
|
|
words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel"]
|
|
for k in range(60):
|
|
n = rng.randint(3, 60)
|
|
t = " ".join(rng.choice(words) for _ in range(n))
|
|
if rng.random() < 0.5:
|
|
t = t + ". " + " ".join(rng.choice(words) for _ in range(rng.randint(2, 12))) + "."
|
|
mt = rng.choice([12, 16, 32, 64])
|
|
ot = rng.choice([0, 2, 4, 8])
|
|
if ot >= mt:
|
|
ot = max(0, mt - 1)
|
|
items.append((f"fuzz-{k}", t, mt, ot))
|
|
return items
|
|
|
|
|
|
def main() -> int:
|
|
cases = corpus()
|
|
n = len(cases)
|
|
fixed_losses = 0
|
|
naive_losses = 0
|
|
print("chunker refactor verification (forge lane)")
|
|
print("=" * 60)
|
|
for name, text, mt, ot in cases:
|
|
chunks = chunk_text(text, max_tokens=mt, overlap_tokens=ot)
|
|
expect = text.strip()
|
|
r_fixed = reconstruct(chunks, ot)
|
|
r_naive = reconstruct_naive(chunks, ot)
|
|
fixed_ok = r_fixed == expect
|
|
naive_ok = r_naive == expect
|
|
fixed_losses += 0 if fixed_ok else 1
|
|
naive_losses += 0 if naive_ok else 1
|
|
status = "OK " if (fixed_ok and not naive_ok) else (
|
|
"LOSSY-BOTH" if (not fixed_ok and not naive_ok) else "lossless-both"
|
|
)
|
|
print(f" [{status:10s}] {name:10s} mt={mt:<3} ot={ot:<2} "
|
|
f"len={len(text):>5} fixed={'ok' if fixed_ok else f'LOSS {len(expect)-len(r_fixed)}'} "
|
|
f"naive={'ok' if naive_ok else f'LOSS {len(expect)-len(r_naive)}'}")
|
|
|
|
print("-" * 60)
|
|
print(f"corpus size : {n}")
|
|
print(f"fixed reconstruct : {n - fixed_losses}/{n} lossless")
|
|
print(f"naive reconstruct : {n - naive_losses}/{n} lossless (pre-refactor algo)")
|
|
|
|
if fixed_losses != 0:
|
|
print("FAIL: the refactored reconstruct is lossy -- refactor is broken.")
|
|
return 1
|
|
if naive_losses == 0:
|
|
print("WARN: naive algorithm is lossless everywhere; fix's necessity not shown.")
|
|
return 2
|
|
print("PASS: fixed=lossless on all cases AND naive=lossy (>=1 case) => "
|
|
"fix is necessary and correct.")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|