#!/usr/bin/env python3 """Independent verification of the chunker refactor (forge-lane Refactor build). The module docstring of ``chunker.py`` claims two things the refactor achieved: 1. The *fixed* ``reconstruct`` is **lossless** on all inputs (including the highly-repetitive / sentence-grouped cases the original broke on). 2. The **pre-refactor** ``reconstruct`` stripped the carried-overlap tail from the previous chunk's content *unconditionally*, which is only correct for genuine hard-split overlaps and is **lossy** for sentence-grouped chunks (where the carried tail is pure context, not duplicated into the chunk's own content). This harness proves BOTH claims with real execution against the genuine ``chunker`` module under test: * ``reconstruct`` (fixed) -- imported from the real module. * ``reconstruct_naive`` -- a faithful reproduction of the pre-fix algorithm (identical to the fixed code except the ``rest.startswith(head)`` structural discriminator is dropped, exactly as the docstring describes the original "strip unconditionally" behaviour). Exit code: 0 fixed is lossless on every corpus case AND naive is lossy on >=1 case 1 fixed is lossy on some case (the refactor is broken) 2 naive is lossless everywhere (cannot demonstrate the fix was necessary) Stdlib only; no external data files. """ from __future__ import annotations import pathlib import random import sys sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) from chunker import ( # the module under test _OVERLAP_SENTINEL, chunk_text, reconstruct, ) def reconstruct_naive(chunks: list, overlap_tokens: int = 32) -> str: """Pre-refactor ``reconstruct``: strip the carried tail unconditionally. This is the fixed algorithm verbatim EXCEPT the ``rest.startswith(head)`` branch discriminator is removed -- i.e. whenever the carried ``head`` is a suffix of the accumulator, it is deleted regardless of whether ``rest`` itself begins with ``head``. That is precisely the lossy behaviour described in chunker.py's module docstring. """ if not chunks: return "" parts: list[str] = [] for i, c in enumerate(chunks): if i == 0: # First chunk never carries an overlap; peel a stray sentinel off # defensively in case callers pass pre-tagged content. parts.append(c.partition(_OVERLAP_SENTINEL)[2] if _OVERLAP_SENTINEL in c else c) continue prev = parts[-1] if _OVERLAP_SENTINEL in c: head, _, rest = c.partition(_OVERLAP_SENTINEL) head_len = len(head) # PRE-FIX: no `rest.startswith(head)` check -> always strip. if head_len and head_len <= len(prev) and prev.endswith(head): parts[-1] = prev[: len(prev) - head_len] else: parts.append(" " + rest if rest else "") continue parts.append(rest) else: parts.append(" " + c) return "".join(parts).strip() def corpus() -> list[tuple[str, str, int, int]]: """(label, text, max_tokens, overlap_tokens) cases to round-trip.""" items = [ # Adversarial repetitive sentences (the exact class the docstring cites). ("repetitive", ("Sentence one. Sentence two. Sentence three. ") * 40, 24, 4), # Varied distinct sentences. ("varied", ("The quick brown fox jumps over the lazy dog. Pack my box with five " "dozen liquor jugs. How vexingly quick daft zebras jump. ") * 20, 64, 8), # Single-character hard-split path (genuine overlap; both must be lossless). ("monorep", "a" * 5000, 32, 4), # Zero overlap (no sentinels inserted; both must be lossless). ("zero-overlap", "One. Two. Three. Four. Five. Six. Seven. Eight. " * 15, 16, 0), # Single chunk. ("single", "Hello world. This is a test. A test of chunker. ", 100, 4), # Tiny max_tokens to force many sentence-grouped chunks. ("tiny", ("The quick brown fox. " * 40), 12, 4), ] rng = random.Random(2026) words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel"] for k in range(60): n = rng.randint(3, 60) t = " ".join(rng.choice(words) for _ in range(n)) if rng.random() < 0.5: t = t + ". " + " ".join(rng.choice(words) for _ in range(rng.randint(2, 12))) + "." mt = rng.choice([12, 16, 32, 64]) ot = rng.choice([0, 2, 4, 8]) if ot >= mt: ot = max(0, mt - 1) items.append((f"fuzz-{k}", t, mt, ot)) return items def main() -> int: cases = corpus() n = len(cases) fixed_losses = 0 naive_losses = 0 print("chunker refactor verification (forge lane)") print("=" * 60) for name, text, mt, ot in cases: chunks = chunk_text(text, max_tokens=mt, overlap_tokens=ot) expect = text.strip() r_fixed = reconstruct(chunks, ot) r_naive = reconstruct_naive(chunks, ot) fixed_ok = r_fixed == expect naive_ok = r_naive == expect fixed_losses += 0 if fixed_ok else 1 naive_losses += 0 if naive_ok else 1 status = "OK " if (fixed_ok and not naive_ok) else ( "LOSSY-BOTH" if (not fixed_ok and not naive_ok) else "lossless-both" ) print(f" [{status:10s}] {name:10s} mt={mt:<3} ot={ot:<2} " f"len={len(text):>5} fixed={'ok' if fixed_ok else f'LOSS {len(expect)-len(r_fixed)}'} " f"naive={'ok' if naive_ok else f'LOSS {len(expect)-len(r_naive)}'}") print("-" * 60) print(f"corpus size : {n}") print(f"fixed reconstruct : {n - fixed_losses}/{n} lossless") print(f"naive reconstruct : {n - naive_losses}/{n} lossless (pre-refactor algo)") if fixed_losses != 0: print("FAIL: the refactored reconstruct is lossy -- refactor is broken.") return 1 if naive_losses == 0: print("WARN: naive algorithm is lossless everywhere; fix's necessity not shown.") return 2 print("PASS: fixed=lossless on all cases AND naive=lossy (>=1 case) => " "fix is necessary and correct.") return 0 if __name__ == "__main__": sys.exit(main())