"""Regression + round-trip tests for the refactored chunker (forge lane build). These mirror the upstream suite in wizard-outputs/null but pin the corrected lossless-round-trip contract that the original ``reconstruct`` violated on repetitive / sentence-grouped input. """ import random import chunker from chunker import _OVERLAP_SENTINEL # --- argument validation (unchanged contract) --- def test_invalid_args(): import pytest with pytest.raises(ValueError): chunker.chunk_text("x", max_tokens=0) with pytest.raises(ValueError): chunker.chunk_text("x", max_tokens=10, overlap_tokens=10) with pytest.raises(ValueError): chunker.chunk_text("x", max_tokens=10, overlap_tokens=-1) # --- chunk-size limits (tokens stay bounded by max + overlap allowance) --- def test_long_splits_bounded(): text = " ".join(f"Sentence number {i}." for i in range(200)) out = chunker.chunk_text(text, max_tokens=64, overlap_tokens=8) assert len(out) > 1 for c in out: assert chunker._estimate_tokens(c) <= 64 + 8 + 2 def test_long_sentence_hard_split_bounded(): out = chunker.chunk_text("a" * 5000, max_tokens=32, overlap_tokens=4) assert len(out) >= 2 for c in out: assert len(c) <= 32 * 4 + 4 * 4 + 5 def test_hard_split_no_empty_chunks(): out = chunker.chunk_text("z" * 200, max_tokens=8, overlap_tokens=2) assert all(c.strip() for c in out) assert len(out) >= 2 # --- round-trip: the property the original bug broke --- def test_roundtrip_varied_sentences(): base = ("The quick brown fox jumps over the lazy dog. Pack my box with five " "dozen liquor jugs. How vexingly quick daft zebras jump. Variant " "sentence here. ") text = (base + "Another distinct sentence. ") * 20 cs = chunker.chunk_text(text, max_tokens=64, overlap_tokens=8) assert chunker.reconstruct(cs, 8) == text.strip() def test_roundtrip_hard_split_single_char(): text = "a" * 5000 cs = chunker.chunk_text(text, max_tokens=32, overlap_tokens=4) assert chunker.reconstruct(cs, 4) == text def test_roundtrip_empty_and_single(): assert chunker.reconstruct([], 4) == "" cs = chunker.chunk_text("Hello world. This is a test.", max_tokens=100, overlap_tokens=4) assert chunker.reconstruct(cs, 4) == "Hello world. This is a test." def test_roundtrip_zero_overlap(): text = "One. Two. Three. Four. Five. Six. Seven. Eight. " * 15 cs = chunker.chunk_text(text, max_tokens=16, overlap_tokens=0) assert chunker.reconstruct(cs, 0) == text.strip() def test_roundtrip_repetitive_sentences_exact(): """Repeated identical sentence groups -- the adversarial case.""" text = ("Sentence one. Sentence two. Sentence three. ") * 40 cs = chunker.chunk_text(text, max_tokens=24, overlap_tokens=4) assert chunker.reconstruct(cs, 4) == text.strip() # --- reconstruct standalone contract --- def test_reconstruct_passthrough(): r = chunker.reconstruct(["alpha beta", "gamma delta"], 0) assert "alpha beta" in r and "gamma delta" in r def test_reconstruct_empty(): assert chunker.reconstruct([], 32) == "" # --- fuzz: lossless round-trip over many random inputs --- def test_fuzz_roundtrip_fifty_hundred(): rng = random.Random(2026) words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot"] fails = 0 for _ in range(500): n = rng.randint(3, 60) text = " ".join(rng.choice(words) for _ in range(n)) # sprinkle sentence punctuation sometimes if rng.random() < 0.5: text = text + ". " + " ".join(rng.choice(words) for _ in range(rng.randint(2, 12))) + "." mt = rng.choice([12, 16, 32, 64]) ot = rng.choice([0, 2, 4, 8]) if ot >= mt: ot = max(0, mt - 1) cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot) r = chunker.reconstruct(cs, ot) if r != text.strip(): fails += 1 assert fails == 0, f"{fails}/500 fuzz trials failed lossless round-trip" def test_fuzz_hard_split_only(): """Pure repeated-token hard-split path under many sizes.""" rng = random.Random(7) fails = 0 for _ in range(500): text = ("x" * rng.randint(50, 2000)) mt = rng.choice([8, 16, 32, 40]) ot = rng.choice([2, 4, 8]) if ot >= mt: ot = max(1, mt - 1) cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot) r = chunker.reconstruct(cs, ot) if r != text: fails += 1 assert fails == 0, f"{fails}/500 hard-split fuzz trials failed"