128 lines
4.5 KiB
Python
128 lines
4.5 KiB
Python
"""Regression + round-trip tests for the refactored chunker (forge lane build).
|
|
|
|
These mirror the upstream suite in wizard-outputs/null but pin the corrected
|
|
lossless-round-trip contract that the original ``reconstruct`` violated on
|
|
repetitive / sentence-grouped input.
|
|
"""
|
|
import random
|
|
|
|
import chunker
|
|
from chunker import _OVERLAP_SENTINEL
|
|
|
|
|
|
# --- argument validation (unchanged contract) ---
|
|
def test_invalid_args():
|
|
import pytest
|
|
with pytest.raises(ValueError):
|
|
chunker.chunk_text("x", max_tokens=0)
|
|
with pytest.raises(ValueError):
|
|
chunker.chunk_text("x", max_tokens=10, overlap_tokens=10)
|
|
with pytest.raises(ValueError):
|
|
chunker.chunk_text("x", max_tokens=10, overlap_tokens=-1)
|
|
|
|
|
|
# --- chunk-size limits (tokens stay bounded by max + overlap allowance) ---
|
|
def test_long_splits_bounded():
|
|
text = " ".join(f"Sentence number {i}." for i in range(200))
|
|
out = chunker.chunk_text(text, max_tokens=64, overlap_tokens=8)
|
|
assert len(out) > 1
|
|
for c in out:
|
|
assert chunker._estimate_tokens(c) <= 64 + 8 + 2
|
|
|
|
|
|
def test_long_sentence_hard_split_bounded():
|
|
out = chunker.chunk_text("a" * 5000, max_tokens=32, overlap_tokens=4)
|
|
assert len(out) >= 2
|
|
for c in out:
|
|
assert len(c) <= 32 * 4 + 4 * 4 + 5
|
|
|
|
|
|
def test_hard_split_no_empty_chunks():
|
|
out = chunker.chunk_text("z" * 200, max_tokens=8, overlap_tokens=2)
|
|
assert all(c.strip() for c in out)
|
|
assert len(out) >= 2
|
|
|
|
|
|
# --- round-trip: the property the original bug broke ---
|
|
def test_roundtrip_varied_sentences():
|
|
base = ("The quick brown fox jumps over the lazy dog. Pack my box with five "
|
|
"dozen liquor jugs. How vexingly quick daft zebras jump. Variant "
|
|
"sentence here. ")
|
|
text = (base + "Another distinct sentence. ") * 20
|
|
cs = chunker.chunk_text(text, max_tokens=64, overlap_tokens=8)
|
|
assert chunker.reconstruct(cs, 8) == text.strip()
|
|
|
|
|
|
def test_roundtrip_hard_split_single_char():
|
|
text = "a" * 5000
|
|
cs = chunker.chunk_text(text, max_tokens=32, overlap_tokens=4)
|
|
assert chunker.reconstruct(cs, 4) == text
|
|
|
|
|
|
def test_roundtrip_empty_and_single():
|
|
assert chunker.reconstruct([], 4) == ""
|
|
cs = chunker.chunk_text("Hello world. This is a test.", max_tokens=100, overlap_tokens=4)
|
|
assert chunker.reconstruct(cs, 4) == "Hello world. This is a test."
|
|
|
|
|
|
def test_roundtrip_zero_overlap():
|
|
text = "One. Two. Three. Four. Five. Six. Seven. Eight. " * 15
|
|
cs = chunker.chunk_text(text, max_tokens=16, overlap_tokens=0)
|
|
assert chunker.reconstruct(cs, 0) == text.strip()
|
|
|
|
|
|
def test_roundtrip_repetitive_sentences_exact():
|
|
"""Repeated identical sentence groups -- the adversarial case."""
|
|
text = ("Sentence one. Sentence two. Sentence three. ") * 40
|
|
cs = chunker.chunk_text(text, max_tokens=24, overlap_tokens=4)
|
|
assert chunker.reconstruct(cs, 4) == text.strip()
|
|
|
|
|
|
# --- reconstruct standalone contract ---
|
|
def test_reconstruct_passthrough():
|
|
r = chunker.reconstruct(["alpha beta", "gamma delta"], 0)
|
|
assert "alpha beta" in r and "gamma delta" in r
|
|
|
|
|
|
def test_reconstruct_empty():
|
|
assert chunker.reconstruct([], 32) == ""
|
|
|
|
|
|
# --- fuzz: lossless round-trip over many random inputs ---
|
|
def test_fuzz_roundtrip_fifty_hundred():
|
|
rng = random.Random(2026)
|
|
words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot"]
|
|
fails = 0
|
|
for _ in range(500):
|
|
n = rng.randint(3, 60)
|
|
text = " ".join(rng.choice(words) for _ in range(n))
|
|
# sprinkle sentence punctuation sometimes
|
|
if rng.random() < 0.5:
|
|
text = text + ". " + " ".join(rng.choice(words) for _ in range(rng.randint(2, 12))) + "."
|
|
mt = rng.choice([12, 16, 32, 64])
|
|
ot = rng.choice([0, 2, 4, 8])
|
|
if ot >= mt:
|
|
ot = max(0, mt - 1)
|
|
cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot)
|
|
r = chunker.reconstruct(cs, ot)
|
|
if r != text.strip():
|
|
fails += 1
|
|
assert fails == 0, f"{fails}/500 fuzz trials failed lossless round-trip"
|
|
|
|
|
|
def test_fuzz_hard_split_only():
|
|
"""Pure repeated-token hard-split path under many sizes."""
|
|
rng = random.Random(7)
|
|
fails = 0
|
|
for _ in range(500):
|
|
text = ("x" * rng.randint(50, 2000))
|
|
mt = rng.choice([8, 16, 32, 40])
|
|
ot = rng.choice([2, 4, 8])
|
|
if ot >= mt:
|
|
ot = max(1, mt - 1)
|
|
cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot)
|
|
r = chunker.reconstruct(cs, ot)
|
|
if r != text:
|
|
fails += 1
|
|
assert fails == 0, f"{fails}/500 hard-split fuzz trials failed"
|