"""Reproduce the chunk-token-overflow bug."""
import sys
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
import chunker

cases = [
    ("Pack my box. ", 3, 0),
    ("Pack my box. ", 3, 1),
    ("Pack my box. ", 3, 2),
    ("The quick brown fox jumps over the lazy dog. ", 11, 0),
    ("The quick brown fox jumps over the lazy dog. ", 11, 5),
]
any_overflow = False
for txt, mt, ot in cases:
    cs = chunker.chunk_text(txt, max_tokens=mt, overlap_tokens=ot)
    for i, c in enumerate(cs):
        body = c.partition(chunker._OVERLAP_SENTINEL)[2] if chunker._OVERLAP_SENTINEL in c else c
        est = chunker.estimate_tokens(body)
        if est > mt and body.strip():
            any_overflow = True
            print(f"OVERFLOW mt={mt} ot={ot} chunk[{i}] body={body!r} est={est} > {mt}")
if not any_overflow:
    print("No overflow found (unexpected).")
sys.exit(1 if any_overflow else 0)