25 lines
900 B
Python
25 lines
900 B
Python
"""Reproduce the chunk-token-overflow bug."""
|
|
import sys
|
|
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
|
import chunker
|
|
|
|
cases = [
|
|
("Pack my box. ", 3, 0),
|
|
("Pack my box. ", 3, 1),
|
|
("Pack my box. ", 3, 2),
|
|
("The quick brown fox jumps over the lazy dog. ", 11, 0),
|
|
("The quick brown fox jumps over the lazy dog. ", 11, 5),
|
|
]
|
|
any_overflow = False
|
|
for txt, mt, ot in cases:
|
|
cs = chunker.chunk_text(txt, max_tokens=mt, overlap_tokens=ot)
|
|
for i, c in enumerate(cs):
|
|
body = c.partition(chunker._OVERLAP_SENTINEL)[2] if chunker._OVERLAP_SENTINEL in c else c
|
|
est = chunker.estimate_tokens(body)
|
|
if est > mt and body.strip():
|
|
any_overflow = True
|
|
print(f"OVERFLOW mt={mt} ot={ot} chunk[{i}] body={body!r} est={est} > {mt}")
|
|
if not any_overflow:
|
|
print("No overflow found (unexpected).")
|
|
sys.exit(1 if any_overflow else 0)
|