46 lines
1.6 KiB
Python
46 lines
1.6 KiB
Python
"""Probe the forge (refactored) chunker round-trip."""
|
|
import sys
|
|
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
|
import chunker
|
|
|
|
base = "The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs. How vexingly quick daft zebras jump. "
|
|
text = (base + "Variant sentence here. ") * 20
|
|
text_stripped = text.strip()
|
|
|
|
cs = chunker.chunk_text(text, max_tokens=64, overlap_tokens=8)
|
|
r = chunker.reconstruct(cs, 8)
|
|
|
|
print("orig_len:", len(text_stripped))
|
|
print("recon_len:", len(r))
|
|
print("equal:", r == text_stripped)
|
|
print("nchunks:", len(cs))
|
|
print()
|
|
|
|
# Show first 5 chunks with sentinel revealed
|
|
S = chunker._OVERLAP_SENTINEL
|
|
for i, c in enumerate(cs[:5]):
|
|
has = S in c
|
|
print(f"chunk[{i}] len={len(c)} sentinel={'YES' if has else 'no'}")
|
|
if has:
|
|
head, _, rest = c.partition(S)
|
|
print(f" head(t)={len(head)}: {head[:20]!r}")
|
|
print(f" rest : {rest[:40]!r}")
|
|
print(f" rest.startswith(head): {rest.startswith(head)}")
|
|
else:
|
|
print(f" full : {c[:40]!r}")
|
|
|
|
# Find first diff
|
|
for i in range(min(len(r), len(text_stripped))):
|
|
if r[i] != text_stripped[i]:
|
|
print()
|
|
print("FIRST DIFF at char", i)
|
|
print(" orig:", repr(text_stripped[max(0, i-20):i+20]))
|
|
print(" recon:", repr(r[max(0, i-20):i+20]))
|
|
break
|
|
else:
|
|
shorter = min(len(r), len(text_stripped))
|
|
print("no char diff in common prefix")
|
|
if len(r) != len(text_stripped):
|
|
print("lengths differ:", len(r), len(text_stripped))
|
|
print("orig tail:", repr(text_stripped[-30:]))
|
|
print("recon tail:", repr(r[-30:])) |