"""Probe the forge (refactored) chunker round-trip."""
import sys
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
import chunker

base = "The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs. How vexingly quick daft zebras jump. "
text = (base + "Variant sentence here. ") * 20
text_stripped = text.strip()

cs = chunker.chunk_text(text, max_tokens=64, overlap_tokens=8)
r = chunker.reconstruct(cs, 8)

print("orig_len:", len(text_stripped))
print("recon_len:", len(r))
print("equal:", r == text_stripped)
print("nchunks:", len(cs))
print()

# Show first 5 chunks with sentinel revealed
S = chunker._OVERLAP_SENTINEL
for i, c in enumerate(cs[:5]):
    has = S in c
    print(f"chunk[{i}] len={len(c)} sentinel={'YES' if has else 'no'}")
    if has:
        head, _, rest = c.partition(S)
        print(f"   head(t)={len(head)}: {head[:20]!r}")
        print(f"   rest   : {rest[:40]!r}")
        print(f"   rest.startswith(head): {rest.startswith(head)}")
    else:
        print(f"   full   : {c[:40]!r}")

# Find first diff
for i in range(min(len(r), len(text_stripped))):
    if r[i] != text_stripped[i]:
        print()
        print("FIRST DIFF at char", i)
        print("  orig:", repr(text_stripped[max(0, i-20):i+20]))
        print("  recon:", repr(r[max(0, i-20):i+20]))
        break
else:
    shorter = min(len(r), len(text_stripped))
    print("no char diff in common prefix")
    if len(r) != len(text_stripped):
        print("lengths differ:", len(r), len(text_stripped))
        print("orig tail:", repr(text_stripped[-30:]))
        print("recon tail:", repr(r[-30:]))