65 lines
2.5 KiB
Python
65 lines
2.5 KiB
Python
"""Adversarial fuzz probe for the forge chunker (evidence-gathering, not a fixture)."""
|
|
import sys
|
|
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
|
import chunker
|
|
|
|
|
|
def roundtrip(text, mt, ot):
|
|
cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot)
|
|
r = chunker.reconstruct(cs, ot)
|
|
exp = text.strip()
|
|
return r == exp, r, exp, cs
|
|
|
|
|
|
cases = []
|
|
for mt in range(2, 20):
|
|
for ot in range(1, mt):
|
|
cases.append(((("Sentence one. Sentence two. Sentence three. ") * 8), mt, ot, f"tiny-mt{mt}-ot{ot}"))
|
|
for mt in range(2, 16):
|
|
for ot in range(1, mt):
|
|
cases.append((("a" * 500), mt, ot, f"hard-mt{mt}-ot{ot}"))
|
|
for mt in range(2, 14):
|
|
for ot in range(0, mt):
|
|
t = ("The quick brown fox jumps over the lazy dog. Pack my box. ") * 5 + "a" * 200
|
|
cases.append((t, mt, ot, f"mixed-mt{mt}-ot{ot}"))
|
|
t_nosp = "Hello.World!How are you?I am fine. " * 10
|
|
for mt in range(4, 40, 4):
|
|
cases.append((t_nosp, mt, (mt // 2 if mt // 2 < mt else 1), f"no-after-space-mt{mt}"))
|
|
t_ws = "Hello world.\tHi now. \n\nNew para. Tab\there. " * 8
|
|
for mt in [8, 16, 32, 64]:
|
|
cases.append((t_ws, mt, mt // 2, f"ws-mt{mt}"))
|
|
cases.append((" \n\t ", 32, 8, "whitespace-only"))
|
|
cases.append(("", 32, 8, "empty-string"))
|
|
cases.append(("Just one.", 4, 1, "tiny-single"))
|
|
cases.append(("Hello. ", 8, 4, "trailing-ws"))
|
|
|
|
fails = 0
|
|
total = 0
|
|
bounds_fail = 0
|
|
for text, mt, ot, label in cases:
|
|
cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot)
|
|
r = chunker.reconstruct(cs, ot)
|
|
exp = text.strip()
|
|
total += 1
|
|
if r != exp:
|
|
fails += 1
|
|
print(f"FAIL {label} mt={mt} ot={ot} len={len(text)} exp_len={len(exp)} got_len={len(r)}")
|
|
for i in range(min(len(r), len(exp))):
|
|
if r[i] != exp[i]:
|
|
print(f" firstdiff@{i} exp={exp[max(0,i-15):i+15]!r} got={r[max(0,i-15):i+15]!r}")
|
|
break
|
|
else:
|
|
print(f" prefix ok, len diff. exp_tail={exp[-20:]!r} got_tail={r[-20:]!r}")
|
|
print(f" nchunks={len(cs)}")
|
|
# bounds check
|
|
for c in cs:
|
|
body = c.partition(chunker._OVERLAP_SENTINEL)[2] if chunker._OVERLAP_SENTINEL in c else c
|
|
est = chunker.estimate_tokens(body)
|
|
if est > mt and body.strip():
|
|
bounds_fail += 1
|
|
print(f"BOUNDS-FAIL {label} mt={mt} chunk est={est} > {mt}: {body[:50]!r}")
|
|
|
|
print(f"\ntotal={total} roundtrip_fails={fails} bounds_failures={bounds_fail}")
|
|
print("RESULT:", "ALL PASS" if fails == 0 and bounds_fail == 0 else "HAS FAILURES")
|
|
sys.exit(0 if fails == 0 and bounds_fail == 0 else 1)
|