17 lines
530 B
Python
17 lines
530 B
Python
"""Verify estimate_tokens is monotonic (needed for _split_into_chunks refactor correctness)."""
|
|
import sys
|
|
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
|
import chunker
|
|
|
|
ok = True
|
|
for la in range(0, 400):
|
|
for lb in range(0, 400):
|
|
if la <= lb:
|
|
if chunker.estimate_tokens("a" * la) > chunker.estimate_tokens("a" * lb):
|
|
ok = False
|
|
print(f"NOT MONOTONIC: len {la} > len {lb}")
|
|
break
|
|
if not ok:
|
|
break
|
|
print("estimate_tokens monotonic:", ok)
|