Cleanup time!

This commit is contained in:
teknium1
2026-02-20 23:23:32 -08:00
parent 9a19fe1f50
commit 70dd3a16dc
38 changed files with 150 additions and 351 deletions

View File

@@ -108,7 +108,7 @@ def _count_tokens_for_entry(entry: Dict) -> Tuple[Dict, int]:
if value:
try:
total += len(_TOKENIZER.encode(value))
except:
except Exception:
# Fallback to character estimate
total += len(value) // 4