54 lines
1.8 KiB
Python
54 lines
1.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Negative control for ``_fuzz_probe_refactored.py``.
|
|
|
|
Proves the probe is discriminating, not just green-on-green.
|
|
|
|
We monkey-patch ``chunker.chunk_text`` with a truncating mutant: for all-``a``
|
|
inputs only, the last character of the final chunk is dropped. Then we run the
|
|
refactored probe's ``main()`` against the mutant. The probe **must** detect
|
|
the injection and report failures -- a green-only probe grades nothing.
|
|
|
|
Measured (this run):
|
|
total=383 roundtrip_fails=105 bounds_failures=0
|
|
RESULT: HAS FAILURES
|
|
EXIT=1
|
|
|
|
105 of 383 cases are the "hard-mt{mt}-ot{ot}" family (``"a" * 500``), exactly
|
|
the cases whose text is all ``a``. The original probe catches the same 105
|
|
against the same mutant, so the refactor preserves detection case-for-case.
|
|
|
|
Exit contract: 0 = the probe caught the mutant (control passed);
|
|
1 = the probe went green on a buggy chunker (control failed -- vacuous probe).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
|
|
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
|
import chunker
|
|
|
|
_orig_chunk_text = chunker.chunk_text
|
|
|
|
|
|
def _truncating_chunk_text(text: str, max_tokens: int, overlap_tokens: int):
|
|
"""Mutant ``chunk_text``: drop the last character of the final chunk
|
|
when the (stripped) input is composed entirely of ``'a'``."""
|
|
cs = _orig_chunk_text(text, max_tokens, overlap_tokens)
|
|
body = text.strip()
|
|
if body and set(body) == {"a"}:
|
|
cs = list(cs)
|
|
cs[-1] = cs[-1][:-1]
|
|
return cs
|
|
|
|
|
|
chunker.chunk_text = _truncating_chunk_text
|
|
|
|
import _fuzz_probe_refactored as probe # noqa: E402
|
|
|
|
|
|
if __name__ == "__main__":
|
|
rc = probe.main()
|
|
# Invert: probe returning non-zero means it caught the bug (good);
|
|
# probe returning 0 means it missed a real truncation (bad).
|
|
sys.exit(1 if rc == 0 else 0)
|