troupe-standup/wizard-work/forge/_fuzz_adversarial.py

#!/usr/bin/env python3
"""Adversarial property test for the refactored chunker.

Targeted at the one invariant the docstring makes a strong claim about:
``reconstruct(chunk_text(t, m, o), o) == t.strip()`` for EVERY input,
including:

  * repetitive sentences whose carried-overlap tail can coincidentally
    equal a prefix of the next chunk's content (the genuine-vs-context
    discriminator in ``reconstruct``);
  * long repetitive-token sentences that hard-split, immediately followed
    by a sentence-grouped chunk whose content begins with the same run
    the overlap tail carries (genuine-overlap branch mis-taken as context);
  * arbitrary inter-sentence whitespace (double space, tab, newline,
    mixed runs) that the original bug collapsed.

If this exits 0, the lossless claim is backed by >= N randomised trials
across all three adversarial shapes. If it exits 1, it prints the first
counterexample with a full structural dump so the bug is reproducible.
"""
from __future__ import annotations

import random
import sys

sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
import chunker  # noqa: E402

S = chunker._OVERLAP_SENTINEL


def shape_long_rep_then_sentence(seed: int) -> str:
    """Hard-split long repetitive sentence immediately followed by a
    sentence chunk whose content begins with the *same* token run as the
    long sentence's tail -- forces the genuine-overlap discriminator to be
    exercised on a context-overlap site."""
    rng = random.Random(seed)
    tok = rng.choice(["word", "x", "abc", "node"])
    # long sentence of repeated tokens (no real EOS until the end)
    long_sent = " ".join([tok] * rng.randint(40, 220)) + "."
    # next sentence repeats the tail token so the overlap tail (last oc chars
    # of the long sentence) coincides with the next chunk's prefix.
    next_sent = (tok + " ") * rng.randint(2, 8) + "tail."
    # vary inter-sentence whitespace
    sep = rng.choice([" ", "  ", "\t", "\n", "\n\n", " \n ", "  \t "])
    return long_sent + sep + next_sent + (sep + "More " + tok + " words.\n\n") * 2


def shape_repetitive_sentences(seed: int) -> str:
    """Repeated sentence groups (the docstring's primary adversarial class)."""
    rng = random.Random(seed)
    groups = []
    for _ in range(rng.randint(10, 60)):
        g = rng.choice(
            [
                "Sentence one. ",
                "Sentence two. ",
                "Sentence three. ",
                "Alpha beta. ",
                "Gamma delta. ",
            ]
        )
        # occasionally insert weird interior whitespace
        if rng.random() < 0.3:
            g = g.replace(" ", rng.choice(["  ", "\t", " \n "]))
        groups.append(g)
    return "".join(groups)


def shape_whitespace_variety(seed: int) -> str:
    """Lots of short sentences joined by weird whitespace runs."""
    rng = random.Random(seed)
    sents = ["Hello world", "How are you", "I am fine", "Test sentence"]
    seps = [" ", "  ", "   ", "\t", "\n", "\n\n", " \n ", "  \t  "]
    parts = []
    for _ in range(rng.randint(5, 40)):
        parts.append(rng.choice(sents))
        parts.append(rng.choice(seps))
    # trailing whitespace to exercise strip boundary
    parts.append(rng.choice(seps))
    return "".join(parts)


def shape_fuzz(seed: int) -> str:
    rng = random.Random(seed)
    words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot",
             "golf", "hotel", "india", "juliet"]
    seps = [" ", "  ", "  ", "\t", "\n", "\n\n"]
    parts = []
    n = rng.randint(1, 8)
    for _ in range(n):
        nw = rng.randint(2, 12)
        parts.append(" ".join(rng.choice(words) for _ in range(nw)))
        parts.append(rng.choice([".", ".", ".", "?", "!", "..."]))
        parts.append(rng.choice(seps))
    return "".join(parts).strip() + rng.choice(["", "", "  ", " \n "])


def check(text: str, mt: int, ot: int) -> bool:
    if not text.strip():
        return True
    expect = text.strip()
    cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot)
    rec = chunker.reconstruct(cs, ot)
    return rec == expect


def main() -> int:
    shapes = [shape_long_rep_then_sentence, shape_repetitive_sentences,
              shape_whitespace_variety, shape_fuzz]
    mts = [8, 10, 12, 16, 20, 24, 32, 40, 48, 64, 72, 100, 128]
    ots = [0, 1, 2, 3, 4, 6, 8, 12, 16]
    trials_per_shape = 600
    total = 0
    fails = 0
    for si, shape in enumerate(shapes):
        for i in range(trials_per_shape):
            seed = si * 100000 + i
            text = shape(seed)
            mt = random.Random(seed).choice(mts)
            ot = random.Random(seed + 1).choice(ots)
            if ot >= mt:
                ot = max(0, mt - 1)
            total += 1
            if not check(text, mt, ot):
                fails += 1
                print(f"FAIL shape={shape.__name__} seed={seed} mt={mt} ot={ot}")
                print(f"  text(:120)={text[:120]!r}... len={len(text)}")
                cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot)
                rec = chunker.reconstruct(cs, ot)
                exp = text.strip()
                for j in range(min(len(rec), len(exp))):
                    if rec[j] != exp[j]:
                        print(f"  FIRST DIFF at {j}")
                        print(f"    exp: {exp[max(0,j-25):j+25]!r}")
                        print(f"    rec: {rec[max(0,j-25):j+25]!r}")
                        break
                else:
                    print(f"  len exp={len(exp)} rec={len(rec)} "
                          f"(common prefix OK, lengths differ)")
                if fails >= 3:
                    print("Stopping after 3 failures.")
                    return 1
    print(f"trials={total} fails={fails}")
    print("PASS: lossless round-trip across all adversarial shapes."
          if fails == 0 else "FAIL: invariant violated.")
    return 0 if fails == 0 else 1


if __name__ == "__main__":
    sys.exit(main())