From 840ef98fe70bf399ad70cc4acfbc2c807e99bd3b Mon Sep 17 00:00:00 2001 From: forge Date: Sat, 24 Oct 2026 13:39:55 +0000 Subject: [PATCH] forge: fuzz probe refactor verification - correct line count/filename citations, add negctrl test, commit evidence files --- .../forge/FUZZ_PROBE_REFACTOR_VERIFICATION.md | 36 ++- wizard-work/forge/_fuzz_adversarial.py | 151 +++++++++ wizard-work/forge/_fuzz_probe.py | 64 ++++ wizard-work/forge/_fuzz_probe_negctrl.py | 53 ++++ .../forge/_fuzz_probe_negctrl_output.txt | 3 + wizard-work/forge/_test_roundtrip.py | 117 +++++++ wizard-work/forge/baseline_fuzz_output.txt | 3 + wizard-work/forge/chunker.py | 288 ++++++++++++++++++ wizard-work/forge/refactored_fuzz_output.txt | 3 + ...rify_path_claim_apply_baseline_refactor.py | 246 +++++++++++++++ 10 files changed, 955 insertions(+), 9 deletions(-) create mode 100644 wizard-work/forge/_fuzz_adversarial.py create mode 100644 wizard-work/forge/_fuzz_probe.py create mode 100644 wizard-work/forge/_fuzz_probe_negctrl.py create mode 100644 wizard-work/forge/_fuzz_probe_negctrl_output.txt create mode 100644 wizard-work/forge/_test_roundtrip.py create mode 100644 wizard-work/forge/baseline_fuzz_output.txt create mode 100644 wizard-work/forge/chunker.py create mode 100644 wizard-work/forge/refactored_fuzz_output.txt create mode 100644 wizard-work/forge/verify_path_claim_apply_baseline_refactor.py diff --git a/wizard-work/forge/FUZZ_PROBE_REFACTOR_VERIFICATION.md b/wizard-work/forge/FUZZ_PROBE_REFACTOR_VERIFICATION.md index 2baf514..3bdf879 100644 --- a/wizard-work/forge/FUZZ_PROBE_REFACTOR_VERIFICATION.md +++ b/wizard-work/forge/FUZZ_PROBE_REFACTOR_VERIFICATION.md @@ -24,7 +24,6 @@ in the main loop with no abstraction. No `if __name__ == "__main__"` guard. - Added `if __name__ == "__main__"` guard. ## Evidence (measured) - Both runs executed against the live chunker at `/root/.hermes/wizard-outputs/forge/chunker.py`. @@ -34,32 +33,50 @@ total=383 roundtrip_fails=0 bounds_failures=0 RESULT: ALL PASS EXIT=0 -$ python3 _fuzz_probe_refactored.py (refactored, 139 lines) +$ python3 _fuzz_probe_refactored.py (refactored, 155 lines) total=383 roundtrip_fails=0 bounds_failures=0 RESULT: ALL PASS EXIT=0 ``` -Byte-for-byte output equivalence: +Byte-for-byte output equivalence (both captured into committed artifacts): ``` $ diff baseline_fuzz_output.txt refactored_fuzz_output.txt -OUTPUT IDENTICAL +$ # no output — files are identical ``` py_compile: ``` $ python3 -m py_compile _fuzz_probe_refactored.py -PY_COMPILE_OK +$ # no output — compiles clean ``` -Cross-check — all other chunker probes still pass with the same chunker: +## Negative control (probe is discriminating) +A green-only probe grades nothing. `_fuzz_probe_negctrl.py` monkey-patches +`chunker.chunk_text` with a truncating mutant — for all-`a` inputs only, the +last character of the final chunk is dropped — then runs the refactored probe +against the mutant. The probe **must** detect the injection: + +``` +$ python3 _fuzz_probe_negctrl.py +total=383 roundtrip_fails=105 bounds_failures=0 +RESULT: HAS FAILURES +``` + +105 of 383 cases fail — every "hard-mt{mt}-ot{ot}" case whose text is `"a"*500`, +which is 1+2+...+14 = 105 (triangular sum over mt=2..15). The original probe +catches the same 105 against the same mutant, so the refactor preserves +detection case-for-case, not just the totals. The negctrl exits 0 = probe +caught the bug; 1 = probe missed a real truncation (control failed). + +## Cross-check — all other chunker probes still pass with the same chunker: ``` $ python3 _edge_sweep_refactored.py total=600 roundtrip_fails=0 bounds_fails=0 RESULT: ALL PASS EXIT=0 -$ python3 _test_roundtrip_refactored.py +$ python3 _test_roundtrip.py === Results: 34 passed, 0 failed === EXIT=0 @@ -73,7 +90,7 @@ EXIT=0 | metric | original | refactored | |-------------------|----------|------------| -| lines | 64 | 139 (incl. docstrings) | +| lines | 64 | 155 (incl. docstrings) | | function defs | 1 (dead) | 4 (all used) | | dead code | 1 unused fn | 0 | | total cases | 383 | 383 | @@ -86,4 +103,5 @@ Refactoring is behavior-preserving and measurably equivalent: 383 cases, identical output, exit 0. The dead `roundtrip()` helper is now wired in, the bounds check is a pure generator, module-global counters are a Stats dataclass, and all 2400 adversarial trials across the wider probe suite -still pass with no regressions. +still pass with no regressions. The negative control proves the probe is +discriminating — it catches a truncating-chunker mutant on 105/383 cases. \ No newline at end of file diff --git a/wizard-work/forge/_fuzz_adversarial.py b/wizard-work/forge/_fuzz_adversarial.py new file mode 100644 index 0000000..5882417 --- /dev/null +++ b/wizard-work/forge/_fuzz_adversarial.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Adversarial property test for the refactored chunker. + +Targeted at the one invariant the docstring makes a strong claim about: +``reconstruct(chunk_text(t, m, o), o) == t.strip()`` for EVERY input, +including: + + * repetitive sentences whose carried-overlap tail can coincidentally + equal a prefix of the next chunk's content (the genuine-vs-context + discriminator in ``reconstruct``); + * long repetitive-token sentences that hard-split, immediately followed + by a sentence-grouped chunk whose content begins with the same run + the overlap tail carries (genuine-overlap branch mis-taken as context); + * arbitrary inter-sentence whitespace (double space, tab, newline, + mixed runs) that the original bug collapsed. + +If this exits 0, the lossless claim is backed by >= N randomised trials +across all three adversarial shapes. If it exits 1, it prints the first +counterexample with a full structural dump so the bug is reproducible. +""" +from __future__ import annotations + +import random +import sys + +sys.path.insert(0, "/root/.hermes/wizard-outputs/forge") +import chunker # noqa: E402 + +S = chunker._OVERLAP_SENTINEL + + +def shape_long_rep_then_sentence(seed: int) -> str: + """Hard-split long repetitive sentence immediately followed by a + sentence chunk whose content begins with the *same* token run as the + long sentence's tail -- forces the genuine-overlap discriminator to be + exercised on a context-overlap site.""" + rng = random.Random(seed) + tok = rng.choice(["word", "x", "abc", "node"]) + # long sentence of repeated tokens (no real EOS until the end) + long_sent = " ".join([tok] * rng.randint(40, 220)) + "." + # next sentence repeats the tail token so the overlap tail (last oc chars + # of the long sentence) coincides with the next chunk's prefix. + next_sent = (tok + " ") * rng.randint(2, 8) + "tail." + # vary inter-sentence whitespace + sep = rng.choice([" ", " ", "\t", "\n", "\n\n", " \n ", " \t "]) + return long_sent + sep + next_sent + (sep + "More " + tok + " words.\n\n") * 2 + + +def shape_repetitive_sentences(seed: int) -> str: + """Repeated sentence groups (the docstring's primary adversarial class).""" + rng = random.Random(seed) + groups = [] + for _ in range(rng.randint(10, 60)): + g = rng.choice( + [ + "Sentence one. ", + "Sentence two. ", + "Sentence three. ", + "Alpha beta. ", + "Gamma delta. ", + ] + ) + # occasionally insert weird interior whitespace + if rng.random() < 0.3: + g = g.replace(" ", rng.choice([" ", "\t", " \n "])) + groups.append(g) + return "".join(groups) + + +def shape_whitespace_variety(seed: int) -> str: + """Lots of short sentences joined by weird whitespace runs.""" + rng = random.Random(seed) + sents = ["Hello world", "How are you", "I am fine", "Test sentence"] + seps = [" ", " ", " ", "\t", "\n", "\n\n", " \n ", " \t "] + parts = [] + for _ in range(rng.randint(5, 40)): + parts.append(rng.choice(sents)) + parts.append(rng.choice(seps)) + # trailing whitespace to exercise strip boundary + parts.append(rng.choice(seps)) + return "".join(parts) + + +def shape_fuzz(seed: int) -> str: + rng = random.Random(seed) + words = ["alpha", "bravo", "charlie", "delta", "echo", "foxtrot", + "golf", "hotel", "india", "juliet"] + seps = [" ", " ", " ", "\t", "\n", "\n\n"] + parts = [] + n = rng.randint(1, 8) + for _ in range(n): + nw = rng.randint(2, 12) + parts.append(" ".join(rng.choice(words) for _ in range(nw))) + parts.append(rng.choice([".", ".", ".", "?", "!", "..."])) + parts.append(rng.choice(seps)) + return "".join(parts).strip() + rng.choice(["", "", " ", " \n "]) + + +def check(text: str, mt: int, ot: int) -> bool: + if not text.strip(): + return True + expect = text.strip() + cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot) + rec = chunker.reconstruct(cs, ot) + return rec == expect + + +def main() -> int: + shapes = [shape_long_rep_then_sentence, shape_repetitive_sentences, + shape_whitespace_variety, shape_fuzz] + mts = [8, 10, 12, 16, 20, 24, 32, 40, 48, 64, 72, 100, 128] + ots = [0, 1, 2, 3, 4, 6, 8, 12, 16] + trials_per_shape = 600 + total = 0 + fails = 0 + for si, shape in enumerate(shapes): + for i in range(trials_per_shape): + seed = si * 100000 + i + text = shape(seed) + mt = random.Random(seed).choice(mts) + ot = random.Random(seed + 1).choice(ots) + if ot >= mt: + ot = max(0, mt - 1) + total += 1 + if not check(text, mt, ot): + fails += 1 + print(f"FAIL shape={shape.__name__} seed={seed} mt={mt} ot={ot}") + print(f" text(:120)={text[:120]!r}... len={len(text)}") + cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot) + rec = chunker.reconstruct(cs, ot) + exp = text.strip() + for j in range(min(len(rec), len(exp))): + if rec[j] != exp[j]: + print(f" FIRST DIFF at {j}") + print(f" exp: {exp[max(0,j-25):j+25]!r}") + print(f" rec: {rec[max(0,j-25):j+25]!r}") + break + else: + print(f" len exp={len(exp)} rec={len(rec)} " + f"(common prefix OK, lengths differ)") + if fails >= 3: + print("Stopping after 3 failures.") + return 1 + print(f"trials={total} fails={fails}") + print("PASS: lossless round-trip across all adversarial shapes." + if fails == 0 else "FAIL: invariant violated.") + return 0 if fails == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wizard-work/forge/_fuzz_probe.py b/wizard-work/forge/_fuzz_probe.py new file mode 100644 index 0000000..9a4f1a9 --- /dev/null +++ b/wizard-work/forge/_fuzz_probe.py @@ -0,0 +1,64 @@ +"""Adversarial fuzz probe for the forge chunker (evidence-gathering, not a fixture).""" +import sys +sys.path.insert(0, "/root/.hermes/wizard-outputs/forge") +import chunker + + +def roundtrip(text, mt, ot): + cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot) + r = chunker.reconstruct(cs, ot) + exp = text.strip() + return r == exp, r, exp, cs + + +cases = [] +for mt in range(2, 20): + for ot in range(1, mt): + cases.append(((("Sentence one. Sentence two. Sentence three. ") * 8), mt, ot, f"tiny-mt{mt}-ot{ot}")) +for mt in range(2, 16): + for ot in range(1, mt): + cases.append((("a" * 500), mt, ot, f"hard-mt{mt}-ot{ot}")) +for mt in range(2, 14): + for ot in range(0, mt): + t = ("The quick brown fox jumps over the lazy dog. Pack my box. ") * 5 + "a" * 200 + cases.append((t, mt, ot, f"mixed-mt{mt}-ot{ot}")) +t_nosp = "Hello.World!How are you?I am fine. " * 10 +for mt in range(4, 40, 4): + cases.append((t_nosp, mt, (mt // 2 if mt // 2 < mt else 1), f"no-after-space-mt{mt}")) +t_ws = "Hello world.\tHi now. \n\nNew para. Tab\there. " * 8 +for mt in [8, 16, 32, 64]: + cases.append((t_ws, mt, mt // 2, f"ws-mt{mt}")) +cases.append((" \n\t ", 32, 8, "whitespace-only")) +cases.append(("", 32, 8, "empty-string")) +cases.append(("Just one.", 4, 1, "tiny-single")) +cases.append(("Hello. ", 8, 4, "trailing-ws")) + +fails = 0 +total = 0 +bounds_fail = 0 +for text, mt, ot, label in cases: + cs = chunker.chunk_text(text, max_tokens=mt, overlap_tokens=ot) + r = chunker.reconstruct(cs, ot) + exp = text.strip() + total += 1 + if r != exp: + fails += 1 + print(f"FAIL {label} mt={mt} ot={ot} len={len(text)} exp_len={len(exp)} got_len={len(r)}") + for i in range(min(len(r), len(exp))): + if r[i] != exp[i]: + print(f" firstdiff@{i} exp={exp[max(0,i-15):i+15]!r} got={r[max(0,i-15):i+15]!r}") + break + else: + print(f" prefix ok, len diff. exp_tail={exp[-20:]!r} got_tail={r[-20:]!r}") + print(f" nchunks={len(cs)}") + # bounds check + for c in cs: + body = c.partition(chunker._OVERLAP_SENTINEL)[2] if chunker._OVERLAP_SENTINEL in c else c + est = chunker.estimate_tokens(body) + if est > mt and body.strip(): + bounds_fail += 1 + print(f"BOUNDS-FAIL {label} mt={mt} chunk est={est} > {mt}: {body[:50]!r}") + +print(f"\ntotal={total} roundtrip_fails={fails} bounds_failures={bounds_fail}") +print("RESULT:", "ALL PASS" if fails == 0 and bounds_fail == 0 else "HAS FAILURES") +sys.exit(0 if fails == 0 and bounds_fail == 0 else 1) diff --git a/wizard-work/forge/_fuzz_probe_negctrl.py b/wizard-work/forge/_fuzz_probe_negctrl.py new file mode 100644 index 0000000..4d5f843 --- /dev/null +++ b/wizard-work/forge/_fuzz_probe_negctrl.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Negative control for ``_fuzz_probe_refactored.py``. + +Proves the probe is discriminating, not just green-on-green. + +We monkey-patch ``chunker.chunk_text`` with a truncating mutant: for all-``a`` +inputs only, the last character of the final chunk is dropped. Then we run the +refactored probe's ``main()`` against the mutant. The probe **must** detect +the injection and report failures -- a green-only probe grades nothing. + +Measured (this run): + total=383 roundtrip_fails=105 bounds_failures=0 + RESULT: HAS FAILURES + EXIT=1 + +105 of 383 cases are the "hard-mt{mt}-ot{ot}" family (``"a" * 500``), exactly +the cases whose text is all ``a``. The original probe catches the same 105 +against the same mutant, so the refactor preserves detection case-for-case. + +Exit contract: 0 = the probe caught the mutant (control passed); +1 = the probe went green on a buggy chunker (control failed -- vacuous probe). +""" +from __future__ import annotations + +import sys + +sys.path.insert(0, "/root/.hermes/wizard-outputs/forge") +import chunker + +_orig_chunk_text = chunker.chunk_text + + +def _truncating_chunk_text(text: str, max_tokens: int, overlap_tokens: int): + """Mutant ``chunk_text``: drop the last character of the final chunk + when the (stripped) input is composed entirely of ``'a'``.""" + cs = _orig_chunk_text(text, max_tokens, overlap_tokens) + body = text.strip() + if body and set(body) == {"a"}: + cs = list(cs) + cs[-1] = cs[-1][:-1] + return cs + + +chunker.chunk_text = _truncating_chunk_text + +import _fuzz_probe_refactored as probe # noqa: E402 + + +if __name__ == "__main__": + rc = probe.main() + # Invert: probe returning non-zero means it caught the bug (good); + # probe returning 0 means it missed a real truncation (bad). + sys.exit(1 if rc == 0 else 0) diff --git a/wizard-work/forge/_fuzz_probe_negctrl_output.txt b/wizard-work/forge/_fuzz_probe_negctrl_output.txt new file mode 100644 index 0000000..2025430 --- /dev/null +++ b/wizard-work/forge/_fuzz_probe_negctrl_output.txt @@ -0,0 +1,3 @@ + +total=383 roundtrip_fails=105 bounds_failures=0 +RESULT: HAS FAILURES diff --git a/wizard-work/forge/_test_roundtrip.py b/wizard-work/forge/_test_roundtrip.py new file mode 100644 index 0000000..e600931 --- /dev/null +++ b/wizard-work/forge/_test_roundtrip.py @@ -0,0 +1,117 @@ +"""Comprehensive round-trip test for the forge (refactored) chunker. + +Covers: + 1. Repetitive sentence-grouped text (the probe case) + 2. Long single sentence -> triggers _hard_split_long + 3. Mixed: some sentences fit, some need hard split + 4. Edge cases: empty, whitespace-only, single chunk, no overlap + 5. Many random-ish inputs with varying params +""" +import sys +import random +import string +sys.path.insert(0, "/root/.hermes/wizard-outputs/forge") +import chunker + +PASS = 0 +FAIL = 0 + +def check(name, text, max_tokens=256, overlap_tokens=32): + global PASS, FAIL + if not text.strip(): + expected = "" + else: + expected = text.strip() + cs = chunker.chunk_text(text, max_tokens=max_tokens, overlap_tokens=overlap_tokens) + r = chunker.reconstruct(cs, overlap_tokens) + ok = r == expected + status = "PASS" if ok else "FAIL" + if ok: + PASS += 1 + else: + FAIL += 1 + print(f"[{status}] {name}") + print(f" orig_len={len(expected)} recon_len={len(r)}") + for i in range(min(len(r), len(expected))): + if r[i] != expected[i]: + print(f" FIRST DIFF at char {i}") + print(f" orig: {expected[max(0,i-30):i+30]!r}") + print(f" recon: {r[max(0,i-30):i+30]!r}") + break + else: + print(f" common prefix OK but lengths differ: {len(r)} vs {len(expected)}") + print(f" orig tail: {expected[-30:]!r}") + print(f" recon tail: {r[-30:]!r}") + print(f" nchunks={len(cs)}") + # show if any chunk exceeded max_tokens + for i, c in enumerate(cs): + # strip the overlap tail for token check + actual = c + if chunker._OVERLAP_SENTINEL in actual: + actual = actual.partition(chunker._OVERLAP_SENTINEL)[2] + est = chunker.estimate_tokens(actual) + if est > max_tokens: + print(f" chunk[{i}] exceeds max_tokens: est={est} > {max_tokens}, content={actual[:60]!r}") + return ok + + +# Case 1: repetitive sentence-grouped text +base = "The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs. How vexingly quick daft zebras jump. " +text = (base + "Variant sentence here. ") * 20 +check("repetitive-sentences", text, max_tokens=64, overlap_tokens=8) + +# Case 2: long single sentence -> hard split +long_sent = "Word" + " word" * 200 + " end." # ~1000 chars, way over 80 +check("long-sentence-hard-split", long_sent, max_tokens=20, overlap_tokens=4) + +# Case 2b: long single sentence with sentence-ending punctuation +long_sent2 = " ".join(["Sentence"] + ["word"]*300 + ["."]) +check("long-sentence-hard-split-v2", long_sent2, max_tokens=30, overlap_tokens=6) + +# Case 3: mixed - some short, some long sentences +mixed = "Short one. " + long_sent + " Another short. " + long_sent2 + " Final." +check("mixed-sizes", mixed, max_tokens=20, overlap_tokens=4) + +# Case 4: edge cases +check("empty-string", "", max_tokens=64, overlap_tokens=8) +check("whitespace-only", " \n\t ", max_tokens=64, overlap_tokens=8) +check("single-sentence", "Just one sentence here.", max_tokens=64, overlap_tokens=8) +check("two-sentences-no-overlap", "First sentence. Second sentence.", max_tokens=256, overlap_tokens=0) +check("small-overlap", "First sentence. Second sentence.", max_tokens=64, overlap_tokens=2) + +# Case 5: multiple sentences exactly at boundary +exactly = " ".join(f"Sentence number {i}." for i in range(10)) +check("boundary-fit", exactly, max_tokens=10, overlap_tokens=2) + +# Case 6: text with special chars, multiple spaces +special = "Hello world. How are you? I am fine. " * 5 +check("multi-space", special, max_tokens=32, overlap_tokens=4) + +# Case 7: text starting/ending with whitespace +ws_text = " The quick brown fox. The lazy dog. " +check("leading-trailing-ws", ws_text, max_tokens=64, overlap_tokens=8) + +# Case 8: single long sentence that needs hard split, with high overlap +long3 = "A" + " b" * 500 + " end." +check("long-with-high-overlap", long3, max_tokens=16, overlap_tokens=8) + +# Case 9: random text fuzz +random.seed(42) +words = ["alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta"] +for trial in range(20): + sents = [] + for _ in range(random.randint(3, 15)): + nwords = random.randint(3, 50) + s = " ".join(random.choice(words) for _ in range(nwords)) + sents.append(s + ".") + text = " ".join(sents) + check(f"random-trial-{trial}", text, max_tokens=random.randint(16, 64), + overlap_tokens=random.randint(0, 8)) + +# Case 10: text with newlines +nl_text = "First paragraph.\n\nSecond paragraph here.\nThird line." +check("newlines", nl_text, max_tokens=64, overlap_tokens=8) + +print(f"\n=== Results: {PASS} passed, {FAIL} failed ===") +if FAIL: + sys.exit(1) diff --git a/wizard-work/forge/baseline_fuzz_output.txt b/wizard-work/forge/baseline_fuzz_output.txt new file mode 100644 index 0000000..d149cb1 --- /dev/null +++ b/wizard-work/forge/baseline_fuzz_output.txt @@ -0,0 +1,3 @@ + +total=383 roundtrip_fails=0 bounds_failures=0 +RESULT: ALL PASS diff --git a/wizard-work/forge/chunker.py b/wizard-work/forge/chunker.py new file mode 100644 index 0000000..16b41a2 --- /dev/null +++ b/wizard-work/forge/chunker.py @@ -0,0 +1,288 @@ +"""chunker.py -- split text into token-estimate-bounded chunks with overlap (stdlib only). + +The overlap carried between chunks is delimited by a unique sentinel so that +``reconstruct`` can invert ``chunk_text`` exactly (lossless round-trip) without +relying on suffix/prefix matching, which is ambiguous on repetitive input. + +Architecture (this build): + + Every overlap is *context-only* -- a verbatim copy of the previous chunk's + tail, prepended behind a sentinel and dropped verbatim by ``reconstruct``. + There is no "genuine-overlap" case (where the next chunk physically contains + the overlap as a real prefix), because ``_hard_split_long`` emits *adjacent, + non-overlapping* pieces. Collapsing the two overlap kinds into one means + ``reconstruct`` needs no content-based discriminator at all: it always drops + the carried ``head`` and appends ``rest``, leaving the previous chunk intact. + That is unambiguous even when the text is highly repetitive and the overlap + ratio is degenerate (e.g. ``overlap_tokens == max_tokens - 1``), where a + ``rest.startswith(head)`` test would conflate genuine and context overlap and + silently strip real content off the accumulator. + +The previous refactor fixed *sentinel discrimination* (genuine vs context) and +removed a ``re.sub(" +", " ")`` collapse, but the round-trip was still lossy +on real whitespace: ``_split_sentences`` discarded the inter-sentence +separators, ``_split_into_chunks`` re-joined sentences with a bare ``" "``, and +``_apply_overlap`` / ``_hard_split_long`` ``.strip()``-ed the carried tails -- so +double spaces, tabs and newlines were silently collapsed to single spaces. + +Fix (this build): + + * ``_split_sentences`` now returns ``(sentence, trailing_ws)`` pairs, so the + concatenation of all pairs reproduces the source verbatim. Whitespace is + part of the data, not an afterthought. + * ``_split_into_chunks`` builds each chunk as an *exact substring* of the + source (the concatenation of its segments); no ``" "`` re-joining and no + ``.strip()`` on accumulated content. A hard-split sentence's trailing + whitespace is included in the text passed to ``_hard_split_long`` (not + appended afterward), so it can never push the final piece past max_chars. + * ``_hard_split_long`` now produces **non-overlapping** pieces (adjacent + exact substrings of the source) instead of pieces that physically overlap + the previous piece. This removes the *genuine-overlap* case entirely: + every overlap is now *context-only*. + * ``_hard_split_long`` guards the token bound: a pure-whitespace fragment + stranded by the word-boundary backtrack is merged into the previous piece + only when the result stays within ``max_chars``; otherwise it is kept as + its own piece (harmless: ``reconstruct`` strips trailing whitespace, and + mid-text whitespace survives in the join). The previous code merged + unconditionally, which could overflow the last content piece past max_tokens. + * ``_apply_overlap`` prepends the previous chunk's tail as pure *context* + (sentinel-delimited) to every chunk after the first. Because hard-split + pieces no longer overlap each other, this context tail is never duplicated + inside the chunk's own content -- it is a copy, not a real prefix. + * ``reconstruct`` therefore needs **no discriminator**: it drops the carried + ``head`` and appends ``rest`` verbatim, leaving the previous chunk intact. + This is unambiguous even on highly-repetitive text, where a + ``rest.startswith(head)`` test would conflate genuine and context overlap + (the prior bug: each misfire stripped chars off the accumulator). No + synthetic ``" "`` is inserted; boundary whitespace lives in the previous + chunk's tail and is preserved. + +Result: ``reconstruct(chunk_text(t, ...), ot) == t.strip()`` for every input, +including highly-repetitive text, degenerate high overlap ratios +(e.g. ``overlap_tokens == max_tokens - 1``) and arbitrary inter-sentence +whitespace (double spaces, tabs, newlines, leading/trailing runs). Each +chunk's estimated token count also satisfies ``estimate_tokens(body) <= max_tokens`` +for non-whitespace content (``body`` = chunk with the carried-overlap head +stripped), which the previous build violated for sentences whose trailing +whitespace was appended post-hoc to a max_chars-saturated piece (e.g. +"Pack my box. " at max_tokens=3). +""" +from __future__ import annotations + +import math +import re + +__all__ = ["chunk_text", "reconstruct", "estimate_tokens"] + +# rough heuristic: ~4 chars per token for English-ish text +CHARS_PER_TOKEN = 4 + +# Sentinel used to mark the carried-overlap region between adjacent chunks. +# It is unlikely to occur in natural text; if it does, it is escaped on +# chunking and unescaped on reconstruction so the round-trip stays lossless. +_OVERLAP_SENTINEL = "\x00OVL\x00" + +# Matches the whitespace run that follows an end-of-sentence punctuation mark. +# The run is captured so separators survive splitting (lossless reconstruction). +_SENT_RE = re.compile(r"(?<=[.!?])(\s+)") + + +def estimate_tokens(s: str) -> int: + """Estimate the token count of ``s`` using a char-per-token heuristic.""" + if not s: + return 0 + return max(1, math.ceil(len(s) / CHARS_PER_TOKEN)) + + +# Backward-compatible alias (used by test_chunker_refactor.py). +_estimate_tokens = estimate_tokens + + +def _split_sentences(text: str) -> list[tuple[str, str]]: + """Split *text* into ``(sentence, trailing_ws)`` pairs. + + The concatenation of every ``sentence + trailing_ws`` reproduces *text* + verbatim (after the outer ``.strip()`` applied to *text* by ``chunk_text``), + so the round-trip never loses inter-sentence whitespace -- double spaces, + tabs, or newlines survive intact. + """ + if not text: + return [] + parts = _SENT_RE.split(text) + # re.split with a capture group interleaves separators: sentence, sep, + # sentence, sep, ... Group them into (sentence, trailing_ws) pairs. + pairs: list[tuple[str, str]] = [] + for i in range(0, len(parts), 2): + sent = parts[i] + sep = parts[i + 1] if i + 1 < len(parts) else "" + if sent or sep: # skip a degenerate all-empty pair at the tail + pairs.append((sent, sep)) + return pairs + + +def _hard_split_long(text: str, max_chars: int) -> list[str]: + """Word-aware hard split of a single over-long sentence into adjacent pieces. + + Pieces are *exact*, **non-overlapping** substrings of *text* (no + ``.strip()``), each no longer than *max_chars* (rounded down to the last + word boundary). They are adjacent -- ``"".join(pieces) == text`` -- so no + genuine-overlap discrimination is needed in ``reconstruct``. Any cross-piece + context is instead supplied uniformly by :func:`_apply_overlap` as + sentinel-delimited *context* overlap, which ``reconstruct`` drops + unconditionally. This is what makes the round-trip lossless even under + highly-repetitive text and high overlap ratios, where a suffix/prefix + discriminator is ambiguous. + """ + pieces: list[str] = [] + n = len(text) + start = 0 + while start < n: + end = min(start + max_chars, n) + if end < n: + boundary = text.rfind(" ", start, end) + if boundary > start: + end = boundary + piece = text[start:end] + if piece.strip(): + pieces.append(piece) + elif pieces and len(pieces[-1]) + len(piece) <= max_chars: + # pure-whitespace fragment (e.g. one space of a double-space run that + # the word-boundary split stranded between two content pieces): merge + # it into the previous piece so the concatenation stays an *exact* + # substring of the source and nothing is silently dropped -- but only + # when the merge stays within max_chars, otherwise the token bound + # would be violated; an un-mergeable whitespace fragment is kept as + # its own piece (reconstruct() .strip()s trailing whitespace, so it + # is harmless, and losslessness is preserved for mid-text whitespace). + pieces[-1] += piece + elif piece: + pieces.append(piece) + # if pieces is empty this is leading whitespace; reconstruct() contracts + # to text.strip(), so it is intentionally not retained here. + if end >= n: + break + start = end + return pieces + + +def _apply_overlap(chunks: list[str], overlap_chars: int) -> list[str]: + """Prepend a sentinel-delimited tail of the previous chunk to each chunk. + + The tail is pure downstream context (it is *not* duplicated inside the chunk's + own content); ``.strip()`` is deliberately NOT applied, because that would + discard the trailing whitespace that makes the round-trip lossless. + """ + if overlap_chars <= 0 or len(chunks) <= 1: + return chunks + out = [chunks[0]] + # Iterate over consecutive (prev, cur) pairs -- no index arithmetic needed; + # each tail is exactly the previous chunk's suffix, which `reconstruct` drops + # unconditionally as pure context. + for prev, cur in zip(chunks, chunks[1:]): + out.append(prev[-overlap_chars:] + _OVERLAP_SENTINEL + cur) + return out + + +def _split_into_chunks( + text: str, + max_tokens: int, + max_chars: int, + overlap_chars: int, +) -> list[str]: + """Core sentence-boundary chunking producing *base* chunks (no overlap). + + Each chunk is the concatenation of whole ``(sentence, sep)`` segments, hence + an exact substring of the source: internal whitespace is preserved, never + re-joined with a synthesized ``" "``. + """ + segments = _split_sentences(text) + chunks: list[str] = [] + cur = "" + for sent, sep in segments: + piece = sent + sep + candidate = cur + piece if cur else piece + if estimate_tokens(candidate) > max_tokens: + if cur: + chunks.append(cur) # cur already ends with its segment's sep + if estimate_tokens(piece) > max_tokens: + chunks.extend(_hard_split_long(piece, max_chars)) + cur = "" + else: + cur = piece + else: + cur = candidate + if cur: + chunks.append(cur) + return _apply_overlap(chunks, overlap_chars) + + +def chunk_text(text: str, max_tokens: int = 256, overlap_tokens: int = 32) -> list[str]: + """Split *text* into chunks each <= *max_tokens* estimated tokens. + + *overlap_tokens* worth of the previous chunk's tail is carried into the + next chunk (sentinel-delimited). Reconstruction via :func:`reconstruct` + reverses this exactly. + + Prefers sentence boundaries; falls back to a word-aware hard split for + sentences that exceed *max_tokens*. + """ + if max_tokens <= 0: + raise ValueError("max_tokens must be positive") + if overlap_tokens < 0 or overlap_tokens >= max_tokens: + raise ValueError("overlap_tokens must be in [0, max_tokens)") + if text is None or not text.strip(): + return [] + + max_chars = max_tokens * CHARS_PER_TOKEN + overlap_chars = overlap_tokens * CHARS_PER_TOKEN + return _split_into_chunks(text, max_tokens, max_chars, overlap_chars) + + +def reconstruct(chunks: list[str], overlap_tokens: int = 32) -> str: + """Invert :func:`chunk_text`: remove sentinel-delimited overlaps and join. + + Exactly reverses chunking for any input (lossless), including highly + repetitive text and arbitrary inter-sentence whitespace. + + Every overlap in this design is *context-only*: the carried ``head`` is a + verbatim copy of the previous chunk's tail, never duplicated inside + ``rest``. ``reconstruct`` therefore unconditionally drops ``head`` and + appends ``rest`` verbatim, leaving the previous chunk intact -- no suffix/ + prefix discriminator is required, which is what makes it unambiguous on + repetitive text where such matching would conflate genuine and context + overlap. + + A single uniform rule applies to *every* chunk (including the first): the + overlap head (everything before the sentinel) is pure context and is + dropped, and ``rest`` (everything after the sentinel) is the genuine + continuation that is appended verbatim. The first chunk never carries a + real overlap by construction, so the rule reduces to "append verbatim"; a + stray sentinel is still stripped defensively as a no-op head. Because an + empty ``chunks`` list simply yields ``"".join([]).strip() == ""``, the + previous explicit empty-guard and the separate ``i == 0`` branch are + redundant and have been folded into one loop. + """ + parts: list[str] = [] + for c in chunks: + if _OVERLAP_SENTINEL in c: + # The carried `head` is pure context (a copy of the previous chunk's + # tail), not a prefix duplicated inside `rest`. Drop it, keep the + # previous chunk intact, and append `rest` (the genuine + # continuation) verbatim. No synthetic separator is inserted: + # boundary whitespace already lives at the previous chunk's tail. + _, _, rest = c.partition(_OVERLAP_SENTINEL) + parts.append(rest) + else: + # No sentinel (e.g. overlap_tokens=0, or the first chunk): the chunk + # is a contiguous source substring; append verbatim. + parts.append(c) + return "".join(parts).strip() + + +if __name__ == "__main__": + sample = " ".join(f"Sentence number {i}." for i in range(50)) + cs = chunk_text(sample, max_tokens=32, overlap_tokens=4) + print("chunks:", len(cs)) + for c in cs: + print("-", c[:60], "...") + print("reconstructed len:", len(reconstruct(cs, 4))) \ No newline at end of file diff --git a/wizard-work/forge/refactored_fuzz_output.txt b/wizard-work/forge/refactored_fuzz_output.txt new file mode 100644 index 0000000..d149cb1 --- /dev/null +++ b/wizard-work/forge/refactored_fuzz_output.txt @@ -0,0 +1,3 @@ + +total=383 roundtrip_fails=0 bounds_failures=0 +RESULT: ALL PASS diff --git a/wizard-work/forge/verify_path_claim_apply_baseline_refactor.py b/wizard-work/forge/verify_path_claim_apply_baseline_refactor.py new file mode 100644 index 0000000..a193826 --- /dev/null +++ b/wizard-work/forge/verify_path_claim_apply_baseline_refactor.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""verify_path_claim_apply_baseline_refactor.py + +Equivalence proof for the behavior-preserving DRY refactor of `apply_baseline` +in `tools/path_claim_check.py` (forge lane: Refactor -- code review and refactors). + +THE SMELL + `apply_baseline(rows, ledger)` walked the SAME `missing` dict FOUR times to + build three independent results, on top of the mutation loop that already + walked it once to attach each row's baseline: + + matched = sum(1 for r in missing.values() if r["baseline"]["known"]) + new = sorted(p for p, r in missing.items() if r["baseline"]["new"]) + tolerated = collections.Counter( + r["baseline"]["class"] for r in missing.values() if r["baseline"]["known"]) + + Four full traversals of one dict (mutation + matched + new + tolerated) -- the + canonical "single-census-pass" smell. + +THE REFACTOR (working tree, uncommitted at HEAD) + A single FUSED pass replaces all four: the mutation that attached each row's + baseline is merged with the census that tallied matched/new/tolerated, so one + `for p, row in missing.items()` loop does both jobs at once: + + matched = 0 + new = [] + tolerated = collections.Counter() + for p, row in missing.items(): + if p in ledger: + cls, reason = ledger[p] + row["baseline"] = {"known": True, "class": cls, + "reason": reason, "new": False} + matched += 1 + tolerated[cls] += 1 + else: + row["baseline"] = {"known": False, "class": None, + "reason": None, "new": True} + new.append(p) + + `known` and `new` are exact complements (a row is either known/new=False or + unknown/new=True), so the tally is updated from the record just written, in the + same iteration -- no second pass over `missing` is needed. The report renders + `new` sorted and `tolerated` via sorted(.items()), so neither traversal order + nor the merge can move a byte of output. + +WHAT THIS VERIFIER PROVES (measured, not claimed) + It materializes the GENUINE pre-refactor source from `git show HEAD:tools/ + path_claim_check.py` -- HEAD already carries the portable DEFAULT_REPO commit, + so this commit is the ONLY delta: the apply_baseline change. It then runs both + versions on the identical repo state (--rev HEAD) and compares every observable + output byte-for-byte: + + [1] selftest: original (HEAD) passes 31/31, rc 0 + [2] selftest: refactored (working tree) passes 31/31, rc 0 + [3] selftest: stdout is byte-identical (byte-for-byte) + [4] live scan --rev HEAD --json: original runs rc 0 + [5] live scan --rev HEAD --json: refactored runs rc 0 + [6] live scan: the ENTIRE --json document is byte-identical, which includes the + `baseline` field -- the direct output of apply_baseline -- so matched, new, + tolerated, cleared, cleared_docrel and dead are all confirmed equal. + [7] non-vacuity: the census found real findings (baseline has 13 entries) + + [8] code-level: the original apply_baseline body contains 4 `missing.` census + scans (mutation loop + matched + new + tolerated); the refactored body + contains 1. The smell is gone, the fix is real. + +Exit 0 iff every check holds, else 1. +""" +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + +# Portable repo root: this file lives in /wizard-work/forge, so the repo +# root is three dirname()s up. Falls back to the hardcoded checkout for any +# one-shot run that isn't on the committed path. +_HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.dirname(os.path.dirname(os.path.dirname(_HERE))) +if not os.path.isdir(os.path.join(REPO, "tools", "path_claim_check.py")): + REPO = "/root/troupe-standup" +TOOL = "tools/path_claim_check.py" +SELTEST_SUMMARY = "selftest: 31 cells, 31 passed, 0 failed" + + +class Verdict(Exception): + pass + + +def git_show(sha: str) -> str: + proc = subprocess.run(["git", "-C", REPO, "show", sha], capture_output=True, + text=True, check=True, timeout=60) + return proc.stdout + + +def run(path: str, args: list[str], cwd: str = REPO, timeout: int = 300): + proc = subprocess.run([sys.executable, path, *args], capture_output=True, + text=True, cwd=cwd, timeout=timeout) + return proc.returncode, proc.stdout, proc.stderr + + +def run_json(path: str, out_path: str, repo: str = REPO, only: str = "missing", + timeout: int = 300) -> tuple[int, str, str]: + proc = subprocess.run([sys.executable, path, "--repo", repo, "--rev", "HEAD", + "--only", only, "--json", out_path], + capture_output=True, text=True, cwd=REPO, timeout=timeout) + return proc.returncode, proc.stdout, proc.stderr + + +def extract_apply_baseline(text: str) -> str: + """Return just the body lines of the apply_baseline function.""" + lines = text.splitlines() + start = None + for i, ln in enumerate(lines): + if ln.startswith("def apply_baseline("): + start = i + 1 + break + if start is None: + raise Verdict("could not locate apply_baseline in source") + body = [] + for ln in lines[start:]: + if ln.startswith("def ") or ln.startswith("@"): + break + body.append(ln) + return "\n".join(body) + + +def census_scans(body: str) -> int: + """Count real `missing.values()` / `missing.items()` census passes in CODE. + + Comment lines and the function docstring are excluded so the prose that + *describes* the smell is never itself counted as a scan -- only executable + traversals of the `missing` dict are. + """ + out = 0 + in_doc = False + for ln in body.splitlines(): + s = ln.strip() + if s.startswith('"""') or s.startswith("'''"): + # A one-line docstring opens AND closes on this line; an unclosed + # triple-quote toggles multi-line docstring state. + if s.count('"""') == 2 or s.count("'''") == 2: + continue + in_doc = not in_doc + continue + if in_doc or s.startswith("#"): + continue + out += len(re.findall(r"missing\.(values|items)\(\)", s)) + return out + + +def main() -> int: + cells: list[tuple[str, bool, str]] = [] + failed = 0 + + def check(label: str, ok: bool, detail: str = "") -> None: + nonlocal failed + cells.append((label, bool(ok), "")) + if not ok: + failed += 1 + print(" [%s] %s%s" % ("ok " if ok else "FAIL", label, + (" [%s]" % detail) if detail and not ok else "")) + + tmpdir = tempfile.mkdtemp(prefix="pc_ab_verify_", dir=os.environ.get("TMPDIR", "/tmp")) + # Materialize the genuine pre-refactor source (HEAD = portable DEFAULT_REPO + # + OLD 4-scan apply_baseline; the only thing the working tree changes). + orig_path = os.path.join(tmpdir, "path_claim_check.py") + with open(orig_path, "w", encoding="utf-8") as fh: + fh.write(git_show("HEAD:" + TOOL)) + ref_path = os.path.join(REPO, TOOL) + + try: + # ---- [1-3] selftest parity ------------------------------------------ + rc_o, out_o, err_o = run(orig_path, ["--selftest"]) + rc_r, out_r, err_r = run(ref_path, ["--selftest"]) + check("selftest: original 31/31 rc 0", + rc_o == 0 and SELTEST_SUMMARY in out_o, "rc=%s" % rc_o) + check("selftest: refactored 31/31 rc 0", + rc_r == 0 and SELTEST_SUMMARY in out_r, "rc=%s" % rc_r) + check("selftest: identical stdout (byte-for-byte)", + out_o == out_r, "len_o=%d len_r=%d" % (len(out_o), len(out_r))) + + # ---- [4-7] live scan equivalence ------------------------------------ + oj_path = os.path.join(tmpdir, "orig.json") + rj_path = os.path.join(tmpdir, "ref.json") + rc_o, _, _ = run_json(orig_path, oj_path) + rc_r, _, _ = run_json(ref_path, rj_path) + check("live scan: original ran rc 0", rc_o == 0, "rc=%s" % rc_o) + check("live scan: refactored ran rc 0", rc_r == 0, "rc=%s" % rc_r) + + if os.path.exists(oj_path) and os.path.exists(rj_path): + with open(oj_path) as fh: + jo = json.load(fh) + with open(rj_path) as fh: + jr = json.load(fh) + # [6] the ENTIRE JSON document is byte-identical -- this includes the + # `baseline` field that apply_baseline() produces. + check("live scan: full --json byte-identical (incl. baseline field)", + json.dumps(jo, sort_keys=True) == json.dumps(jr, sort_keys=True), + "orig baseline=%r ref baseline=%r" % ( + jo.get("baseline"), jr.get("baseline"))) + # [7] non-vacuity: real, populated baseline findings. + check("live scan: non-vacuous (baseline has findings)", + jo.get("baseline") is not None + and jo["baseline"]["entries"] > 0 + and len(jo["paths"]) > 0, + "entries=%r paths=%d" % ( + jo.get("baseline", {}).get("entries"), len(jo.get("paths", [])))) + else: + check("live scan: JSON files produced by both", False, + "orig exists=%s ref exists=%s" % (os.path.exists(oj_path), + os.path.exists(rj_path))) + + # ---- [8] code-level: the smell was real and is gone ------------------ + # Original is the genuine HEAD source; refactored is the working tree. + # Both are stripped of comments/docstrings by census_scans so the prose + # that explains the smell is not itself counted as a scan. + orig_code = git_show("HEAD:" + TOOL) + with open(ref_path, encoding="utf-8") as fh: + ref_code = fh.read() + scans_orig = census_scans(extract_apply_baseline(orig_code)) + scans_ref = census_scans(extract_apply_baseline(ref_code)) + check("code-level: original apply_baseline has 4 census scans", + scans_orig == 4, "got %d" % scans_orig) + check("code-level: refactored apply_baseline has 1 census scan", + scans_ref == 1, "got %d" % scans_ref) + check("code-level: the smell was really reduced (4 -> 1)", + scans_orig == 4 and scans_ref == 1) + except Verdict as e: + print("\nVERDICT: MISMATCH -- %s" % e) + failed += 1 + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + total = len(cells) + passed = total - failed + print("\nverify: %d checks, %d passed, %d failed" % (total, passed, failed)) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main())