forge: probe_forge refactor verification harness — 12/12 checks (add _find_first_diff correctness, dynamic equal, overlap consistency)

wizard-work/forge/verify_probe_forge_refactor.py

@@ -18,6 +18,17 @@ chunker API with the same inputs). Checks:
   8.  Round-trip assertion holds: probe output reports ``equal: True``.
   9.  Non-vacuity: the *original* ``_probe_forge.py`` still exhibits the removed
       smells (no __main__ guard, has ``shorter = min(``, module-level scan/for loop).
+ 10. ``_find_first_diff`` returns correct indices on inputs with known differences
+      (kills ffd_* mutants that neutralize its return value or loop range --
+      mutations invisible to output-comparison because the real round-trip is
+      always lossless so the return-i path is never exercised).
+ 11. ``equal`` output is computed dynamically from ``actual == expected``
+      (static check; kills the hardcode_equal mutant that hides a broken
+      round-trip behind a lie).
+ 12. ``reconstruct`` is called with the same overlap value as ``chunk_text``
+      (AST structural check; kills reconstruct_overlap_zero, which a behavioral
+      test cannot detect because ``reconstruct`` delegates head-stripping to
+      ``chunk_body`` and ignores overlap_tokens).
 """
 from __future__ import annotations
 
@@ -70,6 +81,17 @@ def _code_only(content: str) -> str:
     return "\n".join(kept)
 
 
+def _ensure_path() -> None:
+    if WORK_DIR not in sys.path:
+        sys.path.insert(0, WORK_DIR)
+
+
+def _import_refactored():
+    sys.modules.pop(MOD_NAME, None)
+    _ensure_path()
+    return importlib.import_module(MOD_NAME)
+
+
 def check_py_compile() -> bool:
     r = _run("python3", "-m", "py_compile", REFACTORED)
     if r.returncode != 0:
@@ -139,20 +161,7 @@ def check_helpers_extracted() -> bool:
             return False
     main_func = next(n for n in tree.body
                      if isinstance(n, ast.FunctionDef) and n.name == "main")
-    main_calls = {
+    # Attribute-style calls on `chunker.` (chunk_text, reconstruct).
-        n.func.id for n in ast.walk(main_func)
-        if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
-    }
-    for called in ("_find_first_diff", "_print_chunk_structure",
-                   "chunker.chunk_text", "chunker.reconstruct", "print"):
-        if called not in main_calls and called not in {
-            # chunk_text/reconstruct are called via the chunker attribute
-            *(a.attr for n in ast.walk(main_func)
-              if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
-              for a in [n.func])}:
-            # check attribute-style calls too
-            pass
-    # Verify main() calls the helpers and chunker API (attribute-style).
     attr_calls = {
         n.func.attr for n in ast.walk(main_func)
         if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
@@ -163,37 +172,32 @@ def check_helpers_extracted() -> bool:
         if needed not in attr_calls:
             print(f"FAIL: main() does not call chunker.{needed}()")
             return False
-    if "_find_first_diff" not in main_calls and "_find_first_diff" not in attr_calls:
+    # Name-style calls (_find_first_diff, _print_chunk_structure).
-        # _find_first_diff may be called as a Name
+    name_calls = {
-        name_calls = {
+        n.func.id for n in ast.walk(main_func)
-            n.func.id for n in ast.walk(main_func)
+        if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
-            if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
+    }
-        }
+    for needed in ("_find_first_diff", "_print_chunk_structure"):
-        if "_find_first_diff" not in name_calls:
+        if needed not in name_calls:
-            print("FAIL: main() does not call _find_first_diff()")
+            print(f"FAIL: main() does not call {needed}()")
             return False
-    if "_print_chunk_structure" not in main_calls and "_print_chunk_structure" not in attr_calls:
+    print("OK: _find_first_diff, _print_chunk_structure, main extracted; "
-        name_calls = {
+          "main calls helpers + chunker API")
-            n.func.id for n in ast.walk(main_func)
-            if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
-        }
-        if "_print_chunk_structure" not in name_calls:
-            print("FAIL: main() does not call _print_chunk_structure()")
-            return False
-    print("OK: _find_first_diff, _print_chunk_structure, main extracted; main calls helpers + chunker API")
     return True
 
 
 def check_import_side_effect_free() -> bool:
     """Importing the refactored module must not run the probe or print."""
-    sys.modules.pop(MOD_NAME, None)
+    mod = _import_refactored()
-    if WORK_DIR not in sys.path:
-        sys.path.insert(0, WORK_DIR)
     out, err = io.StringIO(), io.StringIO()
+    # Re-import cleanly to capture any side effects.
+    sys.modules.pop(MOD_NAME, None)
+    _ensure_path()
     with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
         importlib.import_module(MOD_NAME)
     if out.getvalue() or err.getvalue():
-        print(f"FAIL: import produced output: stdout={out.getvalue()!r} stderr={err.getvalue()!r}")
+        print(f"FAIL: import produced output: "
+              f"stdout={out.getvalue()!r} stderr={err.getvalue()!r}")
         return False
     print("OK: import is side-effect-free (no probe output at import)")
     return True
@@ -204,7 +208,8 @@ def check_roundtrip_lossless() -> bool:
     r = _run("python3", REFACTORED)
     combined = r.stdout + r.stderr
     if "equal: True" not in combined:
-        print("FAIL: probe output does not report 'equal: True'; round-trip not lossless")
+        print("FAIL: probe output does not report 'equal: True'; "
+              "round-trip not lossless")
         return False
     print("OK: round-trip lossless (equal: True in probe output)")
     return True
@@ -215,13 +220,10 @@ def check_original_still_smells() -> bool:
     src = _read(ORIGINAL)
     code = _code_only(src)
     problems = []
-    # The original must LACK a __main__ guard (that is the smell we fixed).
     if '__name__ == "__main__"' in src:
         problems.append("original already has __main__ guard (refactor not needed)")
-    # The dead 'shorter' variable must still be present in the original.
     if "shorter =" not in code and "shorter=" not in code:
         problems.append("dead 'shorter' variable missing from original (expected)")
-    # The original must execute the probe at import time: module-level for-loop.
     tree = ast.parse(src)
     has_module_level_for = any(isinstance(n, ast.For) for n in tree.body)
     if not has_module_level_for:
@@ -235,6 +237,111 @@ def check_original_still_smells() -> bool:
     return True
 
 
+def check_find_first_diff_correctness() -> bool:
+    """Independently verify _find_first_diff on inputs with known differences.
+
+    Kills mutation-test survivors that neutralize the return value or loop range
+    of _find_first_diff (return None, return i+1, delete return, skip index 0,
+    always return None).  These mutations are invisible to output-comparison
+    because the real round-trip is always lossless -- no diff is ever found, so
+    the return-i path is never exercised by check_identical_output.  By calling
+    _find_first_diff directly with crafted strings that DO differ, we force the
+    return path and catch the mutants.
+    """
+    mod = _import_refactored()
+
+    cases = [
+        # (actual, expected, expected_result)
+        ("xbcdef", "abcdef", 0),        # differ at index 0
+        ("abcdxf", "abcdef", 4),        # differ at middle index
+        ("abcdeZ", "abcdef", 5),        # differ at last common index
+        ("abcdef", "abcdef", None),     # identical
+        ("abc", "abcdef", None),        # actual is a prefix of expected
+        ("abcdef", "abc", None),        # expected is a prefix of actual
+        ("", "", None),                 # both empty
+        ("a", "b", 0),                  # single char, differ immediately
+        ("a", "", None),                # expected empty
+        ("", "a", None),                # actual empty
+    ]
+    for actual, expected, want in cases:
+        try:
+            got = mod._find_first_diff(actual, expected)
+        except Exception as exc:
+            print(f"FAIL: _find_first_diff({actual!r}, {expected!r}) "
+                  f"raised {type(exc).__name__}: {exc}")
+            return False
+        if got != want:
+            print(f"FAIL: _find_first_diff({actual!r}, {expected!r}) "
+                  f"returned {got!r}, expected {want!r}")
+            return False
+    print(f"OK: _find_first_diff correct on all {len(cases)} cases")
+    return True
+
+
+def check_no_hardcoded_equal() -> bool:
+    """The 'equal' output must be computed from actual == expected, not hard-coded.
+
+    Kills the hardcode_equal mutant (``print("equal:", True)``) that hides a
+    broken round-trip behind a lie -- the verifier's check_roundtrip_lossless
+    only inspects the probe's *printed* output, so a hardcoded 'True' fools it
+    even if the chunker were broken.  This static check ensures the real
+    comparison expression survives in the refactored source.
+    """
+    code = _code_only(_read(REFACTORED))
+    if "actual == expected" not in code:
+        print("FAIL: 'actual == expected' comparison not found in code -- "
+              "equality is likely hard-coded")
+        return False
+    print("OK: 'equal' is computed dynamically from actual == expected")
+    return True
+
+
+def check_overlap_consistency() -> bool:
+    """chunk_text and reconstruct must agree on the overlap_tokens value.
+
+    Kills the reconstruct_overlap_zero mutant (``reconstruct(cs, 0)`` instead
+    of ``reconstruct(cs, 8)``).  Because reconstruct delegates head-stripping
+    to chunk_body and ignores overlap_tokens, a behavioral test cannot detect
+    the mismatch -- so we verify it structurally: the overlap value passed to
+    reconstruct (positional arg 2 or overlap_tokens kwarg) must equal the
+    overlap_tokens kwarg passed to chunk_text.
+    """
+    tree = ast.parse(_read(REFACTORED))
+
+    chunk_overlap = None
+    reconstruct_overlap = None
+
+    for node in ast.walk(tree):
+        if not (isinstance(node, ast.Call)
+                and isinstance(node.func, ast.Attribute)):
+            continue
+        if node.func.attr == "chunk_text":
+            for kw in node.keywords:
+                if kw.arg == "overlap_tokens":
+                    chunk_overlap = ast.literal_eval(kw.value)
+        elif node.func.attr == "reconstruct":
+            if len(node.args) >= 2:
+                reconstruct_overlap = ast.literal_eval(node.args[1])
+            for kw in node.keywords:
+                if kw.arg == "overlap_tokens":
+                    reconstruct_overlap = ast.literal_eval(kw.value)
+
+    if chunk_overlap is None:
+        print("FAIL: no chunk_text call with overlap_tokens found")
+        return False
+    if reconstruct_overlap is None:
+        print("FAIL: no reconstruct call with an overlap argument found")
+        return False
+    if chunk_overlap != reconstruct_overlap:
+        print(f"FAIL: overlap mismatch: "
+              f"chunk_text(overlap_tokens={chunk_overlap}) vs "
+              f"reconstruct(overlap={reconstruct_overlap})")
+        return False
+    print(f"OK: overlap values consistent "
+          f"(chunk_text={chunk_overlap}, reconstruct={reconstruct_overlap})")
+    return True
+
+
 def main() -> None:
     checks = [
         check_py_compile,
@@ -246,6 +353,9 @@ def main() -> None:
         check_import_side_effect_free,
         check_roundtrip_lossless,
         check_original_still_smells,
+        check_find_first_diff_correctness,
+        check_no_hardcoded_equal,
+        check_overlap_consistency,
     ]
     results = [fn() for fn in checks]
     passed = sum(results)