From 496b3e006b40984718c60137b2188913fac51d12 Mon Sep 17 00:00:00 2001 From: forge Date: Mon, 26 Oct 2026 03:49:10 +0000 Subject: [PATCH] =?UTF-8?q?forge:=20probe=5Fforge=20refactor=20verificatio?= =?UTF-8?q?n=20harness=20=E2=80=94=2012/12=20checks=20(add=20=5Ffind=5Ffir?= =?UTF-8?q?st=5Fdiff=20correctness,=20dynamic=20equal,=20overlap=20consist?= =?UTF-8?q?ency)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../forge/verify_probe_forge_refactor.py | 188 ++++++++++++++---- 1 file changed, 149 insertions(+), 39 deletions(-) diff --git a/wizard-work/forge/verify_probe_forge_refactor.py b/wizard-work/forge/verify_probe_forge_refactor.py index 0b9040a..39b4e94 100644 --- a/wizard-work/forge/verify_probe_forge_refactor.py +++ b/wizard-work/forge/verify_probe_forge_refactor.py @@ -18,6 +18,17 @@ chunker API with the same inputs). Checks: 8. Round-trip assertion holds: probe output reports ``equal: True``. 9. Non-vacuity: the *original* ``_probe_forge.py`` still exhibits the removed smells (no __main__ guard, has ``shorter = min(``, module-level scan/for loop). + 10. ``_find_first_diff`` returns correct indices on inputs with known differences + (kills ffd_* mutants that neutralize its return value or loop range -- + mutations invisible to output-comparison because the real round-trip is + always lossless so the return-i path is never exercised). + 11. ``equal`` output is computed dynamically from ``actual == expected`` + (static check; kills the hardcode_equal mutant that hides a broken + round-trip behind a lie). + 12. ``reconstruct`` is called with the same overlap value as ``chunk_text`` + (AST structural check; kills reconstruct_overlap_zero, which a behavioral + test cannot detect because ``reconstruct`` delegates head-stripping to + ``chunk_body`` and ignores overlap_tokens). """ from __future__ import annotations @@ -70,6 +81,17 @@ def _code_only(content: str) -> str: return "\n".join(kept) +def _ensure_path() -> None: + if WORK_DIR not in sys.path: + sys.path.insert(0, WORK_DIR) + + +def _import_refactored(): + sys.modules.pop(MOD_NAME, None) + _ensure_path() + return importlib.import_module(MOD_NAME) + + def check_py_compile() -> bool: r = _run("python3", "-m", "py_compile", REFACTORED) if r.returncode != 0: @@ -139,20 +161,7 @@ def check_helpers_extracted() -> bool: return False main_func = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "main") - main_calls = { - n.func.id for n in ast.walk(main_func) - if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) - } - for called in ("_find_first_diff", "_print_chunk_structure", - "chunker.chunk_text", "chunker.reconstruct", "print"): - if called not in main_calls and called not in { - # chunk_text/reconstruct are called via the chunker attribute - *(a.attr for n in ast.walk(main_func) - if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) - for a in [n.func])}: - # check attribute-style calls too - pass - # Verify main() calls the helpers and chunker API (attribute-style). + # Attribute-style calls on `chunker.` (chunk_text, reconstruct). attr_calls = { n.func.attr for n in ast.walk(main_func) if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) @@ -163,37 +172,32 @@ def check_helpers_extracted() -> bool: if needed not in attr_calls: print(f"FAIL: main() does not call chunker.{needed}()") return False - if "_find_first_diff" not in main_calls and "_find_first_diff" not in attr_calls: - # _find_first_diff may be called as a Name - name_calls = { - n.func.id for n in ast.walk(main_func) - if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) - } - if "_find_first_diff" not in name_calls: - print("FAIL: main() does not call _find_first_diff()") + # Name-style calls (_find_first_diff, _print_chunk_structure). + name_calls = { + n.func.id for n in ast.walk(main_func) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) + } + for needed in ("_find_first_diff", "_print_chunk_structure"): + if needed not in name_calls: + print(f"FAIL: main() does not call {needed}()") return False - if "_print_chunk_structure" not in main_calls and "_print_chunk_structure" not in attr_calls: - name_calls = { - n.func.id for n in ast.walk(main_func) - if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) - } - if "_print_chunk_structure" not in name_calls: - print("FAIL: main() does not call _print_chunk_structure()") - return False - print("OK: _find_first_diff, _print_chunk_structure, main extracted; main calls helpers + chunker API") + print("OK: _find_first_diff, _print_chunk_structure, main extracted; " + "main calls helpers + chunker API") return True def check_import_side_effect_free() -> bool: """Importing the refactored module must not run the probe or print.""" - sys.modules.pop(MOD_NAME, None) - if WORK_DIR not in sys.path: - sys.path.insert(0, WORK_DIR) + mod = _import_refactored() out, err = io.StringIO(), io.StringIO() + # Re-import cleanly to capture any side effects. + sys.modules.pop(MOD_NAME, None) + _ensure_path() with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): importlib.import_module(MOD_NAME) if out.getvalue() or err.getvalue(): - print(f"FAIL: import produced output: stdout={out.getvalue()!r} stderr={err.getvalue()!r}") + print(f"FAIL: import produced output: " + f"stdout={out.getvalue()!r} stderr={err.getvalue()!r}") return False print("OK: import is side-effect-free (no probe output at import)") return True @@ -204,7 +208,8 @@ def check_roundtrip_lossless() -> bool: r = _run("python3", REFACTORED) combined = r.stdout + r.stderr if "equal: True" not in combined: - print("FAIL: probe output does not report 'equal: True'; round-trip not lossless") + print("FAIL: probe output does not report 'equal: True'; " + "round-trip not lossless") return False print("OK: round-trip lossless (equal: True in probe output)") return True @@ -215,13 +220,10 @@ def check_original_still_smells() -> bool: src = _read(ORIGINAL) code = _code_only(src) problems = [] - # The original must LACK a __main__ guard (that is the smell we fixed). if '__name__ == "__main__"' in src: problems.append("original already has __main__ guard (refactor not needed)") - # The dead 'shorter' variable must still be present in the original. if "shorter =" not in code and "shorter=" not in code: problems.append("dead 'shorter' variable missing from original (expected)") - # The original must execute the probe at import time: module-level for-loop. tree = ast.parse(src) has_module_level_for = any(isinstance(n, ast.For) for n in tree.body) if not has_module_level_for: @@ -235,6 +237,111 @@ def check_original_still_smells() -> bool: return True +def check_find_first_diff_correctness() -> bool: + """Independently verify _find_first_diff on inputs with known differences. + + Kills mutation-test survivors that neutralize the return value or loop range + of _find_first_diff (return None, return i+1, delete return, skip index 0, + always return None). These mutations are invisible to output-comparison + because the real round-trip is always lossless -- no diff is ever found, so + the return-i path is never exercised by check_identical_output. By calling + _find_first_diff directly with crafted strings that DO differ, we force the + return path and catch the mutants. + """ + mod = _import_refactored() + + cases = [ + # (actual, expected, expected_result) + ("xbcdef", "abcdef", 0), # differ at index 0 + ("abcdxf", "abcdef", 4), # differ at middle index + ("abcdeZ", "abcdef", 5), # differ at last common index + ("abcdef", "abcdef", None), # identical + ("abc", "abcdef", None), # actual is a prefix of expected + ("abcdef", "abc", None), # expected is a prefix of actual + ("", "", None), # both empty + ("a", "b", 0), # single char, differ immediately + ("a", "", None), # expected empty + ("", "a", None), # actual empty + ] + for actual, expected, want in cases: + try: + got = mod._find_first_diff(actual, expected) + except Exception as exc: + print(f"FAIL: _find_first_diff({actual!r}, {expected!r}) " + f"raised {type(exc).__name__}: {exc}") + return False + if got != want: + print(f"FAIL: _find_first_diff({actual!r}, {expected!r}) " + f"returned {got!r}, expected {want!r}") + return False + print(f"OK: _find_first_diff correct on all {len(cases)} cases") + return True + + +def check_no_hardcoded_equal() -> bool: + """The 'equal' output must be computed from actual == expected, not hard-coded. + + Kills the hardcode_equal mutant (``print("equal:", True)``) that hides a + broken round-trip behind a lie -- the verifier's check_roundtrip_lossless + only inspects the probe's *printed* output, so a hardcoded 'True' fools it + even if the chunker were broken. This static check ensures the real + comparison expression survives in the refactored source. + """ + code = _code_only(_read(REFACTORED)) + if "actual == expected" not in code: + print("FAIL: 'actual == expected' comparison not found in code -- " + "equality is likely hard-coded") + return False + print("OK: 'equal' is computed dynamically from actual == expected") + return True + + +def check_overlap_consistency() -> bool: + """chunk_text and reconstruct must agree on the overlap_tokens value. + + Kills the reconstruct_overlap_zero mutant (``reconstruct(cs, 0)`` instead + of ``reconstruct(cs, 8)``). Because reconstruct delegates head-stripping + to chunk_body and ignores overlap_tokens, a behavioral test cannot detect + the mismatch -- so we verify it structurally: the overlap value passed to + reconstruct (positional arg 2 or overlap_tokens kwarg) must equal the + overlap_tokens kwarg passed to chunk_text. + """ + tree = ast.parse(_read(REFACTORED)) + + chunk_overlap = None + reconstruct_overlap = None + + for node in ast.walk(tree): + if not (isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute)): + continue + if node.func.attr == "chunk_text": + for kw in node.keywords: + if kw.arg == "overlap_tokens": + chunk_overlap = ast.literal_eval(kw.value) + elif node.func.attr == "reconstruct": + if len(node.args) >= 2: + reconstruct_overlap = ast.literal_eval(node.args[1]) + for kw in node.keywords: + if kw.arg == "overlap_tokens": + reconstruct_overlap = ast.literal_eval(kw.value) + + if chunk_overlap is None: + print("FAIL: no chunk_text call with overlap_tokens found") + return False + if reconstruct_overlap is None: + print("FAIL: no reconstruct call with an overlap argument found") + return False + if chunk_overlap != reconstruct_overlap: + print(f"FAIL: overlap mismatch: " + f"chunk_text(overlap_tokens={chunk_overlap}) vs " + f"reconstruct(overlap={reconstruct_overlap})") + return False + print(f"OK: overlap values consistent " + f"(chunk_text={chunk_overlap}, reconstruct={reconstruct_overlap})") + return True + + def main() -> None: checks = [ check_py_compile, @@ -246,6 +353,9 @@ def main() -> None: check_import_side_effect_free, check_roundtrip_lossless, check_original_still_smells, + check_find_first_diff_correctness, + check_no_hardcoded_equal, + check_overlap_consistency, ] results = [fn() for fn in checks] passed = sum(results)