stackchain/troupe-standup
2
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

forge: probe_forge refactor verification harness — 12/12 checks (add _find_first_diff correctness, dynamic equal, overlap consistency)

Browse Source
This commit is contained in:
forge committed 2026-10-26 03:49:10 +00:00
1 parent 73b1bdde4f
commit 496b3e006b
1 file changed
+149 -39
Download Patch File Download Diff File Expand all files Collapse all files
wizard-work/forge/verify_probe_forge_refactor.py
+149 -39
View File
@@ -18,6 +18,17 @@ chunker API with the same inputs). Checks:
8. Round-trip assertion holds: probe output reports ``equal: True``.
9. Non-vacuity: the *original* ``_probe_forge.py`` still exhibits the removed
smells (no __main__ guard, has ``shorter = min(``, module-level scan/for loop).
10. ``_find_first_diff`` returns correct indices on inputs with known differences
(kills ffd_* mutants that neutralize its return value or loop range --
mutations invisible to output-comparison because the real round-trip is
always lossless so the return-i path is never exercised).
11. ``equal`` output is computed dynamically from ``actual == expected``
(static check; kills the hardcode_equal mutant that hides a broken
round-trip behind a lie).
12. ``reconstruct`` is called with the same overlap value as ``chunk_text``
(AST structural check; kills reconstruct_overlap_zero, which a behavioral
test cannot detect because ``reconstruct`` delegates head-stripping to
``chunk_body`` and ignores overlap_tokens).
"""
from __future__ import annotations
@@ -70,6 +81,17 @@ def _code_only(content: str) -> str:
return "\n".join(kept)
def _ensure_path() -> None:
if WORK_DIR not in sys.path:
sys.path.insert(0, WORK_DIR)
def _import_refactored():
sys.modules.pop(MOD_NAME, None)
_ensure_path()
return importlib.import_module(MOD_NAME)
def check_py_compile() -> bool:
r = _run("python3", "-m", "py_compile", REFACTORED)
if r.returncode != 0:
@@ -139,20 +161,7 @@ def check_helpers_extracted() -> bool:
return False
main_func = next(n for n in tree.body
if isinstance(n, ast.FunctionDef) and n.name == "main")
main_calls = {
n.func.id for n in ast.walk(main_func)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
}
for called in ("_find_first_diff", "_print_chunk_structure",
"chunker.chunk_text", "chunker.reconstruct", "print"):
if called not in main_calls and called not in {
# chunk_text/reconstruct are called via the chunker attribute
*(a.attr for n in ast.walk(main_func)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
for a in [n.func])}:
# check attribute-style calls too
pass
# Verify main() calls the helpers and chunker API (attribute-style).
# Attribute-style calls on `chunker.` (chunk_text, reconstruct).
attr_calls = {
n.func.attr for n in ast.walk(main_func)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute)
@@ -163,37 +172,32 @@ def check_helpers_extracted() -> bool:
if needed not in attr_calls:
print(f"FAIL: main() does not call chunker.{needed}()")
return False
if "_find_first_diff" not in main_calls and "_find_first_diff" not in attr_calls:
# _find_first_diff may be called as a Name
name_calls = {
n.func.id for n in ast.walk(main_func)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
}
if "_find_first_diff" not in name_calls:
print("FAIL: main() does not call _find_first_diff()")
# Name-style calls (_find_first_diff, _print_chunk_structure).
name_calls = {
n.func.id for n in ast.walk(main_func)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
}
for needed in ("_find_first_diff", "_print_chunk_structure"):
if needed not in name_calls:
print(f"FAIL: main() does not call {needed}()")
return False
if "_print_chunk_structure" not in main_calls and "_print_chunk_structure" not in attr_calls:
name_calls = {
n.func.id for n in ast.walk(main_func)
if isinstance(n, ast.Call) and isinstance(n.func, ast.Name)
}
if "_print_chunk_structure" not in name_calls:
print("FAIL: main() does not call _print_chunk_structure()")
return False
print("OK: _find_first_diff, _print_chunk_structure, main extracted; main calls helpers + chunker API")
print("OK: _find_first_diff, _print_chunk_structure, main extracted; "
"main calls helpers + chunker API")
return True
def check_import_side_effect_free() -> bool:
"""Importing the refactored module must not run the probe or print."""
sys.modules.pop(MOD_NAME, None)
if WORK_DIR not in sys.path:
sys.path.insert(0, WORK_DIR)
mod = _import_refactored()
out, err = io.StringIO(), io.StringIO()
# Re-import cleanly to capture any side effects.
sys.modules.pop(MOD_NAME, None)
_ensure_path()
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
importlib.import_module(MOD_NAME)
if out.getvalue() or err.getvalue():
print(f"FAIL: import produced output: stdout={out.getvalue()!r} stderr={err.getvalue()!r}")
print(f"FAIL: import produced output: "
f"stdout={out.getvalue()!r} stderr={err.getvalue()!r}")
return False
print("OK: import is side-effect-free (no probe output at import)")
return True
@@ -204,7 +208,8 @@ def check_roundtrip_lossless() -> bool:
r = _run("python3", REFACTORED)
combined = r.stdout + r.stderr
if "equal: True" not in combined:
print("FAIL: probe output does not report 'equal: True'; round-trip not lossless")
print("FAIL: probe output does not report 'equal: True'; "
"round-trip not lossless")
return False
print("OK: round-trip lossless (equal: True in probe output)")
return True
@@ -215,13 +220,10 @@ def check_original_still_smells() -> bool:
src = _read(ORIGINAL)
code = _code_only(src)
problems = []
# The original must LACK a __main__ guard (that is the smell we fixed).
if '__name__ == "__main__"' in src:
problems.append("original already has __main__ guard (refactor not needed)")
# The dead 'shorter' variable must still be present in the original.
if "shorter =" not in code and "shorter=" not in code:
problems.append("dead 'shorter' variable missing from original (expected)")
# The original must execute the probe at import time: module-level for-loop.
tree = ast.parse(src)
has_module_level_for = any(isinstance(n, ast.For) for n in tree.body)
if not has_module_level_for:
@@ -235,6 +237,111 @@ def check_original_still_smells() -> bool:
return True
def check_find_first_diff_correctness() -> bool:
"""Independently verify _find_first_diff on inputs with known differences.
Kills mutation-test survivors that neutralize the return value or loop range
of _find_first_diff (return None, return i+1, delete return, skip index 0,
always return None). These mutations are invisible to output-comparison
because the real round-trip is always lossless -- no diff is ever found, so
the return-i path is never exercised by check_identical_output. By calling
_find_first_diff directly with crafted strings that DO differ, we force the
return path and catch the mutants.
"""
mod = _import_refactored()
cases = [
# (actual, expected, expected_result)
("xbcdef", "abcdef", 0), # differ at index 0
("abcdxf", "abcdef", 4), # differ at middle index
("abcdeZ", "abcdef", 5), # differ at last common index
("abcdef", "abcdef", None), # identical
("abc", "abcdef", None), # actual is a prefix of expected
("abcdef", "abc", None), # expected is a prefix of actual
("", "", None), # both empty
("a", "b", 0), # single char, differ immediately
("a", "", None), # expected empty
("", "a", None), # actual empty
]
for actual, expected, want in cases:
try:
got = mod._find_first_diff(actual, expected)
except Exception as exc:
print(f"FAIL: _find_first_diff({actual!r}, {expected!r}) "
f"raised {type(exc).__name__}: {exc}")
return False
if got != want:
print(f"FAIL: _find_first_diff({actual!r}, {expected!r}) "
f"returned {got!r}, expected {want!r}")
return False
print(f"OK: _find_first_diff correct on all {len(cases)} cases")
return True
def check_no_hardcoded_equal() -> bool:
"""The 'equal' output must be computed from actual == expected, not hard-coded.
Kills the hardcode_equal mutant (``print("equal:", True)``) that hides a
broken round-trip behind a lie -- the verifier's check_roundtrip_lossless
only inspects the probe's *printed* output, so a hardcoded 'True' fools it
even if the chunker were broken. This static check ensures the real
comparison expression survives in the refactored source.
"""
code = _code_only(_read(REFACTORED))
if "actual == expected" not in code:
print("FAIL: 'actual == expected' comparison not found in code -- "
"equality is likely hard-coded")
return False
print("OK: 'equal' is computed dynamically from actual == expected")
return True
def check_overlap_consistency() -> bool:
"""chunk_text and reconstruct must agree on the overlap_tokens value.
Kills the reconstruct_overlap_zero mutant (``reconstruct(cs, 0)`` instead
of ``reconstruct(cs, 8)``). Because reconstruct delegates head-stripping
to chunk_body and ignores overlap_tokens, a behavioral test cannot detect
the mismatch -- so we verify it structurally: the overlap value passed to
reconstruct (positional arg 2 or overlap_tokens kwarg) must equal the
overlap_tokens kwarg passed to chunk_text.
"""
tree = ast.parse(_read(REFACTORED))
chunk_overlap = None
reconstruct_overlap = None
for node in ast.walk(tree):
if not (isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)):
continue
if node.func.attr == "chunk_text":
for kw in node.keywords:
if kw.arg == "overlap_tokens":
chunk_overlap = ast.literal_eval(kw.value)
elif node.func.attr == "reconstruct":
if len(node.args) >= 2:
reconstruct_overlap = ast.literal_eval(node.args[1])
for kw in node.keywords:
if kw.arg == "overlap_tokens":
reconstruct_overlap = ast.literal_eval(kw.value)
if chunk_overlap is None:
print("FAIL: no chunk_text call with overlap_tokens found")
return False
if reconstruct_overlap is None:
print("FAIL: no reconstruct call with an overlap argument found")
return False
if chunk_overlap != reconstruct_overlap:
print(f"FAIL: overlap mismatch: "
f"chunk_text(overlap_tokens={chunk_overlap}) vs "
f"reconstruct(overlap={reconstruct_overlap})")
return False
print(f"OK: overlap values consistent "
f"(chunk_text={chunk_overlap}, reconstruct={reconstruct_overlap})")
return True
def main() -> None:
checks = [
check_py_compile,
@@ -246,6 +353,9 @@ def main() -> None:
check_import_side_effect_free,
check_roundtrip_lossless,
check_original_still_smells,
check_find_first_diff_correctness,
check_no_hardcoded_equal,
check_overlap_consistency,
]
results = [fn() for fn in checks]
passed = sum(results)