#!/usr/bin/env python3 """mutation_check.py -- does test_base32c.py actually have teeth? Copies base32c.py + test_base32c.py into a scratch dir, injects one fault at a time, runs the suite against the mutant, and reports whether the mutant was KILLED (suite exit != 0). A suite that passes everything, including mutants, is worthless. Run: python3 mutation_check.py # exit 0 iff the control passes and every mutant dies """ from __future__ import annotations import os import shutil import subprocess import sys import tempfile HERE = os.path.dirname(os.path.abspath(__file__)) MODULE = os.path.join(HERE, "base32c.py") TEST = os.path.join(HERE, "test_base32c.py") MUTATIONS = [ ( "alphabet: letter J replaced by excluded I", '"0123456789ABCDEFGHJKMNPQRSTVWXYZ"', '"0123456789ABCDEFGHIKMNPQRSTVWXYZ"', ), ( "check symbol: modulus 37 -> 36", "return CHECK_ALPHABET[value % 37]", "return CHECK_ALPHABET[value % 36]", ), ( "check verify: compares raw value instead of value mod 37", "if got != value % 37:", "if got != value:", ), ( "decode: hyphens no longer stripped", 'def decode(text: str) -> int:\n """Decode a symbol string (hyphens ignored, case folded, i/l/o aliases honoured)."""\n if not isinstance(text, str):\n raise TypeError("text must be a str")\n body = text.replace("-", "")', 'def decode(text: str) -> int:\n """Decode a symbol string (hyphens ignored, case folded, i/l/o aliases honoured)."""\n if not isinstance(text, str):\n raise TypeError("text must be a str")\n body = text', ), ( "encode: 5-bit slicing becomes 4-bit slicing", "for i in range(0, len(bits), 5)", "for i in range(0, len(bits), 4)", ), ( "decode aliases: i/l/o tolerance removed", 'for _alias, _target in (("I", "1"), ("L", "1"), ("O", "0")):\n _SYMBOL_VALUE[_alias] = _SYMBOL_VALUE[_target]', 'for _alias, _target in (("I", "1"), ("L", "1"), ("O", "0")):\n pass', ), ] def run_suite(directory: str) -> tuple[int, str]: proc = subprocess.run( [sys.executable, os.path.join(directory, "test_base32c.py")], capture_output=True, text=True, timeout=300, cwd=directory, ) return proc.returncode, (proc.stdout + proc.stderr).strip().splitlines()[-1] if (proc.stdout + proc.stderr).strip() else "" def main() -> int: root = tempfile.mkdtemp(prefix="base32c-mutants-", dir=os.environ.get("TMPDIR", "/tmp")) failures = [] control = os.path.join(root, "control") os.makedirs(control) shutil.copy(MODULE, control) shutil.copy(TEST, control) code, tail = run_suite(control) print(f"CONTROL (unmutated copy): exit {code} -- {tail}") if code != 0: failures.append("control copy of the suite failed; results below are meaningless") source = open(MODULE, encoding="utf-8").read() killed = 0 for index, (label, old, new) in enumerate(MUTATIONS, start=1): if source.count(old) != 1: failures.append(f"mutation {index} anchor matched {source.count(old)} times, expected 1: {label}") print(f"MUTANT {index} ANCHOR-BROKEN: {label}") continue directory = os.path.join(root, f"mutant{index}") os.makedirs(directory) shutil.copy(TEST, directory) with open(os.path.join(directory, "base32c.py"), "w", encoding="utf-8") as handle: handle.write(source.replace(old, new)) code, tail = run_suite(directory) status = "KILLED" if code != 0 else "SURVIVED" if code != 0: killed += 1 else: failures.append(f"mutant survived: {label}") print(f"MUTANT {index} {status}: {label} [{tail}]") print() print(f"MUTANTS {killed}/{len(MUTATIONS)} killed") shutil.rmtree(root, ignore_errors=True) if failures: for line in failures: print(f"PROBLEM: {line}") return 1 print("SUITE HAS TEETH: every injected fault was caught") return 0 if __name__ == "__main__": raise SystemExit(main())