114 lines
4.1 KiB
Python
114 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
|
"""mutation_check.py -- does test_base32c.py actually have teeth?
|
|
|
|
Copies base32c.py + test_base32c.py into a scratch dir, injects one fault at a time,
|
|
runs the suite against the mutant, and reports whether the mutant was KILLED
|
|
(suite exit != 0). A suite that passes everything, including mutants, is worthless.
|
|
|
|
Run: python3 mutation_check.py # exit 0 iff the control passes and every mutant dies
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
MODULE = os.path.join(HERE, "base32c.py")
|
|
TEST = os.path.join(HERE, "test_base32c.py")
|
|
|
|
MUTATIONS = [
|
|
(
|
|
"alphabet: letter J replaced by excluded I",
|
|
'"0123456789ABCDEFGHJKMNPQRSTVWXYZ"',
|
|
'"0123456789ABCDEFGHIKMNPQRSTVWXYZ"',
|
|
),
|
|
(
|
|
"check symbol: modulus 37 -> 36",
|
|
"return CHECK_ALPHABET[value % 37]",
|
|
"return CHECK_ALPHABET[value % 36]",
|
|
),
|
|
(
|
|
"check verify: compares raw value instead of value mod 37",
|
|
"if got != value % 37:",
|
|
"if got != value:",
|
|
),
|
|
(
|
|
"decode: hyphens no longer stripped",
|
|
'def decode(text: str) -> int:\n """Decode a symbol string (hyphens ignored, case folded, i/l/o aliases honoured)."""\n if not isinstance(text, str):\n raise TypeError("text must be a str")\n body = text.replace("-", "")',
|
|
'def decode(text: str) -> int:\n """Decode a symbol string (hyphens ignored, case folded, i/l/o aliases honoured)."""\n if not isinstance(text, str):\n raise TypeError("text must be a str")\n body = text',
|
|
),
|
|
(
|
|
"encode: 5-bit slicing becomes 4-bit slicing",
|
|
"for i in range(0, len(bits), 5)",
|
|
"for i in range(0, len(bits), 4)",
|
|
),
|
|
(
|
|
"decode aliases: i/l/o tolerance removed",
|
|
'for _alias, _target in (("I", "1"), ("L", "1"), ("O", "0")):\n _SYMBOL_VALUE[_alias] = _SYMBOL_VALUE[_target]',
|
|
'for _alias, _target in (("I", "1"), ("L", "1"), ("O", "0")):\n pass',
|
|
),
|
|
]
|
|
|
|
|
|
def run_suite(directory: str) -> tuple[int, str]:
|
|
proc = subprocess.run(
|
|
[sys.executable, os.path.join(directory, "test_base32c.py")],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=300,
|
|
cwd=directory,
|
|
)
|
|
return proc.returncode, (proc.stdout + proc.stderr).strip().splitlines()[-1] if (proc.stdout + proc.stderr).strip() else ""
|
|
|
|
|
|
def main() -> int:
|
|
root = tempfile.mkdtemp(prefix="base32c-mutants-", dir=os.environ.get("TMPDIR", "/tmp"))
|
|
failures = []
|
|
|
|
control = os.path.join(root, "control")
|
|
os.makedirs(control)
|
|
shutil.copy(MODULE, control)
|
|
shutil.copy(TEST, control)
|
|
code, tail = run_suite(control)
|
|
print(f"CONTROL (unmutated copy): exit {code} -- {tail}")
|
|
if code != 0:
|
|
failures.append("control copy of the suite failed; results below are meaningless")
|
|
|
|
source = open(MODULE, encoding="utf-8").read()
|
|
killed = 0
|
|
for index, (label, old, new) in enumerate(MUTATIONS, start=1):
|
|
if source.count(old) != 1:
|
|
failures.append(f"mutation {index} anchor matched {source.count(old)} times, expected 1: {label}")
|
|
print(f"MUTANT {index} ANCHOR-BROKEN: {label}")
|
|
continue
|
|
directory = os.path.join(root, f"mutant{index}")
|
|
os.makedirs(directory)
|
|
shutil.copy(TEST, directory)
|
|
with open(os.path.join(directory, "base32c.py"), "w", encoding="utf-8") as handle:
|
|
handle.write(source.replace(old, new))
|
|
code, tail = run_suite(directory)
|
|
status = "KILLED" if code != 0 else "SURVIVED"
|
|
if code != 0:
|
|
killed += 1
|
|
else:
|
|
failures.append(f"mutant survived: {label}")
|
|
print(f"MUTANT {index} {status}: {label} [{tail}]")
|
|
|
|
print()
|
|
print(f"MUTANTS {killed}/{len(MUTATIONS)} killed")
|
|
shutil.rmtree(root, ignore_errors=True)
|
|
if failures:
|
|
for line in failures:
|
|
print(f"PROBLEM: {line}")
|
|
return 1
|
|
print("SUITE HAS TEETH: every injected fault was caught")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|