volt
e61ccd803b
A document rendered after the LEGS table broke is accepted as current by the pre-fix parser with no gate column at all, so mutant-doc-drifts alone would pass a tree that never had the column. The new leg reads that same mutant's fresh render and requires rc 0 with a '-' cell, pinning the failure mode the repair removes. Measured: q132_verify.py 9 legs, 0 fault, rc 0.
202 lines
8.6 KiB
Python
202 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Q132 instrument: `tool_map` must read a `LEGS` table whose field is not a literal.
|
|
|
|
The fault, measured at `d0ba06e` before any edit: `tools/verify.py`'s `LEGS`
|
|
gained `"path": os.path.join("wizard-work", "spark",
|
|
"closed_artifact_guard.py")` when Q130 added the `closed-artifacts` leg. That
|
|
one call made `ast.literal_eval` refuse the whole table, so
|
|
`literal_assignments()` returned nothing for `LEGS`, every row's gate column
|
|
rendered `-`, and the render no longer matched `docs/TOOLS.md`. The gate-leg
|
|
column did not go red; it went BLANK.
|
|
|
|
Legs
|
|
1. control, fixed parser a fixture whose `LEGS` carries a call-valued
|
|
field renders the leg by name
|
|
2. mutant, pre-fix parser the same fixture and document rendered by
|
|
`ast.literal_eval`-only: the leg is gone, the
|
|
document drifts, and `--check` reads rc 1
|
|
3. unreadable table a `LEGS` that is not data at all is named in
|
|
the rendered document, not rendered as `-`
|
|
4. real tree is current `--check docs/TOOLS.md` rc 0 at HEAD and at
|
|
WORKTREE, and the render names 10 gate legs
|
|
5. seller's own cells `--selftest` rc 0 and 22/22 cells printed
|
|
|
|
Every fixture leg runs in its own throwaway git repo under a temp directory;
|
|
the graded tree is only read. Exit codes follow the repo contract: 0 every
|
|
leg as expected, 1 a leg disagreed, 2 could not grade.
|
|
"""
|
|
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
REPO = os.path.dirname(os.path.dirname(HERE))
|
|
TOOL_MAP = os.path.join(REPO, "tools", "tool_map.py")
|
|
|
|
FAIL = []
|
|
COUNT = [0]
|
|
|
|
MUT_OLD = """ val = partial_literal(node.value)
|
|
if val is not _UNREADABLE:
|
|
found[tgt.id] = val
|
|
"""
|
|
MUT_NEW = """ try:
|
|
found[tgt.id] = ast.literal_eval(node.value)
|
|
except (ValueError, SyntaxError):
|
|
continue
|
|
"""
|
|
|
|
FAKE_TOOL = ('#!/usr/bin/env python3\n'
|
|
'"""fake_tool.py -- grades the fake contract, which nothing measured."""\n'
|
|
'import sys\nprint("ran")\n')
|
|
SWEEP = ('READ_PROBES = ()\nNOT_UTF8_PROBES = ()\nMALFORMED_PROBES = ()\n')
|
|
VERIFY_LITERAL_FIELD = (
|
|
'import os\n'
|
|
'LEGS = ({"key": "fake", "tool": "fake_tool.py", "slow": False,\n'
|
|
' "path": os.path.join("tools", "fake_tool.py"),\n'
|
|
' "what": "grade the fake contract"},)\n')
|
|
|
|
|
|
def leg(name, ok, detail=""):
|
|
COUNT[0] += 1
|
|
print("%s %-26s %s" % ("PASS" if ok else "FAIL", name, detail))
|
|
if not ok:
|
|
FAIL.append(name)
|
|
|
|
|
|
def run(argv, cwd=None):
|
|
p = subprocess.run(argv, cwd=cwd, capture_output=True, text=True)
|
|
return p.returncode, p.stdout, p.stderr
|
|
|
|
|
|
def git(repo, *args):
|
|
env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@e",
|
|
GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@e")
|
|
return run(["git", "-C", repo] + list(args))[0], env
|
|
|
|
|
|
def build_fixture(root, verify_src=VERIFY_LITERAL_FIELD):
|
|
"""A three-tool git work tree whose LEGS table is the grading surface."""
|
|
os.makedirs(os.path.join(root, "tools"))
|
|
os.makedirs(os.path.join(root, "docs"))
|
|
for name, body in (("fake_tool.py", FAKE_TOOL),
|
|
("selftest_sweep.py", SWEEP),
|
|
("verify.py", verify_src)):
|
|
with open(os.path.join(root, "tools", name), "w") as fh:
|
|
fh.write(body)
|
|
env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@e",
|
|
GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@e")
|
|
run(["git", "init", "-q", root], cwd=None)
|
|
run(["git", "-C", root, "add", "-A"])
|
|
run(["git", "-C", root, "-c", "user.name=t", "-c", "user.email=t@e",
|
|
"commit", "-qm", "fixture"])
|
|
return env
|
|
|
|
|
|
def gate_cell(doc_text, tool="fake_tool.py"):
|
|
"""The last table cell of the row for `tool`, or None."""
|
|
for line in doc_text.splitlines():
|
|
if line.startswith("| `%s`" % tool):
|
|
return line.rstrip().rstrip("|").split("|")[-1].strip()
|
|
return None
|
|
|
|
|
|
def main():
|
|
if not os.path.isfile(TOOL_MAP):
|
|
print("q132: could not grade -- no tools/tool_map.py at %s" % REPO)
|
|
return 2
|
|
|
|
td = tempfile.mkdtemp(prefix="q132.")
|
|
fixed = os.path.join(td, "tool_map_fixed.py")
|
|
shutil.copyfile(TOOL_MAP, fixed)
|
|
mutant = os.path.join(td, "tool_map_mutant.py")
|
|
with open(TOOL_MAP, encoding="utf-8") as fh:
|
|
src = fh.read()
|
|
if src.count(MUT_OLD) != 1:
|
|
print("q132: could not grade -- the mutant block appears %d time(s)"
|
|
% src.count(MUT_OLD))
|
|
return 2
|
|
with open(mutant, "w", encoding="utf-8") as fh:
|
|
fh.write(src.replace(MUT_OLD, MUT_NEW))
|
|
|
|
# -- 1. control: the fixed parser reads a call-valued field --------------
|
|
fx = os.path.join(td, "fx")
|
|
build_fixture(fx)
|
|
doc = os.path.join(fx, "docs", "TOOLS.md")
|
|
rc, out, err = run([sys.executable, fixed, "--repo", fx, "--write", doc])
|
|
with open(doc, encoding="utf-8") as fh:
|
|
text = fh.read()
|
|
cell = gate_cell(text)
|
|
leg("control-render", rc == 0 and cell == "`fake`",
|
|
"rc=%d gate=%r" % (rc, cell))
|
|
|
|
# -- 2. mutant: the pre-fix parser loses the leg and drifts -------------
|
|
mdoc = os.path.join(fx, "docs", "MUTANT.md")
|
|
rc_m, out_m, err_m = run([sys.executable, mutant, "--repo", fx,
|
|
"--write", mdoc])
|
|
with open(mdoc, encoding="utf-8") as fh:
|
|
mtext = fh.read()
|
|
mcell = gate_cell(mtext)
|
|
rc_c, out_c, err_c = run([sys.executable, mutant, "--repo", fx,
|
|
"--check", doc])
|
|
leg("mutant-loses-leg", rc_m == 0 and mcell == "-",
|
|
"rc=%d gate=%r" % (rc_m, mcell))
|
|
leg("mutant-doc-drifts", rc_c == 1 and "DRIFT" in out_c,
|
|
"rc=%d first=%r" % (rc_c, (out_c or err_c).splitlines()[:1]))
|
|
# ...and a document rendered AFTER the breakage would have been current
|
|
# and leg-less: rc 0 from the same parser, with no gate column at all.
|
|
rc_s, out_s, err_s = run([sys.executable, mutant, "--repo", fx,
|
|
"--check", mdoc])
|
|
leg("mutant-self-consistent", rc_s == 0 and gate_cell(mtext) == "-",
|
|
"rc=%d (a fresh empty render is 'current' to the pre-fix parser)" % rc_s)
|
|
|
|
# -- 3. a LEGS that is not data is named, not rendered as `-` -----------
|
|
fx2 = os.path.join(td, "fx2")
|
|
build_fixture(fx2, verify_src="LEGS = build_legs()\n")
|
|
doc2 = os.path.join(fx2, "docs", "TOOLS.md")
|
|
rc2, out2, err2 = run([sys.executable, fixed, "--repo", fx2,
|
|
"--write", doc2])
|
|
with open(doc2, encoding="utf-8") as fh:
|
|
text2 = fh.read()
|
|
named = "could not be read as data" in text2
|
|
rc2c, out2c, _ = run([sys.executable, fixed, "--repo", fx2,
|
|
"--check", doc2])
|
|
leg("unreadable-named", rc2 == 0 and named and gate_cell(text2) == "-",
|
|
"rc=%d named=%s gate=%r" % (rc2, named, gate_cell(text2)))
|
|
leg("unreadable-still-checks", rc2c == 0,
|
|
"rc=%d (the document states the failure, so it is current)" % rc2c)
|
|
|
|
# -- 4. the real tree: the column is populated and the map is current ---
|
|
rdoc = os.path.join(td, "TOOLS.real.md")
|
|
rc3, out3, err3 = run([sys.executable, TOOL_MAP, "--repo", REPO,
|
|
"--rev", "WORKTREE", "--write", rdoc])
|
|
told = re.search(r"(\d+) gate legs", out3 or "")
|
|
legs_n = int(told.group(1)) if told else -1
|
|
rc_h, out_h, err_h = run([sys.executable, TOOL_MAP, "--repo", REPO,
|
|
"--check", os.path.join(REPO, "docs", "TOOLS.md")])
|
|
rc_w, out_w, err_w = run([sys.executable, TOOL_MAP, "--repo", REPO,
|
|
"--rev", "WORKTREE", "--check",
|
|
os.path.join(REPO, "docs", "TOOLS.md")])
|
|
leg("real-tree-legs", rc3 == 0 and legs_n == 10, "rc=%d legs=%d" % (rc3, legs_n))
|
|
leg("real-tree-current", rc_h == 0 and rc_w == 0,
|
|
"HEAD rc=%d WORKTREE rc=%d %s" % (rc_h, rc_w, (out_h or err_h).strip()[:80]))
|
|
|
|
# -- 5. the tool's own cells -------------------------------------------
|
|
rc4, out4, err4 = run([sys.executable, TOOL_MAP, "--repo", REPO, "--selftest"])
|
|
m = re.search(r"selftest: (\d+) cells, (\d+) passed, (\d+) failed", out4 or "")
|
|
ok4 = bool(m) and m.group(1) == m.group(2) == "23" and m.group(3) == "0"
|
|
leg("tool-selftest", rc4 == 0 and ok4,
|
|
"rc=%d %s" % (rc4, m.group(0) if m else (err4 or out4).strip()[:90]))
|
|
|
|
print("q132: %d leg(s), %d fault -- %s"
|
|
% (COUNT[0], len(FAIL), "OK" if not FAIL else "FAILED"))
|
|
return 0 if not FAIL else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|