volt: Q132 repair -- tool_map reads a LEGS table whose field is not a literal
Q130's closed-artifacts leg added "path": os.path.join(...) to verify.py's LEGS, so ast.literal_eval refused the whole table and every gate leg vanished from docs/TOOLS.md while --check compared against a document rendered before the field existed: the gate-leg column went blank, not red. partial_literal() keeps every element/field that parses and names what it cannot read (legs_note, probes_note) in the document instead of rendering an empty column; _count_by_tool refuses a row whose name is not a string rather than subscripting it into the dict. Selftest 20 -> 23 cells, pin moved with it, docs/TOOLS.md re-rendered (23 tools, 10 gate legs).
This commit is contained in:
1 parent
232c8af304
commit
e094c913f2
5 files changed
+363
-19
No files matched your search
+2
-1
@@ -32,7 +32,7 @@ module docstring, flags from the tool's own `--help`, probe rows from
|
|||||||
| `selftest_sweep.py` | every instrument proves itself, in one command, and the number it advertises is the number it prints. | `--repo` `--tools-dir` `--timeout` `--json` `--baseline` `--no-baseline` `--write-baseline` `--quiet` `--verbose` `--selftest` | 1 / 0 / 1 | `selftest-sweep` |
|
| `selftest_sweep.py` | every instrument proves itself, in one command, and the number it advertises is the number it prints. | `--repo` `--tools-dir` `--timeout` `--json` `--baseline` `--no-baseline` `--write-baseline` `--quiet` `--verbose` `--selftest` | 1 / 0 / 1 | `selftest-sweep` |
|
||||||
| `served_drift_check.py` | does the host actually serve the tree at this revision? | `--root` `--rev` `--base-url` `--prefix` `--exclude` `--max-bytes` `--timeout` `--json` `--quiet` `--selftest` | 1 / 0 / 1 | - |
|
| `served_drift_check.py` | does the host actually serve the tree at this revision? | `--root` `--rev` `--base-url` `--prefix` `--exclude` `--max-bytes` `--timeout` `--json` `--quiet` `--selftest` | 1 / 0 / 1 | - |
|
||||||
| `tool_map.py` | the instruments get a map, and the map is generated from the tree. | `--repo` `--rev` `--write` `--check` `--json` `--selftest` | 1 / 0 / 1 | - |
|
| `tool_map.py` | the instruments get a map, and the map is generated from the tree. | `--repo` `--rev` `--write` `--check` `--json` `--selftest` | 1 / 0 / 1 | - |
|
||||||
| `verify.py` | the HOME the index gate did not have: one command, nine legs, one exit code, and a row per leg so a partial run cannot look like a clean one. | `--repo` `--queue` `--index` `--checkin-day` `--fast` `--timeout` `--json` `--quiet` `--selftest` | 1 / 0 / 1 | - |
|
| `verify.py` | the HOME the index gate did not have: one command, ten legs, one exit code, and a row per leg so a partial run cannot look like a clean one. | `--repo` `--queue` `--index` `--checkin-day` `--fast` `--timeout` `--json` `--quiet` `--selftest` | 1 / 0 / 1 | - |
|
||||||
| `write_probe.py` | Measure the WRITE end of every tool in tools/: `--json PATH` at a target the host cannot write. | `--repo` `--only` `--timeout` `--json` `--selftest` `--verbose` | 1 / 0 / 1 | `write-axis` |
|
| `write_probe.py` | Measure the WRITE end of every tool in tools/: `--json PATH` at a target the host cannot write. | `--repo` `--only` `--timeout` `--json` `--selftest` `--verbose` | 1 / 0 / 1 | `write-axis` |
|
||||||
|
|
||||||
## The gate
|
## The gate
|
||||||
@@ -52,6 +52,7 @@ dropped by `--fast`.
|
|||||||
| `selftest-sweep` | `selftest_sweep.py` | yes | every tool's --selftest and both read-refusal axes |
|
| `selftest-sweep` | `selftest_sweep.py` | yes | every tool's --selftest and both read-refusal axes |
|
||||||
| `write-axis` | `write_probe.py` | no | a --json target this host cannot write: rc 2, one line naming it, never a traceback -- and the same rule for every caller-path write flag (--write, --save-vocab) |
|
| `write-axis` | `write_probe.py` | no | a --json target this host cannot write: rc 2, one line naming it, never a traceback -- and the same rule for every caller-path write flag (--write, --save-vocab) |
|
||||||
| `import-resolve` | `import_resolve_check.py` | no | every committed test file binds the module it imports to the repo, not to a host path (static sys.path scan plus --import-probe) |
|
| `import-resolve` | `import_resolve_check.py` | no | every committed test file binds the module it imports to the repo, not to a host path (static sys.path scan plus --import-probe) |
|
||||||
|
| `closed-artifacts` | `closed_artifact_guard.py` | no | every repair a closed item landed is still the bytes that landed (sha256 pins for Q109/Q120 and the seven Q126 cells) and its cell still runs green |
|
||||||
|
|
||||||
A tool that appears in no leg is still graded by `selftest-sweep`, which runs
|
A tool that appears in no leg is still graded by `selftest-sweep`, which runs
|
||||||
every tracked tool's `--selftest` and both read-refusal axes -- so the
|
every tracked tool's `--selftest` and both read-refusal axes -- so the
|
||||||
|
|||||||
@@ -29,6 +29,6 @@ queue_index.py 80 cells
|
|||||||
reply_check.py 49 cells
|
reply_check.py 49 cells
|
||||||
selftest_sweep.py 49 cells
|
selftest_sweep.py 49 cells
|
||||||
served_drift_check.py 25 cells
|
served_drift_check.py 25 cells
|
||||||
tool_map.py 20 cells
|
tool_map.py 23 cells
|
||||||
verify.py 59 cells
|
verify.py 59 cells
|
||||||
write_probe.py 15 cells
|
write_probe.py 15 cells
|
||||||
|
+145
-17
@@ -7,7 +7,7 @@ WHY THIS EXISTS
|
|||||||
queue block, in a reply -- and nothing states the whole set: which tool grades
|
queue block, in a reply -- and nothing states the whole set: which tool grades
|
||||||
what, which flags each one actually accepts, which of them the two probe axes
|
what, which flags each one actually accepts, which of them the two probe axes
|
||||||
cover, and which leg of the operator's gate a failure shows up in. The closest
|
cover, and which leg of the operator's gate a failure shows up in. The closest
|
||||||
thing is `verify.py`'s `LEGS` table, which names six of them.
|
thing is `verify.py`'s `LEGS` table, which names ten of them.
|
||||||
|
|
||||||
A hand-written map would be a fourth notation of the defect this suite keeps
|
A hand-written map would be a fourth notation of the defect this suite keeps
|
||||||
grading: prose about the tree, one step from the thing it describes, with no
|
grading: prose about the tree, one step from the thing it describes, with no
|
||||||
@@ -106,7 +106,7 @@ USAGE
|
|||||||
python3 tool_map.py --repo . --check docs/TOOLS.md # grade drift, rc 1 on drift
|
python3 tool_map.py --repo . --check docs/TOOLS.md # grade drift, rc 1 on drift
|
||||||
python3 tool_map.py --repo . --rev WORKTREE --check docs/TOOLS.md # ...as it will be committed
|
python3 tool_map.py --repo . --rev WORKTREE --check docs/TOOLS.md # ...as it will be committed
|
||||||
python3 tool_map.py --repo . --json out.json
|
python3 tool_map.py --repo . --json out.json
|
||||||
python3 tool_map.py --selftest # 20 cells
|
python3 tool_map.py --selftest # 23 cells
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -171,8 +171,51 @@ def tracked_tools(repo, rev):
|
|||||||
return sorted(names)
|
return sorted(names)
|
||||||
|
|
||||||
|
|
||||||
|
_UNREADABLE = object()
|
||||||
|
|
||||||
|
|
||||||
|
def partial_literal(node):
|
||||||
|
"""`ast.literal_eval`, but a container survives a field that is not literal.
|
||||||
|
|
||||||
|
`literal_eval` refuses the WHOLE table when one value is a call, and the
|
||||||
|
caller used to drop that table silently. `verify.py`'s `LEGS` gained
|
||||||
|
`"path": os.path.join(...)` and every gate leg vanished from the map while
|
||||||
|
`--check` still read rc 0 against a document rendered before the field
|
||||||
|
existed. A dict here keeps every key whose value parses, a tuple or list
|
||||||
|
keeps every element that survives, and `_UNREADABLE` is returned only when
|
||||||
|
the node itself is not a container of literals -- the caller decides
|
||||||
|
whether that is fatal, and `collect()` says so in the document.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return ast.literal_eval(node)
|
||||||
|
except (ValueError, SyntaxError):
|
||||||
|
pass
|
||||||
|
if isinstance(node, (ast.Tuple, ast.List)):
|
||||||
|
out = [v for v in (partial_literal(e) for e in node.elts)
|
||||||
|
if v is not _UNREADABLE]
|
||||||
|
if node.elts and not out:
|
||||||
|
return _UNREADABLE # nothing survived: not an empty container
|
||||||
|
return out
|
||||||
|
if isinstance(node, ast.Dict):
|
||||||
|
out = {}
|
||||||
|
for k, v in zip(node.keys, node.values):
|
||||||
|
if k is None: # {**mapping}
|
||||||
|
continue
|
||||||
|
kv, vv = partial_literal(k), partial_literal(v)
|
||||||
|
if kv is _UNREADABLE or vv is _UNREADABLE:
|
||||||
|
continue
|
||||||
|
out[kv] = vv
|
||||||
|
return out
|
||||||
|
return _UNREADABLE
|
||||||
|
|
||||||
|
|
||||||
def literal_assignments(repo, rev, path, names):
|
def literal_assignments(repo, rev, path, names):
|
||||||
"""{name: value} for the module-level assignments whose value is a literal."""
|
"""{name: value} for the module-level assignments whose value is a literal.
|
||||||
|
|
||||||
|
A value that is a container of literals is read even when one field of it
|
||||||
|
is not (see `partial_literal`); a value that cannot be read at all is
|
||||||
|
absent from the result, and `collect()` names that absence in the render.
|
||||||
|
"""
|
||||||
src = read_blob(repo, rev, path)
|
src = read_blob(repo, rev, path)
|
||||||
if src is None:
|
if src is None:
|
||||||
return {}
|
return {}
|
||||||
@@ -187,10 +230,9 @@ def literal_assignments(repo, rev, path, names):
|
|||||||
tgt = node.targets[0]
|
tgt = node.targets[0]
|
||||||
if not isinstance(tgt, ast.Name) or tgt.id not in names:
|
if not isinstance(tgt, ast.Name) or tgt.id not in names:
|
||||||
continue
|
continue
|
||||||
try:
|
val = partial_literal(node.value)
|
||||||
found[tgt.id] = ast.literal_eval(node.value)
|
if val is not _UNREADABLE:
|
||||||
except (ValueError, SyntaxError):
|
found[tgt.id] = val
|
||||||
continue
|
|
||||||
return found
|
return found
|
||||||
|
|
||||||
|
|
||||||
@@ -284,19 +326,25 @@ def flags_of(repo, rev, name):
|
|||||||
|
|
||||||
|
|
||||||
def _count_by_tool(entries):
|
def _count_by_tool(entries):
|
||||||
"""Count probes per tool name (first element of each entry tuple).
|
"""Count probes per tool name: ({tool: rows}, rows that were not readable).
|
||||||
|
|
||||||
All three probe tables share the same shape for tallying: each entry's
|
All three probe tables share the same shape for tallying: each entry's
|
||||||
first element is the tool name, and the rest is payload the tally ignores.
|
first element is the tool name, and the rest is payload the tally ignores.
|
||||||
READ_PROBES / NOT_UTF8_PROBES entries are (tool, argv);
|
READ_PROBES / NOT_UTF8_PROBES entries are (tool, argv);
|
||||||
MALFORMED_PROBES entries are (tool, label, ...). This helper replaces the
|
MALFORMED_PROBES entries are (tool, label, ...). A row whose first element
|
||||||
three identical loops that previously counted each table by hand.
|
is not a string -- an entry built from a name the AST could not resolve --
|
||||||
|
is counted as dropped instead of raising out of the renderer, and
|
||||||
|
`collect()` names the drop in the document so a short count is never read
|
||||||
|
as a table that has no more rows.
|
||||||
"""
|
"""
|
||||||
counts = {}
|
counts, dropped = {}, 0
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
name = entry[0]
|
name = entry[0] if isinstance(entry, (list, tuple)) and entry else None
|
||||||
|
if not isinstance(name, str):
|
||||||
|
dropped += 1
|
||||||
|
continue
|
||||||
counts[name] = counts.get(name, 0) + 1
|
counts[name] = counts.get(name, 0) + 1
|
||||||
return counts
|
return counts, dropped
|
||||||
|
|
||||||
|
|
||||||
def collect(repo, rev):
|
def collect(repo, rev):
|
||||||
@@ -307,10 +355,27 @@ def collect(repo, rev):
|
|||||||
sweep = literal_assignments(repo, rev, "tools/selftest_sweep.py",
|
sweep = literal_assignments(repo, rev, "tools/selftest_sweep.py",
|
||||||
{"READ_PROBES", "NOT_UTF8_PROBES",
|
{"READ_PROBES", "NOT_UTF8_PROBES",
|
||||||
"MALFORMED_PROBES"})
|
"MALFORMED_PROBES"})
|
||||||
read_n = _count_by_tool(sweep.get("READ_PROBES", ()))
|
read_n, read_d = _count_by_tool(sweep.get("READ_PROBES", ()))
|
||||||
utf8_n = _count_by_tool(sweep.get("NOT_UTF8_PROBES", ()))
|
utf8_n, utf8_d = _count_by_tool(sweep.get("NOT_UTF8_PROBES", ()))
|
||||||
mal_n = _count_by_tool(sweep.get("MALFORMED_PROBES", ()))
|
mal_n, mal_d = _count_by_tool(sweep.get("MALFORMED_PROBES", ()))
|
||||||
|
probes_dropped = read_d + utf8_d + mal_d
|
||||||
|
probes_note = ""
|
||||||
|
if probes_dropped:
|
||||||
|
probes_note = ("%d probe row(s) in `tools/selftest_sweep.py` could not be "
|
||||||
|
"read as data -- the probe counts above are lower than the "
|
||||||
|
"tables that carry them" % probes_dropped)
|
||||||
|
leg_src = read_blob(repo, rev, "tools/verify.py") or ""
|
||||||
legs = literal_assignments(repo, rev, "tools/verify.py", {"LEGS"}).get("LEGS", ())
|
legs = literal_assignments(repo, rev, "tools/verify.py", {"LEGS"}).get("LEGS", ())
|
||||||
|
legs_note = ""
|
||||||
|
if not legs and re.search(r"^LEGS\s*=", leg_src, re.M):
|
||||||
|
# An empty column and an unread table render identically, so say which
|
||||||
|
# one this is in the document itself. A `LEGS` that is a call, an
|
||||||
|
# import, or a comprehension cannot be read as data; dropping it
|
||||||
|
# silently is how the gate column went blank while `--check` stayed
|
||||||
|
# green against a document rendered before the table changed.
|
||||||
|
legs_note = ("the `LEGS` table in `tools/verify.py` could not be read as "
|
||||||
|
"data -- the gate column above is empty because the map "
|
||||||
|
"failed to read it, not because no tool has a leg")
|
||||||
leg_of = {}
|
leg_of = {}
|
||||||
for leg in legs:
|
for leg in legs:
|
||||||
if isinstance(leg, dict) and "tool" in leg and "key" in leg:
|
if isinstance(leg, dict) and "tool" in leg and "key" in leg:
|
||||||
@@ -330,7 +395,8 @@ def collect(repo, rev):
|
|||||||
"malformed": mal_n.get(name, 0),
|
"malformed": mal_n.get(name, 0),
|
||||||
"legs": leg_of.get(name, []),
|
"legs": leg_of.get(name, []),
|
||||||
})
|
})
|
||||||
return {"rev": rev, "rows": rows,
|
return {"rev": rev, "rows": rows, "legs_note": legs_note,
|
||||||
|
"probes_note": probes_note,
|
||||||
"legs": [{"key": l.get("key"), "tool": l.get("tool"),
|
"legs": [{"key": l.get("key"), "tool": l.get("tool"),
|
||||||
"slow": bool(l.get("slow")), "what": l.get("what", "")}
|
"slow": bool(l.get("slow")), "what": l.get("what", "")}
|
||||||
for l in legs if isinstance(l, dict)]}
|
for l in legs if isinstance(l, dict)]}
|
||||||
@@ -367,6 +433,8 @@ def render(data):
|
|||||||
if (r["read"] or r["not_utf8"] or r["malformed"]) else "-")
|
if (r["read"] or r["not_utf8"] or r["malformed"]) else "-")
|
||||||
gate = ", ".join(f"`{k}`" for k in r["legs"]) or "-"
|
gate = ", ".join(f"`{k}`" for k in r["legs"]) or "-"
|
||||||
lines.append(f"| `{r['tool']}` | {purpose} | {flags} | {probes} | {gate} |")
|
lines.append(f"| `{r['tool']}` | {purpose} | {flags} | {probes} | {gate} |")
|
||||||
|
if data.get("probes_note"):
|
||||||
|
lines += ["", "*" + data["probes_note"] + "*"]
|
||||||
lines += [
|
lines += [
|
||||||
"",
|
"",
|
||||||
"## The gate",
|
"## The gate",
|
||||||
@@ -381,6 +449,8 @@ def render(data):
|
|||||||
for leg in legs:
|
for leg in legs:
|
||||||
lines.append(f"| `{leg['key']}` | `{leg['tool']}` | "
|
lines.append(f"| `{leg['key']}` | `{leg['tool']}` | "
|
||||||
f"{'yes' if leg['slow'] else 'no'} | {leg['what']} |")
|
f"{'yes' if leg['slow'] else 'no'} | {leg['what']} |")
|
||||||
|
if data.get("legs_note"):
|
||||||
|
lines += ["", "*" + data["legs_note"] + "*"]
|
||||||
lines += [
|
lines += [
|
||||||
"",
|
"",
|
||||||
"A tool that appears in no leg is still graded by `selftest-sweep`, which runs",
|
"A tool that appears in no leg is still graded by `selftest-sweep`, which runs",
|
||||||
@@ -528,6 +598,64 @@ def selftest(repo):
|
|||||||
cell("WORKTREE outside a git work tree: rc 2, not a crash",
|
cell("WORKTREE outside a git work tree: rc 2, not a crash",
|
||||||
rc_wt_norepo == 2, f"rc {rc_wt_norepo}")
|
rc_wt_norepo == 2, f"rc {rc_wt_norepo}")
|
||||||
|
|
||||||
|
# -- a leg field that is not a literal must not erase the leg --------
|
||||||
|
# Q130 gave the closed-artifacts leg `"path": os.path.join(...)`. That
|
||||||
|
# one call made `ast.literal_eval` refuse the whole LEGS table, so every
|
||||||
|
# leg vanished from the map while `--check` stayed green against a
|
||||||
|
# document rendered before the field existed. Both directions are
|
||||||
|
# pinned here: a non-literal FIELD keeps its leg, and a LEGS that is not
|
||||||
|
# data at all is named in the rendered document instead of rendering an
|
||||||
|
# empty column that reads as "no tool has a leg".
|
||||||
|
vpath = os.path.join(fixture, "tools", "verify.py")
|
||||||
|
with open(vpath, "w") as fh:
|
||||||
|
fh.write('import os\n'
|
||||||
|
'LEGS = ({"key": "fake", "tool": "fake_tool.py",\n'
|
||||||
|
' "slow": False,\n'
|
||||||
|
' "path": os.path.join("tools", "fake_tool.py"),\n'
|
||||||
|
' "what": "grade the fake contract"},)\n')
|
||||||
|
commit("non-literal leg field")
|
||||||
|
d2 = collect(fixture, "HEAD")
|
||||||
|
by2 = {r["tool"]: r for r in d2["rows"]}
|
||||||
|
cell("collect: a leg field that is not a literal does not erase the leg",
|
||||||
|
by2["fake_tool.py"]["legs"] == ["fake"]
|
||||||
|
and [l["key"] for l in d2["legs"]] == ["fake"]
|
||||||
|
and d2["legs_note"] == "",
|
||||||
|
str((by2["fake_tool.py"]["legs"], d2["legs"], d2["legs_note"])))
|
||||||
|
|
||||||
|
with open(vpath, "w") as fh:
|
||||||
|
fh.write('LEGS = build_legs()\n')
|
||||||
|
commit("unreadable LEGS")
|
||||||
|
d3 = collect(fixture, "HEAD")
|
||||||
|
txt3 = render(d3)
|
||||||
|
cell("render: an unreadable LEGS table is named, not silently empty",
|
||||||
|
d3["legs"] == [] and d3["legs_note"] != ""
|
||||||
|
and "could not be read as data" in txt3
|
||||||
|
and {r["tool"]: r["legs"] for r in d3["rows"]}["fake_tool.py"] == [],
|
||||||
|
str((d3["legs"], d3["legs_note"])))
|
||||||
|
|
||||||
|
# A probe row whose element is not a literal must not become an empty
|
||||||
|
# element the counter then subscripts: `_count_by_tool` reads entry[0],
|
||||||
|
# so a dropped name raised IndexError out of the renderer.
|
||||||
|
with open(os.path.join(fixture, "tools", "verify.py"), "w") as fh:
|
||||||
|
fh.write('LEGS = ({"key": "fake", "tool": "fake_tool.py",\n'
|
||||||
|
' "slow": False, "what": "grade the fake contract"},)\n')
|
||||||
|
with open(os.path.join(fixture, "tools", "selftest_sweep.py"), "w") as fh:
|
||||||
|
fh.write('READ_PROBES = ((TOOL_NAME, ["--repo", "{nope}"]),)\n'
|
||||||
|
'NOT_UTF8_PROBES = ()\nMALFORMED_PROBES = ()\n')
|
||||||
|
commit("non-literal probe element")
|
||||||
|
try:
|
||||||
|
d4 = collect(fixture, "HEAD")
|
||||||
|
crash = ""
|
||||||
|
read_n4 = {r["tool"]: r["read"] for r in d4["rows"]}["fake_tool.py"]
|
||||||
|
note4 = d4["probes_note"]
|
||||||
|
txt4 = render(d4)
|
||||||
|
except Exception as exc: # noqa: BLE001 - graded
|
||||||
|
crash, read_n4, note4, txt4 = "%s: %s" % (type(exc).__name__, exc), None, "", ""
|
||||||
|
cell("collect: a probe row that is not a literal is dropped, not indexed",
|
||||||
|
crash == "" and read_n4 == 0 and "could not be read as data" in note4
|
||||||
|
and note4 in txt4,
|
||||||
|
crash or f"read={read_n4} note={note4!r}")
|
||||||
|
|
||||||
# -- the currency cell: the reason this document stays true ----------
|
# -- the currency cell: the reason this document stays true ----------
|
||||||
here = os.path.join(repo, "docs", "TOOLS.md")
|
here = os.path.join(repo, "docs", "TOOLS.md")
|
||||||
try:
|
try:
|
||||||
|
|||||||
Executable
+20
@@ -0,0 +1,20 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Q132 CHECK -- tool_map.py must read a LEGS table whose field is not a
|
||||||
|
# literal, must name a table it cannot read, and must stay the render of
|
||||||
|
# this tree.
|
||||||
|
#
|
||||||
|
# rc 0 = every leg as expected; 1 = a graded fault; 2 = could not grade.
|
||||||
|
# The work is done by wizard-work/volt/q132_verify.py beside this file;
|
||||||
|
# every fixture leg runs in a throwaway git repo, so the graded tree is
|
||||||
|
# only ever read. Run it from any checkout.
|
||||||
|
set -u
|
||||||
|
REPO="$(cd "$(dirname "$0")/../.." && pwd)" || exit 2
|
||||||
|
if ! cd "$REPO"; then
|
||||||
|
echo "q132: could not grade -- cannot cd to $REPO"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
if ! command -v python3 >/dev/null 2>&1; then
|
||||||
|
echo "q132: could not grade -- no python3 on PATH"
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
exec python3 wizard-work/volt/q132_verify.py
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Q132 instrument: `tool_map` must read a `LEGS` table whose field is not a literal.
|
||||||
|
|
||||||
|
The fault, measured at `d0ba06e` before any edit: `tools/verify.py`'s `LEGS`
|
||||||
|
gained `"path": os.path.join("wizard-work", "spark",
|
||||||
|
"closed_artifact_guard.py")` when Q130 added the `closed-artifacts` leg. That
|
||||||
|
one call made `ast.literal_eval` refuse the whole table, so
|
||||||
|
`literal_assignments()` returned nothing for `LEGS`, every row's gate column
|
||||||
|
rendered `-`, and the render no longer matched `docs/TOOLS.md`. The gate-leg
|
||||||
|
column did not go red; it went BLANK.
|
||||||
|
|
||||||
|
Legs
|
||||||
|
1. control, fixed parser a fixture whose `LEGS` carries a call-valued
|
||||||
|
field renders the leg by name
|
||||||
|
2. mutant, pre-fix parser the same fixture and document rendered by
|
||||||
|
`ast.literal_eval`-only: the leg is gone, the
|
||||||
|
document drifts, and `--check` reads rc 1
|
||||||
|
3. unreadable table a `LEGS` that is not data at all is named in
|
||||||
|
the rendered document, not rendered as `-`
|
||||||
|
4. real tree is current `--check docs/TOOLS.md` rc 0 at HEAD and at
|
||||||
|
WORKTREE, and the render names 10 gate legs
|
||||||
|
5. seller's own cells `--selftest` rc 0 and 22/22 cells printed
|
||||||
|
|
||||||
|
Every fixture leg runs in its own throwaway git repo under a temp directory;
|
||||||
|
the graded tree is only read. Exit codes follow the repo contract: 0 every
|
||||||
|
leg as expected, 1 a leg disagreed, 2 could not grade.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
REPO = os.path.dirname(os.path.dirname(HERE))
|
||||||
|
TOOL_MAP = os.path.join(REPO, "tools", "tool_map.py")
|
||||||
|
|
||||||
|
FAIL = []
|
||||||
|
COUNT = [0]
|
||||||
|
|
||||||
|
MUT_OLD = """ val = partial_literal(node.value)
|
||||||
|
if val is not _UNREADABLE:
|
||||||
|
found[tgt.id] = val
|
||||||
|
"""
|
||||||
|
MUT_NEW = """ try:
|
||||||
|
found[tgt.id] = ast.literal_eval(node.value)
|
||||||
|
except (ValueError, SyntaxError):
|
||||||
|
continue
|
||||||
|
"""
|
||||||
|
|
||||||
|
FAKE_TOOL = ('#!/usr/bin/env python3\n'
|
||||||
|
'"""fake_tool.py -- grades the fake contract, which nothing measured."""\n'
|
||||||
|
'import sys\nprint("ran")\n')
|
||||||
|
SWEEP = ('READ_PROBES = ()\nNOT_UTF8_PROBES = ()\nMALFORMED_PROBES = ()\n')
|
||||||
|
VERIFY_LITERAL_FIELD = (
|
||||||
|
'import os\n'
|
||||||
|
'LEGS = ({"key": "fake", "tool": "fake_tool.py", "slow": False,\n'
|
||||||
|
' "path": os.path.join("tools", "fake_tool.py"),\n'
|
||||||
|
' "what": "grade the fake contract"},)\n')
|
||||||
|
|
||||||
|
|
||||||
|
def leg(name, ok, detail=""):
|
||||||
|
COUNT[0] += 1
|
||||||
|
print("%s %-26s %s" % ("PASS" if ok else "FAIL", name, detail))
|
||||||
|
if not ok:
|
||||||
|
FAIL.append(name)
|
||||||
|
|
||||||
|
|
||||||
|
def run(argv, cwd=None):
|
||||||
|
p = subprocess.run(argv, cwd=cwd, capture_output=True, text=True)
|
||||||
|
return p.returncode, p.stdout, p.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def git(repo, *args):
|
||||||
|
env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@e",
|
||||||
|
GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@e")
|
||||||
|
return run(["git", "-C", repo] + list(args))[0], env
|
||||||
|
|
||||||
|
|
||||||
|
def build_fixture(root, verify_src=VERIFY_LITERAL_FIELD):
|
||||||
|
"""A three-tool git work tree whose LEGS table is the grading surface."""
|
||||||
|
os.makedirs(os.path.join(root, "tools"))
|
||||||
|
os.makedirs(os.path.join(root, "docs"))
|
||||||
|
for name, body in (("fake_tool.py", FAKE_TOOL),
|
||||||
|
("selftest_sweep.py", SWEEP),
|
||||||
|
("verify.py", verify_src)):
|
||||||
|
with open(os.path.join(root, "tools", name), "w") as fh:
|
||||||
|
fh.write(body)
|
||||||
|
env = dict(os.environ, GIT_AUTHOR_NAME="t", GIT_AUTHOR_EMAIL="t@e",
|
||||||
|
GIT_COMMITTER_NAME="t", GIT_COMMITTER_EMAIL="t@e")
|
||||||
|
run(["git", "init", "-q", root], cwd=None)
|
||||||
|
run(["git", "-C", root, "add", "-A"])
|
||||||
|
run(["git", "-C", root, "-c", "user.name=t", "-c", "user.email=t@e",
|
||||||
|
"commit", "-qm", "fixture"])
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def gate_cell(doc_text, tool="fake_tool.py"):
|
||||||
|
"""The last table cell of the row for `tool`, or None."""
|
||||||
|
for line in doc_text.splitlines():
|
||||||
|
if line.startswith("| `%s`" % tool):
|
||||||
|
return line.rstrip().rstrip("|").split("|")[-1].strip()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if not os.path.isfile(TOOL_MAP):
|
||||||
|
print("q132: could not grade -- no tools/tool_map.py at %s" % REPO)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
td = tempfile.mkdtemp(prefix="q132.")
|
||||||
|
fixed = os.path.join(td, "tool_map_fixed.py")
|
||||||
|
shutil.copyfile(TOOL_MAP, fixed)
|
||||||
|
mutant = os.path.join(td, "tool_map_mutant.py")
|
||||||
|
with open(TOOL_MAP, encoding="utf-8") as fh:
|
||||||
|
src = fh.read()
|
||||||
|
if src.count(MUT_OLD) != 1:
|
||||||
|
print("q132: could not grade -- the mutant block appears %d time(s)"
|
||||||
|
% src.count(MUT_OLD))
|
||||||
|
return 2
|
||||||
|
with open(mutant, "w", encoding="utf-8") as fh:
|
||||||
|
fh.write(src.replace(MUT_OLD, MUT_NEW))
|
||||||
|
|
||||||
|
# -- 1. control: the fixed parser reads a call-valued field --------------
|
||||||
|
fx = os.path.join(td, "fx")
|
||||||
|
build_fixture(fx)
|
||||||
|
doc = os.path.join(fx, "docs", "TOOLS.md")
|
||||||
|
rc, out, err = run([sys.executable, fixed, "--repo", fx, "--write", doc])
|
||||||
|
with open(doc, encoding="utf-8") as fh:
|
||||||
|
text = fh.read()
|
||||||
|
cell = gate_cell(text)
|
||||||
|
leg("control-render", rc == 0 and cell == "`fake`",
|
||||||
|
"rc=%d gate=%r" % (rc, cell))
|
||||||
|
|
||||||
|
# -- 2. mutant: the pre-fix parser loses the leg and drifts -------------
|
||||||
|
mdoc = os.path.join(fx, "docs", "MUTANT.md")
|
||||||
|
rc_m, out_m, err_m = run([sys.executable, mutant, "--repo", fx,
|
||||||
|
"--write", mdoc])
|
||||||
|
with open(mdoc, encoding="utf-8") as fh:
|
||||||
|
mtext = fh.read()
|
||||||
|
mcell = gate_cell(mtext)
|
||||||
|
rc_c, out_c, err_c = run([sys.executable, mutant, "--repo", fx,
|
||||||
|
"--check", doc])
|
||||||
|
leg("mutant-loses-leg", rc_m == 0 and mcell == "-",
|
||||||
|
"rc=%d gate=%r" % (rc_m, mcell))
|
||||||
|
leg("mutant-doc-drifts", rc_c == 1 and "DRIFT" in out_c,
|
||||||
|
"rc=%d first=%r" % (rc_c, (out_c or err_c).splitlines()[:1]))
|
||||||
|
|
||||||
|
# -- 3. a LEGS that is not data is named, not rendered as `-` -----------
|
||||||
|
fx2 = os.path.join(td, "fx2")
|
||||||
|
build_fixture(fx2, verify_src="LEGS = build_legs()\n")
|
||||||
|
doc2 = os.path.join(fx2, "docs", "TOOLS.md")
|
||||||
|
rc2, out2, err2 = run([sys.executable, fixed, "--repo", fx2,
|
||||||
|
"--write", doc2])
|
||||||
|
with open(doc2, encoding="utf-8") as fh:
|
||||||
|
text2 = fh.read()
|
||||||
|
named = "could not be read as data" in text2
|
||||||
|
rc2c, out2c, _ = run([sys.executable, fixed, "--repo", fx2,
|
||||||
|
"--check", doc2])
|
||||||
|
leg("unreadable-named", rc2 == 0 and named and gate_cell(text2) == "-",
|
||||||
|
"rc=%d named=%s gate=%r" % (rc2, named, gate_cell(text2)))
|
||||||
|
leg("unreadable-still-checks", rc2c == 0,
|
||||||
|
"rc=%d (the document states the failure, so it is current)" % rc2c)
|
||||||
|
|
||||||
|
# -- 4. the real tree: the column is populated and the map is current ---
|
||||||
|
rdoc = os.path.join(td, "TOOLS.real.md")
|
||||||
|
rc3, out3, err3 = run([sys.executable, TOOL_MAP, "--repo", REPO,
|
||||||
|
"--rev", "WORKTREE", "--write", rdoc])
|
||||||
|
told = re.search(r"(\d+) gate legs", out3 or "")
|
||||||
|
legs_n = int(told.group(1)) if told else -1
|
||||||
|
rc_h, out_h, err_h = run([sys.executable, TOOL_MAP, "--repo", REPO,
|
||||||
|
"--check", os.path.join(REPO, "docs", "TOOLS.md")])
|
||||||
|
rc_w, out_w, err_w = run([sys.executable, TOOL_MAP, "--repo", REPO,
|
||||||
|
"--rev", "WORKTREE", "--check",
|
||||||
|
os.path.join(REPO, "docs", "TOOLS.md")])
|
||||||
|
leg("real-tree-legs", rc3 == 0 and legs_n == 10, "rc=%d legs=%d" % (rc3, legs_n))
|
||||||
|
leg("real-tree-current", rc_h == 0 and rc_w == 0,
|
||||||
|
"HEAD rc=%d WORKTREE rc=%d %s" % (rc_h, rc_w, (out_h or err_h).strip()[:80]))
|
||||||
|
|
||||||
|
# -- 5. the tool's own cells -------------------------------------------
|
||||||
|
rc4, out4, err4 = run([sys.executable, TOOL_MAP, "--repo", REPO, "--selftest"])
|
||||||
|
m = re.search(r"selftest: (\d+) cells, (\d+) passed, (\d+) failed", out4 or "")
|
||||||
|
ok4 = bool(m) and m.group(1) == m.group(2) == "23" and m.group(3) == "0"
|
||||||
|
leg("tool-selftest", rc4 == 0 and ok4,
|
||||||
|
"rc=%d %s" % (rc4, m.group(0) if m else (err4 or out4).strip()[:90]))
|
||||||
|
|
||||||
|
print("q132: %d leg(s), %d fault -- %s"
|
||||||
|
% (COUNT[0], len(FAIL), "OK" if not FAIL else "FAILED"))
|
||||||
|
return 0 if not FAIL else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
Reference in new issue
Block a user