volt: verify gains a seventh leg -- doc-commands, graded at WORKTREE
The gate had six legs and none of them read a docstring's own command line.
doc_command_check.py existed since 838750c but only as a tool: nothing on the
gate ran it, so a documented flag its parser never learned could land and
nobody read it. This is that leg.
doc-commands tools/doc_command_check.py --rev WORKTREE --fail
WORKTREE, not HEAD: the commit window is where a docstring is
edited. Measured at the worktree this lands in: 67 commands
over 19 oracle tools, 67 OK, rc 0.
doc_command_check.py learns --rev WORKTREE: tracked paths from `git ls-files`
(an untracked tools/x.py is not part of the commit this mode grades), bytes
from disk. Outside a work tree it is rc 2, not an empty clean run. Selftest
15 -> 18 cells: one document edited on disk but not committed is rc 0 at HEAD
and rc 1 at WORKTREE, and the flag it names is the edited one.
Three tools stopped defaulting to a fixed path on this host. verify.py,
selftest_sweep.py, write_probe.py, citation_resolve_check.py,
path_claim_check.py and doc_census_check.py now resolve DEFAULT_REPO from the
file's own location, so a bare run in a clone grades the clone.
Three selftests refuse unreadable input at rc 2 instead of reporting it as
failed content: tool_map (not a work tree), write_probe (no table), and
citation_resolve_check (no corpus). Each cell below them graded `repo`, so an
absent tree read as "your table is wrong".
Evidence, all measured on this tree before the commit:
verify.py --selftest 55 cells, 55 passed (docstring says 55)
doc_command_check.py --selftest 18 cells, 18 passed (baseline pinned 18)
tool_map.py --repo . --rev WORKTREE --check docs/TOOLS.md rc 0
verify.py --repo . 7 legs, 6 pass, 1 fault -- selftest-sweep,
tool_map's drift cell, which renders HEAD
and so cannot pass inside a commit window
tools/selftest_baseline.tsv re-pins doc_command_check.py 15 -> 18 cells.
This commit is contained in:
1 parent
af32c34762
commit
c3d70567a6
10 files changed
+192
-23
No files matched your search
@@ -50,6 +50,13 @@ revision: a command documented at `R` is checked against the tool at `R`. The
|
||||
default is `HEAD`. A revision that does not materialise is rc 2, never an empty
|
||||
clean run.
|
||||
|
||||
`--rev WORKTREE` is not a revision: it reads the files on disk and lists them
|
||||
from the INDEX, so it grades the tree you are about to commit. `HEAD` cannot
|
||||
see an uncommitted docstring, which is the window a lane writes in -- the same
|
||||
blind spot `tool_map.py --rev WORKTREE` exists for. The two are measured
|
||||
apart in the selftest: one document, edited on disk but not committed, is
|
||||
rc 0 at `HEAD` and rc 1 at `WORKTREE`.
|
||||
|
||||
EXIT CODES (the tools/ contract)
|
||||
0 every documented command was adjudicated and none was rejected
|
||||
1 at least one documented command is a fault (FLAG_NOT_ACCEPTED, TOOL_MISSING)
|
||||
@@ -59,10 +66,12 @@ EXIT CODES (the tools/ contract)
|
||||
USAGE
|
||||
python3 doc_command_check.py --repo . [--rev REV] [--tools a.py,b.py]
|
||||
[--json OUT] [--fail]
|
||||
python3 doc_command_check.py --selftest # 15 cells, oracle and verdicts
|
||||
python3 doc_command_check.py --selftest # 18 cells, oracle and verdicts
|
||||
"""
|
||||
import argparse
|
||||
import ast
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
@@ -86,6 +95,11 @@ OPT_RE = re.compile(r"(?<![\w-])(--[A-Za-z][\w-]*|-[A-Za-z])(?![\w-])")
|
||||
|
||||
HELP_TIMEOUT = 60
|
||||
|
||||
# Not a revision: read the files on disk, list them from the index. The same
|
||||
# spelling `tool_map.py` uses, for the same reason -- a commit window is
|
||||
# otherwise ungraded and the defect lands at HEAD with nobody having read it.
|
||||
WORKTREE = "WORKTREE"
|
||||
|
||||
|
||||
class CannotGrade(Exception):
|
||||
"""Input this tool could not read or could not recognise: rc 2, never rc 1.
|
||||
@@ -97,9 +111,17 @@ class CannotGrade(Exception):
|
||||
|
||||
def tracked_tools(repo, rev):
|
||||
"""(rc, stdout) of `git ls-tree` -> the tool paths in the tree at `rev`."""
|
||||
proc = subprocess.run(
|
||||
["git", "-C", repo, "ls-tree", "-r", "--name-only", rev, "--", "tools"],
|
||||
capture_output=True, text=True)
|
||||
if rev == WORKTREE:
|
||||
# The index, not the disk: an untracked `tools/x.py` is not part of the
|
||||
# commit this mode grades, and including it would grade a file that is
|
||||
# about to be left behind. `git ls-files` fails outside a work tree,
|
||||
# which is the rc 2 this mode owes a caller who ran it anywhere else.
|
||||
proc = subprocess.run(["git", "-C", repo, "ls-files", "--", "tools"],
|
||||
capture_output=True, text=True)
|
||||
else:
|
||||
proc = subprocess.run(
|
||||
["git", "-C", repo, "ls-tree", "-r", "--name-only", rev, "--", "tools"],
|
||||
capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
raise CannotGrade("cannot read repo: %s at %s: %s"
|
||||
% (repo, rev, (proc.stderr or "").strip()[:200]))
|
||||
@@ -109,6 +131,18 @@ def tracked_tools(repo, rev):
|
||||
|
||||
def blob(repo, rev, path):
|
||||
"""The bytes of `path` at `rev`, or None when the tree does not hold it."""
|
||||
if rev == WORKTREE:
|
||||
try:
|
||||
with open(os.path.join(repo, *path.split("/")),
|
||||
"r", encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except OSError as exc:
|
||||
raise CannotGrade("cannot read %s at WORKTREE: %s" % (path, exc))
|
||||
except UnicodeDecodeError as exc:
|
||||
raise CannotGrade("cannot read %s at WORKTREE: not UTF-8: %s"
|
||||
% (path, exc))
|
||||
proc = subprocess.run(["git", "-C", repo, "show", "%s:%s" % (rev, path)],
|
||||
capture_output=True)
|
||||
if proc.returncode != 0:
|
||||
@@ -294,6 +328,58 @@ def selftest():
|
||||
harvested = set(OPT_RE.findall("usage: x [-h] [--fail] [--json JSON]\n"))
|
||||
cell("bracketed usage options are harvested", harvested >= {"-h", "--fail", "--json"})
|
||||
|
||||
# WORKTREE: the tree you are about to commit, not the last commit. One
|
||||
# document, edited on disk and NOT committed: HEAD reads the blob (clean),
|
||||
# WORKTREE reads the disk (the documented flag its parser never learned).
|
||||
with tempfile.TemporaryDirectory(prefix="doc-cmd-wt-") as td:
|
||||
repo = os.path.join(td, "repo")
|
||||
os.makedirs(os.path.join(repo, "tools"))
|
||||
tool = os.path.join(repo, "tools", "oracle_tool.py")
|
||||
|
||||
def write_doc(extra=""):
|
||||
with open(tool, "w", encoding="utf-8") as fh:
|
||||
fh.write("#!/usr/bin/env python3\n"
|
||||
'"""USAGE\n'
|
||||
" python3 oracle_tool.py --present\n"
|
||||
+ extra +
|
||||
'"""\n'
|
||||
"import argparse\n"
|
||||
"ap = argparse.ArgumentParser()\n"
|
||||
"ap.add_argument('--present')\n"
|
||||
"ap.parse_args()\n")
|
||||
|
||||
def git(*args):
|
||||
subprocess.run(["git", "-C", repo] + list(args),
|
||||
capture_output=True, text=True)
|
||||
|
||||
write_doc()
|
||||
git("init", "-q")
|
||||
git("add", "-A")
|
||||
git("-c", "user.name=selftest", "-c", "user.email=selftest@invalid",
|
||||
"-c", "commit.gpgsign=false", "commit", "-q", "-m", "fixture")
|
||||
write_doc(" python3 oracle_tool.py --gone-after-edit\n")
|
||||
|
||||
head_rep = census(repo, "HEAD")
|
||||
wt_rep = census(repo, WORKTREE)
|
||||
cell("WORKTREE reads the document on disk, HEAD reads the blob at HEAD",
|
||||
[r["verdict"] for r in wt_rep["rows"]] == ["OK", "FLAG_NOT_ACCEPTED"]
|
||||
and [r["verdict"] for r in head_rep["rows"]] == ["OK"],
|
||||
([r["verdict"] for r in head_rep["rows"]],
|
||||
[r["verdict"] for r in wt_rep["rows"]]))
|
||||
cell("the uncommitted command's flag is the one named",
|
||||
[r["missing"] for r in wt_rep["rows"] if r["missing"]] ==
|
||||
[["--gone-after-edit"]],
|
||||
[r["missing"] for r in wt_rep["rows"] if r["missing"]])
|
||||
|
||||
outsider = os.path.join(td, "not-a-repo")
|
||||
os.makedirs(outsider)
|
||||
buf = io.StringIO()
|
||||
with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(buf):
|
||||
rc_nogit = run(["--repo", outsider, "--rev", WORKTREE, "--fail"])
|
||||
cell("WORKTREE outside a git work tree is rc 2, not a clean run",
|
||||
rc_nogit == 2 and "cannot read repo" in buf.getvalue(),
|
||||
(rc_nogit, buf.getvalue().strip()[:80]))
|
||||
|
||||
# verdicts
|
||||
rep = {"rev": "HEAD", "oracles": 1, "rows": [
|
||||
{"doc": "tools/a.py", "target": "tools/a.py", "verdict": "OK",
|
||||
|
||||
Reference in new issue
Block a user