417 lines
16 KiB
Python
417 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""q169 compile sweep -- every tracked `.py` must PARSE, in the worktree and in
|
|
the commit.
|
|
|
|
WHY THIS EXISTS
|
|
`wizard-work/spark/verify_dir_analyzer.py` sat in the worktree with
|
|
|
|
def _check(cond, msg):
|
|
\"\"\"A failure path `-O` cannot compile away ...\"\"\"
|
|
from pathlib import Path
|
|
if not cond:
|
|
|
|
-- a `from pathlib import Path` line inserted inside the function body by a
|
|
mechanical path refactor, at column 0 between the docstring and the `if`.
|
|
`python3 -m py_compile wizard-work/spark/verify_dir_analyzer.py` answered
|
|
**rc 1**, `Sorry: IndentationError: unexpected indent (verify_dir_analyzer.py,
|
|
line 11)`. In the same worktree `python3 tools/verify.py --fast` read **12
|
|
legs, 11 pass, 0 fault, 1 skipped**, `gate: OK`, **rc 0**.
|
|
|
|
The two readings are both true and that is the gap. Every leg that parses
|
|
Python parses a POPULATION, not the tree: `import_resolve_check.py` grades the
|
|
committed test files (194 at the time), while `git ls-files '*.py'` held
|
|
**1399**. A tracked file the gate never parses can be unimportable, and no leg
|
|
says so -- the class this sweep names.
|
|
|
|
WHAT IT MEASURES -- one row per tracked `*.py`, at one revision
|
|
WORKTREE (default) the files on disk, listed by `git ls-files` -- the tree
|
|
about to be committed, which is what the tick writes.
|
|
<rev> the commit: `git ls-tree -r` names the population and
|
|
`git cat-file --batch` reads the bytes, so no export is
|
|
materialised and the reading is of the commit itself.
|
|
|
|
A file is a FAULT when `ast.parse` cannot read it (SyntaxError, a NUL byte, a
|
|
decode failure) or when a file git tracks at the rev is GONE from the
|
|
worktree -- a file that is not there is not in the tree about to be committed.
|
|
|
|
EXIT CODES (the tools/ contract)
|
|
0 every tracked .py parses
|
|
1 at least one does not (or is missing from the worktree)
|
|
2 could not grade: not a git repository, unknown rev, no tracked .py, git absent
|
|
|
|
Stdlib only; nothing here writes to the tree (`--json` writes the named file).
|
|
"""
|
|
import argparse
|
|
import ast
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
REV_WORKTREE = "WORKTREE"
|
|
FAULT_LIMIT = 40
|
|
|
|
|
|
def run_git(repo, *args, **kw):
|
|
return subprocess.run(["git", "-C", repo] + list(args), capture_output=True,
|
|
text=True, **kw)
|
|
|
|
|
|
def resolve_repo(repo):
|
|
"""(topdir, None) or (None, reason). A refusal names what it read."""
|
|
if not os.path.isdir(repo):
|
|
return None, "could not grade -- %s is not a directory" % repo
|
|
try:
|
|
r = run_git(repo, "rev-parse", "--show-toplevel")
|
|
except OSError as exc:
|
|
return None, "could not grade -- git is not runnable (%s)" % exc
|
|
if r.returncode != 0:
|
|
return None, ("could not grade -- %s is not a git repository (%s)"
|
|
% (repo, (r.stderr or "").strip().splitlines()[-1:][0]
|
|
if r.stderr.strip() else "no reason given"))
|
|
return r.stdout.strip(), None
|
|
|
|
|
|
def tracked(top, rev):
|
|
"""(paths, None) at `rev`, or ([], reason). Sorted, NUL-split, no duplicates.
|
|
|
|
The pathspec is applied here, in Python, and not handed to `ls-tree`:
|
|
`git ls-tree -r HEAD -- '*.py'` matches NOTHING (measured, 0 rows against a
|
|
tree holding 1400), so a sweep built on it would report a clean tree by
|
|
grading no files at all -- the exact shape of a false green this file names.
|
|
|
|
Read as bytes and decoded with `surrogateescape`, so a path that is not
|
|
UTF-8 survives the round trip back to `read_rev`'s request bytes. They are
|
|
sorted as `str`, which is only meaningful for the display order.
|
|
"""
|
|
if rev == REV_WORKTREE:
|
|
r = subprocess.run(["git", "-C", top, "ls-files", "-z"], capture_output=True)
|
|
else:
|
|
r = subprocess.run(
|
|
["git", "-C", top, "ls-tree", "-r", "-z", "--name-only", rev],
|
|
capture_output=True)
|
|
if r.returncode != 0:
|
|
why = r.stderr.decode("utf-8", "replace").strip().splitlines()
|
|
return [], ("could not grade -- revision %s: %s"
|
|
% (rev, why[-1] if why else "git refused"))
|
|
names = r.stdout.decode("utf-8", "surrogateescape").split("\0")
|
|
return sorted({p for p in names if p.endswith(".py")}), None
|
|
|
|
|
|
def read_worktree(top, rel):
|
|
path = os.path.join(top, rel)
|
|
if not os.path.isfile(path):
|
|
return None, "MISSING from the worktree"
|
|
try:
|
|
with open(path, "rb") as fh:
|
|
return fh.read(), None
|
|
except OSError as exc:
|
|
return None, "unreadable (%s)" % exc
|
|
|
|
|
|
def read_rev(top, rev, rels):
|
|
"""{rel: bytes} for every rel at `rev`, read in one `git cat-file --batch`.
|
|
|
|
Requests are NUL-delimited (`-z`), not newline-delimited. A tracked path may
|
|
itself contain a newline -- this repository carried one at `99eba1e`,
|
|
`wizard-work/beam/CHECK\\ncontent">\\n#!...`, 218 bytes -- and a
|
|
newline-delimited request splits it into four extra requests, so every
|
|
later response is mapped to the wrong path. Measured at `99eba1e` over the
|
|
773 tracked `*.py`: the newline form recovered 766 and reported 7 real,
|
|
parseable files as `UNREADABLE`; the `-z` form recovers 773, including the
|
|
newline path byte-exact (185 B). A response shorter than its own header
|
|
size means the framing drifted, so the blob is dropped rather than graded
|
|
on truncated bytes -- a fact, not a silent pass.
|
|
"""
|
|
out = {}
|
|
if not rels:
|
|
return out
|
|
req = b"".join(rev.encode("utf-8", "surrogateescape") + b":" +
|
|
rel.encode("utf-8", "surrogateescape") + b"\0" for rel in rels)
|
|
r = subprocess.run(["git", "-C", top, "cat-file", "--batch", "-z"],
|
|
input=req, capture_output=True)
|
|
if r.returncode != 0:
|
|
return out
|
|
buf, i = r.stdout, 0
|
|
for rel in rels:
|
|
nl = buf.find(b"\n", i)
|
|
if nl < 0:
|
|
break
|
|
header = buf[i:nl].decode("utf-8", "replace")
|
|
i = nl + 1
|
|
parts = header.split()
|
|
if len(parts) < 3 or parts[-1] == "missing":
|
|
continue
|
|
try:
|
|
size = int(parts[2])
|
|
except ValueError:
|
|
continue
|
|
blob = buf[i:i + size]
|
|
i += size + 1
|
|
if len(blob) == size:
|
|
out[rel] = blob
|
|
return out
|
|
|
|
|
|
def grade_bytes(src, rel):
|
|
"""(line, why) or None. `ast.parse` is the question: can Python read it."""
|
|
import warnings
|
|
try:
|
|
with warnings.catch_warnings():
|
|
# An invalid escape sequence is a SyntaxWarning, not a parse
|
|
# failure: the file still compiles, and this sweep's question is
|
|
# "can Python read it", not "is it lint-clean".
|
|
warnings.simplefilter("ignore", SyntaxWarning)
|
|
ast.parse(src, filename=rel)
|
|
return None
|
|
except SyntaxError as exc:
|
|
return (exc.lineno or 0, "%s: %s" % (type(exc).__name__, exc.msg))
|
|
except (ValueError, UnicodeDecodeError) as exc:
|
|
return (0, "%s: %s" % (type(exc).__name__, exc))
|
|
|
|
|
|
def sweep(top, rev):
|
|
"""(files, faults, reason). faults: [{"path","line","why"}]."""
|
|
rels, why = tracked(top, rev)
|
|
if why:
|
|
return 0, [], why
|
|
if not rels:
|
|
return 0, [], "could not grade -- no tracked *.py at %s" % (
|
|
"the worktree" if rev == REV_WORKTREE else rev)
|
|
faults = []
|
|
if rev == REV_WORKTREE:
|
|
for rel in rels:
|
|
src, err = read_worktree(top, rel)
|
|
if err:
|
|
faults.append({"path": rel, "line": 0, "why": err})
|
|
continue
|
|
bad = grade_bytes(src, rel)
|
|
if bad:
|
|
faults.append({"path": rel, "line": bad[0], "why": bad[1]})
|
|
else:
|
|
blobs = read_rev(top, rev, rels)
|
|
for rel in rels:
|
|
if rel not in blobs:
|
|
faults.append({"path": rel, "line": 0,
|
|
"why": "UNREADABLE at %s" % rev})
|
|
continue
|
|
bad = grade_bytes(blobs[rel], rel)
|
|
if bad:
|
|
faults.append({"path": rel, "line": bad[0], "why": bad[1]})
|
|
return len(rels), faults, None
|
|
|
|
|
|
def shown(path):
|
|
"""A path as ONE printable line: newlines and undecodable bytes escaped.
|
|
|
|
A fault line is read by a human and by `grep`; the tracked path at
|
|
`99eba1e` contains four newlines, so printing it raw turned one fault into
|
|
eight lines of bash source in the middle of the report.
|
|
"""
|
|
return (path.encode("utf-8", "backslashreplace")
|
|
.decode("utf-8", "replace").replace("\n", "\\n"))
|
|
|
|
|
|
def report(rev, files, faults, quiet):
|
|
where = "the worktree" if rev == REV_WORKTREE else rev
|
|
if not quiet:
|
|
for f in faults[:FAULT_LIMIT]:
|
|
print("FAULT %s:%d %s" % (shown(f["path"]), f["line"], f["why"]))
|
|
if len(faults) > FAULT_LIMIT:
|
|
print(" ... %d more fault(s) not printed" % (len(faults) - FAULT_LIMIT))
|
|
print("compile_sweep: %d tracked .py at %s, %d fault(s)"
|
|
% (files, where, len(faults)))
|
|
return 1 if faults else 0
|
|
|
|
|
|
# ------------------------------------------------------------------- selftest
|
|
|
|
def _git(repo, *args):
|
|
return subprocess.run(["git", "-C", repo] + list(args), capture_output=True,
|
|
text=True)
|
|
|
|
|
|
def _write(path, text):
|
|
with open(path, "w", encoding="utf-8") as fh:
|
|
fh.write(text)
|
|
|
|
|
|
def _commit(repo, msg="fixture"):
|
|
_git(repo, "add", "-A")
|
|
_git(repo, "-c", "user.name=selftest", "-c", "user.email=selftest@invalid",
|
|
"-c", "commit.gpgsign=false", "commit", "-q", "-m", msg)
|
|
|
|
|
|
GOOD = "def f():\n return 1\n"
|
|
BROKEN = ("def _check(cond, msg):\n"
|
|
" \"\"\"A failure path.\"\"\"\n"
|
|
"from pathlib import Path\n"
|
|
" if not cond:\n"
|
|
" raise RuntimeError(msg)\n")
|
|
|
|
|
|
def selftest():
|
|
cells = passed = 0
|
|
failures = []
|
|
|
|
def check(name, got, want):
|
|
nonlocal cells, passed
|
|
cells += 1
|
|
if got == want:
|
|
passed += 1
|
|
else:
|
|
failures.append("%s: got %r want %r" % (name, got, want))
|
|
|
|
with tempfile.TemporaryDirectory(prefix="q169-selftest-") as td:
|
|
def repo(name, files, do_commit=True):
|
|
p = os.path.join(td, name)
|
|
os.makedirs(p)
|
|
for rel, text in files.items():
|
|
full = os.path.join(p, rel)
|
|
os.makedirs(os.path.dirname(full) or p, exist_ok=True)
|
|
_write(full, text)
|
|
_git(p, "init", "-q", ".")
|
|
if do_commit:
|
|
_commit(p)
|
|
return p
|
|
|
|
# 1 -- a clean tree is clean
|
|
a = repo("clean", {"a.py": GOOD, "pkg/b.py": GOOD})
|
|
files, faults, why = sweep(a, REV_WORKTREE)
|
|
check("1 files counted", files, 2)
|
|
check("1 no fault", (faults, why), ([], None))
|
|
|
|
# 2 -- one unparseable file names itself and its line
|
|
b = repo("broken", {"a.py": GOOD, "bad.py": BROKEN})
|
|
files, faults, why = sweep(b, REV_WORKTREE)
|
|
check("2 one fault", len(faults), 1)
|
|
check("2 names path", faults[0]["path"], "bad.py")
|
|
check("2 names a line", faults[0]["line"] > 0, True)
|
|
|
|
# 3 -- the rev is the axis: HEAD clean, worktree broken
|
|
c = repo("revaxis", {"a.py": GOOD})
|
|
_write(os.path.join(c, "a.py"), BROKEN)
|
|
_, wf, _ = sweep(c, REV_WORKTREE)
|
|
_, hf, _ = sweep(c, "HEAD")
|
|
check("3 worktree faults", len(wf), 1)
|
|
check("3 HEAD clean", (len(hf), hf), (0, []))
|
|
|
|
# 4 -- and the other way: the break is committed, the worktree fixed
|
|
d = repo("revaxis2", {"a.py": BROKEN})
|
|
_write(os.path.join(d, "a.py"), GOOD)
|
|
_, wf2, _ = sweep(d, REV_WORKTREE)
|
|
_, hf2, _ = sweep(d, "HEAD")
|
|
check("4 worktree clean", len(wf2), 0)
|
|
check("4 HEAD faults", [f["path"] for f in hf2], ["a.py"])
|
|
|
|
# 5 -- a tracked file deleted from the worktree is a fault, not a skip
|
|
e = repo("gone", {"a.py": GOOD, "b.py": GOOD})
|
|
os.unlink(os.path.join(e, "b.py"))
|
|
_, gf, _ = sweep(e, REV_WORKTREE)
|
|
check("5 missing is a fault", [(f["path"], f["why"][:7]) for f in gf],
|
|
[("b.py", "MISSING")])
|
|
|
|
# 6 -- a directory that is not a repository is refused, not graded
|
|
plain = os.path.join(td, "plain")
|
|
os.makedirs(plain)
|
|
top, why6 = resolve_repo(plain)
|
|
check("6 refused", (top, "not a git repository" in (why6 or "")),
|
|
(None, True))
|
|
|
|
# 7 -- a repository with no python is a refusal too (rc 2)
|
|
g = repo("nopy", {"README.md": "hi\n"})
|
|
files7, _, why7 = sweep(g, REV_WORKTREE)
|
|
check("7 no .py refused", (files7, "no tracked *.py" in (why7 or "")),
|
|
(0, True))
|
|
|
|
# 8 -- an unknown rev is a refusal, never a clean zero
|
|
_, _, why8 = sweep(a, "no-such-rev")
|
|
check("8 bad rev refused", "revision no-such-rev" in (why8 or ""), True)
|
|
|
|
# 9 -- --json carries the fault and the rc
|
|
jpath = os.path.join(td, "rows.json")
|
|
rc9 = main(["--repo", b, "--json", jpath, "--quiet"])
|
|
with open(jpath) as fh:
|
|
doc = json.load(fh)
|
|
check("9 json rc", (rc9, doc["rc"]), (1, 1))
|
|
check("9 json fault", [f["path"] for f in doc["faults"]], ["bad.py"])
|
|
|
|
# 10 -- an unwritable --json target is rc 2, legible, no traceback
|
|
rc10 = main(["--repo", a, "--quiet", "--json",
|
|
os.path.join(td, "no-such-dir", "x.json")])
|
|
check("10 unwritable json rc 2", rc10, 2)
|
|
|
|
# 11 -- the broken reading is rc 1 through main(), the clean one rc 0
|
|
check("11 main rc", (main(["--repo", a, "--quiet"]),
|
|
main(["--repo", b, "--quiet"])), (0, 1))
|
|
|
|
# 12 -- a tracked path containing a NEWLINE is read at the rev, and the
|
|
# file sorted after it is still read: the batch framing must be NUL.
|
|
# The pre-repair reader (newline-delimited requests) split this path
|
|
# into extra requests and lost `z.py`, which is what this cell kills.
|
|
h = repo("newlinepath", {"a.py": GOOD})
|
|
_write(os.path.join(h, "0weird\nname.py"), GOOD)
|
|
_write(os.path.join(h, "z.py"), GOOD)
|
|
_commit(h, "newline path")
|
|
files12, faults12, why12 = sweep(h, "HEAD")
|
|
check("12 every file read", (files12, why12), (3, None))
|
|
check("12 no fault", [(f["path"], f["why"]) for f in faults12], [])
|
|
check("12 newline path recovered",
|
|
[p for p in read_rev(h, "HEAD", tracked(h, "HEAD")[0])
|
|
if "\n" in p], ["0weird\nname.py"])
|
|
|
|
# 13 -- and a fault on such a path prints as ONE line
|
|
i = repo("newlinefault", {"0weird\nname.py": BROKEN, "z.py": GOOD})
|
|
files13, faults13, _ = sweep(i, "HEAD")
|
|
check("13 fault found", (files13, [f["path"] for f in faults13]),
|
|
(2, ["0weird\nname.py"]))
|
|
check("13 one printable line", shown(faults13[0]["path"]),
|
|
"0weird\\nname.py")
|
|
|
|
print("selftest: %d cells, %d passed, %d failed"
|
|
% (cells, passed, len(failures)))
|
|
for f in failures:
|
|
print("FAIL %s" % f)
|
|
return 0 if passed == cells else 1
|
|
|
|
|
|
# ----------------------------------------------------------------------- main
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(
|
|
description="every tracked *.py must parse, at the worktree or a rev")
|
|
ap.add_argument("--repo", default=".",
|
|
help="repository to grade (default .)")
|
|
ap.add_argument("--rev", default=REV_WORKTREE,
|
|
help="WORKTREE (default) or a git revision")
|
|
ap.add_argument("--json", metavar="PATH", help="write the rows here")
|
|
ap.add_argument("--quiet", action="store_true", help="summary line only")
|
|
ap.add_argument("--selftest", action="store_true", help="run the fixtures")
|
|
a = ap.parse_args(argv)
|
|
if a.selftest:
|
|
return selftest()
|
|
|
|
top, why = resolve_repo(a.repo)
|
|
if why:
|
|
print(why, file=sys.stderr)
|
|
return 2
|
|
files, faults, why = sweep(top, a.rev)
|
|
if why:
|
|
print(why, file=sys.stderr)
|
|
return 2
|
|
rc = report(a.rev, files, faults, a.quiet)
|
|
if a.json:
|
|
try:
|
|
with open(a.json, "w") as fh:
|
|
json.dump({"repo": top, "rev": a.rev, "files": files,
|
|
"faults": faults, "rc": rc}, fh, indent=1)
|
|
except OSError as exc:
|
|
print("cannot write --json %s: %s" % (a.json, exc), file=sys.stderr)
|
|
return 2
|
|
return rc
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|