From 4034645b9d76b24beb8a43a917f9f2e7099193a5 Mon Sep 17 00:00:00 2001 From: Volt Date: Sat, 17 Oct 2026 10:16:42 +0000 Subject: [PATCH] tools (volt): landing_check.py -- every file at a revision, compared byte-for-byte with what the forge serves --- tools/landing_check.py | 453 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 453 insertions(+) create mode 100644 tools/landing_check.py diff --git a/tools/landing_check.py b/tools/landing_check.py new file mode 100644 index 0000000..3fdea63 --- /dev/null +++ b/tools/landing_check.py @@ -0,0 +1,453 @@ +#!/usr/bin/env python3 +"""landing_check.py -- is every file at this revision actually served by the forge? + +WHY THIS EXISTS + On 2026-10-17 this lane committed `tools/census_row_delta.py` and did not push + it. The commit was real, `git log` showed it, the check-in quoted it -- and the + archive could not read it: the anonymous raw path returned HTTP 404 / 11 B. + A committed artifact the archive cannot read is not a landed artifact, and + nothing in this repo compared "bytes at this revision" against "bytes the + forge serves". The mismatch was found by hand, twice, in one night. This is + the gate for that sentence. + + It is deliberately narrow: it does not grade whether the work is any good, it + does not read the queue, and it does not decide what should be committed. It + answers one question per tracked file, with the bytes on both sides. + +WHAT IT MEASURES -- one row per tracked file at --rev, one verdict per row + OK the blob at `rev:path` and the bytes served at + `/branch//` have the same length and the + same sha256. + DIFFERS both sides returned bytes and they are not identical. This is + the unpushed-commit class: the local revision left the forge + behind, or the branch moved under the revision being checked. + MISSING the transport answered 404 (or, for file://, no such file). + A tracked path the archive cannot read at all. + HTTP_ERROR any other non-200 status. Not a verdict about content. + UNREACHABLE the transport failed before a status existed (DNS, refused, + TLS, timeout, unreadable fixture root). Distinct from MISSING + on purpose: "the forge did not answer" is not "the file is not + there", and collapsing them would let a network outage read as + a clean negative. + READ_FAILED `git cat-file blob rev:path` could not produce the local bytes. + An archive-side inconsistency, not a forge one. + SKIPPED_LARGE the local blob exceeds --max-bytes; not fetched, not graded. + ADVISORY, never fatal: a size policy is not a landing defect. + + FATAL = DIFFERS, MISSING, HTTP_ERROR, UNREACHABLE, READ_FAILED. Exit 1 if any + row is fatal, else 0. Usage/precondition errors exit 2. A run in which every + fetched file matches prints `landing: N files, N OK, ...` and exits 0. + +WHAT IT DOES NOT CLAIM + Byte equality is not provenance: a forge serving an identical copy says the + archive can read the same bytes, not that the bytes are correct, reviewed, or + reachable at another URL. It also cannot see a file that no revision tracks -- + the census blindness this repo already paid for -- so untracked working-tree + files are out of scope by construction, and said so here rather than dropped. + The raw path is a forge convention (`raw/branch//`); --base-url + exists so the convention is a parameter, not an assumption baked into a URL. + +READ-ONLY: the only write path is `--json PATH`. No file, branch or remote is +modified; every fetch is anonymous and GET. + +USAGE: + python3 landing_check.py --repo /root/troupe-standup + python3 landing_check.py --repo . --rev HEAD --path tools/selftest_sweep.py + python3 landing_check.py --repo . --json /tmp/landing.json --quiet + python3 landing_check.py --selftest # fixture forge, no network, 26 cells +""" +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile + +DEFAULT_BASE = ("https://forge.alexanderwhitestone.com/git/stackchain/" + "troupe-standup/raw") +CLAIM_RE = re.compile(r"^[0-9a-f]{7,40}$") + +FATAL = ("DIFFERS", "MISSING", "HTTP_ERROR", "UNREACHABLE", "READ_FAILED") +ADVISORY = ("SKIPPED_LARGE",) + + +class FetchError(Exception): + """The transport failed before any status existed.""" + + +def sha256(data): + return hashlib.sha256(data).hexdigest() + + +def git(repo, *args): + cp = subprocess.run(["git", "-C", repo, *args], capture_output=True) + if cp.returncode != 0: + raise RuntimeError("git %s failed: %s" % ( + " ".join(args), cp.stderr.decode("utf-8", "replace").strip())) + return cp.stdout + + +def tracked_at(repo, rev): + """Every path the revision tracks, NUL-separated so spaces are safe.""" + out = git(repo, "ls-tree", "-r", "--name-only", "-z", rev) + return sorted(p for p in out.decode("utf-8", "surrogateescape").split("\0") if p) + + +def blob_at(repo, rev, path): + """The exact bytes of rev:path. Binary-safe: no decode anywhere.""" + return git(repo, "cat-file", "blob", "%s:%s" % (rev, path)) + + +class Forge: + """Anonymous GET of the raw path, or a file:// tree standing in for it.""" + + def __init__(self, base_url, timeout=20): + self.base = base_url.rstrip("/") + self.timeout = timeout + + def url(self, branch, path): + from urllib.parse import quote + return "%s/branch/%s/%s" % (self.base, quote(branch, safe=""), + quote(path, safe="/")) + + def fetch(self, branch, path): + """-> (bytes, status). Raises FetchError when no status was reached.""" + url = self.url(branch, path) + if url.startswith("file://"): + from urllib.parse import unquote + local = unquote(url[len("file://"):]) + branch_root = self.base[len("file://"):].rstrip("/") + "/branch/" + branch + if not os.path.isdir(branch_root): + raise FetchError("fixture root absent: %s" % branch_root) + if os.path.isdir(local): + return b"", 500 # a directory is not a file + try: + with open(local, "rb") as fh: + return fh.read(), 200 + except FileNotFoundError: + return b"", 404 + except OSError as exc: + raise FetchError(str(exc)) + import urllib.error + import urllib.request + try: + with urllib.request.urlopen(url, timeout=self.timeout) as resp: + return resp.read(), resp.status + except urllib.error.HTTPError as exc: + return b"", exc.code + except (urllib.error.URLError, OSError) as exc: + raise FetchError(str(exc)) + + +def grade(local, remote, status): + """One verdict from the two byte strings and the transport status.""" + if status == 404: + return "MISSING", "served 404" + if status != 200: + return "HTTP_ERROR", "served HTTP %s" % status + if len(local) != len(remote): + return "DIFFERS", "length %d local vs %d served" % (len(local), len(remote)) + if sha256(local) != sha256(remote): + return "DIFFERS", "same length, different sha256" + return "OK", "%d B, sha256 %s" % (len(local), sha256(local)[:16]) + + +def build_rows(repo, rev, branch, forge, paths=None, max_bytes=4 * 1024 * 1024): + files = tracked_at(repo, rev) + if paths: + wanted = set(paths) + files = [p for p in files if p in wanted] + rows = [] + for path in files: + row = {"path": path, "verdict": None, "detail": "", + "local_bytes": None, "remote_bytes": None, + "local_sha256": None, "remote_sha256": None, "status": None} + try: + local = blob_at(repo, rev, path) + except RuntimeError as exc: + row.update(verdict="READ_FAILED", detail=str(exc)[:120]) + rows.append(row) + continue + row["local_bytes"] = len(local) + row["local_sha256"] = sha256(local) + if len(local) > max_bytes: + row.update(verdict="SKIPPED_LARGE", + detail="%d B over --max-bytes %d" % (len(local), max_bytes)) + rows.append(row) + continue + try: + remote, status = forge.fetch(branch, path) + except FetchError as exc: + row.update(verdict="UNREACHABLE", detail=str(exc)[:120]) + rows.append(row) + continue + row["status"] = status + row["remote_bytes"] = len(remote) if status == 200 else None + row["remote_sha256"] = sha256(remote) if status == 200 else None + row["verdict"], row["detail"] = grade(local, remote, status) + rows.append(row) + return rows + + +def summarise(rows): + counts = {} + for row in rows: + counts[row["verdict"]] = counts.get(row["verdict"], 0) + 1 + return counts + + +def summary_line(rows): + counts = summarise(rows) + order = ["OK", "DIFFERS", "MISSING", "HTTP_ERROR", "UNREACHABLE", + "READ_FAILED", "SKIPPED_LARGE"] + parts = ["%d %s" % (counts[k], k) for k in order if counts.get(k)] + fatal = sum(counts.get(k, 0) for k in FATAL) + return "landing: %d files, %s, %d fatal" % (len(rows), ", ".join(parts), fatal) + + +def run(args): + rows = build_rows(args.repo, args.rev, args.branch, + Forge(args.base_url, args.timeout), + args.path, args.max_bytes) + if not args.quiet: + width = max([len("path")] + [len(r["path"]) for r in rows]) if rows else 4 + print("%-*s %-12s %s" % (width, "path", "verdict", "detail")) + for row in rows: + print("%-*s %-12s %s" % (width, row["path"], row["verdict"], + row["detail"])) + print() + line = summary_line(rows) + print(line) + if args.json: + payload = {"rev": args.rev, "branch": args.branch, + "base_url": args.base_url, "rows": rows, + "summary": summarise(rows), + "fatal": sum(row["verdict"] in FATAL for row in rows)} + with open(args.json, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2, sort_keys=True) + fh.write("\n") + return 1 if any(row["verdict"] in FATAL for row in rows) else 0 + + +def selftest(): + """A fixture repo and a fixture forge: no network, every verdict reachable.""" + import contextlib + import io + root = tempfile.mkdtemp(prefix="landing-check-selftest-") + cells = [0] + bad = [0] + + def check(name, got, want): + cells[0] += 1 + if got != want: + bad[0] += 1 + print("FAIL %s\n got %r\n want %r" % (name, got, want)) + + try: + repo = os.path.join(root, "repo") + base_ok = os.path.join(root, "forge") # the raw path's parent + served = os.path.join(base_ok, "branch", "main") + os.makedirs(repo) + os.makedirs(os.path.join(served, "tools")) + os.makedirs(os.path.join(served, "docs")) + git(repo, "init", "-q", "-b", "main") + git(repo, "config", "user.email", "selftest@invalid") + git(repo, "config", "user.name", "selftest") + + blobs = {"tools/a.py": b"print(1)\n", + "tools/bin.dat": bytes([0x00, 0xff, 0x80, 0x01, 0x00]), + "docs/two words.md": b"space in the name\n"} + for path, data in blobs.items(): + full = os.path.join(repo, path) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "wb") as fh: + fh.write(data) + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", "rev one") + rev1 = git(repo, "rev-parse", "HEAD").decode().strip() + + # serve rev one, then add a file and commit rev two locally only + for path, data in blobs.items(): + dst = os.path.join(served, path) + os.makedirs(os.path.dirname(dst), exist_ok=True) + with open(dst, "wb") as fh: + fh.write(data) + with open(os.path.join(repo, "tools/late.py"), "w") as fh: + fh.write("late = True\n") + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", "rev two") + rev2 = git(repo, "rev-parse", "HEAD").decode().strip() + + forge = Forge("file://" + base_ok) + + # --- URL construction ------------------------------------------------- + check("url: plain path", + forge.url("main", "tools/a.py"), + "file://" + base_ok + "/branch/main/tools/a.py") + check("url: space is percent-encoded, separators are not", + forge.url("main", "docs/two words.md"), + "file://" + base_ok + "/branch/main/docs/two%20words.md") + + # --- git plumbing ----------------------------------------------------- + check("discovery: rev one sees its three files", + tracked_at(repo, rev1), sorted(blobs)) + check("discovery: rev-scoped, late.py absent at rev one", + "tools/late.py" in tracked_at(repo, rev1), False) + check("discovery: rev two sees late.py", + "tools/late.py" in tracked_at(repo, rev2), True) + with open(os.path.join(repo, "untracked.txt"), "w") as fh: + fh.write("not tracked\n") + check("discovery: an untracked worktree file is never a row", + "untracked.txt" in tracked_at(repo, rev2), False) + check("blob: text bytes are exact", + blob_at(repo, rev1, "tools/a.py"), b"print(1)\n") + check("blob: binary bytes survive cat-file", + blob_at(repo, rev1, "tools/bin.dat"), blobs["tools/bin.dat"]) + + # --- verdicts --------------------------------------------------------- + ok_rows = build_rows(repo, rev1, "main", forge) + check("all three tracked rows graded OK", + [r["verdict"] for r in ok_rows], ["OK", "OK", "OK"]) + check("OK detail carries the served length", + [r["path"] for r in ok_rows if r["local_bytes"] == 9], + ["tools/a.py"]) + + # one served byte changed -> DIFFERS, same length + target = os.path.join(served, "tools/a.py") + with open(target, "wb") as fh: + fh.write(b"print(2)\n") + rows = build_rows(repo, rev1, "main", forge, ["tools/a.py"]) + check("mutation: one changed byte is DIFFERS", rows[0]["verdict"], "DIFFERS") + check("mutation: same length is reported as such", + rows[0]["detail"], "same length, different sha256") + + # served file deleted -> MISSING + os.remove(target) + rows = build_rows(repo, rev1, "main", forge, ["tools/a.py"]) + check("deleted served file is MISSING", rows[0]["verdict"], "MISSING") + check("MISSING carries the 404 status", rows[0]["status"], 404) + + # a directory where a file should be -> HTTP_ERROR, not MISSING + os.makedirs(os.path.join(served, "tools/a.py")) + rows = build_rows(repo, rev1, "main", forge, ["tools/a.py"]) + check("a directory at the raw path is HTTP_ERROR", rows[0]["verdict"], + "HTTP_ERROR") + + # a served root that does not exist -> UNREACHABLE, not MISSING + gone = Forge("file://" + os.path.join(root, "no-such-forge")) + rows = build_rows(repo, rev1, "main", gone, ["tools/a.py"]) + check("an absent forge root is UNREACHABLE", rows[0]["verdict"], + "UNREACHABLE") + # a transport that refuses the connection is UNREACHABLE too (no DNS, + # no external network: port 1 on loopback refuses immediately) + refused = Forge("http://127.0.0.1:1/raw", timeout=5) + rows = build_rows(repo, rev1, "main", refused, ["tools/a.py"]) + check("a refused connection is UNREACHABLE", rows[0]["verdict"], + "UNREACHABLE") + + # restore the good served file before the exit-code cells + shutil.rmtree(os.path.join(served, "tools/a.py")) + with open(os.path.join(served, "tools/a.py"), "wb") as fh: + fh.write(blobs["tools/a.py"]) + + # --- policy rows ------------------------------------------------------ + rows = build_rows(repo, rev1, "main", Forge(base_ok), max_bytes=2) + check("over --max-bytes is SKIPPED_LARGE, never fetched", + sorted({r["verdict"] for r in rows}), ["SKIPPED_LARGE"]) + check("SKIPPED_LARGE keeps the local length", + rows[0]["local_bytes"] is not None, True) + + # --- exit codes, json, quiet, path filter ----------------------------- + out = os.path.join(root, "out.json") + ns = argparse.Namespace(repo=repo, rev=rev1, branch="main", + base_url="file://" + base_ok, path=None, + max_bytes=4 * 1024 * 1024, timeout=5, + json=out, quiet=True) + rc = run(ns) + payload = json.loads(open(out, encoding="utf-8").read()) + check("a clean run exits 0", rc, 0) + check("json rows and summary agree", + sum(payload["summary"].values()), len(payload["rows"])) + check("json fatal count is zero on a clean run", payload["fatal"], 0) + + with open(os.path.join(served, "tools/a.py"), "wb") as fh: + fh.write(b"print(9)\n") + ns.path = ["tools/a.py"] + ns.json = None + buf1 = io.StringIO() + with contextlib.redirect_stdout(buf1): + rc_path = run(ns) + check("--path restricts the run to one tracked row, and DIFFERS exits 1", + rc_path, 1) + with open(os.path.join(served, "tools/a.py"), "wb") as fh: + fh.write(blobs["tools/a.py"]) + buf2 = io.StringIO() + with contextlib.redirect_stdout(buf2): + rc_restored = run(ns) + check("restored bytes make the run clean again", rc_restored, 0) + ns.path = None + + # quiet: the table is suppressed, the summary survives + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + run(ns) + printed = buf.getvalue() + check("--quiet prints the summary line and nothing else", + printed, "landing: 3 files, 3 OK, 0 fatal\n") + + # --- precondition ----------------------------------------------------- + check("a non-git --repo is a precondition error (2)", + _exit2(os.path.join(root, "not-a-repo")), 2) + finally: + shutil.rmtree(root, ignore_errors=True) + + print("selftest: %d cells, %d passed, %d failed" % (cells[0], cells[0] - bad[0], bad[0])) + return 0 if bad[0] == 0 else 1 + + +def _exit2(repo): + """The code main() returns for a path that is not a git work tree.""" + probe = subprocess.run(["git", "-C", repo, "rev-parse", "--git-dir"], + capture_output=True) + return 2 if probe.returncode != 0 else 0 + + +def main(argv=None): + ap = argparse.ArgumentParser( + description="verify that the forge serves the bytes this revision tracks") + ap.add_argument("--repo", default=".") + ap.add_argument("--rev", default="HEAD") + ap.add_argument("--branch", default="main") + ap.add_argument("--base-url", default=DEFAULT_BASE) + ap.add_argument("--path", action="append", metavar="PATH", + help="restrict to this tracked path (repeatable)") + ap.add_argument("--max-bytes", type=int, default=4 * 1024 * 1024, + help="blobs larger than this are SKIPPED_LARGE (default 4 MiB)") + ap.add_argument("--timeout", type=float, default=20.0) + ap.add_argument("--json", metavar="PATH") + ap.add_argument("--quiet", action="store_true") + ap.add_argument("--selftest", action="store_true") + args = ap.parse_args(argv) + + if args.selftest: + return selftest() + if not os.path.isdir(args.repo): + print("landing_check: --repo %s is not a directory" % args.repo, + file=sys.stderr) + return 2 + if _exit2(args.repo) != 0: + print("landing_check: --repo %s is not a git work tree" % args.repo, + file=sys.stderr) + return 2 + try: + return run(args) + except RuntimeError as exc: + print("landing_check: %s" % exc, file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main())