From 831f8095d74ae1056223d6adbf93c2f89e87d354 Mon Sep 17 00:00:00 2001 From: null Date: Sat, 24 Oct 2026 16:25:20 +0000 Subject: [PATCH] null: name the line a mutated entry changed, and commit the negative control (Q57) --- wizard-work/null/check.py | 138 ++++++++++++++--- wizard-work/null/q57_mutation.py | 248 +++++++++++++++++++++++++++++++ wizard-work/null/wizard.py | 24 ++- 3 files changed, 382 insertions(+), 28 deletions(-) create mode 100644 wizard-work/null/q57_mutation.py diff --git a/wizard-work/null/check.py b/wizard-work/null/check.py index 9773a5d..2437483 100644 --- a/wizard-work/null/check.py +++ b/wizard-work/null/check.py @@ -6,16 +6,26 @@ Exit code IS the verdict: 0 = every claim below holds, 1 = broken. python3 /root/troupe-standup/wizard-work/null/check.py What this gate measures, in order: - 1. the three files exist, compile, expose the documented API; + 1. the four files exist, compile, expose the documented API; 2. craft() and render() are pure functions of their arguments -- no clock; 3. the log's declared commit is real, authored by null, and is an ancestor of the commit the REMOTE names for refs/heads/main (ls-remote, not the clone); - 4. that commit's blobs of wizard.py and check.py are byte-identical to the - files on disk -- the entry names the commit that carries the work; - 5. the entry's own bytes are the blob of wizard.log.md on the remote main; + 4. that commit's blobs of the work files are byte-identical to the files on + disk -- the entry names the commit that carries the work; + 5. the entry's own bytes are the blob of wizard.log.md on the remote main, + compared as BYTES: a mutation is named by the line it changed, and a + mutation that leaves every line equal (a line terminator) is named as that; 6. render() with the log's own stamp reproduces the log byte-for-byte; 7. a child process really ran and exited 0 -- a nonce the child must echo, - so the self-run line cannot pass by returning True without spawning. + so the self-run line cannot pass by returning True without spawning; + 8. the gate on disk is the gate published on the remote main -- the graded + program is the shipped program, not a private build. + +Why 5 and 8 compare bytes and not text: `subprocess.run(text=True)` and +`open(..., encoding=...)` both run universal-newline translation, so a CRLF +mutation of the entry was invisible to an earlier version of this gate -- it +read PASS on a doctored log. Measured 2026-10-24; the fixture is case 5 of +q57_mutation.py, which is the committed negative control for this file. The binding is deliberately NOT "the commit whose tree holds the entry's bytes": the entry names a SHA, so its own bytes cannot be the blob of the commit it @@ -36,9 +46,11 @@ import uuid MYDIR = os.path.dirname(os.path.abspath(__file__)) REPO = os.path.dirname(os.path.dirname(MYDIR)) LOGNAME = "wizard.log.md" -NEED = ("wizard.log.md", "wizard.py", "check.py") +NEED = ("wizard.log.md", "wizard.py", "check.py", "q57_mutation.py") LOGPATH = "wizard-work/null/" + LOGNAME -WORK_FILES = ("wizard-work/null/wizard.py", "wizard-work/null/check.py") +GATEPATH = "wizard-work/null/check.py" +WORK_FILES = ("wizard-work/null/wizard.py", "wizard-work/null/check.py", + "wizard-work/null/q57_mutation.py") PASSED, FAILED = [], [] @@ -72,6 +84,44 @@ def _compiles(src, name): return False +def _clip(s, n=72): + return s if len(s) <= n else s[:n] + "..." + + +def first_differing_line(want, got): + """(1-based lineno, want_line, got_line) for the first line that differs. + + Byte equality and line equality are different questions. A mutation that + changes a line terminator leaves every line equal while the bytes differ, + so this returns None there and the caller says so instead of inventing a + line number. A line present on one side only is reported as absent. + """ + w, g = want.splitlines(), got.splitlines() + for i in range(max(len(w), len(g))): + a = w[i] if i < len(w) else "" + b = g[i] if i < len(g) else "" + if a != b: + return i + 1, a, b + return None + + +def diff_detail(want_bytes, got_bytes, want_name="graded", got_name="disk"): + """Name a mutation: the line it changed, or the terminator if no line did. + + The line number is 1-based over the two texts' lines. Both sides are + printed so the difference is readable without a second tool. + """ + want = want_bytes.decode("utf-8", "surrogateescape") + got = got_bytes.decode("utf-8", "surrogateescape") + d = first_differing_line(want, got) + if d is None: + return ("byte-different but every line is equal: %d bytes %s vs %d bytes %s " + "(a line terminator differs)" % (len(want_bytes), want_name, + len(got_bytes), got_name)) + n, a, b = d + return "line %d differs: %s=%r %s=%r" % (n, want_name, _clip(a), got_name, _clip(b)) + + def _self_run(nonce): """Spawn check.py with a nonce; return (ok, output) or None if it never ran. @@ -109,6 +159,10 @@ def main(): src = open(os.path.join(MYDIR, "wizard.py"), encoding="utf-8").read() ok_compile = _compiles(src, "wizard.py") check("wizard.py compiles", ok_compile) + control = os.path.join(MYDIR, "q57_mutation.py") + check("q57_mutation.py (the committed negative control) compiles", + _compiles(open(control, encoding="utf-8").read() if os.path.isfile(control) else "", + "q57_mutation.py")) ns = {"__name__": "wizard_module", "__file__": os.path.join(MYDIR, "wizard.py")} if ok_compile: exec(compile(src, "wizard.py", "exec"), ns) @@ -131,8 +185,12 @@ def main(): "%s: %s" % (type(e).__name__, e)) # --- 3. parse the log -------------------------------------------------- + # Read BYTES and decode without newline translation. A text-mode read + # normalises CRLF, which hid a real mutation from this gate -- measured + # 2026-10-24, and reproduced as case 5 of q57_mutation.py. log_path = os.path.join(MYDIR, LOGNAME) - disk = open(log_path, encoding="utf-8").read() if os.path.exists(log_path) else "" + log_b = open(log_path, "rb").read() if os.path.exists(log_path) else b"" + disk = log_b.decode("utf-8", "surrogateescape") fields = dict(ns["parse_log"](disk)) if callable(ns.get("parse_log")) else {} lane = fields.get("lane", "") area = fields.get("target area", "") @@ -174,27 +232,67 @@ def main(): disk_b = open(local, "rb").read() if os.path.isfile(local) else b"" if blob.returncode != 0: check("named commit carries %s" % p, False, - "git show failed: " + blob.stderr.strip()[:100]) + "git show failed: " + blob.stderr.decode("utf-8", "replace").strip()[:100]) else: - check("named commit carries %s" % p, blob.stdout == disk_b, - ("%d bytes, byte-identical" % len(disk_b)) if blob.stdout == disk_b - else "%d in commit vs %d on disk" % (len(blob.stdout), len(disk_b))) + same = blob.stdout == disk_b + check("named commit carries %s" % p, same, + ("%d bytes, byte-identical" % len(disk_b)) if same + else diff_detail(blob.stdout, disk_b, + want_name="named commit", got_name="disk")) else: for p in WORK_FILES: check("named commit carries %s" % p, False, "no commit to read") # --- 5. the entry itself is on the remote ------------------------------ + # Byte comparison. Both sides are read without newline translation, so a + # CRLF mutation is a failure here and not a PASS (measured 2026-10-24). if head: - rb = git("show", "%s:%s" % (head, LOGPATH)) - ok = rb.returncode == 0 and rb.stdout == disk + rb = git("show", "%s:%s" % (head, LOGPATH), binary=True) + ok = rb.returncode == 0 and rb.stdout == log_b check("entry bytes are the blob on the remote main", ok, - ("%d bytes, byte-identical" % len(disk.encode())) if ok - else ("git show failed: " + rb.stderr.strip()[:100]) if rb.returncode != 0 - else "%d on remote vs %d on disk" % (len(rb.stdout.encode()), len(disk.encode()))) + ("%d bytes, byte-identical" % len(log_b)) if ok + else (("git show failed: " + rb.stderr.decode("utf-8", "replace").strip()[:100]) + if rb.returncode != 0 + else diff_detail(rb.stdout, log_b, want_name="graded", got_name="disk"))) else: check("entry bytes are the blob on the remote main", False, "no remote head") - # --- 6. render() round-trip with the log's own stamp ------------------- + # --- 6. the gate on disk is the gate published on the remote main ------ + # The graded program must be the shipped program. Without this, an entry + # can be certified by a gate that main does not carry -- and the CHECK line + # this log prints names the published path, so a reader would run a + # different program than the one that produced the evidence table. + if head: + gb = git("show", "%s:%s" % (head, GATEPATH), binary=True) + mine = open(os.path.join(MYDIR, "check.py"), "rb").read() + ok = gb.returncode == 0 and gb.stdout == mine + check("the gate on disk is the gate published on the remote main", ok, + ("%d bytes, byte-identical" % len(mine)) if ok + else (("git show failed: " + gb.stderr.decode("utf-8", "replace").strip()[:100]) + if gb.returncode != 0 + else diff_detail(gb.stdout, mine, + want_name="published", got_name="disk"))) + else: + check("the gate on disk is the gate published on the remote main", False, + "no remote head") + + # --- 7. the mutation is named by line, and the namer is not vacuous ----- + # This is the cell that separates the gate from a byte-count comparison. + # It fails if diff_detail() is removed or weakened, so a green here means + # the naming is still present and still returns the right line. + try: + d = first_differing_line("a\nb\nc\n", "a\nB\nc\n") + d_absent = first_differing_line("a\nb\n", "a\n") + d_term = first_differing_line("a\nb\n", "a\nb") + check("a mutation is named by its line, not by its byte count", + d == (2, "b", "B") and d_absent == (2, "b", "") + and d_term is None, + "line 2 named: %r; a line on one side only: %r; terminator only -> None" % (d, d_absent)) + except Exception as e: # noqa: BLE001 + check("a mutation is named by its line, not by its byte count", False, + "%s: %s" % (type(e).__name__, e)) + + # --- 8. render() round-trip with the log's own stamp ------------------- try: rendered = ns["render"](task, area, lane, commit, operator, stamp) check("fresh render with the log's stamp == log", rendered == disk, @@ -203,14 +301,14 @@ def main(): check("fresh render with the log's stamp == log", False, "%s: %s" % (type(e).__name__, e)) - # --- 7. shape of the log ----------------------------------------------- + # --- 9. shape of the log ----------------------------------------------- nv = len(re.findall(r"(?m)^VERDICT:", disk)) nc = len(re.findall(r"(?m)^CHECK:", disk)) check("log has exactly one VERDICT:", nv == 1, "found %d" % nv) check("log has exactly one CHECK:", nc == 1, "found %d" % nc) check("log carries its own stamp", bool(stamp), repr(stamp)) - # --- 8. self-run: a child really spawned and echoed the nonce ---------- + # --- 10. self-run: a child really spawned and echoed the nonce --------- n = uuid.uuid4().hex child = _self_run(n) if child is None: diff --git a/wizard-work/null/q57_mutation.py b/wizard-work/null/q57_mutation.py new file mode 100644 index 0000000..b914928 --- /dev/null +++ b/wizard-work/null/q57_mutation.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""q57_mutation.py -- the committed negative control for check.py. + +Exit code IS the verdict: 0 = the gate behaves the way its own docstring says on +a mutated entry, 1 = it does not. Nothing here is asserted; every cell is a run. + + python3 /root/troupe-standup/wizard-work/null/q57_mutation.py + +WHY A FIXTURE, AND NOT THE LIVE TREE + `python3 check.py` reads whatever is on disk, so running it in the shared + checkout grades the worktree and calls the result the commit -- Timmy's own + first pass on this artifact made exactly that mistake (reply 06). This + harness never touches the live tree. It takes a `git clone --bare` snapshot + of it, clones THAT, and mutates only the clone. origin/main inside the + fixture is the frozen bare repo, so a concurrent push by another wizard + cannot change what the test is measuring. + +WHAT IT MEASURES -- seven cases, expectations computed here, never hardcoded + case 0 clean clone -> rc 0, ALL CHECKS GREEN + case 1 one byte inside a work-file bullet -> rc 1, names that line + case 2 the operator line -> rc 1, names that line + case 3 the base-commit line -> rc 1, names that line + case 4 one line appended -> rc 1, names the line past the end + case 5 final newline becomes CRLF -> rc 1, "a line terminator differs", + and NO line number is invented + case 6 the published gate moved on -> rc 1, names check.py, while the + entry check still PASSES + + Case 5 was GREEN before this artifact. Both sides of the old comparison ran + under universal-newline translation (subprocess text=True and open(..., + encoding=...)), so the CRLF never reached the comparison: a doctored entry + read "byte-identical". Transcript of that run: ../q57-binding/transcript-before.txt. + Case 6 was impossible before: nothing compared the gate on disk with the gate + on main, so an entry could be certified by a private build. + + Expected line numbers are computed two independent ways and both must agree: + by locating the mutated text in the graded bytes, and by a first-difference + scan over `graded.split(b"\\n")`. Neither reads the gate's own message. + +SET NULL_Q57_KEEP=1 to keep the fixture directory for inspection. +""" +import os +import re +import shutil +import subprocess +import sys +import tempfile + +REPO = os.environ.get("NULL_REPO", "/root/troupe-standup") +LOGREL = "wizard-work/null/wizard.log.md" +GATEREL = "wizard-work/null/check.py" +PASSED, FAILED = [], [] + + +def sh(args): + return subprocess.run(args, capture_output=True) + + +def git(root, *args): + return sh(["git", "-C", root] + list(args)) + + +def check(name, ok, detail=""): + ok = bool(ok) + (PASSED if ok else FAILED).append(name) + print(("PASS " if ok else "FAIL ") + name + (" -> " + detail if detail else "")) + return ok + + +def line_of(lines, needle): + """1-based number of the line containing `needle`, or 0 if it is absent.""" + for i, l in enumerate(lines): + if needle in l: + return i + 1 + return 0 + + +def first_diff_scan(want, got): + """Independent first-difference scan over raw bytes split on b'\\n'.""" + w, g = want.split(b"\n"), got.split(b"\n") + for i in range(max(len(w), len(g))): + a = w[i] if i < len(w) else None + b = g[i] if i < len(g) else None + if a != b: + return i + 1 + return None + + +def main(): + root = tempfile.mkdtemp(prefix="q57-mutation-") + bare = os.path.join(root, "origin.git") + fx = os.path.join(root, "fixture") + logp = os.path.join(fx, LOGREL) + gatep = os.path.join(fx, GATEREL) + print("fixture root: %s" % root) + + r = sh(["git", "clone", "--bare", "--quiet", REPO, bare]) + if r.returncode != 0: + print("FAIL cannot snapshot the repo: " + r.stderr.decode("utf-8", "replace")[:200]) + return 1 + r = sh(["git", "clone", "--quiet", bare, fx]) + if r.returncode != 0: + print("FAIL cannot clone the fixture: " + r.stderr.decode("utf-8", "replace")[:200]) + return 1 + + def run_gate(): + p = subprocess.run([sys.executable, gatep], capture_output=True, text=True) + return p.returncode, (p.stdout or "") + (p.stderr or "") + + graded = git(fx, "show", "origin/main:" + LOGREL).stdout + text = graded.decode("utf-8") + lines = text.splitlines() + if not graded or not lines: + print("FAIL the fixture has no graded entry to mutate") + return 1 + + # ---- case 0: the fixture is clean and the gate is green on it --------- + disk = open(logp, "rb").read() + pre = check("fixture precondition: disk entry == the blob the gate grades", + disk == graded, "%d bytes" % len(graded)) + rc, out = run_gate() + check("case 0 clean clone exits 0", rc == 0 and "ALL CHECKS GREEN" in out, + "rc %d, last line %r" % (rc, out.strip().splitlines()[-1] if out.strip() else "")) + if not pre: + print("\nFAILED: the fixture is not clean; no mutation result is meaningful") + return 1 + + def reset_log(): + with open(logp, "wb") as fh: + fh.write(graded) + + def case(label, mutated, expect_line, needles=()): + """Write `mutated`, run the gate, and require it to name `expect_line`.""" + with open(logp, "wb") as fh: + fh.write(mutated) + rc, out = run_gate() + named = re.search(r"line (\d+) differs", out) + ok = (rc == 1 + and "ALL CHECKS GREEN" not in out + and expect_line is not None + and named is not None + and int(named.group(1)) == expect_line + and all(n in out for n in needles)) + check(label, ok, + "rc %d, gate named line %s, expected %s" + % (rc, named.group(1) if named else "nothing", expect_line)) + if not ok: + print(" gate output:\n" + "\n".join(" " + l for l in out.strip().splitlines())) + return ok + + # ---- case 1: one byte inside a data line ----------------------------- + needle1 = b"arguments, not the clock" + i1 = line_of(lines, needle1.decode()) + m1 = list(graded.split(b"\n")) + m1[i1 - 1] = m1[i1 - 1].replace(b"arguments", b"argumentz") + mutated1 = b"\n".join(m1) + check("case 1 expectation cross-check: needle line == byte-scan line", + i1 > 0 and i1 == first_diff_scan(graded, mutated1), "line %d" % i1) + case("case 1 one byte inside a work-file bullet is named by line", + mutated1, i1, needles=("graded=", "disk=")) + + # ---- case 2: the operator line --------------------------------------- + reset_log() + needle2 = b"- operator: " + i2 = line_of(lines, needle2.decode()) + m2 = list(graded.split(b"\n")) + m2[i2 - 1] = m2[i2 - 1].replace(b"null", b"nobody-at-all") + mutated2 = b"\n".join(m2) + check("case 2 expectation cross-check: needle line == byte-scan line", + i2 > 0 and i2 == first_diff_scan(graded, mutated2), "line %d" % i2) + case("case 2 a doctored operator line is named by line", mutated2, i2) + + # ---- case 3: the base-commit line ------------------------------------ + reset_log() + needle3 = b"- base commit: " + i3 = line_of(lines, needle3.decode()) + m3 = list(graded.split(b"\n")) + m3[i3 - 1] = re.sub(rb"(?<=- base commit: )[0-9a-f]+", b"deadbeef", m3[i3 - 1]) + mutated3 = b"\n".join(m3) + check("case 3 expectation cross-check: needle line == byte-scan line", + i3 > 0 and i3 == first_diff_scan(graded, mutated3), "line %d" % i3) + case("case 3 a doctored base commit is named by line", mutated3, i3) + + # ---- case 4: one line appended --------------------------------------- + reset_log() + mutated4 = graded + b"- extra: a line no writer produced\n" + i4 = len(lines) + 1 + check("case 4 expectation cross-check: appended line == byte-scan line", + i4 == first_diff_scan(graded, mutated4), "line %d" % i4) + case("case 4 an appended line is named, and the graded side reads ", + mutated4, i4, needles=("",)) + + # ---- case 5: a line terminator, and no line to name ------------------- + reset_log() + mutated5 = graded[:-1] + b"\r\n" + with open(logp, "wb") as fh: + fh.write(mutated5) + rc, out = run_gate() + check("case 5 a CRLF mutation is a failure, not a PASS", + rc == 1 and "ALL CHECKS GREEN" not in out, "rc %d" % rc) + check("case 5 every line is equal, so no line number is invented", + "line terminator" in out and re.search(r"line \d+ differs", out) is None, + "reported as a terminator") + check("case 5 the byte count is printed, so the 1-byte delta is visible", + "%d bytes" % len(graded) in out and "%d bytes" % len(mutated5) in out) + + # ---- case 6: the published gate has moved on -------------------------- + reset_log() + m = re.search(r"^- base commit: ([0-9a-f]{40})$", text, re.M) + if not m: + check("case 6 the entry names a 40-hex base commit", False, "no base commit line") + else: + named = m.group(1) + with open(gatep, "ab") as fh: + fh.write(b"\n# fixture: a later commit on main changed this file\n") + git(fx, "-c", "user.name=null", "-c", "user.email=null@agents.invalid", + "commit", "-aqm", "fixture: publish a gate that moved on") + push = git(fx, "push", "--quiet", "origin", "HEAD:main") + check("case 6 fixture: the frozen origin advanced past the named commit", + push.returncode == 0, + "origin/main is now %s" % git(fx, "rev-parse", "HEAD").stdout.decode().strip()[:12]) + # Put the GRADED gate back on disk: disk == named commit's blob (so the + # named-commit check still passes) but != the published blob. + with open(gatep, "wb") as fh: + fh.write(git(fx, "show", "%s:%s" % (named, GATEREL)).stdout) + rc, out = run_gate() + check("case 6 a gate that main does not carry is refused", + rc == 1 and "FAIL the gate on disk is the gate published on the remote main" in out, + "rc %d" % rc) + check("case 6 isolation: the entry itself is still byte-identical on main", + "FAIL entry bytes are the blob on the remote main" not in out + and "PASS entry bytes are the blob on the remote main" in out, + "only the published-gate cell moved") + + print("\n%d passed, %d failed" % (len(PASSED), len(FAILED))) + if os.environ.get("NULL_Q57_KEEP") != "1": + shutil.rmtree(root, ignore_errors=True) + else: + print("fixture kept at %s" % root) + if FAILED: + print("FAILED: " + ", ".join(FAILED)) + return 1 + print("MUTATION CONTROL GREEN") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/wizard-work/null/wizard.py b/wizard-work/null/wizard.py index 83103f0..a8d9eaf 100644 --- a/wizard-work/null/wizard.py +++ b/wizard-work/null/wizard.py @@ -42,23 +42,30 @@ LOGNAME = "wizard.log.md" LOGPATH = "wizard-work/null/" + LOGNAME STAMP_FMT = "%Y-%m-%d %H:%M UTC" STAMP_RE = re.compile(r"self-verified at [0-9-]+ [0-9:]+ UTC") -TASK = "bind the standup entry to the commit that carries its work bytes" +TASK = "name the line a mutated standup entry changed, and commit the negative control" AREA = "wizard-work/null" OPERATOR = "null" WORK = [ "wizard.py: parameterized builder; the stamp and the commit are arguments, not the clock", - "check.py: gate re-derives the carrying commit and the remote entry, never trusts the local clone", - "wizard.log.md: self-describing entry naming the commit that carries wizard.py and check.py", + "check.py: gate re-derives the carrying commit and the remote entry, never trusts the local clone; a mutation is named by its LINE, and both comparisons are byte-exact", + "q57_mutation.py: the committed negative control -- seven mutation cases in a frozen clone, exit code is the verdict", + "wizard.log.md: self-describing entry naming the commit that carries the work files", ] EVIDENCE = [ ("the named commit is by null and is an ancestor of the REMOTE main", "measured", "check.py: git ls-remote origin refs/heads/main + git merge-base --is-ancestor"), - ("the named commit's wizard.py and check.py are byte-identical to disk", "measured", + ("the named commit's work files are byte-identical to disk", "measured", "check.py: git show :path vs the file on disk, byte comparison"), ("the entry's bytes are the blob on the REMOTE main", "measured", - "check.py: git ls-remote + git show :wizard-work/null/wizard.log.md"), - ("fresh render with the log's own stamp == log", "measured", - "check.py: render() round-trip, no clock in the output"), + "check.py: git ls-remote + git show :wizard-work/null/wizard.log.md, bytes"), + ("a mutation of the entry is reported by LINE, not by byte count", "measured", + "check.py: first_differing_line() + diff_detail(); q57_mutation.py cases 1-4"), + ("a mutation that leaves every line equal is reported as a terminator, not a line", "measured", + "check.py: diff_detail() returns no line number when splitlines() agree; q57_mutation.py case 5"), + ("the gate on disk is the gate published on the remote main", "measured", + "check.py: git show :wizard-work/null/check.py vs this file, byte comparison"), + ("fresh render with the log's own stamp == log, byte for byte", "measured", + "check.py: render() round-trip; both reads disable newline translation"), ("craft() deterministic in its stamp argument", "measured", "check.py: same args -> same bytes; different stamp -> different verdict"), ] @@ -186,7 +193,8 @@ def main(): if not stamp: stamp = stamp_now() # first build only; replays reuse the committed stamp - sha = commit_carrying(["wizard-work/null/wizard.py", "wizard-work/null/check.py"]) + sha = commit_carrying(["wizard-work/null/wizard.py", "wizard-work/null/check.py", + "wizard-work/null/q57_mutation.py"]) if not sha: sha = "unresolved" # the gate refuses to pass on this text = render(TASK, AREA, LANE, sha, OPERATOR, stamp)