null: name the line a mutated entry changed, and commit the negative control (Q57)
This commit is contained in:
1 parent
b78526b8e9
commit
831f8095d7
3 files changed
+382
-28
No files matched your search
+118
-20
@@ -6,16 +6,26 @@ Exit code IS the verdict: 0 = every claim below holds, 1 = broken.
|
|||||||
python3 /root/troupe-standup/wizard-work/null/check.py
|
python3 /root/troupe-standup/wizard-work/null/check.py
|
||||||
|
|
||||||
What this gate measures, in order:
|
What this gate measures, in order:
|
||||||
1. the three files exist, compile, expose the documented API;
|
1. the four files exist, compile, expose the documented API;
|
||||||
2. craft() and render() are pure functions of their arguments -- no clock;
|
2. craft() and render() are pure functions of their arguments -- no clock;
|
||||||
3. the log's declared commit is real, authored by null, and is an ancestor of
|
3. the log's declared commit is real, authored by null, and is an ancestor of
|
||||||
the commit the REMOTE names for refs/heads/main (ls-remote, not the clone);
|
the commit the REMOTE names for refs/heads/main (ls-remote, not the clone);
|
||||||
4. that commit's blobs of wizard.py and check.py are byte-identical to the
|
4. that commit's blobs of the work files are byte-identical to the files on
|
||||||
files on disk -- the entry names the commit that carries the work;
|
disk -- the entry names the commit that carries the work;
|
||||||
5. the entry's own bytes are the blob of wizard.log.md on the remote main;
|
5. the entry's own bytes are the blob of wizard.log.md on the remote main,
|
||||||
|
compared as BYTES: a mutation is named by the line it changed, and a
|
||||||
|
mutation that leaves every line equal (a line terminator) is named as that;
|
||||||
6. render() with the log's own stamp reproduces the log byte-for-byte;
|
6. render() with the log's own stamp reproduces the log byte-for-byte;
|
||||||
7. a child process really ran and exited 0 -- a nonce the child must echo,
|
7. a child process really ran and exited 0 -- a nonce the child must echo,
|
||||||
so the self-run line cannot pass by returning True without spawning.
|
so the self-run line cannot pass by returning True without spawning;
|
||||||
|
8. the gate on disk is the gate published on the remote main -- the graded
|
||||||
|
program is the shipped program, not a private build.
|
||||||
|
|
||||||
|
Why 5 and 8 compare bytes and not text: `subprocess.run(text=True)` and
|
||||||
|
`open(..., encoding=...)` both run universal-newline translation, so a CRLF
|
||||||
|
mutation of the entry was invisible to an earlier version of this gate -- it
|
||||||
|
read PASS on a doctored log. Measured 2026-10-24; the fixture is case 5 of
|
||||||
|
q57_mutation.py, which is the committed negative control for this file.
|
||||||
|
|
||||||
The binding is deliberately NOT "the commit whose tree holds the entry's bytes":
|
The binding is deliberately NOT "the commit whose tree holds the entry's bytes":
|
||||||
the entry names a SHA, so its own bytes cannot be the blob of the commit it
|
the entry names a SHA, so its own bytes cannot be the blob of the commit it
|
||||||
@@ -36,9 +46,11 @@ import uuid
|
|||||||
MYDIR = os.path.dirname(os.path.abspath(__file__))
|
MYDIR = os.path.dirname(os.path.abspath(__file__))
|
||||||
REPO = os.path.dirname(os.path.dirname(MYDIR))
|
REPO = os.path.dirname(os.path.dirname(MYDIR))
|
||||||
LOGNAME = "wizard.log.md"
|
LOGNAME = "wizard.log.md"
|
||||||
NEED = ("wizard.log.md", "wizard.py", "check.py")
|
NEED = ("wizard.log.md", "wizard.py", "check.py", "q57_mutation.py")
|
||||||
LOGPATH = "wizard-work/null/" + LOGNAME
|
LOGPATH = "wizard-work/null/" + LOGNAME
|
||||||
WORK_FILES = ("wizard-work/null/wizard.py", "wizard-work/null/check.py")
|
GATEPATH = "wizard-work/null/check.py"
|
||||||
|
WORK_FILES = ("wizard-work/null/wizard.py", "wizard-work/null/check.py",
|
||||||
|
"wizard-work/null/q57_mutation.py")
|
||||||
PASSED, FAILED = [], []
|
PASSED, FAILED = [], []
|
||||||
|
|
||||||
|
|
||||||
@@ -72,6 +84,44 @@ def _compiles(src, name):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _clip(s, n=72):
|
||||||
|
return s if len(s) <= n else s[:n] + "..."
|
||||||
|
|
||||||
|
|
||||||
|
def first_differing_line(want, got):
|
||||||
|
"""(1-based lineno, want_line, got_line) for the first line that differs.
|
||||||
|
|
||||||
|
Byte equality and line equality are different questions. A mutation that
|
||||||
|
changes a line terminator leaves every line equal while the bytes differ,
|
||||||
|
so this returns None there and the caller says so instead of inventing a
|
||||||
|
line number. A line present on one side only is reported as absent.
|
||||||
|
"""
|
||||||
|
w, g = want.splitlines(), got.splitlines()
|
||||||
|
for i in range(max(len(w), len(g))):
|
||||||
|
a = w[i] if i < len(w) else "<no such line>"
|
||||||
|
b = g[i] if i < len(g) else "<no such line>"
|
||||||
|
if a != b:
|
||||||
|
return i + 1, a, b
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def diff_detail(want_bytes, got_bytes, want_name="graded", got_name="disk"):
|
||||||
|
"""Name a mutation: the line it changed, or the terminator if no line did.
|
||||||
|
|
||||||
|
The line number is 1-based over the two texts' lines. Both sides are
|
||||||
|
printed so the difference is readable without a second tool.
|
||||||
|
"""
|
||||||
|
want = want_bytes.decode("utf-8", "surrogateescape")
|
||||||
|
got = got_bytes.decode("utf-8", "surrogateescape")
|
||||||
|
d = first_differing_line(want, got)
|
||||||
|
if d is None:
|
||||||
|
return ("byte-different but every line is equal: %d bytes %s vs %d bytes %s "
|
||||||
|
"(a line terminator differs)" % (len(want_bytes), want_name,
|
||||||
|
len(got_bytes), got_name))
|
||||||
|
n, a, b = d
|
||||||
|
return "line %d differs: %s=%r %s=%r" % (n, want_name, _clip(a), got_name, _clip(b))
|
||||||
|
|
||||||
|
|
||||||
def _self_run(nonce):
|
def _self_run(nonce):
|
||||||
"""Spawn check.py with a nonce; return (ok, output) or None if it never ran.
|
"""Spawn check.py with a nonce; return (ok, output) or None if it never ran.
|
||||||
|
|
||||||
@@ -109,6 +159,10 @@ def main():
|
|||||||
src = open(os.path.join(MYDIR, "wizard.py"), encoding="utf-8").read()
|
src = open(os.path.join(MYDIR, "wizard.py"), encoding="utf-8").read()
|
||||||
ok_compile = _compiles(src, "wizard.py")
|
ok_compile = _compiles(src, "wizard.py")
|
||||||
check("wizard.py compiles", ok_compile)
|
check("wizard.py compiles", ok_compile)
|
||||||
|
control = os.path.join(MYDIR, "q57_mutation.py")
|
||||||
|
check("q57_mutation.py (the committed negative control) compiles",
|
||||||
|
_compiles(open(control, encoding="utf-8").read() if os.path.isfile(control) else "",
|
||||||
|
"q57_mutation.py"))
|
||||||
ns = {"__name__": "wizard_module", "__file__": os.path.join(MYDIR, "wizard.py")}
|
ns = {"__name__": "wizard_module", "__file__": os.path.join(MYDIR, "wizard.py")}
|
||||||
if ok_compile:
|
if ok_compile:
|
||||||
exec(compile(src, "wizard.py", "exec"), ns)
|
exec(compile(src, "wizard.py", "exec"), ns)
|
||||||
@@ -131,8 +185,12 @@ def main():
|
|||||||
"%s: %s" % (type(e).__name__, e))
|
"%s: %s" % (type(e).__name__, e))
|
||||||
|
|
||||||
# --- 3. parse the log --------------------------------------------------
|
# --- 3. parse the log --------------------------------------------------
|
||||||
|
# Read BYTES and decode without newline translation. A text-mode read
|
||||||
|
# normalises CRLF, which hid a real mutation from this gate -- measured
|
||||||
|
# 2026-10-24, and reproduced as case 5 of q57_mutation.py.
|
||||||
log_path = os.path.join(MYDIR, LOGNAME)
|
log_path = os.path.join(MYDIR, LOGNAME)
|
||||||
disk = open(log_path, encoding="utf-8").read() if os.path.exists(log_path) else ""
|
log_b = open(log_path, "rb").read() if os.path.exists(log_path) else b""
|
||||||
|
disk = log_b.decode("utf-8", "surrogateescape")
|
||||||
fields = dict(ns["parse_log"](disk)) if callable(ns.get("parse_log")) else {}
|
fields = dict(ns["parse_log"](disk)) if callable(ns.get("parse_log")) else {}
|
||||||
lane = fields.get("lane", "")
|
lane = fields.get("lane", "")
|
||||||
area = fields.get("target area", "")
|
area = fields.get("target area", "")
|
||||||
@@ -174,27 +232,67 @@ def main():
|
|||||||
disk_b = open(local, "rb").read() if os.path.isfile(local) else b""
|
disk_b = open(local, "rb").read() if os.path.isfile(local) else b""
|
||||||
if blob.returncode != 0:
|
if blob.returncode != 0:
|
||||||
check("named commit carries %s" % p, False,
|
check("named commit carries %s" % p, False,
|
||||||
"git show failed: " + blob.stderr.strip()[:100])
|
"git show failed: " + blob.stderr.decode("utf-8", "replace").strip()[:100])
|
||||||
else:
|
else:
|
||||||
check("named commit carries %s" % p, blob.stdout == disk_b,
|
same = blob.stdout == disk_b
|
||||||
("%d bytes, byte-identical" % len(disk_b)) if blob.stdout == disk_b
|
check("named commit carries %s" % p, same,
|
||||||
else "%d in commit vs %d on disk" % (len(blob.stdout), len(disk_b)))
|
("%d bytes, byte-identical" % len(disk_b)) if same
|
||||||
|
else diff_detail(blob.stdout, disk_b,
|
||||||
|
want_name="named commit", got_name="disk"))
|
||||||
else:
|
else:
|
||||||
for p in WORK_FILES:
|
for p in WORK_FILES:
|
||||||
check("named commit carries %s" % p, False, "no commit to read")
|
check("named commit carries %s" % p, False, "no commit to read")
|
||||||
|
|
||||||
# --- 5. the entry itself is on the remote ------------------------------
|
# --- 5. the entry itself is on the remote ------------------------------
|
||||||
|
# Byte comparison. Both sides are read without newline translation, so a
|
||||||
|
# CRLF mutation is a failure here and not a PASS (measured 2026-10-24).
|
||||||
if head:
|
if head:
|
||||||
rb = git("show", "%s:%s" % (head, LOGPATH))
|
rb = git("show", "%s:%s" % (head, LOGPATH), binary=True)
|
||||||
ok = rb.returncode == 0 and rb.stdout == disk
|
ok = rb.returncode == 0 and rb.stdout == log_b
|
||||||
check("entry bytes are the blob on the remote main", ok,
|
check("entry bytes are the blob on the remote main", ok,
|
||||||
("%d bytes, byte-identical" % len(disk.encode())) if ok
|
("%d bytes, byte-identical" % len(log_b)) if ok
|
||||||
else ("git show failed: " + rb.stderr.strip()[:100]) if rb.returncode != 0
|
else (("git show failed: " + rb.stderr.decode("utf-8", "replace").strip()[:100])
|
||||||
else "%d on remote vs %d on disk" % (len(rb.stdout.encode()), len(disk.encode())))
|
if rb.returncode != 0
|
||||||
|
else diff_detail(rb.stdout, log_b, want_name="graded", got_name="disk")))
|
||||||
else:
|
else:
|
||||||
check("entry bytes are the blob on the remote main", False, "no remote head")
|
check("entry bytes are the blob on the remote main", False, "no remote head")
|
||||||
|
|
||||||
# --- 6. render() round-trip with the log's own stamp -------------------
|
# --- 6. the gate on disk is the gate published on the remote main ------
|
||||||
|
# The graded program must be the shipped program. Without this, an entry
|
||||||
|
# can be certified by a gate that main does not carry -- and the CHECK line
|
||||||
|
# this log prints names the published path, so a reader would run a
|
||||||
|
# different program than the one that produced the evidence table.
|
||||||
|
if head:
|
||||||
|
gb = git("show", "%s:%s" % (head, GATEPATH), binary=True)
|
||||||
|
mine = open(os.path.join(MYDIR, "check.py"), "rb").read()
|
||||||
|
ok = gb.returncode == 0 and gb.stdout == mine
|
||||||
|
check("the gate on disk is the gate published on the remote main", ok,
|
||||||
|
("%d bytes, byte-identical" % len(mine)) if ok
|
||||||
|
else (("git show failed: " + gb.stderr.decode("utf-8", "replace").strip()[:100])
|
||||||
|
if gb.returncode != 0
|
||||||
|
else diff_detail(gb.stdout, mine,
|
||||||
|
want_name="published", got_name="disk")))
|
||||||
|
else:
|
||||||
|
check("the gate on disk is the gate published on the remote main", False,
|
||||||
|
"no remote head")
|
||||||
|
|
||||||
|
# --- 7. the mutation is named by line, and the namer is not vacuous -----
|
||||||
|
# This is the cell that separates the gate from a byte-count comparison.
|
||||||
|
# It fails if diff_detail() is removed or weakened, so a green here means
|
||||||
|
# the naming is still present and still returns the right line.
|
||||||
|
try:
|
||||||
|
d = first_differing_line("a\nb\nc\n", "a\nB\nc\n")
|
||||||
|
d_absent = first_differing_line("a\nb\n", "a\n")
|
||||||
|
d_term = first_differing_line("a\nb\n", "a\nb")
|
||||||
|
check("a mutation is named by its line, not by its byte count",
|
||||||
|
d == (2, "b", "B") and d_absent == (2, "b", "<no such line>")
|
||||||
|
and d_term is None,
|
||||||
|
"line 2 named: %r; a line on one side only: %r; terminator only -> None" % (d, d_absent))
|
||||||
|
except Exception as e: # noqa: BLE001
|
||||||
|
check("a mutation is named by its line, not by its byte count", False,
|
||||||
|
"%s: %s" % (type(e).__name__, e))
|
||||||
|
|
||||||
|
# --- 8. render() round-trip with the log's own stamp -------------------
|
||||||
try:
|
try:
|
||||||
rendered = ns["render"](task, area, lane, commit, operator, stamp)
|
rendered = ns["render"](task, area, lane, commit, operator, stamp)
|
||||||
check("fresh render with the log's stamp == log", rendered == disk,
|
check("fresh render with the log's stamp == log", rendered == disk,
|
||||||
@@ -203,14 +301,14 @@ def main():
|
|||||||
check("fresh render with the log's stamp == log", False,
|
check("fresh render with the log's stamp == log", False,
|
||||||
"%s: %s" % (type(e).__name__, e))
|
"%s: %s" % (type(e).__name__, e))
|
||||||
|
|
||||||
# --- 7. shape of the log -----------------------------------------------
|
# --- 9. shape of the log -----------------------------------------------
|
||||||
nv = len(re.findall(r"(?m)^VERDICT:", disk))
|
nv = len(re.findall(r"(?m)^VERDICT:", disk))
|
||||||
nc = len(re.findall(r"(?m)^CHECK:", disk))
|
nc = len(re.findall(r"(?m)^CHECK:", disk))
|
||||||
check("log has exactly one VERDICT:", nv == 1, "found %d" % nv)
|
check("log has exactly one VERDICT:", nv == 1, "found %d" % nv)
|
||||||
check("log has exactly one CHECK:", nc == 1, "found %d" % nc)
|
check("log has exactly one CHECK:", nc == 1, "found %d" % nc)
|
||||||
check("log carries its own stamp", bool(stamp), repr(stamp))
|
check("log carries its own stamp", bool(stamp), repr(stamp))
|
||||||
|
|
||||||
# --- 8. self-run: a child really spawned and echoed the nonce ----------
|
# --- 10. self-run: a child really spawned and echoed the nonce ---------
|
||||||
n = uuid.uuid4().hex
|
n = uuid.uuid4().hex
|
||||||
child = _self_run(n)
|
child = _self_run(n)
|
||||||
if child is None:
|
if child is None:
|
||||||
|
|||||||
@@ -0,0 +1,248 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""q57_mutation.py -- the committed negative control for check.py.
|
||||||
|
|
||||||
|
Exit code IS the verdict: 0 = the gate behaves the way its own docstring says on
|
||||||
|
a mutated entry, 1 = it does not. Nothing here is asserted; every cell is a run.
|
||||||
|
|
||||||
|
python3 /root/troupe-standup/wizard-work/null/q57_mutation.py
|
||||||
|
|
||||||
|
WHY A FIXTURE, AND NOT THE LIVE TREE
|
||||||
|
`python3 check.py` reads whatever is on disk, so running it in the shared
|
||||||
|
checkout grades the worktree and calls the result the commit -- Timmy's own
|
||||||
|
first pass on this artifact made exactly that mistake (reply 06). This
|
||||||
|
harness never touches the live tree. It takes a `git clone --bare` snapshot
|
||||||
|
of it, clones THAT, and mutates only the clone. origin/main inside the
|
||||||
|
fixture is the frozen bare repo, so a concurrent push by another wizard
|
||||||
|
cannot change what the test is measuring.
|
||||||
|
|
||||||
|
WHAT IT MEASURES -- seven cases, expectations computed here, never hardcoded
|
||||||
|
case 0 clean clone -> rc 0, ALL CHECKS GREEN
|
||||||
|
case 1 one byte inside a work-file bullet -> rc 1, names that line
|
||||||
|
case 2 the operator line -> rc 1, names that line
|
||||||
|
case 3 the base-commit line -> rc 1, names that line
|
||||||
|
case 4 one line appended -> rc 1, names the line past the end
|
||||||
|
case 5 final newline becomes CRLF -> rc 1, "a line terminator differs",
|
||||||
|
and NO line number is invented
|
||||||
|
case 6 the published gate moved on -> rc 1, names check.py, while the
|
||||||
|
entry check still PASSES
|
||||||
|
|
||||||
|
Case 5 was GREEN before this artifact. Both sides of the old comparison ran
|
||||||
|
under universal-newline translation (subprocess text=True and open(...,
|
||||||
|
encoding=...)), so the CRLF never reached the comparison: a doctored entry
|
||||||
|
read "byte-identical". Transcript of that run: ../q57-binding/transcript-before.txt.
|
||||||
|
Case 6 was impossible before: nothing compared the gate on disk with the gate
|
||||||
|
on main, so an entry could be certified by a private build.
|
||||||
|
|
||||||
|
Expected line numbers are computed two independent ways and both must agree:
|
||||||
|
by locating the mutated text in the graded bytes, and by a first-difference
|
||||||
|
scan over `graded.split(b"\\n")`. Neither reads the gate's own message.
|
||||||
|
|
||||||
|
SET NULL_Q57_KEEP=1 to keep the fixture directory for inspection.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
REPO = os.environ.get("NULL_REPO", "/root/troupe-standup")
|
||||||
|
LOGREL = "wizard-work/null/wizard.log.md"
|
||||||
|
GATEREL = "wizard-work/null/check.py"
|
||||||
|
PASSED, FAILED = [], []
|
||||||
|
|
||||||
|
|
||||||
|
def sh(args):
|
||||||
|
return subprocess.run(args, capture_output=True)
|
||||||
|
|
||||||
|
|
||||||
|
def git(root, *args):
|
||||||
|
return sh(["git", "-C", root] + list(args))
|
||||||
|
|
||||||
|
|
||||||
|
def check(name, ok, detail=""):
|
||||||
|
ok = bool(ok)
|
||||||
|
(PASSED if ok else FAILED).append(name)
|
||||||
|
print(("PASS " if ok else "FAIL ") + name + (" -> " + detail if detail else ""))
|
||||||
|
return ok
|
||||||
|
|
||||||
|
|
||||||
|
def line_of(lines, needle):
|
||||||
|
"""1-based number of the line containing `needle`, or 0 if it is absent."""
|
||||||
|
for i, l in enumerate(lines):
|
||||||
|
if needle in l:
|
||||||
|
return i + 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def first_diff_scan(want, got):
|
||||||
|
"""Independent first-difference scan over raw bytes split on b'\\n'."""
|
||||||
|
w, g = want.split(b"\n"), got.split(b"\n")
|
||||||
|
for i in range(max(len(w), len(g))):
|
||||||
|
a = w[i] if i < len(w) else None
|
||||||
|
b = g[i] if i < len(g) else None
|
||||||
|
if a != b:
|
||||||
|
return i + 1
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
root = tempfile.mkdtemp(prefix="q57-mutation-")
|
||||||
|
bare = os.path.join(root, "origin.git")
|
||||||
|
fx = os.path.join(root, "fixture")
|
||||||
|
logp = os.path.join(fx, LOGREL)
|
||||||
|
gatep = os.path.join(fx, GATEREL)
|
||||||
|
print("fixture root: %s" % root)
|
||||||
|
|
||||||
|
r = sh(["git", "clone", "--bare", "--quiet", REPO, bare])
|
||||||
|
if r.returncode != 0:
|
||||||
|
print("FAIL cannot snapshot the repo: " + r.stderr.decode("utf-8", "replace")[:200])
|
||||||
|
return 1
|
||||||
|
r = sh(["git", "clone", "--quiet", bare, fx])
|
||||||
|
if r.returncode != 0:
|
||||||
|
print("FAIL cannot clone the fixture: " + r.stderr.decode("utf-8", "replace")[:200])
|
||||||
|
return 1
|
||||||
|
|
||||||
|
def run_gate():
|
||||||
|
p = subprocess.run([sys.executable, gatep], capture_output=True, text=True)
|
||||||
|
return p.returncode, (p.stdout or "") + (p.stderr or "")
|
||||||
|
|
||||||
|
graded = git(fx, "show", "origin/main:" + LOGREL).stdout
|
||||||
|
text = graded.decode("utf-8")
|
||||||
|
lines = text.splitlines()
|
||||||
|
if not graded or not lines:
|
||||||
|
print("FAIL the fixture has no graded entry to mutate")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
# ---- case 0: the fixture is clean and the gate is green on it ---------
|
||||||
|
disk = open(logp, "rb").read()
|
||||||
|
pre = check("fixture precondition: disk entry == the blob the gate grades",
|
||||||
|
disk == graded, "%d bytes" % len(graded))
|
||||||
|
rc, out = run_gate()
|
||||||
|
check("case 0 clean clone exits 0", rc == 0 and "ALL CHECKS GREEN" in out,
|
||||||
|
"rc %d, last line %r" % (rc, out.strip().splitlines()[-1] if out.strip() else ""))
|
||||||
|
if not pre:
|
||||||
|
print("\nFAILED: the fixture is not clean; no mutation result is meaningful")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
def reset_log():
|
||||||
|
with open(logp, "wb") as fh:
|
||||||
|
fh.write(graded)
|
||||||
|
|
||||||
|
def case(label, mutated, expect_line, needles=()):
|
||||||
|
"""Write `mutated`, run the gate, and require it to name `expect_line`."""
|
||||||
|
with open(logp, "wb") as fh:
|
||||||
|
fh.write(mutated)
|
||||||
|
rc, out = run_gate()
|
||||||
|
named = re.search(r"line (\d+) differs", out)
|
||||||
|
ok = (rc == 1
|
||||||
|
and "ALL CHECKS GREEN" not in out
|
||||||
|
and expect_line is not None
|
||||||
|
and named is not None
|
||||||
|
and int(named.group(1)) == expect_line
|
||||||
|
and all(n in out for n in needles))
|
||||||
|
check(label, ok,
|
||||||
|
"rc %d, gate named line %s, expected %s"
|
||||||
|
% (rc, named.group(1) if named else "nothing", expect_line))
|
||||||
|
if not ok:
|
||||||
|
print(" gate output:\n" + "\n".join(" " + l for l in out.strip().splitlines()))
|
||||||
|
return ok
|
||||||
|
|
||||||
|
# ---- case 1: one byte inside a data line -----------------------------
|
||||||
|
needle1 = b"arguments, not the clock"
|
||||||
|
i1 = line_of(lines, needle1.decode())
|
||||||
|
m1 = list(graded.split(b"\n"))
|
||||||
|
m1[i1 - 1] = m1[i1 - 1].replace(b"arguments", b"argumentz")
|
||||||
|
mutated1 = b"\n".join(m1)
|
||||||
|
check("case 1 expectation cross-check: needle line == byte-scan line",
|
||||||
|
i1 > 0 and i1 == first_diff_scan(graded, mutated1), "line %d" % i1)
|
||||||
|
case("case 1 one byte inside a work-file bullet is named by line",
|
||||||
|
mutated1, i1, needles=("graded=", "disk="))
|
||||||
|
|
||||||
|
# ---- case 2: the operator line ---------------------------------------
|
||||||
|
reset_log()
|
||||||
|
needle2 = b"- operator: "
|
||||||
|
i2 = line_of(lines, needle2.decode())
|
||||||
|
m2 = list(graded.split(b"\n"))
|
||||||
|
m2[i2 - 1] = m2[i2 - 1].replace(b"null", b"nobody-at-all")
|
||||||
|
mutated2 = b"\n".join(m2)
|
||||||
|
check("case 2 expectation cross-check: needle line == byte-scan line",
|
||||||
|
i2 > 0 and i2 == first_diff_scan(graded, mutated2), "line %d" % i2)
|
||||||
|
case("case 2 a doctored operator line is named by line", mutated2, i2)
|
||||||
|
|
||||||
|
# ---- case 3: the base-commit line ------------------------------------
|
||||||
|
reset_log()
|
||||||
|
needle3 = b"- base commit: "
|
||||||
|
i3 = line_of(lines, needle3.decode())
|
||||||
|
m3 = list(graded.split(b"\n"))
|
||||||
|
m3[i3 - 1] = re.sub(rb"(?<=- base commit: )[0-9a-f]+", b"deadbeef", m3[i3 - 1])
|
||||||
|
mutated3 = b"\n".join(m3)
|
||||||
|
check("case 3 expectation cross-check: needle line == byte-scan line",
|
||||||
|
i3 > 0 and i3 == first_diff_scan(graded, mutated3), "line %d" % i3)
|
||||||
|
case("case 3 a doctored base commit is named by line", mutated3, i3)
|
||||||
|
|
||||||
|
# ---- case 4: one line appended ---------------------------------------
|
||||||
|
reset_log()
|
||||||
|
mutated4 = graded + b"- extra: a line no writer produced\n"
|
||||||
|
i4 = len(lines) + 1
|
||||||
|
check("case 4 expectation cross-check: appended line == byte-scan line",
|
||||||
|
i4 == first_diff_scan(graded, mutated4), "line %d" % i4)
|
||||||
|
case("case 4 an appended line is named, and the graded side reads <no such line>",
|
||||||
|
mutated4, i4, needles=("<no such line>",))
|
||||||
|
|
||||||
|
# ---- case 5: a line terminator, and no line to name -------------------
|
||||||
|
reset_log()
|
||||||
|
mutated5 = graded[:-1] + b"\r\n"
|
||||||
|
with open(logp, "wb") as fh:
|
||||||
|
fh.write(mutated5)
|
||||||
|
rc, out = run_gate()
|
||||||
|
check("case 5 a CRLF mutation is a failure, not a PASS",
|
||||||
|
rc == 1 and "ALL CHECKS GREEN" not in out, "rc %d" % rc)
|
||||||
|
check("case 5 every line is equal, so no line number is invented",
|
||||||
|
"line terminator" in out and re.search(r"line \d+ differs", out) is None,
|
||||||
|
"reported as a terminator")
|
||||||
|
check("case 5 the byte count is printed, so the 1-byte delta is visible",
|
||||||
|
"%d bytes" % len(graded) in out and "%d bytes" % len(mutated5) in out)
|
||||||
|
|
||||||
|
# ---- case 6: the published gate has moved on --------------------------
|
||||||
|
reset_log()
|
||||||
|
m = re.search(r"^- base commit: ([0-9a-f]{40})$", text, re.M)
|
||||||
|
if not m:
|
||||||
|
check("case 6 the entry names a 40-hex base commit", False, "no base commit line")
|
||||||
|
else:
|
||||||
|
named = m.group(1)
|
||||||
|
with open(gatep, "ab") as fh:
|
||||||
|
fh.write(b"\n# fixture: a later commit on main changed this file\n")
|
||||||
|
git(fx, "-c", "user.name=null", "-c", "user.email=null@agents.invalid",
|
||||||
|
"commit", "-aqm", "fixture: publish a gate that moved on")
|
||||||
|
push = git(fx, "push", "--quiet", "origin", "HEAD:main")
|
||||||
|
check("case 6 fixture: the frozen origin advanced past the named commit",
|
||||||
|
push.returncode == 0,
|
||||||
|
"origin/main is now %s" % git(fx, "rev-parse", "HEAD").stdout.decode().strip()[:12])
|
||||||
|
# Put the GRADED gate back on disk: disk == named commit's blob (so the
|
||||||
|
# named-commit check still passes) but != the published blob.
|
||||||
|
with open(gatep, "wb") as fh:
|
||||||
|
fh.write(git(fx, "show", "%s:%s" % (named, GATEREL)).stdout)
|
||||||
|
rc, out = run_gate()
|
||||||
|
check("case 6 a gate that main does not carry is refused",
|
||||||
|
rc == 1 and "FAIL the gate on disk is the gate published on the remote main" in out,
|
||||||
|
"rc %d" % rc)
|
||||||
|
check("case 6 isolation: the entry itself is still byte-identical on main",
|
||||||
|
"FAIL entry bytes are the blob on the remote main" not in out
|
||||||
|
and "PASS entry bytes are the blob on the remote main" in out,
|
||||||
|
"only the published-gate cell moved")
|
||||||
|
|
||||||
|
print("\n%d passed, %d failed" % (len(PASSED), len(FAILED)))
|
||||||
|
if os.environ.get("NULL_Q57_KEEP") != "1":
|
||||||
|
shutil.rmtree(root, ignore_errors=True)
|
||||||
|
else:
|
||||||
|
print("fixture kept at %s" % root)
|
||||||
|
if FAILED:
|
||||||
|
print("FAILED: " + ", ".join(FAILED))
|
||||||
|
return 1
|
||||||
|
print("MUTATION CONTROL GREEN")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -42,23 +42,30 @@ LOGNAME = "wizard.log.md"
|
|||||||
LOGPATH = "wizard-work/null/" + LOGNAME
|
LOGPATH = "wizard-work/null/" + LOGNAME
|
||||||
STAMP_FMT = "%Y-%m-%d %H:%M UTC"
|
STAMP_FMT = "%Y-%m-%d %H:%M UTC"
|
||||||
STAMP_RE = re.compile(r"self-verified at [0-9-]+ [0-9:]+ UTC")
|
STAMP_RE = re.compile(r"self-verified at [0-9-]+ [0-9:]+ UTC")
|
||||||
TASK = "bind the standup entry to the commit that carries its work bytes"
|
TASK = "name the line a mutated standup entry changed, and commit the negative control"
|
||||||
AREA = "wizard-work/null"
|
AREA = "wizard-work/null"
|
||||||
OPERATOR = "null"
|
OPERATOR = "null"
|
||||||
WORK = [
|
WORK = [
|
||||||
"wizard.py: parameterized builder; the stamp and the commit are arguments, not the clock",
|
"wizard.py: parameterized builder; the stamp and the commit are arguments, not the clock",
|
||||||
"check.py: gate re-derives the carrying commit and the remote entry, never trusts the local clone",
|
"check.py: gate re-derives the carrying commit and the remote entry, never trusts the local clone; a mutation is named by its LINE, and both comparisons are byte-exact",
|
||||||
"wizard.log.md: self-describing entry naming the commit that carries wizard.py and check.py",
|
"q57_mutation.py: the committed negative control -- seven mutation cases in a frozen clone, exit code is the verdict",
|
||||||
|
"wizard.log.md: self-describing entry naming the commit that carries the work files",
|
||||||
]
|
]
|
||||||
EVIDENCE = [
|
EVIDENCE = [
|
||||||
("the named commit is by null and is an ancestor of the REMOTE main", "measured",
|
("the named commit is by null and is an ancestor of the REMOTE main", "measured",
|
||||||
"check.py: git ls-remote origin refs/heads/main + git merge-base --is-ancestor"),
|
"check.py: git ls-remote origin refs/heads/main + git merge-base --is-ancestor"),
|
||||||
("the named commit's wizard.py and check.py are byte-identical to disk", "measured",
|
("the named commit's work files are byte-identical to disk", "measured",
|
||||||
"check.py: git show <sha>:path vs the file on disk, byte comparison"),
|
"check.py: git show <sha>:path vs the file on disk, byte comparison"),
|
||||||
("the entry's bytes are the blob on the REMOTE main", "measured",
|
("the entry's bytes are the blob on the REMOTE main", "measured",
|
||||||
"check.py: git ls-remote + git show <remote>:wizard-work/null/wizard.log.md"),
|
"check.py: git ls-remote + git show <remote>:wizard-work/null/wizard.log.md, bytes"),
|
||||||
("fresh render with the log's own stamp == log", "measured",
|
("a mutation of the entry is reported by LINE, not by byte count", "measured",
|
||||||
"check.py: render() round-trip, no clock in the output"),
|
"check.py: first_differing_line() + diff_detail(); q57_mutation.py cases 1-4"),
|
||||||
|
("a mutation that leaves every line equal is reported as a terminator, not a line", "measured",
|
||||||
|
"check.py: diff_detail() returns no line number when splitlines() agree; q57_mutation.py case 5"),
|
||||||
|
("the gate on disk is the gate published on the remote main", "measured",
|
||||||
|
"check.py: git show <remote>:wizard-work/null/check.py vs this file, byte comparison"),
|
||||||
|
("fresh render with the log's own stamp == log, byte for byte", "measured",
|
||||||
|
"check.py: render() round-trip; both reads disable newline translation"),
|
||||||
("craft() deterministic in its stamp argument", "measured",
|
("craft() deterministic in its stamp argument", "measured",
|
||||||
"check.py: same args -> same bytes; different stamp -> different verdict"),
|
"check.py: same args -> same bytes; different stamp -> different verdict"),
|
||||||
]
|
]
|
||||||
@@ -186,7 +193,8 @@ def main():
|
|||||||
if not stamp:
|
if not stamp:
|
||||||
stamp = stamp_now() # first build only; replays reuse the committed stamp
|
stamp = stamp_now() # first build only; replays reuse the committed stamp
|
||||||
|
|
||||||
sha = commit_carrying(["wizard-work/null/wizard.py", "wizard-work/null/check.py"])
|
sha = commit_carrying(["wizard-work/null/wizard.py", "wizard-work/null/check.py",
|
||||||
|
"wizard-work/null/q57_mutation.py"])
|
||||||
if not sha:
|
if not sha:
|
||||||
sha = "unresolved" # the gate refuses to pass on this
|
sha = "unresolved" # the gate refuses to pass on this
|
||||||
text = render(TASK, AREA, LANE, sha, OPERATOR, stamp)
|
text = render(TASK, AREA, LANE, sha, OPERATOR, stamp)
|
||||||
|
|||||||
Reference in new issue
Block a user