180 lines
7.0 KiB
Python
180 lines
7.0 KiB
Python
#!/usr/bin/env python3
|
|
"""selftest.py -- Q140: nine legs over audit.py, run by CHECK-q140.
|
|
|
|
Every leg drives the instrument through its command line, so a leg cannot pass
|
|
because it shares the instrument's bugs:
|
|
|
|
1 pin the pinned rev reads rc 0 with the pinned counts
|
|
2 rev-move a different rev reads rc 1 and names the rev, not just counts
|
|
3 worktree-free a --shared --no-checkout clone of the repo reads identical counts
|
|
4 mutant-drift one sha character changed in a tracked row -> rc 1
|
|
5 mutant-drop one tracked row deleted -> rc 1
|
|
6 mutant-orphan one row appended for a path that is not at the rev -> rc 1
|
|
|
|
Legs 4-6 are why the pin is not vacuous: the same instrument that prints OK on
|
|
the real census prints FAULT with the predicted count move on a mutated one.
|
|
"""
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
AUDIT = os.path.join(HERE, "audit.py")
|
|
REPO = os.path.abspath(os.path.join(HERE, "..", "..", ".."))
|
|
sys.path.insert(0, HERE)
|
|
import audit # noqa: E402 (pin and csv path only; no derivation is reused)
|
|
|
|
CSV = audit.DEFAULT_CSV
|
|
PIN_REV = audit.PIN["rev"]
|
|
OTHER_REV = "2e97e4d46242224cf56f3d24f3771054bad22eac" # census commit before the rebuild
|
|
FAULTS = []
|
|
LEGS = 0
|
|
|
|
|
|
def run(args):
|
|
proc = subprocess.run([sys.executable, AUDIT] + args, cwd=REPO,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
return proc.returncode, proc.stdout.decode("utf-8", "replace")
|
|
|
|
|
|
def leg(name, rc, out, want_rc, must=()):
|
|
global LEGS
|
|
LEGS += 1
|
|
bad = []
|
|
if rc != want_rc:
|
|
bad.append("rc %d, wanted %d" % (rc, want_rc))
|
|
for needle in must:
|
|
if needle not in out:
|
|
bad.append("missing %r" % needle)
|
|
if bad:
|
|
FAULTS.append(name)
|
|
print("FAIL %-16s %s" % (name, "; ".join(bad)))
|
|
for line in out.strip().splitlines()[-6:]:
|
|
print(" | " + line)
|
|
return False
|
|
print("PASS %-16s rc=%d %s" % (name, rc, must[-1] if must else ""))
|
|
return True
|
|
|
|
|
|
def git(*args, stdin=None):
|
|
proc = subprocess.run(["git", "-C", REPO, *args], input=stdin,
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
|
if proc.returncode != 0:
|
|
raise SystemExit("git %s rc %d: %s" % (" ".join(args), proc.returncode,
|
|
proc.stderr.decode()[:300]))
|
|
return proc.stdout
|
|
|
|
|
|
def census_text():
|
|
blob = git("rev-parse", "%s:%s" % (PIN_REV, CSV)).decode().strip()
|
|
return git("cat-file", "blob", blob).decode("utf-8")
|
|
|
|
|
|
def is_tracked(path):
|
|
return subprocess.run(["git", "-C", REPO, "cat-file", "-e",
|
|
"%s:%s" % (PIN_REV, path)],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL).returncode == 0
|
|
|
|
|
|
def first_tracked_row(lines):
|
|
for i, line in enumerate(lines):
|
|
if not line.strip():
|
|
continue
|
|
path = line.split(",", 1)[0]
|
|
if is_tracked(path):
|
|
return i
|
|
raise SystemExit("no tracked row in the census")
|
|
|
|
|
|
def write_mutant(dirpath, text):
|
|
path = os.path.join(dirpath, "mutant-census.csv")
|
|
with open(path, "w", encoding="utf-8", newline="") as f:
|
|
f.write(text)
|
|
return path
|
|
|
|
|
|
def main():
|
|
scratch = tempfile.mkdtemp(prefix="q140-", dir=os.environ.get("TMPDIR") or "/tmp")
|
|
try:
|
|
# 1 -- the pinned reading
|
|
rc, out = run(["--repo", REPO, "--rev", PIN_REV, "--check"])
|
|
leg("pin", rc, out, 0, ["CHECK OK: 594 rows = 574 tracked + 20 ignored",
|
|
"0 unrowed", "counts match the pin"])
|
|
|
|
# 2 -- the pin compares the rev, so a moved rev is a named fault
|
|
rc, out = run(["--repo", REPO, "--rev", OTHER_REV, "--check"])
|
|
leg("rev-move", rc, out, 1, ["rev moved %s" % PIN_REV[:12]])
|
|
|
|
# 3 -- the derivation is a function of git objects, not of the worktree
|
|
global LEGS
|
|
LEGS += 1
|
|
rc, out = run(["--repo", REPO, "--rev", PIN_REV, "--json"])
|
|
host = json.loads(out)["counts"]
|
|
clone = os.path.join(scratch, "clone.git")
|
|
subprocess.run(["git", "clone", "--quiet", "--shared", "--no-checkout",
|
|
REPO, clone], check=True,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE)
|
|
rc2, out2 = run(["--repo", clone, "--rev", PIN_REV, "--json"])
|
|
ok = rc2 == 0
|
|
try:
|
|
theirs = json.loads(out2)["counts"]
|
|
except Exception:
|
|
theirs = None
|
|
same = theirs == host
|
|
if not (rc == 0 and ok and same):
|
|
FAULTS.append("worktree-free")
|
|
print("FAIL worktree-free rc host=%d clone=%d same_counts=%s"
|
|
% (rc, rc2, same))
|
|
for line in out2.strip().splitlines()[-4:]:
|
|
print(" | " + line)
|
|
else:
|
|
print("PASS worktree-free rc=0 clone counts identical to host: %s"
|
|
% json.dumps(host, sort_keys=True))
|
|
|
|
# 4 -- mutation: one sha character in a tracked row
|
|
lines = census_text().splitlines(keepends=True)
|
|
i = first_tracked_row(lines)
|
|
parts = lines[i].rstrip("\r\n").split(",")
|
|
parts[2] = ("0" if parts[2][0] != "0" else "1") + parts[2][1:]
|
|
lines[i] = ",".join(parts) + "\n"
|
|
path = write_mutant(scratch, "".join(lines))
|
|
rc, out = run(["--repo", REPO, "--rev", PIN_REV, "--csv-file", path])
|
|
leg("mutant-drift", rc, out, 0, ["tracked drift: 1"])
|
|
rc, out = run(["--repo", REPO, "--rev", PIN_REV, "--csv-file", path, "--check"])
|
|
leg("mutant-drift-check", rc, out, 1, ["count tracked_drift moved 0 -> 1"])
|
|
|
|
# 5 -- mutation: a tracked row removed
|
|
lines = census_text().splitlines(keepends=True)
|
|
i = first_tracked_row(lines)
|
|
del lines[i]
|
|
path = write_mutant(scratch, "".join(lines))
|
|
rc, out = run(["--repo", REPO, "--rev", PIN_REV, "--csv-file", path])
|
|
leg("mutant-drop", rc, out, 0, ["tracked blobs with no row: 1"])
|
|
rc, out = run(["--repo", REPO, "--rev", PIN_REV, "--csv-file", path, "--check"])
|
|
leg("mutant-drop-check", rc, out, 1, ["count missing moved 0 -> 1"])
|
|
|
|
# 6 -- mutation: a row for a path that is not at the rev
|
|
ghost = "wizard-work/ledger/ghost-q140.md,3," + "0" * 64 + "\n"
|
|
path = write_mutant(scratch, census_text().rstrip("\r\n") + "\n" + ghost)
|
|
rc, out = run(["--repo", REPO, "--rev", PIN_REV, "--csv-file", path])
|
|
leg("mutant-orphan", rc, out, 0, ["untracked (not ignored): 1"])
|
|
rc, out = run(["--repo", REPO, "--rev", PIN_REV, "--csv-file", path, "--check"])
|
|
leg("mutant-orphan-check", rc, out, 1, ["count untracked_other moved 0 -> 1"])
|
|
finally:
|
|
shutil.rmtree(scratch, ignore_errors=True)
|
|
|
|
print("\n%d leg(s), %d failed" % (LEGS, len(FAULTS)))
|
|
if FAULTS:
|
|
print("FAILED: " + ", ".join(FAULTS))
|
|
return 1
|
|
print("Q140 OK")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|