175 lines
7.4 KiB
Python
175 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
"""test_check_provenance.py -- black-box grader for wizard-work/sol/CHECK-2026-09-28.
|
|
|
|
WHY THIS EXISTS
|
|
Q106: CHECK-2026-09-28 pinned `ROOT = "/root/troupe-standup"`, so it graded the
|
|
host worktree it was born in and ignored the tree it shipped with. Measured
|
|
before the repair: a clone whose own manifest was doctored (one item
|
|
`archive_fix` -> `app_pr`, aggregate kept consistent) still read
|
|
`VERDICT TRUE ... categories={'archive_fix': 7, ...}` rc 0 -- the host's cells,
|
|
not the clone's 6/4/2/0.
|
|
|
|
This module runs the CHECK as a subprocess and imports nothing from it, so no
|
|
cell can agree with the code by sharing its constants. It builds its own
|
|
substrate: a minimal tree (seed + manifest + CHECK) under a temporary root.
|
|
|
|
CELLS
|
|
1 tree-ships-green the shipped CHECK is green and names the tree it lives in
|
|
2 poisoned-copy-is-red a doctored copy reads FALSE, rc 1, and prints the COPY's cells
|
|
3 clean-copy-foreign-cwd green from a cwd that is neither tree, subject = the copy
|
|
4 root-override --root grades the tree it names (poisoned -> rc 1)
|
|
5 no-host-pin the file carries no absolute /root/ path and no ROOT literal
|
|
6 usage-error-is-2 an unknown flag exits 2 (could not grade, not a verdict)
|
|
7 missing-manifest-is-2 a tree without its manifest exits 2 naming the path, no traceback
|
|
8 host-pin-mutant-revived restoring the pin over a poisoned copy reads TRUE again --
|
|
the falsification cell: cell 2 fails on the exact old defect
|
|
|
|
Run from anywhere:
|
|
python3 wizard-work/sol/test_check_provenance.py
|
|
"""
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
REPO = HERE.parents[1]
|
|
CHECK = HERE / "CHECK-2026-09-28"
|
|
SEED_REL = Path("inbox") / "seed-2026-09-26.json"
|
|
MANIFEST_REL = Path("wizard-work") / "sol" / "Triage-2026-09-28-classify.json"
|
|
CHECK_REL = Path("wizard-work") / "sol" / "CHECK-2026-09-28"
|
|
|
|
PIN_LINE = 'DEFAULT_ROOT = os.path.dirname(os.path.dirname(HERE))'
|
|
HOST_PIN = 'DEFAULT_ROOT = "/root/troupe-standup"'
|
|
|
|
|
|
def run_check(check_path, args=(), cwd=None):
|
|
return subprocess.run(
|
|
[sys.executable, str(check_path), *args],
|
|
cwd=str(cwd) if cwd else None,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120,
|
|
)
|
|
|
|
|
|
def build_tree(dest):
|
|
"""A minimal tree the CHECK can grade: seed + manifest + the CHECK itself."""
|
|
root = Path(dest)
|
|
for rel in (SEED_REL, MANIFEST_REL):
|
|
target = root / rel
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(REPO / rel, target)
|
|
(root / CHECK_REL).parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(CHECK, root / CHECK_REL)
|
|
os.chmod(root / CHECK_REL, 0o755)
|
|
return root
|
|
|
|
|
|
def poison(root):
|
|
"""Flip one item's category and keep the aggregate honest: 6/4/2/0."""
|
|
path = Path(root) / MANIFEST_REL
|
|
manifest = json.loads(path.read_text(encoding="utf-8"))
|
|
item = next(i for i in manifest["items"] if i["category"] == "archive_fix")
|
|
item["category"] = "app_pr"
|
|
counts = {"archive_fix": 0, "app_pr": 0, "general": 0, "other": 0}
|
|
for i in manifest["items"]:
|
|
counts[i["category"]] += 1
|
|
manifest["categories"] = counts
|
|
path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
|
|
return item["id"], counts
|
|
|
|
|
|
class CheckProvenance(unittest.TestCase):
|
|
def setUp(self):
|
|
self._tmp = tempfile.TemporaryDirectory(prefix="q106-")
|
|
self.tmp = Path(self._tmp.name)
|
|
|
|
def tearDown(self):
|
|
self._tmp.cleanup()
|
|
|
|
def test_1_tree_ships_green(self):
|
|
proc = run_check(CHECK, cwd=self.tmp)
|
|
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
|
|
self.assertIn("VERDICT TRUE", proc.stdout)
|
|
self.assertIn(f"subject root={REPO}", proc.stdout)
|
|
|
|
def test_2_poisoned_copy_is_graded_as_poisoned(self):
|
|
root = build_tree(self.tmp / "copy")
|
|
item_id, counts = poison(root)
|
|
self.assertEqual(counts, {"archive_fix": 6, "app_pr": 4, "general": 2, "other": 0})
|
|
proc = run_check(root / CHECK_REL, cwd=self.tmp)
|
|
self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr)
|
|
self.assertIn(f"subject root={root}", proc.stdout)
|
|
self.assertNotIn("VERDICT TRUE", proc.stdout)
|
|
self.assertIn("VERDICT FALSE", proc.stdout)
|
|
# the graded side is the COPY's manifest (6/4), not the host's (7/3)
|
|
self.assertIn("categories {'archive_fix': 6, 'app_pr': 4, 'general': 2, "
|
|
"'other': 0}", proc.stdout)
|
|
self.assertIn("per-item categories differ", proc.stdout)
|
|
self.assertIn(f"({item_id}, 'app_pr')", proc.stdout)
|
|
|
|
def test_3_clean_copy_green_from_foreign_cwd(self):
|
|
root = build_tree(self.tmp / "clean")
|
|
elsewhere = self.tmp / "elsewhere"
|
|
elsewhere.mkdir()
|
|
proc = run_check(root / CHECK_REL, cwd=elsewhere)
|
|
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
|
|
self.assertIn("VERDICT TRUE", proc.stdout)
|
|
self.assertIn(f"subject root={root}", proc.stdout)
|
|
|
|
def test_4_root_override_grades_the_named_tree(self):
|
|
root = build_tree(self.tmp / "override")
|
|
poison(root)
|
|
proc = run_check(CHECK, args=["--root", str(root)], cwd=self.tmp)
|
|
self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr)
|
|
self.assertIn(f"subject root={root}", proc.stdout)
|
|
self.assertIn("VERDICT FALSE", proc.stdout)
|
|
|
|
def test_5_no_host_pin(self):
|
|
blob = CHECK.read_bytes()
|
|
self.assertNotIn(b"/root/", blob)
|
|
self.assertNotIn(b'ROOT = "', blob)
|
|
self.assertIn(PIN_LINE.encode(), blob)
|
|
|
|
def test_6_usage_error_is_2(self):
|
|
proc = run_check(CHECK, args=["--bogus"], cwd=self.tmp)
|
|
self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr)
|
|
|
|
def test_7_missing_manifest_is_2(self):
|
|
root = build_tree(self.tmp / "gone")
|
|
(root / MANIFEST_REL).unlink()
|
|
proc = run_check(root / CHECK_REL, cwd=self.tmp)
|
|
self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr)
|
|
self.assertIn("CANNOT GRADE", proc.stdout)
|
|
self.assertIn(str(root / MANIFEST_REL), proc.stdout)
|
|
self.assertNotIn("Traceback", proc.stdout + proc.stderr)
|
|
|
|
def test_8_host_pin_mutant_revives_the_fault(self):
|
|
"""The cell that keeps cell 2 from being vacuous: reinstate the pin."""
|
|
root = build_tree(self.tmp / "mutant")
|
|
poison(root)
|
|
mutant = (root / CHECK_REL).with_name("CHECK-mutant-pinned")
|
|
blob = (root / CHECK_REL).read_text(encoding="utf-8")
|
|
self.assertIn(PIN_LINE, blob)
|
|
mutant.write_text(blob.replace(PIN_LINE, HOST_PIN), encoding="utf-8")
|
|
self.assertIn("/root/", mutant.read_text(encoding="utf-8"))
|
|
|
|
green_under_fault = run_check(mutant, cwd=self.tmp)
|
|
red_over_copy = run_check(root / CHECK_REL, cwd=self.tmp) # repaired line
|
|
|
|
self.assertEqual(green_under_fault.returncode, 0,
|
|
"the mutant should read the host and go green:\n"
|
|
+ green_under_fault.stdout)
|
|
self.assertIn("VERDICT TRUE", green_under_fault.stdout)
|
|
self.assertEqual(red_over_copy.returncode, 1)
|
|
self.assertNotEqual(green_under_fault.returncode, red_over_copy.returncode)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|