#!/usr/bin/env python3 """Tests for the auditable corpus manifest: provenance, tombstones, and lifecycle rules. Issue #25 promotes the ingest script into a corpus pipeline with an append-only manifest, revocation tombstones, and lifecycle transitions that cannot be walked backwards. Every fixture here is synthetic; no real photo bytes are read or written. """ from __future__ import annotations import hashlib import importlib.util import json import os import re import subprocess import sys import tempfile from pathlib import Path import unittest from PIL import Image ROOT = Path(__file__).resolve().parents[1] INGEST = ROOT / "scripts" / "ingest_training_photo.py" FIXTURE = ROOT / "tests" / "fixtures" / "synthetic-type4.jpg" CONTRIBUTION_ID = "6f1d0c9e-2c4a-4f3b-9d1e-8a7b6c5d4e3f" ISO_UTC = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?\+00:00$") def load_structures(): spec = importlib.util.spec_from_file_location("corpus_manifest", ROOT / "scripts" / "corpus_manifest.py") assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module structures = load_structures() def active_image_paths(out: Path) -> list[str]: return [record["relativePath"] for record in structures.active_records(structures.load_manifest(out / "manifest.jsonl"))] def ingest(args: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess: return subprocess.run( [sys.executable, str(INGEST), *args], cwd=ROOT, capture_output=True, text=True, env={**os.environ, "TIMMY_DATASET_SALT": "test-only-salt", **(env or {})}, ) def ingest_default(out: Path, **overrides: str) -> subprocess.CompletedProcess: args = [ "--input", str(FIXTURE), "--output-dir", str(out), "--subject-id", "subject-123", "--consent-version", "research-v1", "--contribution-id", CONTRIBUTION_ID, "--bristol-type", "4", "--color", "brown", "--quality", "good", "--reviewer", "user-confirmed", ] for key, value in overrides.items(): flag = "--" + key.replace("_", "-") if value is None: index = args.index(flag) del args[index:index + 2] continue args[args.index(flag) + 1] = value return ingest(args) def manifest_lines(out: Path) -> list[dict]: text = (out / "manifest.jsonl").read_text(encoding="utf-8") return [json.loads(line) for line in text.splitlines() if line.strip()] class IngestProvenanceTests(unittest.TestCase): def test_ingest_records_consent_source_hashes_derivation_review_split_and_lifecycle(self): with tempfile.TemporaryDirectory() as raw: out = Path(raw) run = ingest_default(out) self.assertEqual(run.returncode, 0, run.stderr) record = json.loads(run.stdout) raw_bytes = FIXTURE.read_bytes() self.assertEqual(record["schemaVersion"], 2) self.assertEqual(record["contributionId"], CONTRIBUTION_ID) self.assertEqual(record["consentVersion"], "research-v1") self.assertEqual(record["consent"], {"version": "research-v1", "recordedAt": record["ingestedAt"]}) self.assertRegex(record["consent"]["recordedAt"], ISO_UTC) # Source is named, and the raw input only ever appears as a digest. self.assertEqual(record["source"], { "kind": "cli-ingest", "inputSha256": hashlib.sha256(raw_bytes).hexdigest(), }) self.assertEqual(record["hashes"]["rawInput"], hashlib.sha256(raw_bytes).hexdigest()) self.assertEqual(record["hashes"]["derived"], record["imageId"]) self.assertEqual(record["derivation"], { "format": "jpeg", "maxDimension": 1200, "metadataStripped": True, "pipeline": "timmy-ingest/2", }) self.assertEqual(record["review"], {"status": "user-confirmed", "reviewedAt": record["ingestedAt"]}) self.assertIn(record["split"], ("train", "validation", "test")) self.assertEqual(record["lifecycle"], {"state": "active", "updatedAt": record["ingestedAt"]}) self.assertEqual(record["recordType"], "sample") # Provenance never leaks the caller's path or the source bytes. self.assertNotIn("inputPath", record) on_disk = (out / "manifest.jsonl").read_text(encoding="utf-8") self.assertNotIn(str(FIXTURE), on_disk) self.assertNotIn(str(out), on_disk) self.assertEqual(manifest_lines(out), [record]) self.assertTrue((out / record["relativePath"]).is_file()) class RevocationTests(unittest.TestCase): def test_revocation_appends_a_tombstone_and_removes_the_bytes_from_active_datasets(self): with tempfile.TemporaryDirectory() as raw: out = Path(raw) self.assertEqual(ingest_default(out).returncode, 0) sample = manifest_lines(out)[0] derived = out / sample["relativePath"] self.assertTrue(derived.is_file()) # Some pipelines keep the pre-derivation copy; revocation must clear that too. raw_dir = out / "raw" raw_dir.mkdir() raw_copy = raw_dir / f"{sample['hashes']['rawInput']}.jpg" raw_copy.write_bytes(FIXTURE.read_bytes()) run = ingest(["--revoke", CONTRIBUTION_ID, "--output-dir", str(out), "--reason", "subject withdrawal"]) self.assertEqual(run.returncode, 0, run.stderr) tombstone = json.loads(run.stdout) self.assertEqual(tombstone["schemaVersion"], 2) self.assertEqual(tombstone["recordType"], "tombstone") self.assertEqual(tombstone["contributionId"], CONTRIBUTION_ID) self.assertEqual(tombstone["imageId"], sample["imageId"]) self.assertEqual(tombstone["lifecycle"]["state"], "revoked") self.assertRegex(tombstone["lifecycle"]["updatedAt"], ISO_UTC) self.assertEqual(tombstone["lifecycle"]["reason"], "subject withdrawal") # The tombstone addresses every derived artifact by contribution id, and says what went. self.assertEqual(tombstone["deletedArtifacts"], [ {"relativePath": sample["relativePath"], "sha256": sample["hashes"]["derived"], "deleted": True}, {"relativePath": f"raw/{sample['hashes']['rawInput']}.jpg", "sha256": sample["hashes"]["rawInput"], "deleted": True}, ]) self.assertEqual(tombstone["contributionId"], CONTRIBUTION_ID) # No byte-bearing copy survives anywhere in the dataset tree. self.assertFalse(derived.exists()) self.assertFalse(raw_copy.exists()) survivors = [] for path in out.rglob("*"): if path.is_file() and path.name != "manifest.jsonl": survivors.append((str(path.relative_to(out)), hashlib.sha256(path.read_bytes()).hexdigest())) self.assertEqual(survivors, []) # History is append-only: the sample record is untouched, the tombstone follows it. history = manifest_lines(out) self.assertEqual(len(history), 2) self.assertEqual(history[0], sample) self.assertEqual(history[1], tombstone) # A deletion receipt keeps identifiers and hashes, never the label or subject key. self.assertNotIn("labels", tombstone) self.assertNotIn("subjectKey", tombstone) self.assertNotIn("relativePath", tombstone) class ActiveRecordTests(unittest.TestCase): def test_only_unrevoked_samples_are_active(self): with tempfile.TemporaryDirectory() as raw: out = Path(raw) self.assertEqual(ingest_default(out).returncode, 0) self.assertEqual(active_image_paths(out), [f"images/{manifest_lines(out)[0]['imageId']}.jpg"]) self.assertEqual(ingest(["--revoke", CONTRIBUTION_ID, "--output-dir", str(out), "--reason", "gone"]).returncode, 0) self.assertEqual(active_image_paths(out), []) class IllegalTransitionTests(unittest.TestCase): def test_a_second_revocation_is_refused_and_appends_nothing(self): with tempfile.TemporaryDirectory() as raw: out = Path(raw) self.assertEqual(ingest_default(out).returncode, 0) self.assertEqual(ingest(["--revoke", CONTRIBUTION_ID, "--output-dir", str(out), "--reason", "first"]).returncode, 0) history = manifest_lines(out) self.assertEqual(len(history), 2) again = ingest(["--revoke", CONTRIBUTION_ID, "--output-dir", str(out), "--reason", "again"]) self.assertNotEqual(again.returncode, 0) self.assertRegex(again.stderr, "already revoked") self.assertEqual(manifest_lines(out), history) def test_revoking_an_unknown_contribution_is_refused(self): with tempfile.TemporaryDirectory() as raw: out = Path(raw) self.assertEqual(ingest_default(out).returncode, 0) history = manifest_lines(out) unknown = ingest(["--revoke", "11111111-2222-3333-4444-555555555555", "--output-dir", str(out), "--reason", "nope"]) self.assertNotEqual(unknown.returncode, 0) self.assertRegex(unknown.stderr, "no ingested sample carries contribution") self.assertEqual(manifest_lines(out), history) self.assertTrue((out / history[0]["relativePath"]).is_file()) def test_a_contribution_cannot_be_ingested_twice(self): with tempfile.TemporaryDirectory() as raw: out = Path(raw) self.assertEqual(ingest_default(out).returncode, 0) history = manifest_lines(out) duplicate = ingest_default(out, color="green") self.assertNotEqual(duplicate.returncode, 0) self.assertRegex(duplicate.stderr, "already ingested") self.assertEqual(manifest_lines(out), history) def test_a_refused_duplicate_leaves_no_orphan_image_behind(self): with tempfile.TemporaryDirectory() as raw: out = Path(raw) self.assertEqual(ingest_default(out).returncode, 0) history = manifest_lines(out) # Same contribution id, different pixels: the derived digest differs, so a guard that # runs after the write would strand a second image in the active dataset. variant = out.parent / "variant-type4.jpg" with Image.open(FIXTURE) as source: changed = source.convert("RGB") changed.putpixel((4, 4), (255, 0, 0)) changed.save(variant, "JPEG", quality=95) duplicate = ingest_default(out, input=str(variant)) self.assertNotEqual(duplicate.returncode, 0) self.assertRegex(duplicate.stderr, "already ingested") self.assertEqual(manifest_lines(out), history) images = sorted(path.name for path in (out / "images").iterdir()) self.assertEqual(images, [f"{history[0]['imageId']}.jpg"]) def test_lifecycle_transitions_cannot_walk_backwards(self): with tempfile.TemporaryDirectory() as raw: out = Path(raw) self.assertEqual(ingest_default(out).returncode, 0) sample = manifest_lines(out)[0] at = sample["ingestedAt"] revoked = structures.transition(sample, "revoked", at, "withdrawal") self.assertEqual(revoked["lifecycle"]["state"], "revoked") self.assertEqual(revoked["lifecycle"]["reason"], "withdrawal") self.assertEqual(sample["lifecycle"]["state"], "active", "transition must not mutate its input") for args, expected in ( ((revoked, "active", at), "cannot return to the active dataset"), ((sample, "revoked", at, " "), "recorded reason"), ((sample, "archived", at), "unknown lifecycle state"), (({**sample, "lifecycle": {"state": "active", "updatedAt": at}}, "active", at), "already active"), ): with self.assertRaises(structures.IllegalTransition) as raised: structures.transition(*args) self.assertRegex(str(raised.exception), expected) if __name__ == "__main__": unittest.main(verbosity=2)