timmy
578cc9f129
Quality gates / quality (pull_request) Successful in 13m1s
CI's diff-hygiene gate rejects a trailing blank line at EOF; the local check:diff run happened before the first commit existed, so it checked the previous range and missed it.
274 lines
12 KiB
Python
274 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Tests for the auditable corpus manifest: provenance, tombstones, and lifecycle rules.
|
|
|
|
Issue #25 promotes the ingest script into a corpus pipeline with an append-only manifest,
|
|
revocation tombstones, and lifecycle transitions that cannot be walked backwards. Every
|
|
fixture here is synthetic; no real photo bytes are read or written.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
import unittest
|
|
|
|
from PIL import Image
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
INGEST = ROOT / "scripts" / "ingest_training_photo.py"
|
|
FIXTURE = ROOT / "tests" / "fixtures" / "synthetic-type4.jpg"
|
|
CONTRIBUTION_ID = "6f1d0c9e-2c4a-4f3b-9d1e-8a7b6c5d4e3f"
|
|
ISO_UTC = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?\+00:00$")
|
|
|
|
|
|
def load_structures():
|
|
spec = importlib.util.spec_from_file_location("corpus_manifest", ROOT / "scripts" / "corpus_manifest.py")
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
structures = load_structures()
|
|
|
|
|
|
def active_image_paths(out: Path) -> list[str]:
|
|
return [record["relativePath"] for record in structures.active_records(structures.load_manifest(out / "manifest.jsonl"))]
|
|
|
|
|
|
|
|
def ingest(args: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
[sys.executable, str(INGEST), *args],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
env={**os.environ, "TIMMY_DATASET_SALT": "test-only-salt", **(env or {})},
|
|
)
|
|
|
|
|
|
def ingest_default(out: Path, **overrides: str) -> subprocess.CompletedProcess:
|
|
args = [
|
|
"--input", str(FIXTURE),
|
|
"--output-dir", str(out),
|
|
"--subject-id", "subject-123",
|
|
"--consent-version", "research-v1",
|
|
"--contribution-id", CONTRIBUTION_ID,
|
|
"--bristol-type", "4",
|
|
"--color", "brown",
|
|
"--quality", "good",
|
|
"--reviewer", "user-confirmed",
|
|
]
|
|
for key, value in overrides.items():
|
|
flag = "--" + key.replace("_", "-")
|
|
if value is None:
|
|
index = args.index(flag)
|
|
del args[index:index + 2]
|
|
continue
|
|
args[args.index(flag) + 1] = value
|
|
return ingest(args)
|
|
|
|
|
|
def manifest_lines(out: Path) -> list[dict]:
|
|
text = (out / "manifest.jsonl").read_text(encoding="utf-8")
|
|
return [json.loads(line) for line in text.splitlines() if line.strip()]
|
|
|
|
|
|
class IngestProvenanceTests(unittest.TestCase):
|
|
def test_ingest_records_consent_source_hashes_derivation_review_split_and_lifecycle(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
out = Path(raw)
|
|
run = ingest_default(out)
|
|
self.assertEqual(run.returncode, 0, run.stderr)
|
|
record = json.loads(run.stdout)
|
|
raw_bytes = FIXTURE.read_bytes()
|
|
|
|
self.assertEqual(record["schemaVersion"], 2)
|
|
self.assertEqual(record["contributionId"], CONTRIBUTION_ID)
|
|
self.assertEqual(record["consentVersion"], "research-v1")
|
|
self.assertEqual(record["consent"], {"version": "research-v1", "recordedAt": record["ingestedAt"]})
|
|
self.assertRegex(record["consent"]["recordedAt"], ISO_UTC)
|
|
|
|
# Source is named, and the raw input only ever appears as a digest.
|
|
self.assertEqual(record["source"], {
|
|
"kind": "cli-ingest",
|
|
"inputSha256": hashlib.sha256(raw_bytes).hexdigest(),
|
|
})
|
|
self.assertEqual(record["hashes"]["rawInput"], hashlib.sha256(raw_bytes).hexdigest())
|
|
self.assertEqual(record["hashes"]["derived"], record["imageId"])
|
|
|
|
self.assertEqual(record["derivation"], {
|
|
"format": "jpeg",
|
|
"maxDimension": 1200,
|
|
"metadataStripped": True,
|
|
"pipeline": "timmy-ingest/2",
|
|
})
|
|
self.assertEqual(record["review"], {"status": "user-confirmed", "reviewedAt": record["ingestedAt"]})
|
|
self.assertIn(record["split"], ("train", "validation", "test"))
|
|
self.assertEqual(record["lifecycle"], {"state": "active", "updatedAt": record["ingestedAt"]})
|
|
self.assertEqual(record["recordType"], "sample")
|
|
|
|
# Provenance never leaks the caller's path or the source bytes.
|
|
self.assertNotIn("inputPath", record)
|
|
on_disk = (out / "manifest.jsonl").read_text(encoding="utf-8")
|
|
self.assertNotIn(str(FIXTURE), on_disk)
|
|
self.assertNotIn(str(out), on_disk)
|
|
self.assertEqual(manifest_lines(out), [record])
|
|
self.assertTrue((out / record["relativePath"]).is_file())
|
|
|
|
|
|
class RevocationTests(unittest.TestCase):
|
|
def test_revocation_appends_a_tombstone_and_removes_the_bytes_from_active_datasets(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
out = Path(raw)
|
|
self.assertEqual(ingest_default(out).returncode, 0)
|
|
sample = manifest_lines(out)[0]
|
|
derived = out / sample["relativePath"]
|
|
self.assertTrue(derived.is_file())
|
|
# Some pipelines keep the pre-derivation copy; revocation must clear that too.
|
|
raw_dir = out / "raw"
|
|
raw_dir.mkdir()
|
|
raw_copy = raw_dir / f"{sample['hashes']['rawInput']}.jpg"
|
|
raw_copy.write_bytes(FIXTURE.read_bytes())
|
|
|
|
run = ingest(["--revoke", CONTRIBUTION_ID, "--output-dir", str(out), "--reason", "subject withdrawal"])
|
|
self.assertEqual(run.returncode, 0, run.stderr)
|
|
tombstone = json.loads(run.stdout)
|
|
|
|
self.assertEqual(tombstone["schemaVersion"], 2)
|
|
self.assertEqual(tombstone["recordType"], "tombstone")
|
|
self.assertEqual(tombstone["contributionId"], CONTRIBUTION_ID)
|
|
self.assertEqual(tombstone["imageId"], sample["imageId"])
|
|
self.assertEqual(tombstone["lifecycle"]["state"], "revoked")
|
|
self.assertRegex(tombstone["lifecycle"]["updatedAt"], ISO_UTC)
|
|
self.assertEqual(tombstone["lifecycle"]["reason"], "subject withdrawal")
|
|
|
|
# The tombstone addresses every derived artifact by contribution id, and says what went.
|
|
self.assertEqual(tombstone["deletedArtifacts"], [
|
|
{"relativePath": sample["relativePath"], "sha256": sample["hashes"]["derived"], "deleted": True},
|
|
{"relativePath": f"raw/{sample['hashes']['rawInput']}.jpg", "sha256": sample["hashes"]["rawInput"], "deleted": True},
|
|
])
|
|
self.assertEqual(tombstone["contributionId"], CONTRIBUTION_ID)
|
|
|
|
# No byte-bearing copy survives anywhere in the dataset tree.
|
|
self.assertFalse(derived.exists())
|
|
self.assertFalse(raw_copy.exists())
|
|
survivors = []
|
|
for path in out.rglob("*"):
|
|
if path.is_file() and path.name != "manifest.jsonl":
|
|
survivors.append((str(path.relative_to(out)), hashlib.sha256(path.read_bytes()).hexdigest()))
|
|
self.assertEqual(survivors, [])
|
|
|
|
# History is append-only: the sample record is untouched, the tombstone follows it.
|
|
history = manifest_lines(out)
|
|
self.assertEqual(len(history), 2)
|
|
self.assertEqual(history[0], sample)
|
|
self.assertEqual(history[1], tombstone)
|
|
|
|
# A deletion receipt keeps identifiers and hashes, never the label or subject key.
|
|
self.assertNotIn("labels", tombstone)
|
|
self.assertNotIn("subjectKey", tombstone)
|
|
self.assertNotIn("relativePath", tombstone)
|
|
|
|
|
|
class ActiveRecordTests(unittest.TestCase):
|
|
def test_only_unrevoked_samples_are_active(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
out = Path(raw)
|
|
self.assertEqual(ingest_default(out).returncode, 0)
|
|
self.assertEqual(active_image_paths(out), [f"images/{manifest_lines(out)[0]['imageId']}.jpg"])
|
|
self.assertEqual(ingest(["--revoke", CONTRIBUTION_ID, "--output-dir", str(out), "--reason", "gone"]).returncode, 0)
|
|
self.assertEqual(active_image_paths(out), [])
|
|
|
|
|
|
class IllegalTransitionTests(unittest.TestCase):
|
|
def test_a_second_revocation_is_refused_and_appends_nothing(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
out = Path(raw)
|
|
self.assertEqual(ingest_default(out).returncode, 0)
|
|
self.assertEqual(ingest(["--revoke", CONTRIBUTION_ID, "--output-dir", str(out), "--reason", "first"]).returncode, 0)
|
|
history = manifest_lines(out)
|
|
self.assertEqual(len(history), 2)
|
|
|
|
again = ingest(["--revoke", CONTRIBUTION_ID, "--output-dir", str(out), "--reason", "again"])
|
|
self.assertNotEqual(again.returncode, 0)
|
|
self.assertRegex(again.stderr, "already revoked")
|
|
self.assertEqual(manifest_lines(out), history)
|
|
|
|
def test_revoking_an_unknown_contribution_is_refused(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
out = Path(raw)
|
|
self.assertEqual(ingest_default(out).returncode, 0)
|
|
history = manifest_lines(out)
|
|
|
|
unknown = ingest(["--revoke", "11111111-2222-3333-4444-555555555555", "--output-dir", str(out), "--reason", "nope"])
|
|
self.assertNotEqual(unknown.returncode, 0)
|
|
self.assertRegex(unknown.stderr, "no ingested sample carries contribution")
|
|
self.assertEqual(manifest_lines(out), history)
|
|
self.assertTrue((out / history[0]["relativePath"]).is_file())
|
|
|
|
def test_a_contribution_cannot_be_ingested_twice(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
out = Path(raw)
|
|
self.assertEqual(ingest_default(out).returncode, 0)
|
|
history = manifest_lines(out)
|
|
|
|
duplicate = ingest_default(out, color="green")
|
|
self.assertNotEqual(duplicate.returncode, 0)
|
|
self.assertRegex(duplicate.stderr, "already ingested")
|
|
self.assertEqual(manifest_lines(out), history)
|
|
|
|
def test_a_refused_duplicate_leaves_no_orphan_image_behind(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
out = Path(raw)
|
|
self.assertEqual(ingest_default(out).returncode, 0)
|
|
history = manifest_lines(out)
|
|
|
|
# Same contribution id, different pixels: the derived digest differs, so a guard that
|
|
# runs after the write would strand a second image in the active dataset.
|
|
variant = out.parent / "variant-type4.jpg"
|
|
with Image.open(FIXTURE) as source:
|
|
changed = source.convert("RGB")
|
|
changed.putpixel((4, 4), (255, 0, 0))
|
|
changed.save(variant, "JPEG", quality=95)
|
|
duplicate = ingest_default(out, input=str(variant))
|
|
self.assertNotEqual(duplicate.returncode, 0)
|
|
self.assertRegex(duplicate.stderr, "already ingested")
|
|
|
|
self.assertEqual(manifest_lines(out), history)
|
|
images = sorted(path.name for path in (out / "images").iterdir())
|
|
self.assertEqual(images, [f"{history[0]['imageId']}.jpg"])
|
|
|
|
|
|
def test_lifecycle_transitions_cannot_walk_backwards(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
out = Path(raw)
|
|
self.assertEqual(ingest_default(out).returncode, 0)
|
|
sample = manifest_lines(out)[0]
|
|
at = sample["ingestedAt"]
|
|
|
|
revoked = structures.transition(sample, "revoked", at, "withdrawal")
|
|
self.assertEqual(revoked["lifecycle"]["state"], "revoked")
|
|
self.assertEqual(revoked["lifecycle"]["reason"], "withdrawal")
|
|
self.assertEqual(sample["lifecycle"]["state"], "active", "transition must not mutate its input")
|
|
|
|
for args, expected in (
|
|
((revoked, "active", at), "cannot return to the active dataset"),
|
|
((sample, "revoked", at, " "), "recorded reason"),
|
|
((sample, "archived", at), "unknown lifecycle state"),
|
|
(({**sample, "lifecycle": {"state": "active", "updatedAt": at}}, "active", at), "already active"),
|
|
):
|
|
with self.assertRaises(structures.IllegalTransition) as raised:
|
|
structures.transition(*args)
|
|
self.assertRegex(str(raised.exception), expected)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|