#!/usr/bin/env python3 """Tests for the published dataset card and the contributor-level split policy. Issue #27 asks for three things a stranger can inspect before trusting a model: a dataset card that reports who is in the corpus, *the* labeling guide, and a contributor-level split policy that keeps a person's photos in exactly one split. The card is generated from the append-only manifest by [`scripts/dataset_card.py`](../scripts/dataset_card.py) instead of being written by hand, so a number on it is a claim the corpus can be re-checked against and an unknown fact can only appear as ``pending`` — never as a plausible-looking estimate. Every fixture here is synthetic: records are built through the real [`scripts/corpus_manifest.py`](../scripts/corpus_manifest.py) builder and no image bytes are read or written. """ from __future__ import annotations import importlib.util import json import os import re import subprocess import sys import tempfile from pathlib import Path import unittest ROOT = Path(__file__).resolve().parents[1] CARD_SCRIPT = ROOT / "scripts" / "dataset_card.py" COMMITTED_CARD = ROOT / "docs" / "DATASET_CARD.md" LABELING_GUIDE = ROOT / "docs" / "LABELING_GUIDE.md" CANONICAL_CORPUS = "/srv/timmy-corpus" # Subject keys chosen so the ingestion rule (first byte of the key) puts each contributor in a known # split: < 205 train, < 230 validation, otherwise test. TRAIN_A = "0a" + "1" * 14 TRAIN_B = "0b" + "2" * 14 VALIDATION_C = "d0" + "3" * 14 TEST_D = "ee" + "4" * 14 def load_module(): spec = importlib.util.spec_from_file_location("corpus_manifest", ROOT / "scripts" / "corpus_manifest.py") assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module structures = load_module() def digest(seed: str) -> str: import hashlib return hashlib.sha256(seed.encode()).hexdigest() def sample( *, contribution_id: str, subject_key: str, split: str, bristol: int = 4, color: str = "brown", quality: str = "good", reviewer: str = "user-confirmed", at: str = "2026-08-01T10:00:00+00:00", ) -> dict: return structures.build_record( contribution_id=contribution_id, consent_version="training-consent-v1", subject_key=subject_key, split=split, labels={"bristolType": bristol, "color": color, "quality": quality}, review_status=reviewer, derived_sha256=digest(contribution_id + "-derived"), raw_sha256=digest(contribution_id + "-raw"), ingested_at=at, ) def write_manifest(dataset: Path, records: list[dict]) -> Path: manifest = dataset / "manifest.jsonl" manifest.parent.mkdir(parents=True, exist_ok=True) with manifest.open("w", encoding="utf-8") as handle: for record in records: handle.write(json.dumps(record, separators=(",", ":")) + "\n") return manifest def run_card(args: list[str]) -> subprocess.CompletedProcess: return subprocess.run( [sys.executable, str(CARD_SCRIPT), *args], cwd=ROOT, capture_output=True, text=True, env={**os.environ}, ) def build_corpus(dataset: Path) -> None: """Four active samples across three contributors, plus one withdrawal.""" records = [ sample(contribution_id="contrib-train-a1", subject_key=TRAIN_A, split="train", bristol=4), sample(contribution_id="contrib-train-a2", subject_key=TRAIN_A, split="train", bristol=3, color="green"), sample(contribution_id="contrib-train-b1", subject_key=TRAIN_B, split="train", bristol=6, quality="fair"), sample( contribution_id="contrib-validation-c1", subject_key=VALIDATION_C, split="validation", bristol=4, reviewer="clinician-reviewed", ), sample(contribution_id="contrib-test-d1", subject_key=TEST_D, split="test", bristol=1, quality="poor"), ] revoked = structures.transition( sample(contribution_id="contrib-withdrawn-1", subject_key=TEST_D, split="test", bristol=7, color="black"), "revoked", "2026-08-02T09:00:00+00:00", "participant withdrew consent", ) write_manifest(dataset, [*records, revoked]) class DatasetCardTest(unittest.TestCase): def setUp(self) -> None: self.tmp = tempfile.TemporaryDirectory() self.dataset = Path(self.tmp.name) / "corpus" self.dataset.mkdir(parents=True, exist_ok=True) def tearDown(self) -> None: self.tmp.cleanup() def test_card_reports_the_real_corpus_counts_from_the_manifest(self) -> None: build_corpus(self.dataset) result = run_card(["--corpus", str(self.dataset)]) self.assertEqual(result.returncode, 0, result.stderr) card = result.stdout # Five active samples from four contributors; the withdrawal is reported separately. self.assertIn("| Active samples | 5 |", card) self.assertIn("| Contributors | 4 |", card) self.assertIn("| Withdrawals | 1 |", card) # Contributor-level split policy: the contributor count per split is what a reader needs to # see that one person cannot be in two splits. self.assertIn("| `train` | 2 | 3 |", card) self.assertIn("| `validation` | 1 | 1 |", card) self.assertIn("| `test` | 1 | 1 |", card) # Class balance is counted, not described. self.assertIn("| Type 4 | 2 |", card) self.assertIn("| Type 1 | 1 |", card) self.assertIn("| `black` | 0 |", card, "a withdrawn sample's label must not be counted") self.assertIn("| `clinician-reviewed` | 1 |", card) def test_card_marks_uncollected_facts_pending_instead_of_estimating_them(self) -> None: build_corpus(self.dataset) card = run_card(["--corpus", str(self.dataset)]).stdout for fact in ("Devices", "Contexts", "Licenses"): line = next((row for row in card.splitlines() if row.startswith(f"| {fact} |")), None) self.assertIsNotNone(line, f"the card must name {fact}") self.assertIn("pending", line.lower(), f"{fact} cannot be reported until it is collected") self.assertIn("rather than estimated", card.lower()) self.assertIn("nothing here is invented", card.lower()) def test_card_names_withdrawals_without_exposing_the_person_or_their_labels(self) -> None: build_corpus(self.dataset) card = run_card(["--corpus", str(self.dataset)]).stdout self.assertIn("contrib-withdrawn-1", card) self.assertIn("participant withdrew consent", card) for subject in (TRAIN_A, TRAIN_B, VALIDATION_C, TEST_D): self.assertNotIn(subject, card, "the published card must never carry a subject key") self.assertNotIn(digest("contrib-withdrawn-1" + "-derived"), card) def test_card_refuses_a_contributor_spread_across_two_splits(self) -> None: write_manifest( self.dataset, [ sample(contribution_id="contrib-leak-1", subject_key=TEST_D, split="test"), # Same person, relabelled into training: the leak the policy exists to stop. sample(contribution_id="contrib-leak-2", subject_key=TEST_D, split="train"), ], ) out = Path(self.tmp.name) / "card.md" result = run_card(["--corpus", str(self.dataset), "--out", str(out)]) self.assertNotEqual(result.returncode, 0) self.assertRegex(result.stderr, r"contributor [0-9a-f]{16} (leaks across|is in two splits)") self.assertFalse(out.exists(), "a refused card must not be written") def test_card_refuses_a_split_the_subject_key_does_not_derive(self) -> None: write_manifest( self.dataset, [sample(contribution_id="contrib-mislabelled-1", subject_key=TEST_D, split="train")], ) result = run_card(["--corpus", str(self.dataset)]) self.assertNotEqual(result.returncode, 0) self.assertRegex(result.stderr, r"not reproducible") def test_card_states_the_split_rule_it_applies(self) -> None: build_corpus(self.dataset) card = run_card(["--corpus", str(self.dataset)]).stdout # The thresholds are the ingestion rule; a card that states a different rule is a different # policy, so both edges are named on the page. self.assertIn("205", card) self.assertIn("230", card) self.assertRegex(card, r"subject key") def test_card_is_byte_identical_between_runs(self) -> None: build_corpus(self.dataset) first = run_card(["--corpus", str(self.dataset)]) second = run_card(["--corpus", str(self.dataset)]) # A rendered card, not two matching empty strings: the run has to succeed and say something. self.assertEqual(first.returncode, 0, first.stderr) self.assertIn("| Active samples | 5 |", first.stdout) self.assertEqual(first.stdout, second.stdout) def test_card_is_honest_when_there_is_no_corpus(self) -> None: missing = Path(self.tmp.name) / "absent" result = run_card(["--corpus", str(missing)]) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("| Active samples | 0 |", result.stdout) self.assertIn("| Contributors | 0 |", result.stdout) self.assertRegex(result.stdout, r"no consented images") self.assertIn("pending", result.stdout.lower(), "an empty corpus reports pending facts, not blanks") def test_card_reports_a_withdrawal_count_without_inventing_a_denominator(self) -> None: build_corpus(self.dataset) summary = json.loads(run_card(["--corpus", str(self.dataset), "--json"]).stdout) self.assertEqual(summary["activeSamples"], 5) self.assertEqual(summary["contributors"], 4) self.assertEqual(summary["withdrawals"], 1) self.assertEqual(summary["bySplit"]["train"], {"contributors": 2, "samples": 3}) self.assertEqual(summary["pending"], ["contexts", "devices", "licenses"]) self.assertEqual([entry["contributionId"] for entry in summary["withdrawalsDetail"]], ["contrib-withdrawn-1"]) self.assertNotIn("subjectKey", json.dumps(summary)) def test_committed_card_is_the_generator_output_not_a_hand_written_page(self) -> None: card = COMMITTED_CARD.read_text(encoding="utf-8") result = run_card(["--corpus", CANONICAL_CORPUS]) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual( card, result.stdout, "docs/DATASET_CARD.md must be regenerated with scripts/dataset_card.py, never hand-edited", ) self.assertIn(f"--corpus {CANONICAL_CORPUS}", card) def test_check_mode_refuses_a_hand_edited_card(self) -> None: build_corpus(self.dataset) stale = Path(self.tmp.name) / "stale-card.md" stale.write_text("# Dataset card\n\n| Active samples | 9000 |\n", encoding="utf-8") result = run_card(["--corpus", str(self.dataset), "--check", str(stale)]) self.assertNotEqual(result.returncode, 0) self.assertRegex(result.stderr, r"out of date|differs") def test_labeling_guide_separates_visible_color_from_blood_and_forbids_diagnosis(self) -> None: guide = LABELING_GUIDE.read_text(encoding="utf-8") lowered = guide.lower() self.assertIn("color", lowered) # Visible color is not bleeding: the guide has to say so in the reviewer's own vocabulary. self.assertIn("not blood", lowered) for forbidden in ("diagnose", "diagnosis"): self.assertIn(forbidden, lowered, "the guide must name what a label may never assert") self.assertRegex(lowered, r"do not diagnose|never diagnos") # Exclusions and the split policy are part of the guide, not a separate folklore. self.assertIn("exclude", lowered) self.assertRegex(lowered, r"one split|same split") # The card and the guide point at each other, so a reader lands on both. self.assertIn("DATASET_CARD.md", guide) self.assertIn("LABELING_GUIDE.md", COMMITTED_CARD.read_text(encoding="utf-8")) def test_labeling_guide_never_claims_a_model_decides_the_label(self) -> None: guide = LABELING_GUIDE.read_text(encoding="utf-8") lowered = guide.lower() self.assertRegex(lowered, r"human|reviewer") # A model suggestion is provisional input to a human decision, never the decision. self.assertRegex(lowered, r"provisional|suggestion") self.assertNotRegex(lowered, r"model (decides|determines) the label") def test_readme_points_a_newcomer_at_the_card_and_the_guide(self) -> None: readme = (ROOT / "README.md").read_text(encoding="utf-8") # A published card nobody links to is not published: both pages are reachable from the front page. self.assertIn("docs/DATASET_CARD.md", readme) self.assertIn("docs/LABELING_GUIDE.md", readme) self.assertIn("scripts/dataset_card.py", readme) if __name__ == "__main__": unittest.main(verbosity=2)