timmy
9dabad495b
Quality gates / quality (pull_request) Canceled after 0s
The corpus had an append-only manifest, a reviewer tool, and an evaluation harness, but nothing a stranger could read: what is in the corpus, what a label means, and how leakage is prevented lived in code and in reviewers' heads. scripts/dataset_card.py generates docs/DATASET_CARD.md from manifest.jsonl. Every count is read back from real records — active samples, contributors, per-split contributor and sample counts, class balance over active samples only — and every fact the pipeline does not collect (devices, contexts, licenses) is printed as pending rather than estimated. The generator refuses to render a card for a corpus where one contributor straddles two splits or where a recorded split is not the one the subject key derives, and it never publishes a subject key or a withdrawn sample's labels. --check fails when the committed page is not the generator's own output, so the card cannot drift into a hand-written document. docs/LABELING_GUIDE.md is the reviewer's rulebook: visible form, broad visible color, and photo quality; visible color is not blood; nothing is diagnosed; a model suggestion is provisional input to a human decision. Exclusions are named, and the split policy is stated where it is enforced. The demo's new DATASET CARD scene builds a throwaway corpus with the real ingest command from committed synthetic fixtures, publishes the card, and shows the refusal a planted contributor earns, then re-checks the committed page. Its captions are release gates in build_release.py, which also runs tests/dataset-card.test.py before a release exists. tests/dataset-card.test.py covers the counts, the pending facts, the withdrawal receipt, both refusals, determinism, the empty-corpus case, the committed-page identity, and both documents' prohibitions. Every fixture is synthetic. Closes #27
301 lines
13 KiB
Python
301 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""Tests for the published dataset card and the contributor-level split policy.
|
|
|
|
Issue #27 asks for three things a stranger can inspect before trusting a model: a dataset card that
|
|
reports who is in the corpus, *the* labeling guide, and a contributor-level split policy that keeps a
|
|
person's photos in exactly one split. The card is generated from the append-only manifest by
|
|
[`scripts/dataset_card.py`](../scripts/dataset_card.py) instead of being written by hand, so a number
|
|
on it is a claim the corpus can be re-checked against and an unknown fact can only appear as
|
|
``pending`` — never as a plausible-looking estimate.
|
|
|
|
Every fixture here is synthetic: records are built through the real
|
|
[`scripts/corpus_manifest.py`](../scripts/corpus_manifest.py) builder and no image bytes are read or
|
|
written.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
import unittest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
CARD_SCRIPT = ROOT / "scripts" / "dataset_card.py"
|
|
COMMITTED_CARD = ROOT / "docs" / "DATASET_CARD.md"
|
|
LABELING_GUIDE = ROOT / "docs" / "LABELING_GUIDE.md"
|
|
CANONICAL_CORPUS = "/srv/timmy-corpus"
|
|
|
|
# Subject keys chosen so the ingestion rule (first byte of the key) puts each contributor in a known
|
|
# split: < 205 train, < 230 validation, otherwise test.
|
|
TRAIN_A = "0a" + "1" * 14
|
|
TRAIN_B = "0b" + "2" * 14
|
|
VALIDATION_C = "d0" + "3" * 14
|
|
TEST_D = "ee" + "4" * 14
|
|
|
|
|
|
def load_module():
|
|
spec = importlib.util.spec_from_file_location("corpus_manifest", ROOT / "scripts" / "corpus_manifest.py")
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
structures = load_module()
|
|
|
|
|
|
def digest(seed: str) -> str:
|
|
import hashlib
|
|
|
|
return hashlib.sha256(seed.encode()).hexdigest()
|
|
|
|
|
|
def sample(
|
|
*,
|
|
contribution_id: str,
|
|
subject_key: str,
|
|
split: str,
|
|
bristol: int = 4,
|
|
color: str = "brown",
|
|
quality: str = "good",
|
|
reviewer: str = "user-confirmed",
|
|
at: str = "2026-08-01T10:00:00+00:00",
|
|
) -> dict:
|
|
return structures.build_record(
|
|
contribution_id=contribution_id,
|
|
consent_version="training-consent-v1",
|
|
subject_key=subject_key,
|
|
split=split,
|
|
labels={"bristolType": bristol, "color": color, "quality": quality},
|
|
review_status=reviewer,
|
|
derived_sha256=digest(contribution_id + "-derived"),
|
|
raw_sha256=digest(contribution_id + "-raw"),
|
|
ingested_at=at,
|
|
)
|
|
|
|
|
|
def write_manifest(dataset: Path, records: list[dict]) -> Path:
|
|
manifest = dataset / "manifest.jsonl"
|
|
manifest.parent.mkdir(parents=True, exist_ok=True)
|
|
with manifest.open("w", encoding="utf-8") as handle:
|
|
for record in records:
|
|
handle.write(json.dumps(record, separators=(",", ":")) + "\n")
|
|
return manifest
|
|
|
|
|
|
def run_card(args: list[str]) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
[sys.executable, str(CARD_SCRIPT), *args],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
env={**os.environ},
|
|
)
|
|
|
|
|
|
def build_corpus(dataset: Path) -> None:
|
|
"""Four active samples across three contributors, plus one withdrawal."""
|
|
records = [
|
|
sample(contribution_id="contrib-train-a1", subject_key=TRAIN_A, split="train", bristol=4),
|
|
sample(contribution_id="contrib-train-a2", subject_key=TRAIN_A, split="train", bristol=3, color="green"),
|
|
sample(contribution_id="contrib-train-b1", subject_key=TRAIN_B, split="train", bristol=6, quality="fair"),
|
|
sample(
|
|
contribution_id="contrib-validation-c1",
|
|
subject_key=VALIDATION_C,
|
|
split="validation",
|
|
bristol=4,
|
|
reviewer="clinician-reviewed",
|
|
),
|
|
sample(contribution_id="contrib-test-d1", subject_key=TEST_D, split="test", bristol=1, quality="poor"),
|
|
]
|
|
revoked = structures.transition(
|
|
sample(contribution_id="contrib-withdrawn-1", subject_key=TEST_D, split="test", bristol=7, color="black"),
|
|
"revoked",
|
|
"2026-08-02T09:00:00+00:00",
|
|
"participant withdrew consent",
|
|
)
|
|
write_manifest(dataset, [*records, revoked])
|
|
|
|
|
|
class DatasetCardTest(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.tmp = tempfile.TemporaryDirectory()
|
|
self.dataset = Path(self.tmp.name) / "corpus"
|
|
self.dataset.mkdir(parents=True, exist_ok=True)
|
|
|
|
def tearDown(self) -> None:
|
|
self.tmp.cleanup()
|
|
|
|
def test_card_reports_the_real_corpus_counts_from_the_manifest(self) -> None:
|
|
build_corpus(self.dataset)
|
|
result = run_card(["--corpus", str(self.dataset)])
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
card = result.stdout
|
|
|
|
# Five active samples from four contributors; the withdrawal is reported separately.
|
|
self.assertIn("| Active samples | 5 |", card)
|
|
self.assertIn("| Contributors | 4 |", card)
|
|
self.assertIn("| Withdrawals | 1 |", card)
|
|
# Contributor-level split policy: the contributor count per split is what a reader needs to
|
|
# see that one person cannot be in two splits.
|
|
self.assertIn("| `train` | 2 | 3 |", card)
|
|
self.assertIn("| `validation` | 1 | 1 |", card)
|
|
self.assertIn("| `test` | 1 | 1 |", card)
|
|
# Class balance is counted, not described.
|
|
self.assertIn("| Type 4 | 2 |", card)
|
|
self.assertIn("| Type 1 | 1 |", card)
|
|
self.assertIn("| `black` | 0 |", card, "a withdrawn sample's label must not be counted")
|
|
self.assertIn("| `clinician-reviewed` | 1 |", card)
|
|
|
|
def test_card_marks_uncollected_facts_pending_instead_of_estimating_them(self) -> None:
|
|
build_corpus(self.dataset)
|
|
card = run_card(["--corpus", str(self.dataset)]).stdout
|
|
|
|
for fact in ("Devices", "Contexts", "Licenses"):
|
|
line = next((row for row in card.splitlines() if row.startswith(f"| {fact} |")), None)
|
|
self.assertIsNotNone(line, f"the card must name {fact}")
|
|
self.assertIn("pending", line.lower(), f"{fact} cannot be reported until it is collected")
|
|
self.assertIn("rather than estimated", card.lower())
|
|
self.assertIn("nothing here is invented", card.lower())
|
|
|
|
def test_card_names_withdrawals_without_exposing_the_person_or_their_labels(self) -> None:
|
|
build_corpus(self.dataset)
|
|
card = run_card(["--corpus", str(self.dataset)]).stdout
|
|
|
|
self.assertIn("contrib-withdrawn-1", card)
|
|
self.assertIn("participant withdrew consent", card)
|
|
for subject in (TRAIN_A, TRAIN_B, VALIDATION_C, TEST_D):
|
|
self.assertNotIn(subject, card, "the published card must never carry a subject key")
|
|
self.assertNotIn(digest("contrib-withdrawn-1" + "-derived"), card)
|
|
|
|
def test_card_refuses_a_contributor_spread_across_two_splits(self) -> None:
|
|
write_manifest(
|
|
self.dataset,
|
|
[
|
|
sample(contribution_id="contrib-leak-1", subject_key=TEST_D, split="test"),
|
|
# Same person, relabelled into training: the leak the policy exists to stop.
|
|
sample(contribution_id="contrib-leak-2", subject_key=TEST_D, split="train"),
|
|
],
|
|
)
|
|
out = Path(self.tmp.name) / "card.md"
|
|
result = run_card(["--corpus", str(self.dataset), "--out", str(out)])
|
|
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertRegex(result.stderr, r"contributor [0-9a-f]{16} (leaks across|is in two splits)")
|
|
self.assertFalse(out.exists(), "a refused card must not be written")
|
|
|
|
def test_card_refuses_a_split_the_subject_key_does_not_derive(self) -> None:
|
|
write_manifest(
|
|
self.dataset,
|
|
[sample(contribution_id="contrib-mislabelled-1", subject_key=TEST_D, split="train")],
|
|
)
|
|
result = run_card(["--corpus", str(self.dataset)])
|
|
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertRegex(result.stderr, r"not reproducible")
|
|
|
|
def test_card_states_the_split_rule_it_applies(self) -> None:
|
|
build_corpus(self.dataset)
|
|
card = run_card(["--corpus", str(self.dataset)]).stdout
|
|
|
|
# The thresholds are the ingestion rule; a card that states a different rule is a different
|
|
# policy, so both edges are named on the page.
|
|
self.assertIn("205", card)
|
|
self.assertIn("230", card)
|
|
self.assertRegex(card, r"subject key")
|
|
|
|
def test_card_is_byte_identical_between_runs(self) -> None:
|
|
build_corpus(self.dataset)
|
|
first = run_card(["--corpus", str(self.dataset)])
|
|
second = run_card(["--corpus", str(self.dataset)])
|
|
# A rendered card, not two matching empty strings: the run has to succeed and say something.
|
|
self.assertEqual(first.returncode, 0, first.stderr)
|
|
self.assertIn("| Active samples | 5 |", first.stdout)
|
|
self.assertEqual(first.stdout, second.stdout)
|
|
|
|
def test_card_is_honest_when_there_is_no_corpus(self) -> None:
|
|
missing = Path(self.tmp.name) / "absent"
|
|
result = run_card(["--corpus", str(missing)])
|
|
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertIn("| Active samples | 0 |", result.stdout)
|
|
self.assertIn("| Contributors | 0 |", result.stdout)
|
|
self.assertRegex(result.stdout, r"no consented images")
|
|
self.assertIn("pending", result.stdout.lower(), "an empty corpus reports pending facts, not blanks")
|
|
|
|
def test_card_reports_a_withdrawal_count_without_inventing_a_denominator(self) -> None:
|
|
build_corpus(self.dataset)
|
|
summary = json.loads(run_card(["--corpus", str(self.dataset), "--json"]).stdout)
|
|
|
|
self.assertEqual(summary["activeSamples"], 5)
|
|
self.assertEqual(summary["contributors"], 4)
|
|
self.assertEqual(summary["withdrawals"], 1)
|
|
self.assertEqual(summary["bySplit"]["train"], {"contributors": 2, "samples": 3})
|
|
self.assertEqual(summary["pending"], ["contexts", "devices", "licenses"])
|
|
self.assertEqual([entry["contributionId"] for entry in summary["withdrawalsDetail"]], ["contrib-withdrawn-1"])
|
|
self.assertNotIn("subjectKey", json.dumps(summary))
|
|
|
|
def test_committed_card_is_the_generator_output_not_a_hand_written_page(self) -> None:
|
|
card = COMMITTED_CARD.read_text(encoding="utf-8")
|
|
result = run_card(["--corpus", CANONICAL_CORPUS])
|
|
self.assertEqual(result.returncode, 0, result.stderr)
|
|
self.assertEqual(
|
|
card,
|
|
result.stdout,
|
|
"docs/DATASET_CARD.md must be regenerated with scripts/dataset_card.py, never hand-edited",
|
|
)
|
|
self.assertIn(f"--corpus {CANONICAL_CORPUS}", card)
|
|
|
|
def test_check_mode_refuses_a_hand_edited_card(self) -> None:
|
|
build_corpus(self.dataset)
|
|
stale = Path(self.tmp.name) / "stale-card.md"
|
|
stale.write_text("# Dataset card\n\n| Active samples | 9000 |\n", encoding="utf-8")
|
|
|
|
result = run_card(["--corpus", str(self.dataset), "--check", str(stale)])
|
|
self.assertNotEqual(result.returncode, 0)
|
|
self.assertRegex(result.stderr, r"out of date|differs")
|
|
|
|
def test_labeling_guide_separates_visible_color_from_blood_and_forbids_diagnosis(self) -> None:
|
|
guide = LABELING_GUIDE.read_text(encoding="utf-8")
|
|
lowered = guide.lower()
|
|
|
|
self.assertIn("color", lowered)
|
|
# Visible color is not bleeding: the guide has to say so in the reviewer's own vocabulary.
|
|
self.assertIn("not blood", lowered)
|
|
for forbidden in ("diagnose", "diagnosis"):
|
|
self.assertIn(forbidden, lowered, "the guide must name what a label may never assert")
|
|
self.assertRegex(lowered, r"do not diagnose|never diagnos")
|
|
# Exclusions and the split policy are part of the guide, not a separate folklore.
|
|
self.assertIn("exclude", lowered)
|
|
self.assertRegex(lowered, r"one split|same split")
|
|
# The card and the guide point at each other, so a reader lands on both.
|
|
self.assertIn("DATASET_CARD.md", guide)
|
|
self.assertIn("LABELING_GUIDE.md", COMMITTED_CARD.read_text(encoding="utf-8"))
|
|
|
|
def test_labeling_guide_never_claims_a_model_decides_the_label(self) -> None:
|
|
guide = LABELING_GUIDE.read_text(encoding="utf-8")
|
|
lowered = guide.lower()
|
|
|
|
self.assertRegex(lowered, r"human|reviewer")
|
|
# A model suggestion is provisional input to a human decision, never the decision.
|
|
self.assertRegex(lowered, r"provisional|suggestion")
|
|
self.assertNotRegex(lowered, r"model (decides|determines) the label")
|
|
|
|
|
|
def test_readme_points_a_newcomer_at_the_card_and_the_guide(self) -> None:
|
|
readme = (ROOT / "README.md").read_text(encoding="utf-8")
|
|
|
|
# A published card nobody links to is not published: both pages are reachable from the front page.
|
|
self.assertIn("docs/DATASET_CARD.md", readme)
|
|
self.assertIn("docs/LABELING_GUIDE.md", readme)
|
|
self.assertIn("scripts/dataset_card.py", readme)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|