Timmy
c5fa2123e7
Quality gates / quality (pull_request) Canceled after 0s
The corpus manifest records what was ingested and whether the person still allows it; it does not record whether the labels are right. This adds the labeling control the data-flow threat model named: scripts/review_corpus.py, a command-line tool with no app screen. - The review queue shows the contributor's proposal and carries no model field at all: blindness is structural, because build_queue() never receives predictions. - reveal() refuses a contribution with no committed review, so the model answer cannot steer the human who labels it. Revealing appends nothing. - A review is one append-only line carrying the proposal it judged; a correction must change it. An adjudication is a second reviewer's line that names the record it resolves; the review it overrules survives untouched. - A withdrawn contribution cannot be queued, reviewed, revealed, adjudicated, or trained on, and the refusal says it was withdrawn rather than that it is already reviewed. - trainable_records() returns only active samples with a human decision that was not an exclusion, so unreviewed proposals are never training data. Tests are written first against every refusal (tests/corpus-review.test.py, 21 cases); the release builder runs the suite and gates four on-camera captions, and the demo now runs the real queue -> review -> reveal -> adjudicate -> withdraw sequence against the committed synthetic sample and draws the evidence card from that output. docs/REVIEWER-TOOL.md is the contract; no model was measured.
420 lines
19 KiB
Python
420 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Tests for the blind reviewer and adjudication tool (issue #26).
|
|
|
|
The tool decides what a training sample is allowed to say, so these tests attack the two
|
|
properties that make the labels trustworthy:
|
|
|
|
* a reviewer never sees the model's answer before committing their own, and
|
|
* a withdrawn contribution can never be shown, reviewed, or trained on.
|
|
|
|
Every fixture is the committed synthetic image; no real photo bytes are read or written.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
INGEST = ROOT / "scripts" / "ingest_training_photo.py"
|
|
FIXTURE = ROOT / "tests" / "fixtures" / "synthetic-type4.jpg"
|
|
FIRST = "6f1d0c9e-2c4a-4f3b-9d1e-8a7b6c5d4e3f"
|
|
SECOND = "0b9a8c7d-6e5f-4a3b-2c1d-0e9f8a7b6c5d"
|
|
|
|
|
|
def load_tool():
|
|
spec = importlib.util.spec_from_file_location("review_corpus", ROOT / "scripts" / "review_corpus.py")
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
tool = load_tool()
|
|
|
|
|
|
def ingest(dataset: Path, contribution: str, *, bristol: str = "4", color: str = "brown", quality: str = "good") -> dict:
|
|
done = subprocess.run(
|
|
[
|
|
sys.executable, str(INGEST),
|
|
"--input", str(FIXTURE),
|
|
"--output-dir", str(dataset),
|
|
"--subject-id", "subject-123",
|
|
"--consent-version", "research-v1",
|
|
"--contribution-id", contribution,
|
|
"--bristol-type", bristol,
|
|
"--color", color,
|
|
"--quality", quality,
|
|
"--reviewer", "user-confirmed",
|
|
],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
env={**os.environ, "TIMMY_DATASET_SALT": "review-test-salt"},
|
|
)
|
|
assert done.returncode == 0, done.stderr
|
|
return json.loads(done.stdout)
|
|
|
|
|
|
def revoke(dataset: Path, contribution: str) -> dict:
|
|
done = subprocess.run(
|
|
[sys.executable, str(INGEST), "--output-dir", str(dataset), "--revoke", contribution, "--reason", "participant withdrew consent"],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
env={**os.environ, "TIMMY_DATASET_SALT": "review-test-salt"},
|
|
)
|
|
assert done.returncode == 0, done.stderr
|
|
return json.loads(done.stdout)
|
|
|
|
|
|
def ledger(dataset: Path) -> list[dict]:
|
|
path = dataset / "reviews.jsonl"
|
|
if not path.exists():
|
|
return []
|
|
return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
|
|
|
|
PREDICTIONS = {
|
|
"version": "synthetic-model-placeholder",
|
|
"answers": {
|
|
FIRST: {"bristolType": 6, "color": "brown", "quality": "good", "confidence": 0.71},
|
|
SECOND: {"bristolType": 4, "color": "brown", "quality": "good", "confidence": 0.9},
|
|
},
|
|
}
|
|
|
|
|
|
class QueueBlindnessTests(unittest.TestCase):
|
|
def test_the_queue_shows_the_proposal_and_withholds_every_model_field(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
sample = ingest(dataset, FIRST)
|
|
|
|
queue = tool.build_queue(dataset)
|
|
|
|
self.assertEqual(len(queue), 1)
|
|
packet = queue[0]
|
|
self.assertEqual(packet["contributionId"], FIRST)
|
|
self.assertEqual(packet["imageId"], sample["imageId"])
|
|
self.assertEqual(packet["relativePath"], sample["relativePath"])
|
|
self.assertEqual(packet["proposed"], {"bristolType": 4, "color": "brown", "quality": "good"})
|
|
self.assertTrue(packet["blindToModel"])
|
|
|
|
# Not a naming convention: nothing model-shaped and nothing identity-bearing may ride along.
|
|
text = json.dumps(packet)
|
|
for forbidden in ("confidence", "modelSuggestion", "prediction", "subjectKey", "data:image", "base64"):
|
|
self.assertNotIn(forbidden, text)
|
|
|
|
def test_resolved_samples_leave_the_queue(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
self.assertEqual(len(tool.build_queue(dataset)), 1)
|
|
|
|
tool.commit_review(dataset, FIRST, "reviewer-a", "agreed")
|
|
|
|
self.assertEqual(tool.build_queue(dataset), [])
|
|
|
|
def test_the_queue_refuses_an_active_sample_whose_image_is_gone(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
sample = ingest(dataset, FIRST)
|
|
(dataset / sample["relativePath"]).unlink()
|
|
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
tool.build_queue(dataset)
|
|
self.assertRegex(str(raised.exception), "image is missing")
|
|
|
|
|
|
class BlindRevealTests(unittest.TestCase):
|
|
def test_the_model_answer_stays_hidden_until_a_review_is_committed(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
tool.reveal(dataset, FIRST, PREDICTIONS)
|
|
self.assertRegex(str(raised.exception), "hidden until a review is committed")
|
|
|
|
def test_reveal_reports_the_disagreement_after_the_human_commits(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
tool.commit_review(dataset, FIRST, "reviewer-a", "corrected", {"bristolType": 5, "color": "green", "quality": "fair"})
|
|
|
|
revealed = tool.reveal(dataset, FIRST, PREDICTIONS)
|
|
|
|
self.assertEqual(revealed["human"]["labels"], {"bristolType": 5, "color": "green", "quality": "fair"})
|
|
self.assertEqual(revealed["model"]["labels"], {"bristolType": 6, "color": "brown", "quality": "good"})
|
|
self.assertEqual(revealed["disagreeFields"], ["bristolType", "color", "quality"])
|
|
self.assertTrue(revealed["disagreement"])
|
|
# Revealing is a read: it must not quietly rewrite the review ledger.
|
|
self.assertEqual(len(ledger(dataset)), 1)
|
|
|
|
|
|
class CommitTests(unittest.TestCase):
|
|
def test_a_committed_review_keeps_the_proposal_and_the_human_decision(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
sample = ingest(dataset, FIRST)
|
|
|
|
record = tool.commit_review(dataset, FIRST, "reviewer-a", "corrected", {"bristolType": 2, "color": "brown", "quality": "poor"}, at="2026-10-27T09:00:00+00:00")
|
|
|
|
self.assertEqual(record["recordType"], "review")
|
|
self.assertEqual(record["contributionId"], FIRST)
|
|
self.assertEqual(record["imageId"], sample["imageId"])
|
|
self.assertEqual(record["reviewer"], "reviewer-a")
|
|
self.assertEqual(record["decision"], "corrected")
|
|
self.assertEqual(record["labels"], {"bristolType": 2, "color": "brown", "quality": "poor"})
|
|
self.assertEqual(record["proposed"], {"bristolType": 4, "color": "brown", "quality": "good"})
|
|
self.assertEqual(record["committedAt"], "2026-10-27T09:00:00+00:00")
|
|
self.assertEqual(ledger(dataset), [record])
|
|
self.assertNotIn("subjectKey", json.dumps(record))
|
|
|
|
def test_a_contribution_cannot_be_reviewed_twice(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
tool.commit_review(dataset, FIRST, "reviewer-a", "agreed")
|
|
history = ledger(dataset)
|
|
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
tool.commit_review(dataset, FIRST, "reviewer-b", "corrected", {"bristolType": 2, "color": "black", "quality": "poor"})
|
|
self.assertRegex(str(raised.exception), "already reviewed")
|
|
self.assertEqual(ledger(dataset), history)
|
|
|
|
def test_a_correction_that_changes_nothing_is_refused(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
tool.commit_review(dataset, FIRST, "reviewer-a", "corrected", {"bristolType": 4, "color": "brown", "quality": "good"})
|
|
self.assertRegex(str(raised.exception), "must change the proposed label")
|
|
self.assertEqual(ledger(dataset), [])
|
|
|
|
def test_invalid_labels_and_blank_reviewers_are_refused(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
|
|
for decision, labels, reviewer, expected in (
|
|
("corrected", {"bristolType": 9, "color": "brown", "quality": "good"}, "reviewer-a", "bristolType must be 1-7"),
|
|
("corrected", {"bristolType": 4, "color": "purple", "quality": "good"}, "reviewer-a", "color and quality"),
|
|
("agreed", None, " ", "reviewer is required"),
|
|
("excluded", None, "reviewer-a", "exclusion needs a recorded reason"),
|
|
("approved", None, "reviewer-a", "decision must be one of"),
|
|
):
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
tool.commit_review(dataset, FIRST, reviewer, decision, labels)
|
|
self.assertRegex(str(raised.exception), expected)
|
|
self.assertEqual(ledger(dataset), [])
|
|
|
|
def test_reviewing_an_unknown_contribution_is_refused(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
tool.commit_review(dataset, SECOND, "reviewer-a", "agreed")
|
|
self.assertRegex(str(raised.exception), "no active sample carries contribution")
|
|
self.assertEqual(ledger(dataset), [])
|
|
|
|
|
|
class WithdrawnContributionTests(unittest.TestCase):
|
|
def test_a_withdrawn_contribution_never_reaches_the_queue_review_or_the_trainer(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
tool.commit_review(dataset, FIRST, "reviewer-a", "corrected", {"bristolType": 5, "color": "brown", "quality": "good"})
|
|
self.assertEqual(len(tool.trainable_records(dataset)), 1)
|
|
|
|
revoke(dataset, FIRST)
|
|
|
|
self.assertEqual(tool.build_queue(dataset), [])
|
|
self.assertEqual(tool.trainable_records(dataset), [])
|
|
for call in (
|
|
lambda: tool.commit_review(dataset, FIRST, "reviewer-b", "agreed"),
|
|
lambda: tool.adjudicate(dataset, FIRST, "reviewer-b", "upheld"),
|
|
lambda: tool.reveal(dataset, FIRST, PREDICTIONS),
|
|
):
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
call()
|
|
self.assertRegex(str(raised.exception), "withdrawn")
|
|
|
|
|
|
class AdjudicationTests(unittest.TestCase):
|
|
def test_adjudication_is_appended_and_the_first_review_is_preserved(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
first = tool.commit_review(dataset, FIRST, "reviewer-a", "corrected", {"bristolType": 5, "color": "brown", "quality": "good"}, at="2026-10-27T09:00:00+00:00")
|
|
|
|
ruling = tool.adjudicate(dataset, FIRST, "reviewer-b", "corrected", {"bristolType": 4, "color": "brown", "quality": "fair"}, at="2026-10-27T10:00:00+00:00")
|
|
|
|
self.assertEqual(ruling["recordType"], "adjudication")
|
|
self.assertEqual(ruling["decision"], "corrected")
|
|
self.assertEqual(ruling["labels"], {"bristolType": 4, "color": "brown", "quality": "fair"})
|
|
self.assertEqual(ruling["resolvesRecord"], 1)
|
|
self.assertEqual(ruling["resolves"], {"reviewer": "reviewer-a", "decision": "corrected"})
|
|
|
|
history = ledger(dataset)
|
|
self.assertEqual(len(history), 2)
|
|
self.assertEqual(history[0], first, "the review being overruled must survive untouched")
|
|
self.assertEqual(history[1], ruling)
|
|
|
|
# The trainer follows the ruling, not the review it replaced.
|
|
trainable = tool.trainable_records(dataset)
|
|
self.assertEqual(len(trainable), 1)
|
|
self.assertEqual(trainable[0]["labels"], {"bristolType": 4, "color": "brown", "quality": "fair"})
|
|
self.assertEqual(trainable[0]["adjudicatedBy"], "reviewer-b")
|
|
|
|
def test_an_upheld_review_keeps_the_label_and_records_the_second_reader(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
tool.commit_review(dataset, FIRST, "reviewer-a", "agreed")
|
|
|
|
ruling = tool.adjudicate(dataset, FIRST, "reviewer-b", "upheld")
|
|
|
|
self.assertEqual(ruling["labels"], {"bristolType": 4, "color": "brown", "quality": "good"})
|
|
self.assertEqual(ruling["resolves"], {"reviewer": "reviewer-a", "decision": "agreed"})
|
|
|
|
def test_a_reviewer_cannot_adjudicate_their_own_review(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
tool.commit_review(dataset, FIRST, "reviewer-a", "agreed")
|
|
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
tool.adjudicate(dataset, FIRST, "reviewer-a", "corrected", {"bristolType": 2, "color": "brown", "quality": "good"})
|
|
self.assertRegex(str(raised.exception), "second reviewer")
|
|
self.assertEqual(len(ledger(dataset)), 1)
|
|
|
|
def test_a_second_adjudication_is_refused(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
tool.commit_review(dataset, FIRST, "reviewer-a", "agreed")
|
|
tool.adjudicate(dataset, FIRST, "reviewer-b", "upheld")
|
|
history = ledger(dataset)
|
|
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
tool.adjudicate(dataset, FIRST, "reviewer-c", "excluded", exclusion_reason="unclear")
|
|
self.assertRegex(str(raised.exception), "already adjudicated")
|
|
self.assertEqual(ledger(dataset), history)
|
|
|
|
def test_adjudicating_an_unreviewed_contribution_is_refused(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
tool.adjudicate(dataset, FIRST, "reviewer-b", "upheld")
|
|
self.assertRegex(str(raised.exception), "no review to adjudicate")
|
|
self.assertEqual(ledger(dataset), [])
|
|
|
|
|
|
class TrainableSetTests(unittest.TestCase):
|
|
def test_only_human_reviewed_and_unexcluded_samples_reach_the_trainer(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
ingest(dataset, SECOND)
|
|
|
|
# Unreviewed samples are not training data, however confident the proposal is.
|
|
self.assertEqual(tool.trainable_records(dataset), [])
|
|
|
|
tool.commit_review(dataset, FIRST, "reviewer-a", "agreed")
|
|
tool.commit_review(dataset, SECOND, "reviewer-a", "excluded", exclusion_reason="cannot tell stool from photo edge")
|
|
|
|
trainable = tool.trainable_records(dataset)
|
|
self.assertEqual([item["contributionId"] for item in trainable], [FIRST])
|
|
self.assertEqual(trainable[0]["labels"], {"bristolType": 4, "color": "brown", "quality": "good"})
|
|
self.assertEqual(trainable[0]["reviewedBy"], "reviewer-a")
|
|
|
|
# Excluded is a labeling judgement, not a withdrawal: the provenance record still exists.
|
|
history = [json.loads(line) for line in (dataset / "manifest.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
self.assertEqual(len(history), 2)
|
|
|
|
|
|
class PredictionFileTests(unittest.TestCase):
|
|
def test_a_pretty_printed_answer_file_is_read_whole_not_line_by_line(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
path = Path(raw) / "answers.json"
|
|
path.write_text(json.dumps({"model": "synthetic answer file", "answers": {FIRST: {"bristolType": 6, "color": "brown", "quality": "good"}}}, indent=2), encoding="utf-8")
|
|
|
|
loaded = tool.load_predictions(path)
|
|
|
|
self.assertEqual(loaded["answers"][FIRST]["bristolType"], 6)
|
|
|
|
def test_a_line_delimited_answer_file_is_read_by_contribution(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
path = Path(raw) / "answers.jsonl"
|
|
path.write_text(
|
|
json.dumps({"contributionId": FIRST, "bristolType": 6, "color": "brown", "quality": "good"}) + "\n"
|
|
+ json.dumps({"contributionId": SECOND, "bristolType": 2, "color": "black", "quality": "poor"}) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
loaded = tool.load_predictions(path)
|
|
|
|
self.assertEqual(loaded["answers"][FIRST]["bristolType"], 6)
|
|
self.assertEqual(loaded["answers"][SECOND]["bristolType"], 2)
|
|
|
|
def test_an_answer_without_an_identifier_is_refused(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
path = Path(raw) / "answers.jsonl"
|
|
path.write_text(json.dumps({"bristolType": 6}) + "\n", encoding="utf-8")
|
|
|
|
with self.assertRaises(tool.ReviewError) as raised:
|
|
tool.load_predictions(path)
|
|
self.assertRegex(str(raised.exception), "contributionId or imageId")
|
|
|
|
|
|
class CliTests(unittest.TestCase):
|
|
def run_cli(self, *args: str) -> subprocess.CompletedProcess:
|
|
return subprocess.run(
|
|
[sys.executable, str(ROOT / "scripts" / "review_corpus.py"), *args],
|
|
cwd=ROOT, capture_output=True, text=True,
|
|
)
|
|
|
|
def test_the_cli_keeps_the_blind_order_and_reports_refusals_non_zero(self):
|
|
with tempfile.TemporaryDirectory() as raw:
|
|
dataset = Path(raw)
|
|
ingest(dataset, FIRST)
|
|
|
|
queue = self.run_cli("queue", "--dataset", str(dataset))
|
|
self.assertEqual(queue.returncode, 0, queue.stderr)
|
|
packet = json.loads(queue.stdout)
|
|
self.assertEqual(packet["count"], 1)
|
|
self.assertNotIn("confidence", queue.stdout)
|
|
|
|
early = self.run_cli("reveal", "--dataset", str(dataset), "--contribution", FIRST)
|
|
self.assertNotEqual(early.returncode, 0)
|
|
self.assertRegex(early.stderr, "hidden until a review is committed")
|
|
|
|
committed = self.run_cli(
|
|
"review", "--dataset", str(dataset), "--contribution", FIRST,
|
|
"--reviewer", "reviewer-a", "--bristol", "5", "--color", "green", "--quality", "fair",
|
|
)
|
|
self.assertEqual(committed.returncode, 0, committed.stderr)
|
|
self.assertEqual(json.loads(committed.stdout)["decision"], "corrected")
|
|
|
|
reveal = self.run_cli("reveal", "--dataset", str(dataset), "--contribution", FIRST)
|
|
self.assertEqual(reveal.returncode, 0, reveal.stderr)
|
|
self.assertEqual(json.loads(reveal.stdout)["model"], None, "no prediction was supplied, so none is invented")
|
|
|
|
trainable = self.run_cli("trainable", "--dataset", str(dataset))
|
|
self.assertEqual(trainable.returncode, 0, trainable.stderr)
|
|
self.assertEqual(json.loads(trainable.stdout)["count"], 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|