timmy
860c84f0fe
Quality gates / quality (pull_request) Failing after 9m42s
Promote the ingest script into an auditable corpus pipeline. Each sample now records consent (version + timestamp), source kind and raw-input digest, both hashes, the derivation pipeline, review status, split, and lifecycle state, with the caller's path never written down. Withdrawal is a new `--revoke <contribution-id> --reason` operation: it deletes the derived image and any raw pre-derivation copy, then appends a tombstone that addresses each removed artifact by contribution id and digest while dropping labels, subject key, and paths. History is append-only, so the original sample record survives the withdrawal it documents. Illegal moves are refused instead of silently applied: revoked -> active, revoking an unknown or already-revoked contribution, an unknown state, a duplicate contribution id, and a second label set for the same derived image. The duplicate guard runs before any byte is written, so a refused ingest cannot strand an image in the active dataset. Tests written first and watched fail (tests/corpus-manifest.test.py): provenance, tombstone + byte removal, active-record filtering, and the transition guards. The transition helper itself passed on first write and is reported as a hold-out check rather than a RED cycle. Wired into the quality workflow and check:syntax; contract documented in docs/CORPUS-MANIFEST.md. No browser surface changed, so record_release_demo.mjs is untouched and claims nothing new: the nearest honest visible effect is the withdrawn-receipt tombstone already recorded for #24.
126 lines
4.4 KiB
Python
Executable File
126 lines
4.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Create a metadata-free, consent-traceable Timmy training record."""
|
|
import argparse
|
|
import hashlib
|
|
import hmac
|
|
import io
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
from PIL import Image, ImageOps
|
|
|
|
from corpus_manifest import CorpusError, append_record, assert_admissible, build_record, revoke, sha256_hex
|
|
|
|
COLORS = ("brown", "green", "yellow", "pale", "red", "black")
|
|
QUALITIES = ("good", "fair", "poor")
|
|
REVIEWERS = ("user-confirmed", "clinician-reviewed")
|
|
|
|
|
|
def parser() -> argparse.ArgumentParser:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--input")
|
|
p.add_argument("--output-dir", required=True)
|
|
p.add_argument("--subject-id")
|
|
p.add_argument("--consent-version")
|
|
p.add_argument("--contribution-id")
|
|
p.add_argument("--bristol-type", type=int, choices=range(1, 8))
|
|
p.add_argument("--color", choices=COLORS)
|
|
p.add_argument("--quality", choices=QUALITIES)
|
|
p.add_argument("--reviewer", choices=REVIEWERS)
|
|
# Revocation is a corpus operation, not an ingestion: it needs the dataset and a reason.
|
|
p.add_argument("--revoke", metavar="CONTRIBUTION_ID")
|
|
p.add_argument("--reason")
|
|
return p
|
|
|
|
|
|
def require_ingest_arguments(args: argparse.Namespace) -> None:
|
|
if not args.consent_version or not args.consent_version.strip():
|
|
raise SystemExit("consent version is required")
|
|
if not args.reviewer:
|
|
raise SystemExit("a human review status is required")
|
|
missing = [
|
|
flag
|
|
for flag, value in (
|
|
("--input", args.input),
|
|
("--subject-id", args.subject_id),
|
|
("--contribution-id", args.contribution_id),
|
|
("--bristol-type", args.bristol_type),
|
|
("--color", args.color),
|
|
("--quality", args.quality),
|
|
)
|
|
if value is None or value == ""
|
|
]
|
|
if missing:
|
|
raise SystemExit(f"ingesting a sample also needs {', '.join(missing)}")
|
|
|
|
|
|
def main() -> int:
|
|
args = parser().parse_args()
|
|
if args.revoke:
|
|
if not args.reason:
|
|
raise SystemExit("revoking a contribution needs a recorded reason")
|
|
try:
|
|
tombstone = revoke(Path(args.output_dir), args.revoke, datetime.now(timezone.utc).isoformat(), args.reason)
|
|
except CorpusError as exc:
|
|
raise SystemExit(str(exc)) from exc
|
|
print(json.dumps(tombstone))
|
|
return 0
|
|
|
|
require_ingest_arguments(args)
|
|
salt = os.environ.get("TIMMY_DATASET_SALT", "")
|
|
if len(salt) < 8:
|
|
raise SystemExit("TIMMY_DATASET_SALT must contain at least 8 characters")
|
|
|
|
try:
|
|
raw_bytes = Path(args.input).read_bytes()
|
|
with Image.open(io.BytesIO(raw_bytes)) as source:
|
|
image = ImageOps.exif_transpose(source).convert("RGB")
|
|
image.thumbnail((1200, 1200))
|
|
buffer = io.BytesIO()
|
|
image.save(buffer, "JPEG", quality=85, optimize=True, exif=b"")
|
|
except Exception as exc:
|
|
raise SystemExit(f"input is not a decodable image: {exc}") from exc
|
|
|
|
data = buffer.getvalue()
|
|
image_id = hashlib.sha256(data).hexdigest()
|
|
subject_key = hmac.new(salt.encode(), args.subject_id.encode(), hashlib.sha256).hexdigest()[:16]
|
|
bucket = int(subject_key[:2], 16)
|
|
split = "train" if bucket < 205 else "validation" if bucket < 230 else "test"
|
|
|
|
output = Path(args.output_dir)
|
|
images = output / "images"
|
|
relative_path = f"images/{image_id}.jpg"
|
|
target = output / relative_path
|
|
|
|
now = datetime.now(timezone.utc).isoformat()
|
|
record = build_record(
|
|
contribution_id=args.contribution_id,
|
|
consent_version=args.consent_version,
|
|
subject_key=subject_key,
|
|
split=split,
|
|
labels={"bristolType": args.bristol_type, "color": args.color, "quality": args.quality},
|
|
review_status=args.reviewer,
|
|
derived_sha256=image_id,
|
|
raw_sha256=sha256_hex(raw_bytes),
|
|
ingested_at=now,
|
|
)
|
|
try:
|
|
# Refuse a duplicate contribution before any byte lands in the active dataset.
|
|
assert_admissible(output / "manifest.jsonl", record)
|
|
images.mkdir(parents=True, exist_ok=True)
|
|
if not target.exists():
|
|
target.write_bytes(data)
|
|
append_record(output / "manifest.jsonl", record)
|
|
except CorpusError as exc:
|
|
raise SystemExit(str(exc)) from exc
|
|
print(json.dumps(record))
|
|
return 0
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|