timmy
5c759078c4
Quality gates / quality (pull_request) Failing after 14s
The blank-first ordering let one blank field short-circuit the whole reply, so a response carrying an impossible value came back as `unusable_visual_fields`: the user was told the photo was unreadable when the model had in fact returned something Timmy must never apply. Blank fields still abstain and still prefill nothing. A filled-in field is a claim, and a claim outside the contract now fails closed even when the other field came back blank. The corpus earns its label too. `out-of-range-fields` drops from confidence 1.4 to 0.9 so the case reaches the field-range check instead of dying at the confidence gate, and a new `blank-plus-out-of-range` case covers the class the old ordering let through. The release demo replays that case on camera and the fail-closed 503 budget moves from one to two, so the scene cannot be dropped silently. Closes #73
294 lines
11 KiB
Python
294 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Generate the deterministic synthetic vision corpus for issue #18.
|
|
|
|
Every fixture is drawn from fixed integer math: no camera, no human subject, no
|
|
third-party content, and no randomness. That keeps the corpus redistributable and
|
|
keeps the recorded SHA-256 hashes meaningful as an integrity receipt.
|
|
|
|
Usage:
|
|
python3 scripts/build_vision_corpus.py # (re)write fixtures and manifest.json
|
|
python3 scripts/build_vision_corpus.py --verify # verify committed fixtures against the manifest
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
from PIL import Image, ImageDraw, ImageFilter
|
|
|
|
LICENSE = "CC0-1.0"
|
|
CORPUS_DIR = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "vision-corpus"
|
|
WIDTH, HEIGHT = 640, 480
|
|
PROVENANCE = "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content."
|
|
|
|
OBSERVATION_STRINGS = {
|
|
"smooth": "Smooth, formed appearance.",
|
|
"flatware": "Flatware on a tray.",
|
|
"unclear": "Form is unclear.",
|
|
"highlight": "Large specular highlight hides the surface.",
|
|
"container": "Empty container.",
|
|
"confident": "Confident.",
|
|
"partial": "Form read, colour not.",
|
|
}
|
|
|
|
|
|
def background(color: tuple[int, int, int]) -> Image.Image:
|
|
"""A flat canvas with a coarse, deterministic shade band."""
|
|
image = Image.new("RGB", (WIDTH, HEIGHT), color)
|
|
draw = ImageDraw.Draw(image)
|
|
for index in range(4):
|
|
shade = tuple(min(255, channel + 6 * index) for channel in color)
|
|
draw.rectangle([0, index * 120, WIDTH, index * 120 + 119], fill=shade)
|
|
return image
|
|
|
|
|
|
def blob(image: Image.Image, box: tuple[int, int, int, int], tone: tuple[int, int, int]) -> None:
|
|
draw = ImageDraw.Draw(image)
|
|
draw.ellipse(box, fill=tone)
|
|
left, top, right, bottom = box
|
|
inset = max(4, (right - left) // 6)
|
|
draw.ellipse([left + inset, top + inset, right - inset, bottom - inset],
|
|
fill=tuple(min(255, channel + 10) for channel in tone))
|
|
|
|
|
|
def stool_like() -> Image.Image:
|
|
image = background((226, 222, 212))
|
|
blob(image, (170, 150, 470, 330), (108, 76, 48))
|
|
return image
|
|
|
|
|
|
def non_stool_scene() -> Image.Image:
|
|
image = background((198, 202, 208))
|
|
draw = ImageDraw.Draw(image)
|
|
draw.ellipse([120, 90, 520, 390], fill=(244, 244, 240))
|
|
draw.rectangle([60, 200, 300, 224], fill=(120, 126, 134))
|
|
draw.rectangle([340, 200, 580, 224], fill=(120, 126, 134))
|
|
return image
|
|
|
|
|
|
def blur_low_detail() -> Image.Image:
|
|
return stool_like().filter(ImageFilter.GaussianBlur(5))
|
|
|
|
|
|
def glare_overexposed() -> Image.Image:
|
|
image = background((222, 218, 208))
|
|
blob(image, (150, 140, 430, 300), (104, 74, 46))
|
|
draw = ImageDraw.Draw(image)
|
|
draw.polygon([(380, 200), (640, 120), (640, 360), (420, 330)], fill=(252, 252, 250))
|
|
draw.ellipse([470, 210, 620, 320], fill=(255, 255, 255))
|
|
return image
|
|
|
|
|
|
def empty_bowl() -> Image.Image:
|
|
image = background((214, 210, 202))
|
|
draw = ImageDraw.Draw(image)
|
|
draw.ellipse([130, 90, 510, 400], fill=(238, 236, 230))
|
|
draw.ellipse([175, 130, 465, 360], fill=(206, 204, 198))
|
|
draw.ellipse([220, 170, 420, 320], fill=(232, 230, 224))
|
|
return image
|
|
|
|
|
|
def unusable_tone() -> Image.Image:
|
|
image = background((202, 206, 198))
|
|
blob(image, (190, 160, 450, 320), (128, 118, 92))
|
|
return image
|
|
|
|
|
|
def shaded_tone() -> Image.Image:
|
|
image = background((216, 214, 206))
|
|
blob(image, (180, 155, 460, 325), (96, 84, 52))
|
|
draw = ImageDraw.Draw(image)
|
|
draw.rectangle([0, 300, WIDTH, 306], fill=(180, 178, 170))
|
|
return image
|
|
|
|
|
|
def half_read_tone() -> Image.Image:
|
|
"""A fixture for the reply shape that fills one field and blanks the other (issue #73)."""
|
|
image = background((208, 212, 204))
|
|
blob(image, (175, 150, 455, 322), (112, 96, 66))
|
|
draw = ImageDraw.Draw(image)
|
|
draw.rectangle([0, 200, WIDTH // 2, 206], fill=(196, 194, 186))
|
|
draw.rectangle([WIDTH // 2, 260, WIDTH, 266], fill=(170, 168, 160))
|
|
return image
|
|
|
|
|
|
def content(payload: dict[str, object] | str) -> str:
|
|
return payload if isinstance(payload, str) else json.dumps(payload, separators=(",", ":"))
|
|
|
|
|
|
def cases() -> list[dict[str, Any]]:
|
|
return [
|
|
{
|
|
"id": "stool-like-formed",
|
|
"caseClass": "stool",
|
|
"image": stool_like,
|
|
"expectedOutcome": "suggestion",
|
|
"expectedReason": "schema_valid_suggestion",
|
|
"providerContent": content({
|
|
"isStool": True, "bristolType": 4, "color": "brown", "confidence": 0.83,
|
|
"imageQuality": "good", "observations": OBSERVATION_STRINGS["smooth"],
|
|
}),
|
|
},
|
|
{
|
|
"id": "non-stool-scene",
|
|
"caseClass": "non-stool",
|
|
"image": non_stool_scene,
|
|
"expectedOutcome": "abstained",
|
|
"expectedReason": "not_stool",
|
|
"providerContent": content({
|
|
"isStool": False, "bristolType": None, "color": None, "confidence": 0.97,
|
|
"imageQuality": "good", "observations": OBSERVATION_STRINGS["flatware"],
|
|
}),
|
|
},
|
|
{
|
|
"id": "blur-low-detail",
|
|
"caseClass": "blur",
|
|
"image": blur_low_detail,
|
|
"expectedOutcome": "abstained",
|
|
"expectedReason": "low_confidence",
|
|
"providerContent": content({
|
|
"isStool": True, "bristolType": 4, "color": "brown", "confidence": 0.41,
|
|
"imageQuality": "poor", "observations": OBSERVATION_STRINGS["unclear"],
|
|
}),
|
|
},
|
|
{
|
|
"id": "glare-overexposed",
|
|
"caseClass": "glare",
|
|
"image": glare_overexposed,
|
|
"expectedOutcome": "abstained",
|
|
"expectedReason": "unusable_visual_fields",
|
|
"providerContent": content({
|
|
"isStool": True, "bristolType": None, "color": None, "confidence": 0.66,
|
|
"imageQuality": "poor", "observations": OBSERVATION_STRINGS["highlight"],
|
|
}),
|
|
},
|
|
{
|
|
"id": "empty-bowl",
|
|
"caseClass": "empty-bowl",
|
|
"image": empty_bowl,
|
|
"expectedOutcome": "abstained",
|
|
"expectedReason": "not_stool",
|
|
"providerContent": content({
|
|
"isStool": False, "bristolType": None, "color": None, "confidence": 0.93,
|
|
"imageQuality": "good", "observations": OBSERVATION_STRINGS["container"],
|
|
}),
|
|
},
|
|
{
|
|
"id": "malformed-response",
|
|
"caseClass": "malformed-response",
|
|
"image": unusable_tone,
|
|
"expectedOutcome": "fail_closed",
|
|
"expectedReason": "unparsable_content",
|
|
"providerContent": "Sure! It looks like a Type 4 to me. Hope that helps.",
|
|
},
|
|
{
|
|
"id": "out-of-range-fields",
|
|
"caseClass": "malformed-response",
|
|
"image": shaded_tone,
|
|
"expectedOutcome": "fail_closed",
|
|
"expectedReason": "out_of_range_or_missing_fields",
|
|
# Confidence stays inside the contract so this reply reaches the field-range check. At 1.4 it
|
|
# died at the confidence gate instead, which credited the `out_of_range_or_missing_fields`
|
|
# label to a guard the case never exercised (issue #73).
|
|
"providerContent": content({
|
|
"isStool": True, "bristolType": 9, "color": "purple", "confidence": 0.9,
|
|
"imageQuality": "excellent", "observations": OBSERVATION_STRINGS["confident"],
|
|
}),
|
|
},
|
|
{
|
|
"id": "blank-plus-out-of-range",
|
|
"caseClass": "malformed-response",
|
|
"image": half_read_tone,
|
|
"expectedOutcome": "fail_closed",
|
|
"expectedReason": "out_of_range_or_missing_fields",
|
|
# The class the blank-first ordering used to let through (issue #73): one field is blank and
|
|
# the other is a filled-in, impossible value. A filled-in field is a claim, so this must fail
|
|
# closed rather than abstain as `unusable_visual_fields`.
|
|
"providerContent": content({
|
|
"isStool": True, "bristolType": 9, "color": None, "confidence": 0.9,
|
|
"imageQuality": "good", "observations": OBSERVATION_STRINGS["partial"],
|
|
}),
|
|
},
|
|
]
|
|
|
|
|
|
def write_fixture(definition: dict[str, Any], directory: Path) -> dict[str, Any]:
|
|
name = f"{definition['id']}.png"
|
|
target = directory / name
|
|
factory: Callable[[], Image.Image] = definition["image"]
|
|
factory().save(target, "PNG", optimize=True)
|
|
data = target.read_bytes()
|
|
return {
|
|
"id": definition["id"],
|
|
"caseClass": definition["caseClass"],
|
|
"file": name,
|
|
"bytes": len(data),
|
|
"sha256": hashlib.sha256(data).hexdigest(),
|
|
"width": WIDTH,
|
|
"height": HEIGHT,
|
|
"format": "png",
|
|
"synthetic": True,
|
|
"license": LICENSE,
|
|
"provenance": PROVENANCE,
|
|
"expectedOutcome": definition["expectedOutcome"],
|
|
"expectedReason": definition["expectedReason"],
|
|
"providerContent": definition["providerContent"],
|
|
}
|
|
|
|
|
|
def manifest(entries: list[dict[str, object]]) -> dict[str, object]:
|
|
return {
|
|
"schemaVersion": 1,
|
|
"generator": "scripts/build_vision_corpus.py",
|
|
"license": LICENSE,
|
|
"note": "Synthetic fixtures only. No real medical imagery, no personal data, and no third-party content.",
|
|
"cases": entries,
|
|
}
|
|
|
|
|
|
def write(directory: Path) -> int:
|
|
directory.mkdir(parents=True, exist_ok=True)
|
|
entries = [write_fixture(definition, directory) for definition in cases()]
|
|
(directory / "manifest.json").write_text(json.dumps(manifest(entries), indent=2) + "\n", encoding="utf-8")
|
|
print(f"wrote {len(entries)} fixtures to {directory}")
|
|
return 0
|
|
|
|
|
|
def verify(directory: Path) -> int:
|
|
manifest_path = directory / "manifest.json"
|
|
recorded = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
problems: list[str] = []
|
|
for entry in recorded.get("cases", []):
|
|
path = directory / str(entry.get("file"))
|
|
if not path.is_file():
|
|
problems.append(f"{entry.get('id')}: missing fixture {entry.get('file')}")
|
|
continue
|
|
data = path.read_bytes()
|
|
if hashlib.sha256(data).hexdigest() != entry.get("sha256"):
|
|
problems.append(f"{entry.get('id')}: bytes do not match the recorded sha256")
|
|
if len(data) != entry.get("bytes"):
|
|
problems.append(f"{entry.get('id')}: byte length does not match the manifest")
|
|
if problems:
|
|
for problem in problems:
|
|
print(problem, file=sys.stderr)
|
|
return 1
|
|
print(f"verified {len(recorded.get('cases', []))} corpus fixtures against manifest.json")
|
|
return 0
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--verify", action="store_true", help="check committed fixtures against the manifest instead of rewriting")
|
|
parser.add_argument("--directory", default=str(CORPUS_DIR))
|
|
args = parser.parse_args()
|
|
directory = Path(args.directory)
|
|
return verify(directory) if args.verify else write(directory)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|