timmy
5c759078c4
Quality gates / quality (pull_request) Failing after 14s
The blank-first ordering let one blank field short-circuit the whole reply, so a response carrying an impossible value came back as `unusable_visual_fields`: the user was told the photo was unreadable when the model had in fact returned something Timmy must never apply. Blank fields still abstain and still prefill nothing. A filled-in field is a claim, and a claim outside the contract now fails closed even when the other field came back blank. The corpus earns its label too. `out-of-range-fields` drops from confidence 1.4 to 0.9 so the case reaches the field-range check instead of dying at the confidence gate, and a new `blank-plus-out-of-range` case covers the class the old ordering let through. The release demo replays that case on camera and the fail-closed 503 budget moves from one to two, so the scene cannot be dropped silently. Closes #73
137 lines
6.0 KiB
JSON
137 lines
6.0 KiB
JSON
{
|
|
"schemaVersion": 1,
|
|
"generator": "scripts/build_vision_corpus.py",
|
|
"license": "CC0-1.0",
|
|
"note": "Synthetic fixtures only. No real medical imagery, no personal data, and no third-party content.",
|
|
"cases": [
|
|
{
|
|
"id": "stool-like-formed",
|
|
"caseClass": "stool",
|
|
"file": "stool-like-formed.png",
|
|
"bytes": 2414,
|
|
"sha256": "4bb50ec6fc4ef7e9c83ed3b7ba69cdf5e73ef74b9f05e97898892bc852df970a",
|
|
"width": 640,
|
|
"height": 480,
|
|
"format": "png",
|
|
"synthetic": true,
|
|
"license": "CC0-1.0",
|
|
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
|
"expectedOutcome": "suggestion",
|
|
"expectedReason": "schema_valid_suggestion",
|
|
"providerContent": "{\"isStool\":true,\"bristolType\":4,\"color\":\"brown\",\"confidence\":0.83,\"imageQuality\":\"good\",\"observations\":\"Smooth, formed appearance.\"}"
|
|
},
|
|
{
|
|
"id": "non-stool-scene",
|
|
"caseClass": "non-stool",
|
|
"file": "non-stool-scene.png",
|
|
"bytes": 2406,
|
|
"sha256": "4d12150073856c1d7dfb9332049182ee0145da6ad50f6964a7273c8c61f21f91",
|
|
"width": 640,
|
|
"height": 480,
|
|
"format": "png",
|
|
"synthetic": true,
|
|
"license": "CC0-1.0",
|
|
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
|
"expectedOutcome": "abstained",
|
|
"expectedReason": "not_stool",
|
|
"providerContent": "{\"isStool\":false,\"bristolType\":null,\"color\":null,\"confidence\":0.97,\"imageQuality\":\"good\",\"observations\":\"Flatware on a tray.\"}"
|
|
},
|
|
{
|
|
"id": "blur-low-detail",
|
|
"caseClass": "blur",
|
|
"file": "blur-low-detail.png",
|
|
"bytes": 21598,
|
|
"sha256": "17000163662a42fd434bb2b2428d9e5a2156da08cbc82210e41dfc6fa7e24852",
|
|
"width": 640,
|
|
"height": 480,
|
|
"format": "png",
|
|
"synthetic": true,
|
|
"license": "CC0-1.0",
|
|
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
|
"expectedOutcome": "abstained",
|
|
"expectedReason": "low_confidence",
|
|
"providerContent": "{\"isStool\":true,\"bristolType\":4,\"color\":\"brown\",\"confidence\":0.41,\"imageQuality\":\"poor\",\"observations\":\"Form is unclear.\"}"
|
|
},
|
|
{
|
|
"id": "glare-overexposed",
|
|
"caseClass": "glare",
|
|
"file": "glare-overexposed.png",
|
|
"bytes": 3140,
|
|
"sha256": "1dc87a80951dd244c9831cba8b352e24718cf5f103f4dfd0a30e8c8c1c599f95",
|
|
"width": 640,
|
|
"height": 480,
|
|
"format": "png",
|
|
"synthetic": true,
|
|
"license": "CC0-1.0",
|
|
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
|
"expectedOutcome": "abstained",
|
|
"expectedReason": "unusable_visual_fields",
|
|
"providerContent": "{\"isStool\":true,\"bristolType\":null,\"color\":null,\"confidence\":0.66,\"imageQuality\":\"poor\",\"observations\":\"Large specular highlight hides the surface.\"}"
|
|
},
|
|
{
|
|
"id": "empty-bowl",
|
|
"caseClass": "empty-bowl",
|
|
"file": "empty-bowl.png",
|
|
"bytes": 3479,
|
|
"sha256": "ab443d1d9d28338e5c9f48054e7989cfce59ae25e2a9008be31cd9223be6abc4",
|
|
"width": 640,
|
|
"height": 480,
|
|
"format": "png",
|
|
"synthetic": true,
|
|
"license": "CC0-1.0",
|
|
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
|
"expectedOutcome": "abstained",
|
|
"expectedReason": "not_stool",
|
|
"providerContent": "{\"isStool\":false,\"bristolType\":null,\"color\":null,\"confidence\":0.93,\"imageQuality\":\"good\",\"observations\":\"Empty container.\"}"
|
|
},
|
|
{
|
|
"id": "malformed-response",
|
|
"caseClass": "malformed-response",
|
|
"file": "malformed-response.png",
|
|
"bytes": 2341,
|
|
"sha256": "634d16337879783b307d0ecf84348cfdc07950f6893095267901383f2efc83c2",
|
|
"width": 640,
|
|
"height": 480,
|
|
"format": "png",
|
|
"synthetic": true,
|
|
"license": "CC0-1.0",
|
|
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
|
"expectedOutcome": "fail_closed",
|
|
"expectedReason": "unparsable_content",
|
|
"providerContent": "Sure! It looks like a Type 4 to me. Hope that helps."
|
|
},
|
|
{
|
|
"id": "out-of-range-fields",
|
|
"caseClass": "malformed-response",
|
|
"file": "out-of-range-fields.png",
|
|
"bytes": 2375,
|
|
"sha256": "ac20ca9f3c9cbd118d3a2957110505d049a85861f97d361dcfbc4787960a5f8f",
|
|
"width": 640,
|
|
"height": 480,
|
|
"format": "png",
|
|
"synthetic": true,
|
|
"license": "CC0-1.0",
|
|
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
|
"expectedOutcome": "fail_closed",
|
|
"expectedReason": "out_of_range_or_missing_fields",
|
|
"providerContent": "{\"isStool\":true,\"bristolType\":9,\"color\":\"purple\",\"confidence\":0.9,\"imageQuality\":\"excellent\",\"observations\":\"Confident.\"}"
|
|
},
|
|
{
|
|
"id": "blank-plus-out-of-range",
|
|
"caseClass": "malformed-response",
|
|
"file": "blank-plus-out-of-range.png",
|
|
"bytes": 2403,
|
|
"sha256": "d45048cd1f5bfb1be1667238c9c21fca0cd1cae8b55e12d4f325203625eeafbd",
|
|
"width": 640,
|
|
"height": 480,
|
|
"format": "png",
|
|
"synthetic": true,
|
|
"license": "CC0-1.0",
|
|
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
|
"expectedOutcome": "fail_closed",
|
|
"expectedReason": "out_of_range_or_missing_fields",
|
|
"providerContent": "{\"isStool\":true,\"bristolType\":9,\"color\":null,\"confidence\":0.9,\"imageQuality\":\"good\",\"observations\":\"Form read, colour not.\"}"
|
|
}
|
|
]
|
|
}
|