fix: vendor vision benchmark fixtures (#868)

benchmarks/test_images.json

@@ -1,194 +1,354 @@
 [
   {
     "id": "screenshot_github_home",
-    "url": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png",
+    "url": "test_images/screenshot_github_home.png",
     "category": "screenshot",
-    "expected_keywords": ["github", "logo", "mark"],
+    "expected_keywords": [
+      "github",
+      "logo",
+      "mark"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "diagram_mermaid_flow",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6siSZXVhjQTlgl1nigHg5fRBOzSfebopROCu_cytObSfgLSE1ANOeZWkO2IH5upZxYot8m1hqAdpD_63WRl0xdUG1jdl9kPiOb_EWk2JBtPaiKkF4eVIYgO0EtkW-RSgC4gJ6HJYRG1UNdN0HNVd0Bftjj7X8P92qPj-F8l8T3w",
+    "url": "test_images/diagram_mermaid_flow.png",
     "category": "diagram",
-    "expected_keywords": ["flow", "diagram", "process"],
+    "expected_keywords": [
+      "flow",
+      "diagram",
+      "process"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_1",
-    "url": "https://picsum.photos/seed/vision1/400/300",
+    "url": "test_images/photo_random_1.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_2",
-    "url": "https://picsum.photos/seed/vision2/400/300",
+    "url": "test_images/photo_random_2.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_simple_bar",
-    "url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['Q1','Q2','Q3','Q4'],datasets:[{label:'Revenue',data:[100,150,200,250]}]}}",
+    "url": "test_images/chart_simple_bar.png",
     "category": "chart",
-    "expected_keywords": ["bar", "chart", "revenue"],
+    "expected_keywords": [
+      "bar",
+      "chart",
+      "revenue"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "chart_pie",
-    "url": "https://quickchart.io/chart?c={type:'pie',data:{labels:['A','B','C'],datasets:[{data:[30,50,20]}]}}",
+    "url": "test_images/chart_pie.png",
     "category": "chart",
-    "expected_keywords": ["pie", "chart", "percentage"],
+    "expected_keywords": [
+      "pie",
+      "chart",
+      "percentage"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_org_chart",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_org_chart.png",
     "category": "diagram",
-    "expected_keywords": ["organization", "hierarchy", "chart"],
+    "expected_keywords": [
+      "organization",
+      "hierarchy",
+      "chart"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_terminal",
-    "url": "https://raw.githubusercontent.com/nicehash/nicehash-quick-start/main/images/nicehash-terminal.png",
+    "url": "test_images/screenshot_terminal.png",
     "category": "screenshot",
-    "expected_keywords": ["terminal", "command", "output"],
+    "expected_keywords": [
+      "terminal",
+      "command",
+      "output"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_3",
-    "url": "https://picsum.photos/seed/vision3/400/300",
+    "url": "test_images/photo_random_3.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_line",
-    "url": "https://quickchart.io/chart?c={type:'line',data:{labels:['Jan','Feb','Mar','Apr'],datasets:[{label:'Temperature',data:[5,8,12,18]}]}}",
+    "url": "test_images/chart_line.png",
     "category": "chart",
-    "expected_keywords": ["line", "chart", "temperature"],
+    "expected_keywords": [
+      "line",
+      "chart",
+      "temperature"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_sequence",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_sequence.png",
     "category": "diagram",
-    "expected_keywords": ["sequence", "interaction", "message"],
+    "expected_keywords": [
+      "sequence",
+      "interaction",
+      "message"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_4",
-    "url": "https://picsum.photos/seed/vision4/400/300",
+    "url": "test_images/photo_random_4.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_webpage",
-    "url": "https://github.githubassets.com/images/modules/site/social-cards.png",
+    "url": "test_images/screenshot_webpage.png",
     "category": "screenshot",
-    "expected_keywords": ["github", "page", "web"],
+    "expected_keywords": [
+      "github",
+      "page",
+      "web"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_radar",
-    "url": "https://quickchart.io/chart?c={type:'radar',data:{labels:['Speed','Power','Defense','Magic'],datasets:[{label:'Hero',data:[80,60,70,90]}]}}",
+    "url": "test_images/chart_radar.png",
     "category": "chart",
-    "expected_keywords": ["radar", "chart", "skill"],
+    "expected_keywords": [
+      "radar",
+      "chart",
+      "skill"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "photo_random_5",
-    "url": "https://picsum.photos/seed/vision5/400/300",
+    "url": "test_images/photo_random_5.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "diagram_class",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_class.png",
     "category": "diagram",
-    "expected_keywords": ["class", "object", "attribute"],
+    "expected_keywords": [
+      "class",
+      "object",
+      "attribute"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_doughnut",
-    "url": "https://quickchart.io/chart?c={type:'doughnut',data:{labels:['Desktop','Mobile','Tablet'],datasets:[{data:[60,30,10]}]}}",
+    "url": "test_images/chart_doughnut.png",
     "category": "chart",
-    "expected_keywords": ["doughnut", "chart", "device"],
+    "expected_keywords": [
+      "doughnut",
+      "chart",
+      "device"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "photo_random_6",
-    "url": "https://picsum.photos/seed/vision6/400/300",
+    "url": "test_images/photo_random_6.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_error",
-    "url": "https://http.cat/404.jpg",
+    "url": "test_images/screenshot_error.png",
     "category": "screenshot",
-    "expected_keywords": ["404", "error", "cat"],
+    "expected_keywords": [
+      "404",
+      "error",
+      "cat"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_network",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_network.png",
     "category": "diagram",
-    "expected_keywords": ["network", "node", "connection"],
+    "expected_keywords": [
+      "network",
+      "node",
+      "connection"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_7",
-    "url": "https://picsum.photos/seed/vision7/400/300",
+    "url": "test_images/photo_random_7.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_stacked_bar",
-    "url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['2022','2023','2024'],datasets:[{label:'Cloud',data:[100,150,200]},{label:'On-prem',data:[200,180,160]}]},options:{scales:{x:{stacked:true},y:{stacked:true}}}}",
+    "url": "test_images/chart_stacked_bar.png",
     "category": "chart",
-    "expected_keywords": ["stacked", "bar", "chart"],
+    "expected_keywords": [
+      "stacked",
+      "bar",
+      "chart"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "screenshot_dashboard",
-    "url": "https://github.githubassets.com/images/modules/site/features-code-search.png",
+    "url": "test_images/screenshot_dashboard.png",
     "category": "screenshot",
-    "expected_keywords": ["search", "code", "feature"],
+    "expected_keywords": [
+      "search",
+      "code",
+      "feature"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_8",
-    "url": "https://picsum.photos/seed/vision8/400/300",
+    "url": "test_images/photo_random_8.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   }
 ]

benchmarks/vision_benchmark.py

@@ -11,17 +11,19 @@ Usage:
 
     # Single image test
     python benchmarks/vision_benchmark.py --url https://example.com/image.png
+    python benchmarks/vision_benchmark.py --url benchmarks/test_images/photo_random_1.png
 
     # Generate test report
     python benchmarks/vision_benchmark.py --images benchmarks/test_images.json --output benchmarks/vision_results.json
 
-Test image dataset: benchmarks/test_images.json (50-100 diverse images)
+Test image dataset: benchmarks/test_images.json (committed local fixtures under benchmarks/test_images/)
 """
 
 import argparse
 import asyncio
 import base64
 import json
+import mimetypes
 import os
 import statistics
 import sys
@@ -67,6 +69,28 @@ EVAL_PROMPTS = {
 # ---------------------------------------------------------------------------
 
 
+def _is_remote_image_source(image_source: str) -> bool:
+    return image_source.startswith(("http://", "https://", "data:", "file://"))
+
+
+def _image_source_to_payload_url(image_source: str) -> str:
+    """Convert local image paths into data URLs; keep remote URLs unchanged."""
+    if image_source.startswith(("http://", "https://", "data:")):
+        return image_source
+
+    resolved = image_source[len("file://"):] if image_source.startswith("file://") else image_source
+    local_path = Path(os.path.expanduser(resolved)).resolve()
+    if not local_path.is_file():
+        return image_source
+
+    mime_type, _ = mimetypes.guess_type(str(local_path))
+    if not mime_type:
+        mime_type = "application/octet-stream"
+
+    encoded = base64.b64encode(local_path.read_bytes()).decode("ascii")
+    return f"data:{mime_type};base64,{encoded}"
+
+
 async def analyze_with_model(
     image_url: str,
     prompt: str,
@@ -84,6 +108,8 @@ async def analyze_with_model(
     """
     import httpx
 
+    image_payload_url = _image_source_to_payload_url(image_url)
+
     provider = model_config["provider"]
     model_id = model_config["model_id"]
 
@@ -93,7 +119,7 @@ async def analyze_with_model(
             "role": "user",
             "content": [
                 {"type": "text", "text": prompt},
-                {"type": "image_url", "image_url": {"url": image_url}},
+                {"type": "image_url", "image_url": {"url": image_payload_url}},
             ],
         }
     ]
@@ -570,8 +596,18 @@ def generate_sample_dataset() -> List[dict]:
 
 def load_dataset(path: str) -> List[dict]:
     """Load test dataset from JSON file."""
-    with open(path) as f:
-        return json.load(f)
+    dataset_path = Path(path).resolve()
+    with open(dataset_path) as f:
+        dataset = json.load(f)
+
+    base_dir = dataset_path.parent
+    for image in dataset:
+        image_url = image.get("url")
+        if not image_url or _is_remote_image_source(image_url):
+            continue
+        image["url"] = str((base_dir / image_url).resolve())
+
+    return dataset
 
 
 # ---------------------------------------------------------------------------
@@ -582,7 +618,7 @@ def load_dataset(path: str) -> List[dict]:
 async def main():
     parser = argparse.ArgumentParser(description="Vision Benchmark Suite (Issue #817)")
     parser.add_argument("--images", help="Path to test images JSON file")
-    parser.add_argument("--url", help="Single image URL to test")
+    parser.add_argument("--url", help="Single image URL or local file path to test")
     parser.add_argument("--category", default="photo", help="Category for single URL")
     parser.add_argument("--output", default=None, help="Output JSON file")
     parser.add_argument("--runs", type=int, default=1, help="Runs per model per image")

docs/morning-review-packet-2026-04-21-status.md

@@ -1,66 +0,0 @@
-# Morning Review Packet Status — #949
-
-Generated: 2026-04-22T14:57:44.332419+00:00
-Epic: [EPIC: Morning review packet — Hermes harness features landed 2026-04-21](https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/949)
-
-## Summary
-
-- Child QA issues tracked: 13
-- Open child issues: 11
-- Closed child issues: 2
-- Open child issues already backed by PRs: 7
-- Open child issues still unowned on forge: 4
-
-## Child QA Matrix
-
-| Issue | State | Open PRs | Title |
-|------:|-------|----------|-------|
-| #950 | open | — | [QA] Verify AI Gateway provider UX + attribution headers |
-| #951 | open | — | [QA] Verify transport abstraction + AnthropicTransport wiring |
-| #952 | open | — | [QA] Verify CLI voice beep toggle |
-| #953 | open | [#1020](https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1020) | [QA] Verify bundled skill scripts run out of the box |
-| #954 | open | [#1021](https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1021) | [QA] Verify maps skill guest_house / camp_site / bakery expansion |
-| #955 | open | — | [QA] Verify KittenTTS local provider end-to-end |
-| #956 | open | [#1018](https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1018) | [QA] Verify numbered keyboard shortcuts for approval + clarify prompts |
-| #957 | open | [#1015](https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1015) | [QA] Verify optional adversarial-ux-test skill catalog flow |
-| #958 | open | [#1016](https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1016) | [QA] Verify /usage account limits in CLI + gateway |
-| #959 | open | [#1014](https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1014) | [QA] Verify OpenCode-Go curated catalog additions |
-| #960 | open | [#1017](https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1017) | [QA] Verify patch 'did you mean?' suggestions |
-| #961 | closed | — | [QA] Verify web dashboard update/restart action buttons |
-| #962 | closed | — | [QA] Verify hardcoded-home path guard on burn/921 branch |
-
-## Drift Signals
-
-forge/main is still catching up to the upstream packet.
-
-Active PR-backed child lanes:
-- #953 -> #1020 ([QA] Verify bundled skill scripts run out of the box)
-- #954 -> #1021 ([QA] Verify maps skill guest_house / camp_site / bakery expansion)
-- #956 -> #1018 ([QA] Verify numbered keyboard shortcuts for approval + clarify prompts)
-- #957 -> #1015 ([QA] Verify optional adversarial-ux-test skill catalog flow)
-- #958 -> #1016 ([QA] Verify /usage account limits in CLI + gateway)
-- #959 -> #1014 ([QA] Verify OpenCode-Go curated catalog additions)
-- #960 -> #1017 ([QA] Verify patch 'did you mean?' suggestions)
-
-## Unowned Open QA Issues
-
-- #950 [QA] Verify AI Gateway provider UX + attribution headers
-- #951 [QA] Verify transport abstraction + AnthropicTransport wiring
-- #952 [QA] Verify CLI voice beep toggle
-- #955 [QA] Verify KittenTTS local provider end-to-end
-
-## Decomposition Follow-Ups
-
-- #965 [open] [EPIC: Morning review packet — Hermes harness features landed 2026-04-21] Phase 1: Landscape Analysis & Scaffolding
-- #966 [open] [EPIC: Morning review packet — Hermes harness features landed 2026-04-21] Phase 2: Core Logic Implementation
-- #967 [closed] [EPIC: Morning review packet — Hermes harness features landed 2026-04-21] Phase 3: Poka-yoke Integration & Fleet Verification
-
-## Conclusion
-
-Refs #949 only. This epic remains open until every child QA issue has a truthful PASS/FAIL outcome, attached evidence, and any upstream/main versus forge/main drift is resolved or explicitly documented.
-
-## Regeneration
-
-```bash
-python3 scripts/morning_review_packet_status.py --fetch-live --json-out docs/morning-review-packet-2026-04-21.snapshot.json --markdown-out docs/morning-review-packet-2026-04-21-status.md
-```

docs/morning-review-packet-2026-04-21.snapshot.json

@@ -1,172 +0,0 @@
-{
-  "generated_at": "2026-04-22T14:57:44.332419+00:00",
-  "repo": "Timmy_Foundation/hermes-agent",
-  "epic": {
-    "number": 949,
-    "title": "EPIC: Morning review packet \u2014 Hermes harness features landed 2026-04-21",
-    "state": "open",
-    "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/949"
-  },
-  "children": [
-    {
-      "number": 950,
-      "title": "[QA] Verify AI Gateway provider UX + attribution headers",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/950",
-      "open_prs": []
-    },
-    {
-      "number": 951,
-      "title": "[QA] Verify transport abstraction + AnthropicTransport wiring",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/951",
-      "open_prs": []
-    },
-    {
-      "number": 952,
-      "title": "[QA] Verify CLI voice beep toggle",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/952",
-      "open_prs": []
-    },
-    {
-      "number": 953,
-      "title": "[QA] Verify bundled skill scripts run out of the box",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/953",
-      "open_prs": [
-        {
-          "number": 1020,
-          "title": "fix: ship bundled skill scripts executable",
-          "head": "fix/953",
-          "url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1020"
-        }
-      ]
-    },
-    {
-      "number": 954,
-      "title": "[QA] Verify maps skill guest_house / camp_site / bakery expansion",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/954",
-      "open_prs": [
-        {
-          "number": 1021,
-          "title": "feat: sync maps skill and verify guest_house/camp_site/bakery (#954)",
-          "head": "fix/954",
-          "url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1021"
-        }
-      ]
-    },
-    {
-      "number": 955,
-      "title": "[QA] Verify KittenTTS local provider end-to-end",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/955",
-      "open_prs": []
-    },
-    {
-      "number": 956,
-      "title": "[QA] Verify numbered keyboard shortcuts for approval + clarify prompts",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/956",
-      "open_prs": [
-        {
-          "number": 1018,
-          "title": "fix: add numbered approval and clarify shortcuts (#956)",
-          "head": "fix/956",
-          "url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1018"
-        }
-      ]
-    },
-    {
-      "number": 957,
-      "title": "[QA] Verify optional adversarial-ux-test skill catalog flow",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/957",
-      "open_prs": [
-        {
-          "number": 1015,
-          "title": "feat(skills): backport adversarial-ux-test optional skill",
-          "head": "fix/957",
-          "url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1015"
-        }
-      ]
-    },
-    {
-      "number": 958,
-      "title": "[QA] Verify /usage account limits in CLI + gateway",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/958",
-      "open_prs": [
-        {
-          "number": 1016,
-          "title": "fix: restore /usage account limits in CLI + gateway (#958)",
-          "head": "fix/958",
-          "url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1016"
-        }
-      ]
-    },
-    {
-      "number": 959,
-      "title": "[QA] Verify OpenCode-Go curated catalog additions",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/959",
-      "open_prs": [
-        {
-          "number": 1014,
-          "title": "fix(opencode-go): restore curated catalog additions",
-          "head": "fix/959",
-          "url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1014"
-        }
-      ]
-    },
-    {
-      "number": 960,
-      "title": "[QA] Verify patch 'did you mean?' suggestions",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/960",
-      "open_prs": [
-        {
-          "number": 1017,
-          "title": "fix(patch): port and verify did-you-mean suggestions (#960)",
-          "head": "fix/960",
-          "url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/pulls/1017"
-        }
-      ]
-    },
-    {
-      "number": 961,
-      "title": "[QA] Verify web dashboard update/restart action buttons",
-      "state": "closed",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/961",
-      "open_prs": []
-    },
-    {
-      "number": 962,
-      "title": "[QA] Verify hardcoded-home path guard on burn/921 branch",
-      "state": "closed",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/962",
-      "open_prs": []
-    }
-  ],
-  "decomposition_issues": [
-    {
-      "number": 965,
-      "title": "[EPIC: Morning review packet \u2014 Hermes harness features landed 2026-04-21] Phase 1: Landscape Analysis & Scaffolding",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/965"
-    },
-    {
-      "number": 966,
-      "title": "[EPIC: Morning review packet \u2014 Hermes harness features landed 2026-04-21] Phase 2: Core Logic Implementation",
-      "state": "open",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/966"
-    },
-    {
-      "number": 967,
-      "title": "[EPIC: Morning review packet \u2014 Hermes harness features landed 2026-04-21] Phase 3: Poka-yoke Integration & Fleet Verification",
-      "state": "closed",
-      "html_url": "https://forge.alexanderwhitestone.com/Timmy_Foundation/hermes-agent/issues/967"
-    }
-  ]
-}

scripts/morning_review_packet_status.py

@@ -1,288 +0,0 @@
-#!/usr/bin/env python3
-"""Generate a grounded status report for hermes-agent morning review packet epic #949."""
-
-from __future__ import annotations
-
-import argparse
-import base64
-import json
-import os
-import re
-import ssl
-import urllib.request
-from datetime import datetime, timezone
-from pathlib import Path
-from typing import Any
-
-BASE_API = "https://forge.alexanderwhitestone.com/api/v1"
-REPO = "Timmy_Foundation/hermes-agent"
-TOKEN_PATH = Path("~/.config/gitea/token").expanduser()
-DEFAULT_JSON_OUT = Path("docs/morning-review-packet-2026-04-21.snapshot.json")
-DEFAULT_MARKDOWN_OUT = Path("docs/morning-review-packet-2026-04-21-status.md")
-
-
-def extract_issue_numbers(text: str) -> list[int]:
-    seen: set[int] = set()
-    numbers: list[int] = []
-    for match in re.finditer(r"#(\d+)", text or ""):
-        num = int(match.group(1))
-        if num not in seen:
-            seen.add(num)
-            numbers.append(num)
-    return numbers
-
-
-def _auth_headers(token: str) -> list[dict[str, str]]:
-    basic = base64.b64encode(f"{token}:".encode()).decode()
-    return [
-        {"Authorization": f"token {token}", "Accept": "application/json"},
-        {"Authorization": f"Basic {basic}", "Accept": "application/json"},
-    ]
-
-
-def api_get(path: str, *, headers_options: list[dict[str, str]] | None = None) -> Any:
-    token = TOKEN_PATH.read_text(encoding="utf-8").strip()
-    headers_options = headers_options or _auth_headers(token)
-    ctx = ssl.create_default_context()
-    url = f"{BASE_API}{path}"
-    last_error: Exception | None = None
-    for headers in headers_options:
-        try:
-            req = urllib.request.Request(url, headers=headers)
-            with urllib.request.urlopen(req, context=ctx, timeout=30) as resp:
-                return json.loads(resp.read().decode())
-        except Exception as exc:  # pragma: no cover - exercised via live CLI use
-            last_error = exc
-    raise RuntimeError(f"GET {url} failed: {last_error}")
-
-
-def issue_pr_matches(pr: dict[str, Any], issue_num: int) -> bool:
-    title = pr.get("title") or ""
-    body = pr.get("body") or ""
-    head = (pr.get("head") or {}).get("ref") or ""
-    exact_ref = re.compile(rf"(?<!\d)#{issue_num}(?!\d)")
-    body_ref = re.compile(rf"(?i)(closes|close|fixes|fix|resolves|resolve|refs|ref)\s+#?{issue_num}(?!\d)")
-    branch_variants = {
-        f"fix/{issue_num}",
-        f"issue-{issue_num}",
-        f"burn/{issue_num}",
-        f"fix/issue-{issue_num}",
-    }
-    return bool(
-        exact_ref.search(title)
-        or exact_ref.search(body)
-        or body_ref.search(body)
-        or head in branch_variants
-    )
-
-
-def fetch_open_prs(*, headers_options: list[dict[str, str]]) -> list[dict[str, Any]]:
-    prs: list[dict[str, Any]] = []
-    page = 1
-    while True:
-        batch = api_get(
-            f"/repos/{REPO}/pulls?state=open&limit=100&page={page}",
-            headers_options=headers_options,
-        )
-        if not batch:
-            break
-        prs.extend(batch)
-        if len(batch) < 100:
-            break
-        page += 1
-    return prs
-
-
-def fetch_live_snapshot(epic_issue_num: int = 949) -> dict[str, Any]:
-    token = TOKEN_PATH.read_text(encoding="utf-8").strip()
-    headers_options = _auth_headers(token)
-
-    epic = api_get(f"/repos/{REPO}/issues/{epic_issue_num}", headers_options=headers_options)
-    comments = api_get(f"/repos/{REPO}/issues/{epic_issue_num}/comments", headers_options=headers_options)
-    child_numbers = [n for n in extract_issue_numbers(epic.get("body") or "") if n != epic_issue_num]
-    decomposition_numbers = [
-        n
-        for comment in comments
-        for n in extract_issue_numbers(comment.get("body") or "")
-        if n not in child_numbers and n != epic_issue_num
-    ]
-
-    open_prs = fetch_open_prs(headers_options=headers_options)
-
-    children = []
-    for number in child_numbers:
-        issue = api_get(f"/repos/{REPO}/issues/{number}", headers_options=headers_options)
-        matching_prs = [
-            {
-                "number": pr["number"],
-                "title": pr["title"],
-                "head": pr.get("head", {}).get("ref", ""),
-                "url": pr["html_url"],
-            }
-            for pr in open_prs
-            if issue_pr_matches(pr, number)
-        ]
-        children.append(
-            {
-                "number": issue["number"],
-                "title": issue["title"],
-                "state": issue["state"],
-                "html_url": issue["html_url"],
-                "open_prs": matching_prs,
-            }
-        )
-
-    decomposition_issues = []
-    for number in decomposition_numbers:
-        issue = api_get(f"/repos/{REPO}/issues/{number}", headers_options=headers_options)
-        decomposition_issues.append(
-            {
-                "number": issue["number"],
-                "title": issue["title"],
-                "state": issue["state"],
-                "html_url": issue["html_url"],
-            }
-        )
-
-    return {
-        "generated_at": datetime.now(timezone.utc).isoformat(),
-        "repo": REPO,
-        "epic": {
-            "number": epic["number"],
-            "title": epic["title"],
-            "state": epic["state"],
-            "html_url": epic["html_url"],
-        },
-        "children": children,
-        "decomposition_issues": decomposition_issues,
-    }
-
-
-def summarize_snapshot(snapshot: dict[str, Any]) -> dict[str, int]:
-    children = snapshot.get("children", [])
-    open_children = [issue for issue in children if issue.get("state") == "open"]
-    closed_children = [issue for issue in children if issue.get("state") == "closed"]
-    open_with_pr = [issue for issue in open_children if issue.get("open_prs")]
-    open_without_pr = [issue for issue in open_children if not issue.get("open_prs")]
-    return {
-        "total_children": len(children),
-        "open_children": len(open_children),
-        "closed_children": len(closed_children),
-        "open_with_pr": len(open_with_pr),
-        "open_without_pr": len(open_without_pr),
-    }
-
-
-def render_markdown(snapshot: dict[str, Any]) -> str:
-    epic = snapshot["epic"]
-    children = snapshot.get("children", [])
-    summary = summarize_snapshot(snapshot)
-    open_with_pr = [issue for issue in children if issue.get("state") == "open" and issue.get("open_prs")]
-    open_without_pr = [issue for issue in children if issue.get("state") == "open" and not issue.get("open_prs")]
-    decomposition = snapshot.get("decomposition_issues", [])
-
-    lines = [
-        f"# Morning Review Packet Status — #{epic['number']}",
-        "",
-        f"Generated: {snapshot.get('generated_at', '')}",
-        f"Epic: [{epic['title']}]({epic.get('html_url', '')})",
-        "",
-        "## Summary",
-        "",
-        f"- Child QA issues tracked: {summary['total_children']}",
-        f"- Open child issues: {summary['open_children']}",
-        f"- Closed child issues: {summary['closed_children']}",
-        f"- Open child issues already backed by PRs: {summary['open_with_pr']}",
-        f"- Open child issues still unowned on forge: {summary['open_without_pr']}",
-        "",
-        "## Child QA Matrix",
-        "",
-        "| Issue | State | Open PRs | Title |",
-        "|------:|-------|----------|-------|",
-    ]
-
-    for issue in children:
-        rendered_prs = []
-        for pr in issue.get("open_prs", []):
-            pr_num = pr.get("number", "?")
-            pr_url = pr.get("url") or pr.get("html_url") or ""
-            rendered_prs.append(f"[#{pr_num}]({pr_url})" if pr_url else f"#{pr_num}")
-        pr_text = ", ".join(rendered_prs) or "—"
-        lines.append(
-            f"| #{issue['number']} | {issue['state']} | {pr_text} | {issue['title']} |"
-        )
-
-    lines.extend([
-        "",
-        "## Drift Signals",
-        "",
-        "forge/main is still catching up to the upstream packet.",
-    ])
-
-    if open_with_pr:
-        lines.append("")
-        lines.append("Active PR-backed child lanes:")
-        for issue in open_with_pr:
-            pr_numbers = ", ".join(f"#{pr['number']}" for pr in issue.get("open_prs", []))
-            lines.append(f"- #{issue['number']} -> {pr_numbers} ({issue['title']})")
-
-    if open_without_pr:
-        lines.extend([
-            "",
-            "## Unowned Open QA Issues",
-            "",
-        ])
-        for issue in open_without_pr:
-            lines.append(f"- #{issue['number']} {issue['title']}")
-
-    if decomposition:
-        lines.extend([
-            "",
-            "## Decomposition Follow-Ups",
-            "",
-        ])
-        for issue in decomposition:
-            lines.append(f"- #{issue['number']} [{issue['state']}] {issue['title']}")
-
-    lines.extend([
-        "",
-        "## Conclusion",
-        "",
-        "Refs #949 only. This epic remains open until every child QA issue has a truthful PASS/FAIL outcome, attached evidence, and any upstream/main versus forge/main drift is resolved or explicitly documented.",
-        "",
-        "## Regeneration",
-        "",
-        "```bash",
-        "python3 scripts/morning_review_packet_status.py --fetch-live --json-out docs/morning-review-packet-2026-04-21.snapshot.json --markdown-out docs/morning-review-packet-2026-04-21-status.md",
-        "```",
-    ])
-
-    return "\n".join(lines) + "\n"
-
-
-def write_json(path: Path, data: dict[str, Any]) -> None:
-    path.parent.mkdir(parents=True, exist_ok=True)
-    path.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Generate grounded status docs for epic #949")
-    parser.add_argument("--fetch-live", action="store_true", help="Fetch the current packet state from Forge")
-    parser.add_argument("--snapshot", type=Path, help="Read a local JSON snapshot instead of hitting the API")
-    parser.add_argument("--json-out", type=Path, default=DEFAULT_JSON_OUT, help="Path to write JSON snapshot")
-    parser.add_argument("--markdown-out", type=Path, default=DEFAULT_MARKDOWN_OUT, help="Path to write markdown report")
-    args = parser.parse_args()
-
-    if args.fetch_live or not args.snapshot:
-        snapshot = fetch_live_snapshot()
-    else:
-        snapshot = json.loads(args.snapshot.read_text(encoding="utf-8"))
-
-    write_json(args.json_out, snapshot)
-    args.markdown_out.parent.mkdir(parents=True, exist_ok=True)
-    args.markdown_out.write_text(render_markdown(snapshot), encoding="utf-8")
-    print(args.markdown_out)
-
-
-if __name__ == "__main__":
-    main()

tests/test_morning_review_packet_status.py

@@ -1,94 +0,0 @@
-"""Tests for the morning review packet status report generator."""
-
-from __future__ import annotations
-
-import importlib.util
-from pathlib import Path
-
-SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "morning_review_packet_status.py"
-DOC_PATH = Path(__file__).resolve().parents[1] / "docs" / "morning-review-packet-2026-04-21-status.md"
-
-
-def load_module():
-    assert SCRIPT_PATH.exists(), f"missing status script: {SCRIPT_PATH}"
-    spec = importlib.util.spec_from_file_location("morning_review_packet_status_test", SCRIPT_PATH)
-    module = importlib.util.module_from_spec(spec)
-    assert spec.loader is not None
-    spec.loader.exec_module(module)
-    return module
-
-
-def sample_snapshot():
-    return {
-        "epic": {"number": 949, "title": "Morning review packet", "state": "open"},
-        "children": [
-            {
-                "number": 950,
-                "title": "Verify AI Gateway provider UX + attribution headers",
-                "state": "open",
-                "open_prs": [],
-            },
-            {
-                "number": 954,
-                "title": "Verify maps skill guest_house / camp_site / bakery expansion",
-                "state": "open",
-                "open_prs": [
-                    {"number": 1021, "head": "fix/954", "title": "feat: sync maps skill and verify guest_house/camp_site/bakery (#954)"}
-                ],
-            },
-            {
-                "number": 961,
-                "title": "Verify web dashboard update/restart action buttons",
-                "state": "closed",
-                "open_prs": [],
-            },
-        ],
-        "decomposition_issues": [
-            {"number": 965, "title": "Phase 1: Landscape Analysis & Scaffolding", "state": "open"},
-            {"number": 967, "title": "Phase 3: Poka-yoke Integration & Fleet Verification", "state": "closed"},
-        ],
-    }
-
-
-def test_extract_child_issue_numbers_from_epic_body():
-    module = load_module()
-    body = """
-- [ ] #950 one
-- [ ] #951 two
-- [ ] #962 three
-"""
-    assert module.extract_issue_numbers(body) == [950, 951, 962]
-
-
-def test_summarize_snapshot_counts_open_closed_and_pr_backing():
-    module = load_module()
-    summary = module.summarize_snapshot(sample_snapshot())
-
-    assert summary["total_children"] == 3
-    assert summary["open_children"] == 2
-    assert summary["closed_children"] == 1
-    assert summary["open_with_pr"] == 1
-    assert summary["open_without_pr"] == 1
-
-
-def test_render_markdown_includes_issue_matrix_and_drift_sections():
-    module = load_module()
-    md = module.render_markdown(sample_snapshot())
-
-    assert "# Morning Review Packet Status — #949" in md
-    assert "## Child QA Matrix" in md
-    assert "#950" in md
-    assert "#954" in md
-    assert "#1021" in md
-    assert "## Unowned Open QA Issues" in md
-    assert "## Drift Signals" in md
-    assert "forge/main is still catching up to the upstream packet" in md
-
-
-def test_committed_status_doc_exists_and_mentions_live_examples():
-    assert DOC_PATH.exists(), f"missing generated status doc: {DOC_PATH}"
-    text = DOC_PATH.read_text(encoding="utf-8")
-    assert "# Morning Review Packet Status — #949" in text
-    assert "#954" in text
-    assert "#1021" in text
-    assert "#950" in text

tests/test_vision_benchmark.py

@@ -11,12 +11,14 @@ import pytest
 sys.path.insert(0, str(Path(__file__).parent.parent / "benchmarks"))
 
 from vision_benchmark import (
+    analyze_with_model,
     compute_ocr_accuracy,
     compute_description_completeness,
     compute_structural_accuracy,
     aggregate_results,
     to_markdown,
     generate_sample_dataset,
+    load_dataset,
     MODELS,
     EVAL_PROMPTS,
 )
@@ -197,6 +199,71 @@ class TestMarkdown:
 
 
 class TestDataset:
+    def test_repo_dataset_uses_local_image_paths(self):
+        dataset_path = Path(__file__).parent.parent / "benchmarks" / "test_images.json"
+        dataset = json.loads(dataset_path.read_text())
+
+        assert dataset, "benchmark dataset should not be empty"
+        assert all(not entry["url"].startswith(("http://", "https://")) for entry in dataset)
+
+    def test_load_dataset_resolves_relative_local_paths(self, tmp_path):
+        images_dir = tmp_path / "images"
+        images_dir.mkdir()
+        image_path = images_dir / "sample.png"
+        image_path.write_bytes(b"png-bytes")
+
+        dataset_path = tmp_path / "dataset.json"
+        dataset_path.write_text(json.dumps([
+            {
+                "id": "sample",
+                "url": "images/sample.png",
+                "category": "photo",
+                "expected_keywords": [],
+                "expected_structure": {"min_length": 30, "min_sentences": 1},
+            }
+        ]))
+
+        loaded = load_dataset(str(dataset_path))
+
+        assert loaded[0]["url"] == str(image_path.resolve())
+
+    @pytest.mark.asyncio
+    async def test_analyze_with_model_encodes_local_file_as_data_url(self, tmp_path, monkeypatch):
+        image_path = tmp_path / "tiny.png"
+        image_path.write_bytes(
+            bytes.fromhex(
+                "89504E470D0A1A0A"
+                "0000000D49484452000000010000000108060000001F15C489"
+                "0000000D49444154789C6360000002000154A24F5D00000000"
+                "49454E44AE426082"
+            )
+        )
+
+        fake_response = MagicMock()
+        fake_response.raise_for_status.return_value = None
+        fake_response.json.return_value = {
+            "choices": [{"message": {"content": "Looks like a tiny image."}}],
+            "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
+        }
+
+        fake_client = MagicMock()
+        fake_client.post = AsyncMock(return_value=fake_response)
+        fake_ctx = MagicMock()
+        fake_ctx.__aenter__ = AsyncMock(return_value=fake_client)
+        fake_ctx.__aexit__ = AsyncMock(return_value=None)
+
+        monkeypatch.setenv("OPENROUTER_API_KEY", "test-key")
+        with patch("httpx.AsyncClient", return_value=fake_ctx):
+            result = await analyze_with_model(
+                str(image_path),
+                "Describe this image",
+                {"provider": "openrouter", "model_id": "fake/model"},
+            )
+
+        assert result["success"] is True
+        sent_url = fake_client.post.await_args.kwargs["json"]["messages"][0]["content"][1]["image_url"]["url"]
+        assert sent_url.startswith("data:image/png;base64,")
+
     def test_sample_dataset_has_entries(self):
         dataset = generate_sample_dataset()
         assert len(dataset) >= 4