fix: vendor vision benchmark fixtures (#868)

agent/anthropic_adapter.py

@@ -1396,8 +1396,6 @@ def normalize_anthropic_response(
         "tool_use": "tool_calls",
         "max_tokens": "length",
         "stop_sequence": "stop",
-        "refusal": "content_filter",
-        "model_context_window_exceeded": "length",
     }
     finish_reason = stop_reason_map.get(response.stop_reason, "stop")
 
@@ -1411,42 +1409,3 @@ def normalize_anthropic_response(
         ),
         finish_reason,
     )
-
-
-def normalize_anthropic_response_v2(
-    response,
-    strip_tool_prefix: bool = False,
-) -> "NormalizedResponse":
-    """Normalize Anthropic response to NormalizedResponse.
-
-    Wraps the existing normalize_anthropic_response() and maps its output
-    to the shared transport types. This allows incremental migration
-    without disturbing the legacy call sites.
-    """
-    from agent.transports.types import NormalizedResponse, build_tool_call
-
-    assistant_msg, finish_reason = normalize_anthropic_response(response, strip_tool_prefix)
-
-    tool_calls = None
-    if assistant_msg.tool_calls:
-        tool_calls = [
-            build_tool_call(
-                id=tc.id,
-                name=tc.function.name,
-                arguments=tc.function.arguments,
-            )
-            for tc in assistant_msg.tool_calls
-        ]
-
-    provider_data = {}
-    if getattr(assistant_msg, "reasoning_details", None):
-        provider_data["reasoning_details"] = assistant_msg.reasoning_details
-
-    return NormalizedResponse(
-        content=assistant_msg.content,
-        tool_calls=tool_calls,
-        finish_reason=finish_reason,
-        reasoning=getattr(assistant_msg, "reasoning", None),
-        usage=None,
-        provider_data=provider_data or None,
-    )

agent/transports/__init__.py

@@ -1,57 +0,0 @@
-"""Transport layer types and registry for provider response normalization.
-
-Usage:
-    from agent.transports import get_transport
-    transport = get_transport("anthropic_messages")
-    result = transport.normalize_response(raw_response)
-"""
-
-from agent.transports.types import (  # noqa: F401
-    NormalizedResponse,
-    ToolCall,
-    Usage,
-    build_tool_call,
-    map_finish_reason,
-)
-
-_REGISTRY: dict = {}
-
-
-def register_transport(api_mode: str, transport_cls: type) -> None:
-    """Register a transport class for an api_mode string."""
-    _REGISTRY[api_mode] = transport_cls
-
-
-def get_transport(api_mode: str):
-    """Get a transport instance for the given api_mode.
-
-    Returns None if no transport is registered for this api_mode.
-    This allows gradual migration — call sites can check for None
-    and fall back to the legacy code path.
-    """
-    if not _REGISTRY:
-        _discover_transports()
-    cls = _REGISTRY.get(api_mode)
-    if cls is None:
-        return None
-    return cls()
-
-
-def _discover_transports() -> None:
-    """Import all transport modules to trigger auto-registration."""
-    try:
-        import agent.transports.anthropic  # noqa: F401
-    except ImportError:
-        pass
-    try:
-        import agent.transports.codex  # noqa: F401
-    except ImportError:
-        pass
-    try:
-        import agent.transports.chat_completions  # noqa: F401
-    except ImportError:
-        pass
-    try:
-        import agent.transports.bedrock  # noqa: F401
-    except ImportError:
-        pass

agent/transports/anthropic.py

@@ -1,95 +0,0 @@
-"""Anthropic Messages API transport.
-
-Delegates to the existing adapter functions in agent/anthropic_adapter.py.
-This transport owns format conversion and normalization — NOT client lifecycle.
-"""
-
-from typing import Any, Dict, List, Optional
-
-from agent.transports.base import ProviderTransport
-from agent.transports.types import NormalizedResponse
-
-
-class AnthropicTransport(ProviderTransport):
-    """Transport for api_mode='anthropic_messages'."""
-
-    @property
-    def api_mode(self) -> str:
-        return "anthropic_messages"
-
-    def convert_messages(self, messages: List[Dict[str, Any]], **kwargs) -> Any:
-        from agent.anthropic_adapter import convert_messages_to_anthropic
-
-        base_url = kwargs.get("base_url")
-        return convert_messages_to_anthropic(messages, base_url=base_url)
-
-    def convert_tools(self, tools: List[Dict[str, Any]]) -> Any:
-        from agent.anthropic_adapter import convert_tools_to_anthropic
-
-        return convert_tools_to_anthropic(tools)
-
-    def build_kwargs(
-        self,
-        model: str,
-        messages: List[Dict[str, Any]],
-        tools: Optional[List[Dict[str, Any]]] = None,
-        **params,
-    ) -> Dict[str, Any]:
-        from agent.anthropic_adapter import build_anthropic_kwargs
-
-        return build_anthropic_kwargs(
-            model=model,
-            messages=messages,
-            tools=tools,
-            max_tokens=params.get("max_tokens", 16384),
-            reasoning_config=params.get("reasoning_config"),
-            tool_choice=params.get("tool_choice"),
-            is_oauth=params.get("is_oauth", False),
-            preserve_dots=params.get("preserve_dots", False),
-            context_length=params.get("context_length"),
-            base_url=params.get("base_url"),
-            fast_mode=params.get("fast_mode", False),
-        )
-
-    def normalize_response(self, response: Any, **kwargs) -> NormalizedResponse:
-        from agent.anthropic_adapter import normalize_anthropic_response_v2
-
-        strip_tool_prefix = kwargs.get("strip_tool_prefix", False)
-        return normalize_anthropic_response_v2(response, strip_tool_prefix=strip_tool_prefix)
-
-    def validate_response(self, response: Any) -> bool:
-        if response is None:
-            return False
-        content_blocks = getattr(response, "content", None)
-        if not isinstance(content_blocks, list):
-            return False
-        if not content_blocks:
-            return False
-        return True
-
-    def extract_cache_stats(self, response: Any):
-        usage = getattr(response, "usage", None)
-        if usage is None:
-            return None
-        cached = getattr(usage, "cache_read_input_tokens", 0) or 0
-        written = getattr(usage, "cache_creation_input_tokens", 0) or 0
-        if cached or written:
-            return {"cached_tokens": cached, "creation_tokens": written}
-        return None
-
-    _STOP_REASON_MAP = {
-        "end_turn": "stop",
-        "tool_use": "tool_calls",
-        "max_tokens": "length",
-        "stop_sequence": "stop",
-        "refusal": "content_filter",
-        "model_context_window_exceeded": "length",
-    }
-
-    def map_finish_reason(self, raw_reason: str) -> str:
-        return self._STOP_REASON_MAP.get(raw_reason, "stop")
-
-
-from agent.transports import register_transport  # noqa: E402
-
-register_transport("anthropic_messages", AnthropicTransport)

agent/transports/base.py

@@ -1,61 +0,0 @@
-"""Abstract base for provider transports.
-
-A transport owns the data path for one api_mode:
-  convert_messages → convert_tools → build_kwargs → normalize_response
-
-It does NOT own: client construction, streaming, credential refresh,
-prompt caching, interrupt handling, or retry logic. Those stay on AIAgent.
-"""
-
-from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional
-
-from agent.transports.types import NormalizedResponse
-
-
-class ProviderTransport(ABC):
-    """Base class for provider-specific format conversion and normalization."""
-
-    @property
-    @abstractmethod
-    def api_mode(self) -> str:
-        """The api_mode string this transport handles."""
-        ...
-
-    @abstractmethod
-    def convert_messages(self, messages: List[Dict[str, Any]], **kwargs) -> Any:
-        """Convert OpenAI-format messages to provider-native format."""
-        ...
-
-    @abstractmethod
-    def convert_tools(self, tools: List[Dict[str, Any]]) -> Any:
-        """Convert OpenAI-format tool definitions to provider-native format."""
-        ...
-
-    @abstractmethod
-    def build_kwargs(
-        self,
-        model: str,
-        messages: List[Dict[str, Any]],
-        tools: Optional[List[Dict[str, Any]]] = None,
-        **params,
-    ) -> Dict[str, Any]:
-        """Build the complete provider kwargs dict."""
-        ...
-
-    @abstractmethod
-    def normalize_response(self, response: Any, **kwargs) -> NormalizedResponse:
-        """Normalize a raw provider response to the shared NormalizedResponse type."""
-        ...
-
-    def validate_response(self, response: Any) -> bool:
-        """Optional structural validation for raw responses."""
-        return True
-
-    def extract_cache_stats(self, response: Any) -> Optional[Dict[str, int]]:
-        """Optional cache stats extraction."""
-        return None
-
-    def map_finish_reason(self, raw_reason: str) -> str:
-        """Optional stop-reason mapping. Defaults to passthrough."""
-        return raw_reason

agent/transports/types.py

@@ -1,58 +0,0 @@
-"""Shared types for normalized provider responses."""
-
-from __future__ import annotations
-
-import json
-from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional
-
-
-@dataclass
-class ToolCall:
-    """A normalized tool call from any provider."""
-
-    id: Optional[str]
-    name: str
-    arguments: str
-    provider_data: Optional[Dict[str, Any]] = field(default=None, repr=False)
-
-
-@dataclass
-class Usage:
-    """Token usage from an API response."""
-
-    prompt_tokens: int = 0
-    completion_tokens: int = 0
-    total_tokens: int = 0
-    cached_tokens: int = 0
-
-
-@dataclass
-class NormalizedResponse:
-    """Normalized API response from any provider."""
-
-    content: Optional[str]
-    tool_calls: Optional[List[ToolCall]]
-    finish_reason: str
-    reasoning: Optional[str] = None
-    usage: Optional[Usage] = None
-    provider_data: Optional[Dict[str, Any]] = field(default=None, repr=False)
-
-
-def build_tool_call(
-    id: Optional[str],
-    name: str,
-    arguments: Any,
-    **provider_fields: Any,
-) -> ToolCall:
-    """Build a ToolCall, auto-serialising dict arguments."""
-    args_str = json.dumps(arguments) if isinstance(arguments, dict) else str(arguments)
-    provider_data = dict(provider_fields) if provider_fields else None
-    return ToolCall(id=id, name=name, arguments=args_str, provider_data=provider_data)
-
-
-def map_finish_reason(reason: Optional[str], mapping: Dict[str, str]) -> str:
-    """Translate a provider-specific stop reason to the normalized set."""
-    if reason is None:
-        return "stop"
-    return mapping.get(reason, "stop")

benchmarks/test_images.json

@@ -1,194 +1,354 @@
 [
   {
     "id": "screenshot_github_home",
-    "url": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png",
+    "url": "test_images/screenshot_github_home.png",
     "category": "screenshot",
-    "expected_keywords": ["github", "logo", "mark"],
+    "expected_keywords": [
+      "github",
+      "logo",
+      "mark"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "diagram_mermaid_flow",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6siSZXVhjQTlgl1nigHg5fRBOzSfebopROCu_cytObSfgLSE1ANOeZWkO2IH5upZxYot8m1hqAdpD_63WRl0xdUG1jdl9kPiOb_EWk2JBtPaiKkF4eVIYgO0EtkW-RSgC4gJ6HJYRG1UNdN0HNVd0Bftjj7X8P92qPj-F8l8T3w",
+    "url": "test_images/diagram_mermaid_flow.png",
     "category": "diagram",
-    "expected_keywords": ["flow", "diagram", "process"],
+    "expected_keywords": [
+      "flow",
+      "diagram",
+      "process"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_1",
-    "url": "https://picsum.photos/seed/vision1/400/300",
+    "url": "test_images/photo_random_1.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_2",
-    "url": "https://picsum.photos/seed/vision2/400/300",
+    "url": "test_images/photo_random_2.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_simple_bar",
-    "url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['Q1','Q2','Q3','Q4'],datasets:[{label:'Revenue',data:[100,150,200,250]}]}}",
+    "url": "test_images/chart_simple_bar.png",
     "category": "chart",
-    "expected_keywords": ["bar", "chart", "revenue"],
+    "expected_keywords": [
+      "bar",
+      "chart",
+      "revenue"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "chart_pie",
-    "url": "https://quickchart.io/chart?c={type:'pie',data:{labels:['A','B','C'],datasets:[{data:[30,50,20]}]}}",
+    "url": "test_images/chart_pie.png",
     "category": "chart",
-    "expected_keywords": ["pie", "chart", "percentage"],
+    "expected_keywords": [
+      "pie",
+      "chart",
+      "percentage"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_org_chart",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_org_chart.png",
     "category": "diagram",
-    "expected_keywords": ["organization", "hierarchy", "chart"],
+    "expected_keywords": [
+      "organization",
+      "hierarchy",
+      "chart"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_terminal",
-    "url": "https://raw.githubusercontent.com/nicehash/nicehash-quick-start/main/images/nicehash-terminal.png",
+    "url": "test_images/screenshot_terminal.png",
     "category": "screenshot",
-    "expected_keywords": ["terminal", "command", "output"],
+    "expected_keywords": [
+      "terminal",
+      "command",
+      "output"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_3",
-    "url": "https://picsum.photos/seed/vision3/400/300",
+    "url": "test_images/photo_random_3.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_line",
-    "url": "https://quickchart.io/chart?c={type:'line',data:{labels:['Jan','Feb','Mar','Apr'],datasets:[{label:'Temperature',data:[5,8,12,18]}]}}",
+    "url": "test_images/chart_line.png",
     "category": "chart",
-    "expected_keywords": ["line", "chart", "temperature"],
+    "expected_keywords": [
+      "line",
+      "chart",
+      "temperature"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_sequence",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_sequence.png",
     "category": "diagram",
-    "expected_keywords": ["sequence", "interaction", "message"],
+    "expected_keywords": [
+      "sequence",
+      "interaction",
+      "message"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_4",
-    "url": "https://picsum.photos/seed/vision4/400/300",
+    "url": "test_images/photo_random_4.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_webpage",
-    "url": "https://github.githubassets.com/images/modules/site/social-cards.png",
+    "url": "test_images/screenshot_webpage.png",
     "category": "screenshot",
-    "expected_keywords": ["github", "page", "web"],
+    "expected_keywords": [
+      "github",
+      "page",
+      "web"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_radar",
-    "url": "https://quickchart.io/chart?c={type:'radar',data:{labels:['Speed','Power','Defense','Magic'],datasets:[{label:'Hero',data:[80,60,70,90]}]}}",
+    "url": "test_images/chart_radar.png",
     "category": "chart",
-    "expected_keywords": ["radar", "chart", "skill"],
+    "expected_keywords": [
+      "radar",
+      "chart",
+      "skill"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "photo_random_5",
-    "url": "https://picsum.photos/seed/vision5/400/300",
+    "url": "test_images/photo_random_5.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "diagram_class",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_class.png",
     "category": "diagram",
-    "expected_keywords": ["class", "object", "attribute"],
+    "expected_keywords": [
+      "class",
+      "object",
+      "attribute"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_doughnut",
-    "url": "https://quickchart.io/chart?c={type:'doughnut',data:{labels:['Desktop','Mobile','Tablet'],datasets:[{data:[60,30,10]}]}}",
+    "url": "test_images/chart_doughnut.png",
     "category": "chart",
-    "expected_keywords": ["doughnut", "chart", "device"],
+    "expected_keywords": [
+      "doughnut",
+      "chart",
+      "device"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "photo_random_6",
-    "url": "https://picsum.photos/seed/vision6/400/300",
+    "url": "test_images/photo_random_6.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_error",
-    "url": "https://http.cat/404.jpg",
+    "url": "test_images/screenshot_error.png",
     "category": "screenshot",
-    "expected_keywords": ["404", "error", "cat"],
+    "expected_keywords": [
+      "404",
+      "error",
+      "cat"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_network",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_network.png",
     "category": "diagram",
-    "expected_keywords": ["network", "node", "connection"],
+    "expected_keywords": [
+      "network",
+      "node",
+      "connection"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_7",
-    "url": "https://picsum.photos/seed/vision7/400/300",
+    "url": "test_images/photo_random_7.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_stacked_bar",
-    "url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['2022','2023','2024'],datasets:[{label:'Cloud',data:[100,150,200]},{label:'On-prem',data:[200,180,160]}]},options:{scales:{x:{stacked:true},y:{stacked:true}}}}",
+    "url": "test_images/chart_stacked_bar.png",
     "category": "chart",
-    "expected_keywords": ["stacked", "bar", "chart"],
+    "expected_keywords": [
+      "stacked",
+      "bar",
+      "chart"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "screenshot_dashboard",
-    "url": "https://github.githubassets.com/images/modules/site/features-code-search.png",
+    "url": "test_images/screenshot_dashboard.png",
     "category": "screenshot",
-    "expected_keywords": ["search", "code", "feature"],
+    "expected_keywords": [
+      "search",
+      "code",
+      "feature"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_8",
-    "url": "https://picsum.photos/seed/vision8/400/300",
+    "url": "test_images/photo_random_8.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   }
 ]

benchmarks/vision_benchmark.py

@@ -11,17 +11,19 @@ Usage:
 
     # Single image test
     python benchmarks/vision_benchmark.py --url https://example.com/image.png
+    python benchmarks/vision_benchmark.py --url benchmarks/test_images/photo_random_1.png
 
     # Generate test report
     python benchmarks/vision_benchmark.py --images benchmarks/test_images.json --output benchmarks/vision_results.json
 
-Test image dataset: benchmarks/test_images.json (50-100 diverse images)
+Test image dataset: benchmarks/test_images.json (committed local fixtures under benchmarks/test_images/)
 """
 
 import argparse
 import asyncio
 import base64
 import json
+import mimetypes
 import os
 import statistics
 import sys
@@ -67,6 +69,28 @@ EVAL_PROMPTS = {
 # ---------------------------------------------------------------------------
 
 
+def _is_remote_image_source(image_source: str) -> bool:
+    return image_source.startswith(("http://", "https://", "data:", "file://"))
+
+
+def _image_source_to_payload_url(image_source: str) -> str:
+    """Convert local image paths into data URLs; keep remote URLs unchanged."""
+    if image_source.startswith(("http://", "https://", "data:")):
+        return image_source
+
+    resolved = image_source[len("file://"):] if image_source.startswith("file://") else image_source
+    local_path = Path(os.path.expanduser(resolved)).resolve()
+    if not local_path.is_file():
+        return image_source
+
+    mime_type, _ = mimetypes.guess_type(str(local_path))
+    if not mime_type:
+        mime_type = "application/octet-stream"
+
+    encoded = base64.b64encode(local_path.read_bytes()).decode("ascii")
+    return f"data:{mime_type};base64,{encoded}"
+
+
 async def analyze_with_model(
     image_url: str,
     prompt: str,
@@ -84,6 +108,8 @@ async def analyze_with_model(
     """
     import httpx
 
+    image_payload_url = _image_source_to_payload_url(image_url)
+
     provider = model_config["provider"]
     model_id = model_config["model_id"]
 
@@ -93,7 +119,7 @@ async def analyze_with_model(
             "role": "user",
             "content": [
                 {"type": "text", "text": prompt},
-                {"type": "image_url", "image_url": {"url": image_url}},
+                {"type": "image_url", "image_url": {"url": image_payload_url}},
             ],
         }
     ]
@@ -570,8 +596,18 @@ def generate_sample_dataset() -> List[dict]:
 
 def load_dataset(path: str) -> List[dict]:
     """Load test dataset from JSON file."""
-    with open(path) as f:
-        return json.load(f)
+    dataset_path = Path(path).resolve()
+    with open(dataset_path) as f:
+        dataset = json.load(f)
+
+    base_dir = dataset_path.parent
+    for image in dataset:
+        image_url = image.get("url")
+        if not image_url or _is_remote_image_source(image_url):
+            continue
+        image["url"] = str((base_dir / image_url).resolve())
+
+    return dataset
 
 
 # ---------------------------------------------------------------------------
@@ -582,7 +618,7 @@ def load_dataset(path: str) -> List[dict]:
 async def main():
     parser = argparse.ArgumentParser(description="Vision Benchmark Suite (Issue #817)")
     parser.add_argument("--images", help="Path to test images JSON file")
-    parser.add_argument("--url", help="Single image URL to test")
+    parser.add_argument("--url", help="Single image URL or local file path to test")
     parser.add_argument("--category", default="photo", help="Category for single URL")
     parser.add_argument("--output", default=None, help="Output JSON file")
     parser.add_argument("--runs", type=int, default=1, help="Runs per model per image")

tests/agent/test_anthropic_normalize_v2.py

@@ -1,213 +0,0 @@
-"""Regression tests: normalize_anthropic_response_v2 vs v1.
-
-Constructs mock Anthropic responses and asserts that the v2 function
-(returning NormalizedResponse) produces identical field values to the
-original v1 function (returning SimpleNamespace + finish_reason).
-"""
-
-from types import SimpleNamespace
-
-import pytest
-
-from agent.anthropic_adapter import (
-    normalize_anthropic_response,
-    normalize_anthropic_response_v2,
-)
-from agent.transports.types import NormalizedResponse
-
-
-def _text_block(text: str):
-    return SimpleNamespace(type="text", text=text)
-
-
-def _thinking_block(thinking: str, signature: str = "sig_abc"):
-    return SimpleNamespace(type="thinking", thinking=thinking, signature=signature)
-
-
-def _tool_use_block(id: str, name: str, input: dict):
-    return SimpleNamespace(type="tool_use", id=id, name=name, input=input)
-
-
-def _response(content_blocks, stop_reason="end_turn"):
-    return SimpleNamespace(
-        content=content_blocks,
-        stop_reason=stop_reason,
-        usage=SimpleNamespace(input_tokens=10, output_tokens=5),
-    )
-
-
-class TestTextOnly:
-    def setup_method(self):
-        self.resp = _response([_text_block("Hello world")])
-        self.v1_msg, self.v1_finish = normalize_anthropic_response(self.resp)
-        self.v2 = normalize_anthropic_response_v2(self.resp)
-
-    def test_type(self):
-        assert isinstance(self.v2, NormalizedResponse)
-
-    def test_content_matches(self):
-        assert self.v2.content == self.v1_msg.content
-
-    def test_finish_reason_matches(self):
-        assert self.v2.finish_reason == self.v1_finish
-
-    def test_no_tool_calls(self):
-        assert self.v2.tool_calls is None
-        assert self.v1_msg.tool_calls is None
-
-    def test_no_reasoning(self):
-        assert self.v2.reasoning is None
-        assert self.v1_msg.reasoning is None
-
-
-class TestWithToolCalls:
-    def setup_method(self):
-        self.resp = _response(
-            [
-                _text_block("I'll check that"),
-                _tool_use_block("toolu_abc", "terminal", {"command": "ls"}),
-                _tool_use_block("toolu_def", "read_file", {"path": "/tmp"}),
-            ],
-            stop_reason="tool_use",
-        )
-        self.v1_msg, self.v1_finish = normalize_anthropic_response(self.resp)
-        self.v2 = normalize_anthropic_response_v2(self.resp)
-
-    def test_finish_reason(self):
-        assert self.v2.finish_reason == "tool_calls"
-        assert self.v1_finish == "tool_calls"
-
-    def test_tool_call_count(self):
-        assert len(self.v2.tool_calls) == 2
-        assert len(self.v1_msg.tool_calls) == 2
-
-    def test_tool_call_ids_match(self):
-        for i in range(2):
-            assert self.v2.tool_calls[i].id == self.v1_msg.tool_calls[i].id
-
-    def test_tool_call_names_match(self):
-        assert self.v2.tool_calls[0].name == "terminal"
-        assert self.v2.tool_calls[1].name == "read_file"
-        for i in range(2):
-            assert self.v2.tool_calls[i].name == self.v1_msg.tool_calls[i].function.name
-
-    def test_tool_call_arguments_match(self):
-        for i in range(2):
-            assert self.v2.tool_calls[i].arguments == self.v1_msg.tool_calls[i].function.arguments
-
-    def test_content_preserved(self):
-        assert self.v2.content == self.v1_msg.content
-        assert "check that" in self.v2.content
-
-
-class TestWithThinking:
-    def setup_method(self):
-        self.resp = _response([
-            _thinking_block("Let me think about this carefully..."),
-            _text_block("The answer is 42."),
-        ])
-        self.v1_msg, self.v1_finish = normalize_anthropic_response(self.resp)
-        self.v2 = normalize_anthropic_response_v2(self.resp)
-
-    def test_reasoning_matches(self):
-        assert self.v2.reasoning == self.v1_msg.reasoning
-        assert "think about this" in self.v2.reasoning
-
-    def test_reasoning_details_in_provider_data(self):
-        v1_details = self.v1_msg.reasoning_details
-        v2_details = self.v2.provider_data.get("reasoning_details") if self.v2.provider_data else None
-        assert v1_details is not None
-        assert v2_details is not None
-        assert len(v2_details) == len(v1_details)
-
-    def test_content_excludes_thinking(self):
-        assert self.v2.content == "The answer is 42."
-
-
-class TestMixed:
-    def setup_method(self):
-        self.resp = _response(
-            [
-                _thinking_block("Planning my approach..."),
-                _text_block("I'll run the command"),
-                _tool_use_block("toolu_xyz", "terminal", {"command": "pwd"}),
-            ],
-            stop_reason="tool_use",
-        )
-        self.v1_msg, self.v1_finish = normalize_anthropic_response(self.resp)
-        self.v2 = normalize_anthropic_response_v2(self.resp)
-
-    def test_all_fields_present(self):
-        assert self.v2.content is not None
-        assert self.v2.tool_calls is not None
-        assert self.v2.reasoning is not None
-        assert self.v2.finish_reason == "tool_calls"
-
-    def test_content_matches(self):
-        assert self.v2.content == self.v1_msg.content
-
-    def test_reasoning_matches(self):
-        assert self.v2.reasoning == self.v1_msg.reasoning
-
-    def test_tool_call_matches(self):
-        assert self.v2.tool_calls[0].id == self.v1_msg.tool_calls[0].id
-        assert self.v2.tool_calls[0].name == self.v1_msg.tool_calls[0].function.name
-
-
-class TestStopReasons:
-    @pytest.mark.parametrize("stop_reason,expected", [
-        ("end_turn", "stop"),
-        ("tool_use", "tool_calls"),
-        ("max_tokens", "length"),
-        ("stop_sequence", "stop"),
-        ("refusal", "content_filter"),
-        ("model_context_window_exceeded", "length"),
-        ("unknown_future_reason", "stop"),
-    ])
-    def test_stop_reason_mapping(self, stop_reason, expected):
-        resp = _response([_text_block("x")], stop_reason=stop_reason)
-        _v1_msg, v1_finish = normalize_anthropic_response(resp)
-        v2 = normalize_anthropic_response_v2(resp)
-        assert v2.finish_reason == v1_finish == expected
-
-
-class TestStripToolPrefix:
-    def test_prefix_stripped(self):
-        resp = _response(
-            [_tool_use_block("toolu_1", "mcp_terminal", {"cmd": "ls"})],
-            stop_reason="tool_use",
-        )
-        v1_msg, _ = normalize_anthropic_response(resp, strip_tool_prefix=True)
-        v2 = normalize_anthropic_response_v2(resp, strip_tool_prefix=True)
-        assert v1_msg.tool_calls[0].function.name == "terminal"
-        assert v2.tool_calls[0].name == "terminal"
-
-    def test_prefix_kept(self):
-        resp = _response(
-            [_tool_use_block("toolu_1", "mcp_terminal", {"cmd": "ls"})],
-            stop_reason="tool_use",
-        )
-        v1_msg, _ = normalize_anthropic_response(resp, strip_tool_prefix=False)
-        v2 = normalize_anthropic_response_v2(resp, strip_tool_prefix=False)
-        assert v1_msg.tool_calls[0].function.name == "mcp_terminal"
-        assert v2.tool_calls[0].name == "mcp_terminal"
-
-
-class TestEdgeCases:
-    def test_empty_content_blocks(self):
-        resp = _response([])
-        v1_msg, _v1_finish = normalize_anthropic_response(resp)
-        v2 = normalize_anthropic_response_v2(resp)
-        assert v2.content == v1_msg.content
-        assert v2.content is None
-
-    def test_no_reasoning_details_means_none_provider_data(self):
-        resp = _response([_text_block("hi")])
-        v2 = normalize_anthropic_response_v2(resp)
-        assert v2.provider_data is None
-
-    def test_v2_returns_dataclass_not_namespace(self):
-        resp = _response([_text_block("hi")])
-        v2 = normalize_anthropic_response_v2(resp)
-        assert isinstance(v2, NormalizedResponse)
-        assert not isinstance(v2, SimpleNamespace)

tests/agent/transports/test_transport.py

@@ -1,208 +0,0 @@
-"""Tests for the transport ABC, registry, and AnthropicTransport."""
-
-from types import SimpleNamespace
-
-import pytest
-
-from agent.transports import _REGISTRY, get_transport, register_transport
-from agent.transports.base import ProviderTransport
-from agent.transports.types import NormalizedResponse
-
-
-class TestProviderTransportABC:
-    def test_cannot_instantiate_abc(self):
-        with pytest.raises(TypeError):
-            ProviderTransport()
-
-    def test_concrete_must_implement_all_abstract(self):
-        class Incomplete(ProviderTransport):
-            @property
-            def api_mode(self):
-                return "test"
-
-        with pytest.raises(TypeError):
-            Incomplete()
-
-    def test_minimal_concrete(self):
-        class Minimal(ProviderTransport):
-            @property
-            def api_mode(self):
-                return "test_minimal"
-
-            def convert_messages(self, messages, **kw):
-                return messages
-
-            def convert_tools(self, tools):
-                return tools
-
-            def build_kwargs(self, model, messages, tools=None, **params):
-                return {"model": model, "messages": messages}
-
-            def normalize_response(self, response, **kw):
-                return NormalizedResponse(content="ok", tool_calls=None, finish_reason="stop")
-
-        t = Minimal()
-        assert t.api_mode == "test_minimal"
-        assert t.validate_response(None) is True
-        assert t.extract_cache_stats(None) is None
-        assert t.map_finish_reason("end_turn") == "end_turn"
-
-
-class TestTransportRegistry:
-    def test_get_unregistered_returns_none(self):
-        assert get_transport("nonexistent_mode") is None
-
-    def test_anthropic_registered_on_import(self):
-        import agent.transports.anthropic  # noqa: F401
-
-        t = get_transport("anthropic_messages")
-        assert t is not None
-        assert t.api_mode == "anthropic_messages"
-
-    def test_register_and_get(self):
-        class DummyTransport(ProviderTransport):
-            @property
-            def api_mode(self):
-                return "dummy_test"
-
-            def convert_messages(self, messages, **kw):
-                return messages
-
-            def convert_tools(self, tools):
-                return tools
-
-            def build_kwargs(self, model, messages, tools=None, **params):
-                return {}
-
-            def normalize_response(self, response, **kw):
-                return NormalizedResponse(content=None, tool_calls=None, finish_reason="stop")
-
-        register_transport("dummy_test", DummyTransport)
-        t = get_transport("dummy_test")
-        assert t.api_mode == "dummy_test"
-        _REGISTRY.pop("dummy_test", None)
-
-
-class TestAnthropicTransport:
-    @pytest.fixture
-    def transport(self):
-        import agent.transports.anthropic  # noqa: F401
-
-        return get_transport("anthropic_messages")
-
-    def test_api_mode(self, transport):
-        assert transport.api_mode == "anthropic_messages"
-
-    def test_convert_tools_simple(self, transport):
-        tools = [{
-            "type": "function",
-            "function": {
-                "name": "test_tool",
-                "description": "A test",
-                "parameters": {"type": "object", "properties": {}},
-            },
-        }]
-        result = transport.convert_tools(tools)
-        assert len(result) == 1
-        assert result[0]["name"] == "test_tool"
-        assert "input_schema" in result[0]
-
-    def test_validate_response_none(self, transport):
-        assert transport.validate_response(None) is False
-
-    def test_validate_response_empty_content(self, transport):
-        r = SimpleNamespace(content=[])
-        assert transport.validate_response(r) is False
-
-    def test_validate_response_valid(self, transport):
-        r = SimpleNamespace(content=[SimpleNamespace(type="text", text="hello")])
-        assert transport.validate_response(r) is True
-
-    def test_map_finish_reason(self, transport):
-        assert transport.map_finish_reason("end_turn") == "stop"
-        assert transport.map_finish_reason("tool_use") == "tool_calls"
-        assert transport.map_finish_reason("max_tokens") == "length"
-        assert transport.map_finish_reason("stop_sequence") == "stop"
-        assert transport.map_finish_reason("refusal") == "content_filter"
-        assert transport.map_finish_reason("model_context_window_exceeded") == "length"
-        assert transport.map_finish_reason("unknown") == "stop"
-
-    def test_extract_cache_stats_none_usage(self, transport):
-        r = SimpleNamespace(usage=None)
-        assert transport.extract_cache_stats(r) is None
-
-    def test_extract_cache_stats_with_cache(self, transport):
-        usage = SimpleNamespace(cache_read_input_tokens=100, cache_creation_input_tokens=50)
-        r = SimpleNamespace(usage=usage)
-        result = transport.extract_cache_stats(r)
-        assert result == {"cached_tokens": 100, "creation_tokens": 50}
-
-    def test_extract_cache_stats_zero(self, transport):
-        usage = SimpleNamespace(cache_read_input_tokens=0, cache_creation_input_tokens=0)
-        r = SimpleNamespace(usage=usage)
-        assert transport.extract_cache_stats(r) is None
-
-    def test_normalize_response_text(self, transport):
-        r = SimpleNamespace(
-            content=[SimpleNamespace(type="text", text="Hello world")],
-            stop_reason="end_turn",
-            usage=SimpleNamespace(input_tokens=10, output_tokens=5),
-            model="claude-sonnet-4-6",
-        )
-        nr = transport.normalize_response(r)
-        assert isinstance(nr, NormalizedResponse)
-        assert nr.content == "Hello world"
-        assert nr.tool_calls is None or nr.tool_calls == []
-        assert nr.finish_reason == "stop"
-
-    def test_normalize_response_tool_calls(self, transport):
-        r = SimpleNamespace(
-            content=[
-                SimpleNamespace(type="tool_use", id="toolu_123", name="terminal", input={"command": "ls"}),
-            ],
-            stop_reason="tool_use",
-            usage=SimpleNamespace(input_tokens=10, output_tokens=20),
-            model="claude-sonnet-4-6",
-        )
-        nr = transport.normalize_response(r)
-        assert nr.finish_reason == "tool_calls"
-        assert len(nr.tool_calls) == 1
-        tc = nr.tool_calls[0]
-        assert tc.name == "terminal"
-        assert tc.id == "toolu_123"
-        assert '"command"' in tc.arguments
-
-    def test_normalize_response_thinking(self, transport):
-        r = SimpleNamespace(
-            content=[
-                SimpleNamespace(type="thinking", thinking="Let me think..."),
-                SimpleNamespace(type="text", text="The answer is 42"),
-            ],
-            stop_reason="end_turn",
-            usage=SimpleNamespace(input_tokens=10, output_tokens=15),
-            model="claude-sonnet-4-6",
-        )
-        nr = transport.normalize_response(r)
-        assert nr.content == "The answer is 42"
-        assert nr.reasoning == "Let me think..."
-
-    def test_build_kwargs_returns_dict(self, transport):
-        messages = [{"role": "user", "content": "Hello"}]
-        kw = transport.build_kwargs(
-            model="claude-sonnet-4-6",
-            messages=messages,
-            max_tokens=1024,
-        )
-        assert isinstance(kw, dict)
-        assert "model" in kw
-        assert "max_tokens" in kw
-        assert "messages" in kw
-
-    def test_convert_messages_extracts_system(self, transport):
-        messages = [
-            {"role": "system", "content": "You are helpful."},
-            {"role": "user", "content": "Hi"},
-        ]
-        system, msgs = transport.convert_messages(messages)
-        assert system is not None
-        assert len(msgs) >= 1

tests/agent/transports/test_types.py

@@ -1,130 +0,0 @@
-"""Tests for agent/transports/types.py — dataclass construction + helpers."""
-
-import json
-
-from agent.transports.types import (
-    NormalizedResponse,
-    ToolCall,
-    Usage,
-    build_tool_call,
-    map_finish_reason,
-)
-
-
-class TestToolCall:
-    def test_basic_construction(self):
-        tc = ToolCall(id="call_abc", name="terminal", arguments='{"cmd": "ls"}')
-        assert tc.id == "call_abc"
-        assert tc.name == "terminal"
-        assert tc.arguments == '{"cmd": "ls"}'
-        assert tc.provider_data is None
-
-    def test_none_id(self):
-        tc = ToolCall(id=None, name="read_file", arguments="{}")
-        assert tc.id is None
-
-    def test_provider_data(self):
-        tc = ToolCall(
-            id="call_x",
-            name="t",
-            arguments="{}",
-            provider_data={"call_id": "call_x", "response_item_id": "fc_x"},
-        )
-        assert tc.provider_data["call_id"] == "call_x"
-        assert tc.provider_data["response_item_id"] == "fc_x"
-
-
-class TestUsage:
-    def test_defaults(self):
-        u = Usage()
-        assert u.prompt_tokens == 0
-        assert u.completion_tokens == 0
-        assert u.total_tokens == 0
-        assert u.cached_tokens == 0
-
-    def test_explicit(self):
-        u = Usage(prompt_tokens=100, completion_tokens=50, total_tokens=150, cached_tokens=80)
-        assert u.total_tokens == 150
-
-
-class TestNormalizedResponse:
-    def test_text_only(self):
-        r = NormalizedResponse(content="hello", tool_calls=None, finish_reason="stop")
-        assert r.content == "hello"
-        assert r.tool_calls is None
-        assert r.finish_reason == "stop"
-        assert r.reasoning is None
-        assert r.usage is None
-        assert r.provider_data is None
-
-    def test_with_tool_calls(self):
-        tcs = [ToolCall(id="call_1", name="terminal", arguments='{"cmd":"pwd"}')]
-        r = NormalizedResponse(content=None, tool_calls=tcs, finish_reason="tool_calls")
-        assert r.finish_reason == "tool_calls"
-        assert len(r.tool_calls) == 1
-        assert r.tool_calls[0].name == "terminal"
-
-    def test_with_reasoning(self):
-        r = NormalizedResponse(
-            content="answer",
-            tool_calls=None,
-            finish_reason="stop",
-            reasoning="I thought about it",
-        )
-        assert r.reasoning == "I thought about it"
-
-    def test_with_provider_data(self):
-        r = NormalizedResponse(
-            content=None,
-            tool_calls=None,
-            finish_reason="stop",
-            provider_data={"reasoning_details": [{"type": "thinking", "thinking": "hmm"}]},
-        )
-        assert r.provider_data["reasoning_details"][0]["type"] == "thinking"
-
-
-class TestBuildToolCall:
-    def test_dict_arguments_serialized(self):
-        tc = build_tool_call(id="call_1", name="terminal", arguments={"cmd": "ls"})
-        assert tc.arguments == json.dumps({"cmd": "ls"})
-        assert tc.provider_data is None
-
-    def test_string_arguments_passthrough(self):
-        tc = build_tool_call(id="call_2", name="read_file", arguments='{"path": "/tmp"}')
-        assert tc.arguments == '{"path": "/tmp"}'
-
-    def test_provider_fields(self):
-        tc = build_tool_call(
-            id="call_3",
-            name="terminal",
-            arguments="{}",
-            call_id="call_3",
-            response_item_id="fc_3",
-        )
-        assert tc.provider_data == {"call_id": "call_3", "response_item_id": "fc_3"}
-
-    def test_none_id(self):
-        tc = build_tool_call(id=None, name="t", arguments="{}")
-        assert tc.id is None
-
-
-class TestMapFinishReason:
-    ANTHROPIC_MAP = {
-        "end_turn": "stop",
-        "tool_use": "tool_calls",
-        "max_tokens": "length",
-        "stop_sequence": "stop",
-        "refusal": "content_filter",
-    }
-
-    def test_known_reason(self):
-        assert map_finish_reason("end_turn", self.ANTHROPIC_MAP) == "stop"
-        assert map_finish_reason("tool_use", self.ANTHROPIC_MAP) == "tool_calls"
-        assert map_finish_reason("max_tokens", self.ANTHROPIC_MAP) == "length"
-        assert map_finish_reason("refusal", self.ANTHROPIC_MAP) == "content_filter"
-
-    def test_unknown_reason_defaults_to_stop(self):
-        assert map_finish_reason("something_new", self.ANTHROPIC_MAP) == "stop"
-
-    def test_none_reason(self):
-        assert map_finish_reason(None, self.ANTHROPIC_MAP) == "stop"

tests/test_vision_benchmark.py

@@ -11,12 +11,14 @@ import pytest
 sys.path.insert(0, str(Path(__file__).parent.parent / "benchmarks"))
 
 from vision_benchmark import (
+    analyze_with_model,
     compute_ocr_accuracy,
     compute_description_completeness,
     compute_structural_accuracy,
     aggregate_results,
     to_markdown,
     generate_sample_dataset,
+    load_dataset,
     MODELS,
     EVAL_PROMPTS,
 )
@@ -197,6 +199,71 @@ class TestMarkdown:
 
 
 class TestDataset:
+    def test_repo_dataset_uses_local_image_paths(self):
+        dataset_path = Path(__file__).parent.parent / "benchmarks" / "test_images.json"
+        dataset = json.loads(dataset_path.read_text())
+
+        assert dataset, "benchmark dataset should not be empty"
+        assert all(not entry["url"].startswith(("http://", "https://")) for entry in dataset)
+
+    def test_load_dataset_resolves_relative_local_paths(self, tmp_path):
+        images_dir = tmp_path / "images"
+        images_dir.mkdir()
+        image_path = images_dir / "sample.png"
+        image_path.write_bytes(b"png-bytes")
+
+        dataset_path = tmp_path / "dataset.json"
+        dataset_path.write_text(json.dumps([
+            {
+                "id": "sample",
+                "url": "images/sample.png",
+                "category": "photo",
+                "expected_keywords": [],
+                "expected_structure": {"min_length": 30, "min_sentences": 1},
+            }
+        ]))
+
+        loaded = load_dataset(str(dataset_path))
+
+        assert loaded[0]["url"] == str(image_path.resolve())
+
+    @pytest.mark.asyncio
+    async def test_analyze_with_model_encodes_local_file_as_data_url(self, tmp_path, monkeypatch):
+        image_path = tmp_path / "tiny.png"
+        image_path.write_bytes(
+            bytes.fromhex(
+                "89504E470D0A1A0A"
+                "0000000D49484452000000010000000108060000001F15C489"
+                "0000000D49444154789C6360000002000154A24F5D00000000"
+                "49454E44AE426082"
+            )
+        )
+
+        fake_response = MagicMock()
+        fake_response.raise_for_status.return_value = None
+        fake_response.json.return_value = {
+            "choices": [{"message": {"content": "Looks like a tiny image."}}],
+            "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
+        }
+
+        fake_client = MagicMock()
+        fake_client.post = AsyncMock(return_value=fake_response)
+        fake_ctx = MagicMock()
+        fake_ctx.__aenter__ = AsyncMock(return_value=fake_client)
+        fake_ctx.__aexit__ = AsyncMock(return_value=None)
+
+        monkeypatch.setenv("OPENROUTER_API_KEY", "test-key")
+        with patch("httpx.AsyncClient", return_value=fake_ctx):
+            result = await analyze_with_model(
+                str(image_path),
+                "Describe this image",
+                {"provider": "openrouter", "model_id": "fake/model"},
+            )
+
+        assert result["success"] is True
+        sent_url = fake_client.post.await_args.kwargs["json"]["messages"][0]["content"][1]["image_url"]["url"]
+        assert sent_url.startswith("data:image/png;base64,")
+
     def test_sample_dataset_has_entries(self):
         dataset = generate_sample_dataset()
         assert len(dataset) >= 4