fix: vendor vision benchmark fixtures (#868)

benchmarks/test_images.json

@@ -1,194 +1,354 @@
 [
   {
     "id": "screenshot_github_home",
-    "url": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png",
+    "url": "test_images/screenshot_github_home.png",
     "category": "screenshot",
-    "expected_keywords": ["github", "logo", "mark"],
+    "expected_keywords": [
+      "github",
+      "logo",
+      "mark"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "diagram_mermaid_flow",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6siSZXVhjQTlgl1nigHg5fRBOzSfebopROCu_cytObSfgLSE1ANOeZWkO2IH5upZxYot8m1hqAdpD_63WRl0xdUG1jdl9kPiOb_EWk2JBtPaiKkF4eVIYgO0EtkW-RSgC4gJ6HJYRG1UNdN0HNVd0Bftjj7X8P92qPj-F8l8T3w",
+    "url": "test_images/diagram_mermaid_flow.png",
     "category": "diagram",
-    "expected_keywords": ["flow", "diagram", "process"],
+    "expected_keywords": [
+      "flow",
+      "diagram",
+      "process"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_1",
-    "url": "https://picsum.photos/seed/vision1/400/300",
+    "url": "test_images/photo_random_1.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_2",
-    "url": "https://picsum.photos/seed/vision2/400/300",
+    "url": "test_images/photo_random_2.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_simple_bar",
-    "url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['Q1','Q2','Q3','Q4'],datasets:[{label:'Revenue',data:[100,150,200,250]}]}}",
+    "url": "test_images/chart_simple_bar.png",
     "category": "chart",
-    "expected_keywords": ["bar", "chart", "revenue"],
+    "expected_keywords": [
+      "bar",
+      "chart",
+      "revenue"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "chart_pie",
-    "url": "https://quickchart.io/chart?c={type:'pie',data:{labels:['A','B','C'],datasets:[{data:[30,50,20]}]}}",
+    "url": "test_images/chart_pie.png",
     "category": "chart",
-    "expected_keywords": ["pie", "chart", "percentage"],
+    "expected_keywords": [
+      "pie",
+      "chart",
+      "percentage"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_org_chart",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_org_chart.png",
     "category": "diagram",
-    "expected_keywords": ["organization", "hierarchy", "chart"],
+    "expected_keywords": [
+      "organization",
+      "hierarchy",
+      "chart"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_terminal",
-    "url": "https://raw.githubusercontent.com/nicehash/nicehash-quick-start/main/images/nicehash-terminal.png",
+    "url": "test_images/screenshot_terminal.png",
     "category": "screenshot",
-    "expected_keywords": ["terminal", "command", "output"],
+    "expected_keywords": [
+      "terminal",
+      "command",
+      "output"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_3",
-    "url": "https://picsum.photos/seed/vision3/400/300",
+    "url": "test_images/photo_random_3.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_line",
-    "url": "https://quickchart.io/chart?c={type:'line',data:{labels:['Jan','Feb','Mar','Apr'],datasets:[{label:'Temperature',data:[5,8,12,18]}]}}",
+    "url": "test_images/chart_line.png",
     "category": "chart",
-    "expected_keywords": ["line", "chart", "temperature"],
+    "expected_keywords": [
+      "line",
+      "chart",
+      "temperature"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_sequence",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_sequence.png",
     "category": "diagram",
-    "expected_keywords": ["sequence", "interaction", "message"],
+    "expected_keywords": [
+      "sequence",
+      "interaction",
+      "message"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_4",
-    "url": "https://picsum.photos/seed/vision4/400/300",
+    "url": "test_images/photo_random_4.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_webpage",
-    "url": "https://github.githubassets.com/images/modules/site/social-cards.png",
+    "url": "test_images/screenshot_webpage.png",
     "category": "screenshot",
-    "expected_keywords": ["github", "page", "web"],
+    "expected_keywords": [
+      "github",
+      "page",
+      "web"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_radar",
-    "url": "https://quickchart.io/chart?c={type:'radar',data:{labels:['Speed','Power','Defense','Magic'],datasets:[{label:'Hero',data:[80,60,70,90]}]}}",
+    "url": "test_images/chart_radar.png",
     "category": "chart",
-    "expected_keywords": ["radar", "chart", "skill"],
+    "expected_keywords": [
+      "radar",
+      "chart",
+      "skill"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "photo_random_5",
-    "url": "https://picsum.photos/seed/vision5/400/300",
+    "url": "test_images/photo_random_5.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "diagram_class",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_class.png",
     "category": "diagram",
-    "expected_keywords": ["class", "object", "attribute"],
+    "expected_keywords": [
+      "class",
+      "object",
+      "attribute"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_doughnut",
-    "url": "https://quickchart.io/chart?c={type:'doughnut',data:{labels:['Desktop','Mobile','Tablet'],datasets:[{data:[60,30,10]}]}}",
+    "url": "test_images/chart_doughnut.png",
     "category": "chart",
-    "expected_keywords": ["doughnut", "chart", "device"],
+    "expected_keywords": [
+      "doughnut",
+      "chart",
+      "device"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "photo_random_6",
-    "url": "https://picsum.photos/seed/vision6/400/300",
+    "url": "test_images/photo_random_6.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_error",
-    "url": "https://http.cat/404.jpg",
+    "url": "test_images/screenshot_error.png",
     "category": "screenshot",
-    "expected_keywords": ["404", "error", "cat"],
+    "expected_keywords": [
+      "404",
+      "error",
+      "cat"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_network",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_network.png",
     "category": "diagram",
-    "expected_keywords": ["network", "node", "connection"],
+    "expected_keywords": [
+      "network",
+      "node",
+      "connection"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_7",
-    "url": "https://picsum.photos/seed/vision7/400/300",
+    "url": "test_images/photo_random_7.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_stacked_bar",
-    "url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['2022','2023','2024'],datasets:[{label:'Cloud',data:[100,150,200]},{label:'On-prem',data:[200,180,160]}]},options:{scales:{x:{stacked:true},y:{stacked:true}}}}",
+    "url": "test_images/chart_stacked_bar.png",
     "category": "chart",
-    "expected_keywords": ["stacked", "bar", "chart"],
+    "expected_keywords": [
+      "stacked",
+      "bar",
+      "chart"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "screenshot_dashboard",
-    "url": "https://github.githubassets.com/images/modules/site/features-code-search.png",
+    "url": "test_images/screenshot_dashboard.png",
     "category": "screenshot",
-    "expected_keywords": ["search", "code", "feature"],
+    "expected_keywords": [
+      "search",
+      "code",
+      "feature"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_8",
-    "url": "https://picsum.photos/seed/vision8/400/300",
+    "url": "test_images/photo_random_8.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   }
 ]

benchmarks/vision_benchmark.py

@@ -11,17 +11,19 @@ Usage:
 
     # Single image test
     python benchmarks/vision_benchmark.py --url https://example.com/image.png
+    python benchmarks/vision_benchmark.py --url benchmarks/test_images/photo_random_1.png
 
     # Generate test report
     python benchmarks/vision_benchmark.py --images benchmarks/test_images.json --output benchmarks/vision_results.json
 
-Test image dataset: benchmarks/test_images.json (50-100 diverse images)
+Test image dataset: benchmarks/test_images.json (committed local fixtures under benchmarks/test_images/)
 """
 
 import argparse
 import asyncio
 import base64
 import json
+import mimetypes
 import os
 import statistics
 import sys
@@ -67,6 +69,28 @@ EVAL_PROMPTS = {
 # ---------------------------------------------------------------------------
 
 
+def _is_remote_image_source(image_source: str) -> bool:
+    return image_source.startswith(("http://", "https://", "data:", "file://"))
+
+
+def _image_source_to_payload_url(image_source: str) -> str:
+    """Convert local image paths into data URLs; keep remote URLs unchanged."""
+    if image_source.startswith(("http://", "https://", "data:")):
+        return image_source
+
+    resolved = image_source[len("file://"):] if image_source.startswith("file://") else image_source
+    local_path = Path(os.path.expanduser(resolved)).resolve()
+    if not local_path.is_file():
+        return image_source
+
+    mime_type, _ = mimetypes.guess_type(str(local_path))
+    if not mime_type:
+        mime_type = "application/octet-stream"
+
+    encoded = base64.b64encode(local_path.read_bytes()).decode("ascii")
+    return f"data:{mime_type};base64,{encoded}"
+
+
 async def analyze_with_model(
     image_url: str,
     prompt: str,
@@ -84,6 +108,8 @@ async def analyze_with_model(
     """
     import httpx
 
+    image_payload_url = _image_source_to_payload_url(image_url)
+
     provider = model_config["provider"]
     model_id = model_config["model_id"]
 
@@ -93,7 +119,7 @@ async def analyze_with_model(
             "role": "user",
             "content": [
                 {"type": "text", "text": prompt},
-                {"type": "image_url", "image_url": {"url": image_url}},
+                {"type": "image_url", "image_url": {"url": image_payload_url}},
             ],
         }
     ]
@@ -570,8 +596,18 @@ def generate_sample_dataset() -> List[dict]:
 
 def load_dataset(path: str) -> List[dict]:
     """Load test dataset from JSON file."""
-    with open(path) as f:
-        return json.load(f)
+    dataset_path = Path(path).resolve()
+    with open(dataset_path) as f:
+        dataset = json.load(f)
+
+    base_dir = dataset_path.parent
+    for image in dataset:
+        image_url = image.get("url")
+        if not image_url or _is_remote_image_source(image_url):
+            continue
+        image["url"] = str((base_dir / image_url).resolve())
+
+    return dataset
 
 
 # ---------------------------------------------------------------------------
@@ -582,7 +618,7 @@ def load_dataset(path: str) -> List[dict]:
 async def main():
     parser = argparse.ArgumentParser(description="Vision Benchmark Suite (Issue #817)")
     parser.add_argument("--images", help="Path to test images JSON file")
-    parser.add_argument("--url", help="Single image URL to test")
+    parser.add_argument("--url", help="Single image URL or local file path to test")
     parser.add_argument("--category", default="photo", help="Category for single URL")
     parser.add_argument("--output", default=None, help="Output JSON file")
     parser.add_argument("--runs", type=int, default=1, help="Runs per model per image")

cli.py

@@ -589,7 +589,6 @@ from tools.terminal_tool import set_sudo_password_callback, set_approval_callbac
 from tools.skills_tool import set_secret_capture_callback
 from hermes_cli.callbacks import prompt_for_secret
 from tools.browser_tool import _emergency_cleanup_all_sessions as _cleanup_all_browsers
-from utils import repair_and_load_json
 
 # Guard to prevent cleanup from running multiple times on exit
 _cleanup_done = False
@@ -3570,11 +3569,7 @@ class HermesCLI:
                 result_json = _asyncio.run(
                     vision_analyze_tool(image_url=str(img_path), user_prompt=analysis_prompt)
                 )
-                result = repair_and_load_json(
-                    result_json,
-                    default={},
-                    context="cli_image_analysis",
-                ) if isinstance(result_json, str) else {}
+                result = _json.loads(result_json)
                 if result.get("success"):
                     description = result.get("analysis", "")
                     enriched_parts.append(
@@ -4965,14 +4960,7 @@ class HermesCLI:
         from tools.cronjob_tools import cronjob as cronjob_tool
 
         def _cron_api(**kwargs):
-            result = repair_and_load_json(
-                cronjob_tool(**kwargs),
-                default=None,
-                context="cli_cron_command",
-            )
-            if isinstance(result, dict):
-                return result
-            return {"success": False, "error": "Invalid JSON from cronjob tool"}
+            return json.loads(cronjob_tool(**kwargs))
 
         def _normalize_skills(values):
             normalized = []

tests/cli/test_cli_json_repair.py

@@ -1,62 +0,0 @@
-import sys
-import types
-from unittest.mock import patch
-
-
-def _stub_auxiliary_client():
-    stub = types.ModuleType("agent.auxiliary_client")
-    stub.call_llm = lambda *args, **kwargs: None
-    stub.resolve_provider_client = lambda *args, **kwargs: (None, None)
-    stub.get_text_auxiliary_client = lambda *args, **kwargs: (None, None)
-    stub.async_call_llm = lambda *args, **kwargs: None
-    stub.extract_content_or_reasoning = lambda *args, **kwargs: ""
-    stub._OR_HEADERS = {}
-    stub._get_task_timeout = lambda *args, **kwargs: 30
-    sys.modules["agent.auxiliary_client"] = stub
-
-
-def _stub_vision_tools(vision_analyze_tool):
-    stub = types.ModuleType("tools.vision_tools")
-    stub.vision_analyze_tool = vision_analyze_tool
-    sys.modules["tools.vision_tools"] = stub
-
-
-def test_preprocess_images_with_vision_repairs_malformed_json(tmp_path):
-    _stub_auxiliary_client()
-    from cli import HermesCLI
-
-    cli_obj = HermesCLI.__new__(HermesCLI)
-    image_path = tmp_path / "test.png"
-    image_path.write_bytes(b"fake-image-bytes")
-
-    async def fake_vision(**kwargs):
-        return "{'success': true, 'analysis': 'Recovered image description',}"
-
-    _stub_vision_tools(fake_vision)
-    result = HermesCLI._preprocess_images_with_vision(
-        cli_obj,
-        "Describe this",
-        [image_path],
-        announce=False,
-    )
-
-    assert "Recovered image description" in result
-    assert "Describe this" in result
-    assert str(image_path) in result
-
-
-def test_handle_cron_command_repairs_malformed_json(capsys):
-    _stub_auxiliary_client()
-    from cli import HermesCLI
-
-    cli_obj = HermesCLI.__new__(HermesCLI)
-    malformed_result = """{'success': true, 'jobs': [{'job_id': 'job-1234567890ab', 'name': 'Nightly Check', 'state': 'scheduled', 'schedule': 'every 1h', 'repeat': 'forever', 'prompt_preview': 'Check server status', 'skills': ['blogwatcher',], 'next_run_at': '2026-04-22T01:00:00Z',},],}"""
-
-    with patch("tools.cronjob_tools.cronjob", return_value=malformed_result):
-        HermesCLI._handle_cron_command(cli_obj, "/cron list")
-
-    out = capsys.readouterr().out
-    assert "Scheduled Jobs:" in out
-    assert "job-1234567890ab" in out
-    assert "Nightly Check" in out
-    assert "blogwatcher" in out

tests/test_vision_benchmark.py

@@ -11,12 +11,14 @@ import pytest
 sys.path.insert(0, str(Path(__file__).parent.parent / "benchmarks"))
 
 from vision_benchmark import (
+    analyze_with_model,
     compute_ocr_accuracy,
     compute_description_completeness,
     compute_structural_accuracy,
     aggregate_results,
     to_markdown,
     generate_sample_dataset,
+    load_dataset,
     MODELS,
     EVAL_PROMPTS,
 )
@@ -197,6 +199,71 @@ class TestMarkdown:
 
 
 class TestDataset:
+    def test_repo_dataset_uses_local_image_paths(self):
+        dataset_path = Path(__file__).parent.parent / "benchmarks" / "test_images.json"
+        dataset = json.loads(dataset_path.read_text())
+
+        assert dataset, "benchmark dataset should not be empty"
+        assert all(not entry["url"].startswith(("http://", "https://")) for entry in dataset)
+
+    def test_load_dataset_resolves_relative_local_paths(self, tmp_path):
+        images_dir = tmp_path / "images"
+        images_dir.mkdir()
+        image_path = images_dir / "sample.png"
+        image_path.write_bytes(b"png-bytes")
+
+        dataset_path = tmp_path / "dataset.json"
+        dataset_path.write_text(json.dumps([
+            {
+                "id": "sample",
+                "url": "images/sample.png",
+                "category": "photo",
+                "expected_keywords": [],
+                "expected_structure": {"min_length": 30, "min_sentences": 1},
+            }
+        ]))
+
+        loaded = load_dataset(str(dataset_path))
+
+        assert loaded[0]["url"] == str(image_path.resolve())
+
+    @pytest.mark.asyncio
+    async def test_analyze_with_model_encodes_local_file_as_data_url(self, tmp_path, monkeypatch):
+        image_path = tmp_path / "tiny.png"
+        image_path.write_bytes(
+            bytes.fromhex(
+                "89504E470D0A1A0A"
+                "0000000D49484452000000010000000108060000001F15C489"
+                "0000000D49444154789C6360000002000154A24F5D00000000"
+                "49454E44AE426082"
+            )
+        )
+
+        fake_response = MagicMock()
+        fake_response.raise_for_status.return_value = None
+        fake_response.json.return_value = {
+            "choices": [{"message": {"content": "Looks like a tiny image."}}],
+            "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
+        }
+
+        fake_client = MagicMock()
+        fake_client.post = AsyncMock(return_value=fake_response)
+        fake_ctx = MagicMock()
+        fake_ctx.__aenter__ = AsyncMock(return_value=fake_client)
+        fake_ctx.__aexit__ = AsyncMock(return_value=None)
+
+        monkeypatch.setenv("OPENROUTER_API_KEY", "test-key")
+        with patch("httpx.AsyncClient", return_value=fake_ctx):
+            result = await analyze_with_model(
+                str(image_path),
+                "Describe this image",
+                {"provider": "openrouter", "model_id": "fake/model"},
+            )
+
+        assert result["success"] is True
+        sent_url = fake_client.post.await_args.kwargs["json"]["messages"][0]["content"][1]["image_url"]["url"]
+        assert sent_url.startswith("data:image/png;base64,")
+
     def test_sample_dataset_has_entries(self):
         dataset = generate_sample_dataset()
         assert len(dataset) >= 4

tests/tools/test_browser_json_repair.py

@@ -1,108 +0,0 @@
-import io
-import json
-import sys
-import types
-from unittest.mock import MagicMock, patch
-
-
-def _stub_auxiliary_client():
-    stub = types.ModuleType("agent.auxiliary_client")
-    stub.call_llm = lambda *args, **kwargs: None
-    stub.resolve_provider_client = lambda *args, **kwargs: (None, None)
-    stub.get_text_auxiliary_client = lambda *args, **kwargs: (None, None)
-    stub.async_call_llm = lambda *args, **kwargs: None
-    stub.extract_content_or_reasoning = lambda *args, **kwargs: ""
-    stub._OR_HEADERS = {}
-    stub._get_task_timeout = lambda *args, **kwargs: 30
-    sys.modules["agent.auxiliary_client"] = stub
-
-
-def test_run_browser_command_repairs_malformed_stdout_envelope(tmp_path):
-    _stub_auxiliary_client()
-    from tools.browser_tool import _run_browser_command
-
-    mock_proc = MagicMock()
-    mock_proc.returncode = 0
-    mock_proc.wait.return_value = 0
-    fake_session = {
-        "session_name": "test-session",
-        "session_id": "test-id",
-        "cdp_url": None,
-    }
-    malformed_stdout = "{'success': true, 'data': {'url': 'https://example.com',},}"
-
-    def fake_open(path, mode="r", *args, **kwargs):
-        path = str(path)
-        if path.endswith("_stdout_navigate"):
-            return io.StringIO(malformed_stdout)
-        if path.endswith("_stderr_navigate"):
-            return io.StringIO("")
-        raise FileNotFoundError(path)
-
-    with (
-        patch("tools.browser_tool._find_agent_browser", return_value="/usr/bin/agent-browser"),
-        patch("tools.browser_tool._get_session_info", return_value=fake_session),
-        patch("tools.browser_tool._socket_safe_tmpdir", return_value=str(tmp_path)),
-        patch("tools.browser_tool._merge_browser_path", side_effect=lambda p: p),
-        patch("tools.interrupt.is_interrupted", return_value=False),
-        patch("subprocess.Popen", return_value=mock_proc),
-        patch("os.open", return_value=99),
-        patch("os.close"),
-        patch("os.unlink"),
-        patch("builtins.open", side_effect=fake_open),
-    ):
-        result = _run_browser_command("task-1", "navigate", ["https://example.com"])
-
-    assert result["success"] is True
-    assert result["data"]["url"] == "https://example.com"
-
-
-def test_agent_browser_eval_repairs_malformed_json_result():
-    _stub_auxiliary_client()
-    from tools.browser_tool import _browser_eval
-
-    with patch(
-        "tools.browser_tool._run_browser_command",
-        return_value={"success": True, "data": {"result": "{'items': ['a', 'b',],}"}},
-    ):
-        result = json.loads(_browser_eval("document.body.innerText", task_id="test"))
-
-    assert result["success"] is True
-    assert result["result"] == {"items": ["a", "b"]}
-    assert result["result_type"] == "dict"
-
-
-def test_camofox_eval_repairs_malformed_json_result():
-    _stub_auxiliary_client()
-    from tools.browser_tool import _camofox_eval
-
-    with (
-        patch("tools.browser_camofox._ensure_tab", return_value={"tab_id": "tab-1", "user_id": "user-1"}),
-        patch("tools.browser_camofox._post", return_value={"result": "{'count': 3,}"}),
-    ):
-        result = json.loads(_camofox_eval("2+1", task_id="test"))
-
-    assert result["success"] is True
-    assert result["result"] == {"count": 3}
-    assert result["result_type"] == "dict"
-
-
-def test_browser_get_images_repairs_malformed_json_result():
-    _stub_auxiliary_client()
-    from tools.browser_tool import browser_get_images
-
-    with patch(
-        "tools.browser_tool._run_browser_command",
-        return_value={
-            "success": True,
-            "data": {
-                "result": "[{\"src\": \"https://example.com/cat.png\", \"alt\": \"cat\",}]"
-            },
-        },
-    ):
-        result = json.loads(browser_get_images(task_id="test"))
-
-    assert result["success"] is True
-    assert result["count"] == 1
-    assert result["images"] == [{"src": "https://example.com/cat.png", "alt": "cat"}]
-    assert "warning" not in result

tools/browser_tool.py

@@ -67,7 +67,6 @@ from typing import Dict, Any, Optional, List
 from pathlib import Path
 from agent.auxiliary_client import call_llm
 from hermes_constants import get_hermes_home
-from utils import repair_and_load_json
 
 try:
     from tools.website_policy import check_website_access
@@ -1172,12 +1171,8 @@ def _run_browser_command(
             return {"success": False, "error": f"Browser command '{command}' returned no output"}
 
         if stdout_text:
-            parsed = repair_and_load_json(
-                stdout_text,
-                default=None,
-                context=f"browser_{command}_stdout",
-            )
-            if isinstance(parsed, dict):
+            try:
+                parsed = json.loads(stdout_text)
                 # Warn if snapshot came back empty (common sign of daemon/CDP issues)
                 if command == "snapshot" and parsed.get("success"):
                     snap_data = parsed.get("data", {})
@@ -1186,35 +1181,35 @@ def _run_browser_command(
                                        "Possible stale daemon or CDP connection issue. "
                                        "returncode=%s", returncode)
                 return parsed
+            except json.JSONDecodeError:
+                raw = stdout_text[:2000]
+                logger.warning("browser '%s' returned non-JSON output (rc=%s): %s",
+                               command, returncode, raw[:500])
 
-            raw = stdout_text[:2000]
-            logger.warning("browser '%s' returned non-JSON output (rc=%s): %s",
-                           command, returncode, raw[:500])
-
-            if command == "screenshot":
-                stderr_text = (stderr or "").strip()
-                combined_text = "\n".join(
-                    part for part in [stdout_text, stderr_text] if part
-                )
-                recovered_path = _extract_screenshot_path_from_text(combined_text)
-
-                if recovered_path and Path(recovered_path).exists():
-                    logger.info(
-                        "browser 'screenshot' recovered file from non-JSON output: %s",
-                        recovered_path,
+                if command == "screenshot":
+                    stderr_text = (stderr or "").strip()
+                    combined_text = "\n".join(
+                        part for part in [stdout_text, stderr_text] if part
                     )
-                    return {
-                        "success": True,
-                        "data": {
-                            "path": recovered_path,
-                            "raw": raw,
-                        },
-                    }
+                    recovered_path = _extract_screenshot_path_from_text(combined_text)
 
-            return {
-                "success": False,
-                "error": f"Non-JSON output from agent-browser for '{command}': {raw}"
-            }
+                    if recovered_path and Path(recovered_path).exists():
+                        logger.info(
+                            "browser 'screenshot' recovered file from non-JSON output: %s",
+                            recovered_path,
+                        )
+                        return {
+                            "success": True,
+                            "data": {
+                                "path": recovered_path,
+                                "raw": raw,
+                            },
+                        }
+
+                return {
+                    "success": False,
+                    "error": f"Non-JSON output from agent-browser for '{command}': {raw}"
+                }
         
         # Check for errors
         if returncode != 0:
@@ -1782,11 +1777,10 @@ def _browser_eval(expression: str, task_id: Optional[str] = None) -> str:
     # is valid JSON, parse it so the model gets structured data.
     parsed = raw_result
     if isinstance(raw_result, str):
-        parsed = repair_and_load_json(
-            raw_result,
-            default=raw_result,
-            context="browser_eval_result",
-        )
+        try:
+            parsed = json.loads(raw_result)
+        except (json.JSONDecodeError, ValueError):
+            pass  # keep as string
 
     return json.dumps({
         "success": True,
@@ -1807,11 +1801,10 @@ def _camofox_eval(expression: str, task_id: Optional[str] = None) -> str:
         raw_result = resp.get("result") if isinstance(resp, dict) else resp
         parsed = raw_result
         if isinstance(raw_result, str):
-            parsed = repair_and_load_json(
-                raw_result,
-                default=raw_result,
-                context="camofox_eval_result",
-            )
+            try:
+                parsed = json.loads(raw_result)
+            except (json.JSONDecodeError, ValueError):
+                pass
 
         return json.dumps({
             "success": True,
@@ -1911,29 +1904,26 @@ def browser_get_images(task_id: Optional[str] = None) -> str:
     if result.get("success"):
         data = result.get("data", {})
         raw_result = data.get("result", "[]")
-
-        warning = None
-        if isinstance(raw_result, str):
-            images = repair_and_load_json(
-                raw_result,
-                default=None,
-                context="browser_get_images_result",
-            )
-        else:
-            images = raw_result
-
-        if not isinstance(images, list):
-            images = []
-            warning = "Could not parse image data"
-
-        payload = {
-            "success": True,
-            "images": images,
-            "count": len(images),
-        }
-        if warning:
-            payload["warning"] = warning
-        return json.dumps(payload, ensure_ascii=False)
+        
+        try:
+            # Parse the JSON string returned by JavaScript
+            if isinstance(raw_result, str):
+                images = json.loads(raw_result)
+            else:
+                images = raw_result
+            
+            return json.dumps({
+                "success": True,
+                "images": images,
+                "count": len(images)
+            }, ensure_ascii=False)
+        except json.JSONDecodeError:
+            return json.dumps({
+                "success": True,
+                "images": [],
+                "count": 0,
+                "warning": "Could not parse image data"
+            }, ensure_ascii=False)
     else:
         return json.dumps({
             "success": False,