fix: vendor vision benchmark fixtures (#868)

benchmarks/test_images.json

@@ -1,194 +1,354 @@
 [
   {
     "id": "screenshot_github_home",
-    "url": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png",
+    "url": "test_images/screenshot_github_home.png",
     "category": "screenshot",
-    "expected_keywords": ["github", "logo", "mark"],
+    "expected_keywords": [
+      "github",
+      "logo",
+      "mark"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "diagram_mermaid_flow",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6siSZXVhjQTlgl1nigHg5fRBOzSfebopROCu_cytObSfgLSE1ANOeZWkO2IH5upZxYot8m1hqAdpD_63WRl0xdUG1jdl9kPiOb_EWk2JBtPaiKkF4eVIYgO0EtkW-RSgC4gJ6HJYRG1UNdN0HNVd0Bftjj7X8P92qPj-F8l8T3w",
+    "url": "test_images/diagram_mermaid_flow.png",
     "category": "diagram",
-    "expected_keywords": ["flow", "diagram", "process"],
+    "expected_keywords": [
+      "flow",
+      "diagram",
+      "process"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_1",
-    "url": "https://picsum.photos/seed/vision1/400/300",
+    "url": "test_images/photo_random_1.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_2",
-    "url": "https://picsum.photos/seed/vision2/400/300",
+    "url": "test_images/photo_random_2.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_simple_bar",
-    "url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['Q1','Q2','Q3','Q4'],datasets:[{label:'Revenue',data:[100,150,200,250]}]}}",
+    "url": "test_images/chart_simple_bar.png",
     "category": "chart",
-    "expected_keywords": ["bar", "chart", "revenue"],
+    "expected_keywords": [
+      "bar",
+      "chart",
+      "revenue"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "chart_pie",
-    "url": "https://quickchart.io/chart?c={type:'pie',data:{labels:['A','B','C'],datasets:[{data:[30,50,20]}]}}",
+    "url": "test_images/chart_pie.png",
     "category": "chart",
-    "expected_keywords": ["pie", "chart", "percentage"],
+    "expected_keywords": [
+      "pie",
+      "chart",
+      "percentage"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_org_chart",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_org_chart.png",
     "category": "diagram",
-    "expected_keywords": ["organization", "hierarchy", "chart"],
+    "expected_keywords": [
+      "organization",
+      "hierarchy",
+      "chart"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_terminal",
-    "url": "https://raw.githubusercontent.com/nicehash/nicehash-quick-start/main/images/nicehash-terminal.png",
+    "url": "test_images/screenshot_terminal.png",
     "category": "screenshot",
-    "expected_keywords": ["terminal", "command", "output"],
+    "expected_keywords": [
+      "terminal",
+      "command",
+      "output"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_3",
-    "url": "https://picsum.photos/seed/vision3/400/300",
+    "url": "test_images/photo_random_3.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_line",
-    "url": "https://quickchart.io/chart?c={type:'line',data:{labels:['Jan','Feb','Mar','Apr'],datasets:[{label:'Temperature',data:[5,8,12,18]}]}}",
+    "url": "test_images/chart_line.png",
     "category": "chart",
-    "expected_keywords": ["line", "chart", "temperature"],
+    "expected_keywords": [
+      "line",
+      "chart",
+      "temperature"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_sequence",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_sequence.png",
     "category": "diagram",
-    "expected_keywords": ["sequence", "interaction", "message"],
+    "expected_keywords": [
+      "sequence",
+      "interaction",
+      "message"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_4",
-    "url": "https://picsum.photos/seed/vision4/400/300",
+    "url": "test_images/photo_random_4.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_webpage",
-    "url": "https://github.githubassets.com/images/modules/site/social-cards.png",
+    "url": "test_images/screenshot_webpage.png",
     "category": "screenshot",
-    "expected_keywords": ["github", "page", "web"],
+    "expected_keywords": [
+      "github",
+      "page",
+      "web"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_radar",
-    "url": "https://quickchart.io/chart?c={type:'radar',data:{labels:['Speed','Power','Defense','Magic'],datasets:[{label:'Hero',data:[80,60,70,90]}]}}",
+    "url": "test_images/chart_radar.png",
     "category": "chart",
-    "expected_keywords": ["radar", "chart", "skill"],
+    "expected_keywords": [
+      "radar",
+      "chart",
+      "skill"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "photo_random_5",
-    "url": "https://picsum.photos/seed/vision5/400/300",
+    "url": "test_images/photo_random_5.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "diagram_class",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_class.png",
     "category": "diagram",
-    "expected_keywords": ["class", "object", "attribute"],
+    "expected_keywords": [
+      "class",
+      "object",
+      "attribute"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_doughnut",
-    "url": "https://quickchart.io/chart?c={type:'doughnut',data:{labels:['Desktop','Mobile','Tablet'],datasets:[{data:[60,30,10]}]}}",
+    "url": "test_images/chart_doughnut.png",
     "category": "chart",
-    "expected_keywords": ["doughnut", "chart", "device"],
+    "expected_keywords": [
+      "doughnut",
+      "chart",
+      "device"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "photo_random_6",
-    "url": "https://picsum.photos/seed/vision6/400/300",
+    "url": "test_images/photo_random_6.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_error",
-    "url": "https://http.cat/404.jpg",
+    "url": "test_images/screenshot_error.png",
     "category": "screenshot",
-    "expected_keywords": ["404", "error", "cat"],
+    "expected_keywords": [
+      "404",
+      "error",
+      "cat"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_network",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_network.png",
     "category": "diagram",
-    "expected_keywords": ["network", "node", "connection"],
+    "expected_keywords": [
+      "network",
+      "node",
+      "connection"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_7",
-    "url": "https://picsum.photos/seed/vision7/400/300",
+    "url": "test_images/photo_random_7.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_stacked_bar",
-    "url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['2022','2023','2024'],datasets:[{label:'Cloud',data:[100,150,200]},{label:'On-prem',data:[200,180,160]}]},options:{scales:{x:{stacked:true},y:{stacked:true}}}}",
+    "url": "test_images/chart_stacked_bar.png",
     "category": "chart",
-    "expected_keywords": ["stacked", "bar", "chart"],
+    "expected_keywords": [
+      "stacked",
+      "bar",
+      "chart"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "screenshot_dashboard",
-    "url": "https://github.githubassets.com/images/modules/site/features-code-search.png",
+    "url": "test_images/screenshot_dashboard.png",
     "category": "screenshot",
-    "expected_keywords": ["search", "code", "feature"],
+    "expected_keywords": [
+      "search",
+      "code",
+      "feature"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_8",
-    "url": "https://picsum.photos/seed/vision8/400/300",
+    "url": "test_images/photo_random_8.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   }
 ]

benchmarks/vision_benchmark.py

@@ -11,17 +11,19 @@ Usage:
 
     # Single image test
     python benchmarks/vision_benchmark.py --url https://example.com/image.png
+    python benchmarks/vision_benchmark.py --url benchmarks/test_images/photo_random_1.png
 
     # Generate test report
     python benchmarks/vision_benchmark.py --images benchmarks/test_images.json --output benchmarks/vision_results.json
 
-Test image dataset: benchmarks/test_images.json (50-100 diverse images)
+Test image dataset: benchmarks/test_images.json (committed local fixtures under benchmarks/test_images/)
 """
 
 import argparse
 import asyncio
 import base64
 import json
+import mimetypes
 import os
 import statistics
 import sys
@@ -67,6 +69,28 @@ EVAL_PROMPTS = {
 # ---------------------------------------------------------------------------
 
 
+def _is_remote_image_source(image_source: str) -> bool:
+    return image_source.startswith(("http://", "https://", "data:", "file://"))
+
+
+def _image_source_to_payload_url(image_source: str) -> str:
+    """Convert local image paths into data URLs; keep remote URLs unchanged."""
+    if image_source.startswith(("http://", "https://", "data:")):
+        return image_source
+
+    resolved = image_source[len("file://"):] if image_source.startswith("file://") else image_source
+    local_path = Path(os.path.expanduser(resolved)).resolve()
+    if not local_path.is_file():
+        return image_source
+
+    mime_type, _ = mimetypes.guess_type(str(local_path))
+    if not mime_type:
+        mime_type = "application/octet-stream"
+
+    encoded = base64.b64encode(local_path.read_bytes()).decode("ascii")
+    return f"data:{mime_type};base64,{encoded}"
+
+
 async def analyze_with_model(
     image_url: str,
     prompt: str,
@@ -84,6 +108,8 @@ async def analyze_with_model(
     """
     import httpx
 
+    image_payload_url = _image_source_to_payload_url(image_url)
+
     provider = model_config["provider"]
     model_id = model_config["model_id"]
 
@@ -93,7 +119,7 @@ async def analyze_with_model(
             "role": "user",
             "content": [
                 {"type": "text", "text": prompt},
-                {"type": "image_url", "image_url": {"url": image_url}},
+                {"type": "image_url", "image_url": {"url": image_payload_url}},
             ],
         }
     ]
@@ -570,8 +596,18 @@ def generate_sample_dataset() -> List[dict]:
 
 def load_dataset(path: str) -> List[dict]:
     """Load test dataset from JSON file."""
-    with open(path) as f:
-        return json.load(f)
+    dataset_path = Path(path).resolve()
+    with open(dataset_path) as f:
+        dataset = json.load(f)
+
+    base_dir = dataset_path.parent
+    for image in dataset:
+        image_url = image.get("url")
+        if not image_url or _is_remote_image_source(image_url):
+            continue
+        image["url"] = str((base_dir / image_url).resolve())
+
+    return dataset
 
 
 # ---------------------------------------------------------------------------
@@ -582,7 +618,7 @@ def load_dataset(path: str) -> List[dict]:
 async def main():
     parser = argparse.ArgumentParser(description="Vision Benchmark Suite (Issue #817)")
     parser.add_argument("--images", help="Path to test images JSON file")
-    parser.add_argument("--url", help="Single image URL to test")
+    parser.add_argument("--url", help="Single image URL or local file path to test")
     parser.add_argument("--category", default="photo", help="Category for single URL")
     parser.add_argument("--output", default=None, help="Output JSON file")
     parser.add_argument("--runs", type=int, default=1, help="Runs per model per image")

research_human_confirmation_firewall.md

@@ -1,515 +0,0 @@
-# Human Confirmation Firewall: Research Report
-## Implementation Patterns for Hermes Agent
-
-**Issue:** #878  
-**Parent:** #659  
-**Priority:** P0  
-**Scope:** Human-in-the-loop safety patterns for tool calls, crisis handling, and irreversible actions
-
----
-
-## Executive Summary
-
-Hermes already has a partial human confirmation firewall, but it is narrow.
-
-Current repo state shows:
-- a real **pre-execution gate** for dangerous terminal commands in `tools/approval.py`
-- a partial **confidence-threshold path** via `_smart_approve()` in `tools/approval.py`
-- gateway support for blocking approval resolution in `gateway/run.py`
-
-What is still missing is the core recommendation from this research issue:
-- **confidence scoring on all tool calls**, not just terminal commands that already matched a dangerous regex
-- a **hard pre-execution human gate for crisis interventions**, especially any action that would auto-respond to suicidal content
-- a consistent way to classify actions into:
-  1. pre-execution gate
-  2. post-execution review
-  3. confidence-threshold execution
-
-Recommendation:
-- use **Pattern 1: Pre-Execution Gate** for crisis interventions and irreversible/high-impact actions
-- use **Pattern 3: Confidence Threshold** for normal operations
-- reserve **Pattern 2: Post-Execution Review** only for low-risk and reversible actions
-
-The next implementation step should be a **tool-call risk assessment layer** that runs before dispatch in `model_tools.handle_function_call()`, assigns a score and pattern to every tool call, and routes only the highest-risk calls into mandatory human confirmation.
-
----
-
-## 1. The Three Proven Patterns
-
-### Pattern 1: Pre-Execution Gate
-
-Definition:
-- halt before execution
-- show the proposed action to the human
-- require explicit approval or denial
-
-Best for:
-- destructive actions
-- irreversible side effects
-- crisis interventions
-- actions that affect another human's safety, money, infrastructure, or private data
-
-Strengths:
-- strongest safety guarantee
-- simplest audit story
-- prevents the most catastrophic failure mode: acting first and apologizing later
-
-Weaknesses:
-- adds latency
-- creates operator burden if overused
-- should not be applied to every ordinary tool call
-
-### Pattern 2: Post-Execution Review
-
-Definition:
-- execute first
-- expose result to human
-- allow rollback or follow-up correction
-
-Best for:
-- reversible operations
-- low-risk actions with fast recovery
-- tasks where human review matters but immediate execution is acceptable
-
-Strengths:
-- low friction
-- fast iteration
-- useful when rollback is practical
-
-Weaknesses:
-- unsafe for crisis or destructive actions
-- only works when rollback actually exists
-- a poor fit for external communication or life-safety contexts
-
-### Pattern 3: Confidence Threshold
-
-Definition:
-- compute a risk/confidence score before execution
-- auto-execute high-confidence safe actions
-- request confirmation for lower-confidence or higher-risk actions
-
-Best for:
-- mixed-risk tool ecosystems
-- day-to-day operations where always-confirm would be too expensive
-- systems with a large volume of ordinary, safe reads and edits
-
-Strengths:
-- best balance of speed and safety
-- scales across many tool types
-- allows targeted human attention where it matters most
-
-Weaknesses:
-- depends on a good scoring model
-- weak scoring creates false negatives or unnecessary prompts
-- must remain inspectable and debuggable
-
----
-
-## 2. What Hermes Already Has
-
-## 2.1 Existing Pre-Execution Gate for Dangerous Terminal Commands
-
-`tools/approval.py` already implements a real pre-execution confirmation path for dangerous shell commands.
-
-Observed components:
-- `DANGEROUS_PATTERNS`
-- `detect_dangerous_command()`
-- `prompt_dangerous_approval()`
-- `check_dangerous_command()`
-- gateway queueing and resolution support in the same module
-
-This is already Pattern 1.
-
-Current behavior:
-- dangerous terminal commands are detected before execution
-- the user can allow once / session / always / deny
-- gateway sessions can block until approval resolves
-
-This is a strong foundation, but it is limited to a subset of terminal commands.
-
-## 2.2 Partial Confidence Threshold via Smart Approvals
-
-Hermes also already has a partial Pattern 3.
-
-Observed component:
-- `_smart_approve()` in `tools/approval.py`
-
-Current behavior:
-- only runs **after** a command has already been flagged by dangerous-pattern detection
-- uses the auxiliary LLM to decide:
-  - approve
-  - deny
-  - escalate
-
-This means Hermes has a confidence-threshold mechanism, but only for **already-flagged dangerous terminal commands**.
-
-What it does not yet do:
-- score all tool calls
-- classify non-terminal tools
-- distinguish crisis interventions from normal ops
-- produce a shared risk model across the tool surface
-
-## 2.3 Blocking Approval UX in Gateway
-
-`gateway/run.py` already routes `/approve` and `/deny` into the blocking approval path.
-
-This means the infrastructure for a true human confirmation firewall already exists in messaging contexts.
-
-That is important because the missing work is not "invent human approval from zero."
-The missing work is:
-- expand the scope from dangerous shell commands to **all tool calls that matter**
-- make the routing policy explicit and inspectable
-
----
-
-## 3. What Hermes Still Lacks
-
-## 3.1 No Universal Tool-Call Risk Assessment
-
-The current approval system is command-pattern-centric.
-It is not yet a tool-call firewall.
-
-Missing capability:
-- before dispatch, every tool call should receive a structured assessment:
-  - tool name
-  - side-effect class
-  - reversibility
-  - human-impact potential
-  - crisis relevance
-  - confidence score
-  - recommended confirmation pattern
-
-Natural insertion point:
-- `model_tools.handle_function_call()`
-
-That function already sits at the central dispatch boundary.
-It is the right place to add a pre-dispatch classifier.
-
-## 3.2 No Hard Crisis Gate for Outbound Intervention
-
-Issue #878 explicitly recommends:
-- Pattern 1 for crisis interventions
-- never auto-respond to suicidal content
-
-That recommendation is not yet codified as a global firewall rule.
-
-Missing rule:
-- if a tool call would directly intervene in a crisis context or send outward guidance in response to suicidal content, it must require explicit human confirmation before execution
-
-Examples that should hard-gate:
-- outbound `send_message` content aimed at a suicidal user
-- any future tool that places calls, escalates emergencies, or contacts third parties about a crisis
-- any autonomous action that claims a person should or should not take a life-safety step
-
-## 3.3 No First-Class Post-Execution Review Policy
-
-Hermes has approval and denial, but it does not yet have a formal policy for when Pattern 2 is acceptable.
-
-Without a policy, post-execution review tends to get used implicitly rather than intentionally.
-
-That is risky.
-
-Hermes should define Pattern 2 narrowly:
-- only for actions that are both low-risk and reversible
-- only when the system can show the human exactly what happened
-- never for crisis, finance, destructive config, or sensitive comms
-
----
-
-## 4. Recommended Architecture for Hermes
-
-## 4.1 Add a Tool-Call Assessment Layer
-
-Add a pre-dispatch assessment object for every tool call.
-
-Suggested shape:
-
-```python
-@dataclass
-class ToolCallAssessment:
-    tool_name: str
-    risk_score: float          # 0.0 to 1.0
-    confidence: float          # confidence in the assessment itself
-    pattern: str               # pre_execution_gate | post_execution_review | confidence_threshold
-    requires_human: bool
-    reasons: list[str]
-    reversible: bool
-    crisis_sensitive: bool
-```
-
-Suggested execution point:
-- inside `model_tools.handle_function_call()` before `orchestrator.dispatch()`
-
-Why here:
-- one place covers all tools
-- one place can emit traces
-- one place can remain model-agnostic
-- one place lets plugins observe or override the assessment
-
-## 4.2 Classify Tool Calls by Side-Effect Class
-
-Suggested first-pass taxonomy:
-
-### A. Read-only
-Examples:
-- `read_file`
-- `search_files`
-- `browser_snapshot`
-- `browser_console` read-only inspection
-
-Pattern:
-- confidence threshold
-- almost always auto-execute
-- human confirmation normally unnecessary
-
-### B. Local reversible edits
-Examples:
-- `patch`
-- `write_file`
-- `todo`
-
-Pattern:
-- confidence threshold
-- human confirmation only when risk score rises because of path sensitivity or scope breadth
-
-### C. External side effects
-Examples:
-- `send_message`
-- `cronjob`
-- `delegate_task`
-- smart-home actuation tools
-
-Pattern:
-- confidence threshold by default
-- pre-execution gate when score exceeds threshold or when context is sensitive
-
-### D. Critical / destructive / crisis-sensitive
-Examples:
-- dangerous `terminal`
-- financial actions
-- deletion / kill / restart / deployment in sensitive paths
-- outbound crisis intervention
-
-Pattern:
-- pre-execution gate
-- never auto-execute on confidence alone
-
-## 4.3 Crisis Override Rule
-
-Add a hard override:
-
-```text
-If tool call is crisis-sensitive AND outbound or irreversible:
-    requires_human = True
-    pattern = pre_execution_gate
-```
-
-This is the most important rule in the issue.
-
-The model may draft the message.
-The human must confirm before the system sends it.
-
-## 4.4 Use Confidence Threshold for Normal Ops
-
-For non-crisis operations, use Pattern 3.
-
-Suggested logic:
-- low risk + high assessment confidence -> auto-execute
-- medium risk or medium confidence -> ask human
-- high risk -> always ask human
-
-Key point:
-- confidence is not just "how sure the LLM is"
-- confidence should combine:
-  - tool type certainty
-  - argument clarity
-  - path sensitivity
-  - external side effects
-  - crisis indicators
-
----
-
-## 5. Recommended Initial Scoring Factors
-
-A simple initial scorer is enough.
-It does not need to be fancy.
-
-Suggested factors:
-
-### 5.1 Tool class risk
-- read-only tools: very low base risk
-- local mutation tools: moderate base risk
-- external communication / automation tools: higher base risk
-- shell execution: variable, often high
-
-### 5.2 Target sensitivity
-Examples:
-- `/tmp` or local scratch paths -> lower
-- repo files under git -> medium
-- system config, credentials, secrets, gateway lifecycle -> high
-- human-facing channels -> high if message content is sensitive
-
-### 5.3 Reversibility
-- reversible -> lower
-- difficult but possible to undo -> medium
-- practically irreversible -> high
-
-### 5.4 Human-impact content
-- no direct human impact -> low
-- administrative impact -> medium
-- crisis / safety / emotional intervention -> critical
-
-### 5.5 Context certainty
-- arguments are explicit and narrow -> higher confidence
-- arguments are vague, inferred, or broad -> lower confidence
-
----
-
-## 6. Implementation Plan
-
-## Phase 1: Assessment Without Behavior Change
-
-Goal:
-- score all tool calls
-- log assessment decisions
-- emit traces for review
-- do not yet block new tool categories
-
-Files to touch:
-- `tools/approval.py`
-- `model_tools.py`
-- tests for assessment coverage
-
-Output:
-- risk/confidence trace for every tool call
-- pattern recommendation for every tool call
-
-Why first:
-- lets us calibrate before changing runtime behavior
-- avoids breaking existing workflows blindly
-
-## Phase 2: Hard-Gate Crisis-Sensitive Outbound Actions
-
-Goal:
-- enforce Pattern 1 for crisis interventions
-
-Likely surfaces:
-- `send_message`
-- any future telephony / call / escalation tools
-- other tools with direct human intervention side effects
-
-Rule:
-- never auto-send crisis intervention content without human confirmation
-
-## Phase 3: General Confidence Threshold for Normal Ops
-
-Goal:
-- apply Pattern 3 to all tool calls
-- auto-run clearly safe actions
-- escalate ambiguous or medium-risk actions
-
-Likely thresholds:
-- score < 0.25 -> auto
-- 0.25 to 0.60 -> confirm if confidence is weak
-- > 0.60 -> confirm
-- crisis-sensitive -> always confirm
-
-## Phase 4: Optional Post-Execution Review Lane
-
-Goal:
-- allow Pattern 2 only for explicitly reversible operations
-
-Examples:
-- maybe low-risk messaging drafts saved locally
-- maybe reversible UI actions in specific environments
-
-Important:
-- this phase is optional
-- Hermes should not rely on Pattern 2 for safety-critical flows
-
----
-
-## 7. Verification Criteria for the Future Implementation
-
-The eventual implementation should prove all of the following:
-
-1. every tool call receives a scored assessment before dispatch
-2. crisis-sensitive outbound actions always require human confirmation
-3. dangerous terminal commands still preserve their current pre-execution gate
-4. clearly safe read-only tool calls are not slowed by unnecessary prompts
-5. assessment traces can be inspected after a run
-6. approval decisions remain session-safe across CLI and gateway contexts
-
----
-
-## 8. Concrete Recommendations
-
-### Recommendation 1
-Do **not** replace the current dangerous-command approval path.
-Generalize above it.
-
-Why:
-- existing terminal Pattern 1 already works
-- this is the strongest piece of the current firewall
-
-### Recommendation 2
-Add a universal scorer in `model_tools.handle_function_call()`.
-
-Why:
-- that is the first point where Hermes knows the tool name and structured arguments
-- it is the cleanest place to classify all tool calls uniformly
-
-### Recommendation 3
-Treat crisis-sensitive outbound intervention as a separate safety class.
-
-Why:
-- issue #878 explicitly calls for Pattern 1 here
-- this matches Timmy's SOUL-level safety requirements
-
-### Recommendation 4
-Ship scoring traces before enforcement expansion.
-
-Why:
-- you cannot tune thresholds you cannot inspect
-- false positives will otherwise frustrate normal usage
-
-### Recommendation 5
-Use Pattern 3 as the default policy for normal operations.
-
-Why:
-- full manual confirmation on every tool call is too expensive
-- full autonomy is too risky
-- Pattern 3 is the practical middle ground
-
----
-
-## 9. Bottom Line
-
-Hermes should implement a **two-track human confirmation firewall**:
-
-1. **Pattern 1: Pre-Execution Gate**
-   - crisis interventions
-   - destructive terminal actions
-   - irreversible or safety-critical tool calls
-
-2. **Pattern 3: Confidence Threshold**
-   - all ordinary tool calls
-   - driven by a universal tool-call assessment layer
-   - integrated at the central dispatch boundary
-
-Pattern 2 should remain optional and narrow.
-It is not the primary answer for Hermes.
-
-The repo already contains the beginnings of this system.
-The next step is not new theory.
-It is to turn the existing approval path into a true **tool-call-wide human confirmation firewall**.
-
----
-
-## References
-
-- Issue #878 — Human Confirmation Firewall Implementation Patterns
-- Issue #659 — Critical Research Tasks
-- `tools/approval.py` — current dangerous-command approval flow and smart approvals
-- `model_tools.py` — central tool dispatch boundary
-- `gateway/run.py` — blocking approval handling for messaging sessions

tests/test_vision_benchmark.py

@@ -11,12 +11,14 @@ import pytest
 sys.path.insert(0, str(Path(__file__).parent.parent / "benchmarks"))
 
 from vision_benchmark import (
+    analyze_with_model,
     compute_ocr_accuracy,
     compute_description_completeness,
     compute_structural_accuracy,
     aggregate_results,
     to_markdown,
     generate_sample_dataset,
+    load_dataset,
     MODELS,
     EVAL_PROMPTS,
 )
@@ -197,6 +199,71 @@ class TestMarkdown:
 
 
 class TestDataset:
+    def test_repo_dataset_uses_local_image_paths(self):
+        dataset_path = Path(__file__).parent.parent / "benchmarks" / "test_images.json"
+        dataset = json.loads(dataset_path.read_text())
+
+        assert dataset, "benchmark dataset should not be empty"
+        assert all(not entry["url"].startswith(("http://", "https://")) for entry in dataset)
+
+    def test_load_dataset_resolves_relative_local_paths(self, tmp_path):
+        images_dir = tmp_path / "images"
+        images_dir.mkdir()
+        image_path = images_dir / "sample.png"
+        image_path.write_bytes(b"png-bytes")
+
+        dataset_path = tmp_path / "dataset.json"
+        dataset_path.write_text(json.dumps([
+            {
+                "id": "sample",
+                "url": "images/sample.png",
+                "category": "photo",
+                "expected_keywords": [],
+                "expected_structure": {"min_length": 30, "min_sentences": 1},
+            }
+        ]))
+
+        loaded = load_dataset(str(dataset_path))
+
+        assert loaded[0]["url"] == str(image_path.resolve())
+
+    @pytest.mark.asyncio
+    async def test_analyze_with_model_encodes_local_file_as_data_url(self, tmp_path, monkeypatch):
+        image_path = tmp_path / "tiny.png"
+        image_path.write_bytes(
+            bytes.fromhex(
+                "89504E470D0A1A0A"
+                "0000000D49484452000000010000000108060000001F15C489"
+                "0000000D49444154789C6360000002000154A24F5D00000000"
+                "49454E44AE426082"
+            )
+        )
+
+        fake_response = MagicMock()
+        fake_response.raise_for_status.return_value = None
+        fake_response.json.return_value = {
+            "choices": [{"message": {"content": "Looks like a tiny image."}}],
+            "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
+        }
+
+        fake_client = MagicMock()
+        fake_client.post = AsyncMock(return_value=fake_response)
+        fake_ctx = MagicMock()
+        fake_ctx.__aenter__ = AsyncMock(return_value=fake_client)
+        fake_ctx.__aexit__ = AsyncMock(return_value=None)
+
+        monkeypatch.setenv("OPENROUTER_API_KEY", "test-key")
+        with patch("httpx.AsyncClient", return_value=fake_ctx):
+            result = await analyze_with_model(
+                str(image_path),
+                "Describe this image",
+                {"provider": "openrouter", "model_id": "fake/model"},
+            )
+
+        assert result["success"] is True
+        sent_url = fake_client.post.await_args.kwargs["json"]["messages"][0]["content"][1]["image_url"]["url"]
+        assert sent_url.startswith("data:image/png;base64,")
+
     def test_sample_dataset_has_entries(self):
         dataset = generate_sample_dataset()
         assert len(dataset) >= 4