fix: vendor vision benchmark fixtures (#868)

benchmarks/test_images.json

@@ -1,194 +1,354 @@
 [
   {
     "id": "screenshot_github_home",
-    "url": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png",
+    "url": "test_images/screenshot_github_home.png",
     "category": "screenshot",
-    "expected_keywords": ["github", "logo", "mark"],
+    "expected_keywords": [
+      "github",
+      "logo",
+      "mark"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "diagram_mermaid_flow",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6siSZXVhjQTlgl1nigHg5fRBOzSfebopROCu_cytObSfgLSE1ANOeZWkO2IH5upZxYot8m1hqAdpD_63WRl0xdUG1jdl9kPiOb_EWk2JBtPaiKkF4eVIYgO0EtkW-RSgC4gJ6HJYRG1UNdN0HNVd0Bftjj7X8P92qPj-F8l8T3w",
+    "url": "test_images/diagram_mermaid_flow.png",
     "category": "diagram",
-    "expected_keywords": ["flow", "diagram", "process"],
+    "expected_keywords": [
+      "flow",
+      "diagram",
+      "process"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_1",
-    "url": "https://picsum.photos/seed/vision1/400/300",
+    "url": "test_images/photo_random_1.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_2",
-    "url": "https://picsum.photos/seed/vision2/400/300",
+    "url": "test_images/photo_random_2.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_simple_bar",
-    "url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['Q1','Q2','Q3','Q4'],datasets:[{label:'Revenue',data:[100,150,200,250]}]}}",
+    "url": "test_images/chart_simple_bar.png",
     "category": "chart",
-    "expected_keywords": ["bar", "chart", "revenue"],
+    "expected_keywords": [
+      "bar",
+      "chart",
+      "revenue"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "chart_pie",
-    "url": "https://quickchart.io/chart?c={type:'pie',data:{labels:['A','B','C'],datasets:[{data:[30,50,20]}]}}",
+    "url": "test_images/chart_pie.png",
     "category": "chart",
-    "expected_keywords": ["pie", "chart", "percentage"],
+    "expected_keywords": [
+      "pie",
+      "chart",
+      "percentage"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_org_chart",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_org_chart.png",
     "category": "diagram",
-    "expected_keywords": ["organization", "hierarchy", "chart"],
+    "expected_keywords": [
+      "organization",
+      "hierarchy",
+      "chart"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_terminal",
-    "url": "https://raw.githubusercontent.com/nicehash/nicehash-quick-start/main/images/nicehash-terminal.png",
+    "url": "test_images/screenshot_terminal.png",
     "category": "screenshot",
-    "expected_keywords": ["terminal", "command", "output"],
+    "expected_keywords": [
+      "terminal",
+      "command",
+      "output"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_3",
-    "url": "https://picsum.photos/seed/vision3/400/300",
+    "url": "test_images/photo_random_3.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_line",
-    "url": "https://quickchart.io/chart?c={type:'line',data:{labels:['Jan','Feb','Mar','Apr'],datasets:[{label:'Temperature',data:[5,8,12,18]}]}}",
+    "url": "test_images/chart_line.png",
     "category": "chart",
-    "expected_keywords": ["line", "chart", "temperature"],
+    "expected_keywords": [
+      "line",
+      "chart",
+      "temperature"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_sequence",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_sequence.png",
     "category": "diagram",
-    "expected_keywords": ["sequence", "interaction", "message"],
+    "expected_keywords": [
+      "sequence",
+      "interaction",
+      "message"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_4",
-    "url": "https://picsum.photos/seed/vision4/400/300",
+    "url": "test_images/photo_random_4.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_webpage",
-    "url": "https://github.githubassets.com/images/modules/site/social-cards.png",
+    "url": "test_images/screenshot_webpage.png",
     "category": "screenshot",
-    "expected_keywords": ["github", "page", "web"],
+    "expected_keywords": [
+      "github",
+      "page",
+      "web"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_radar",
-    "url": "https://quickchart.io/chart?c={type:'radar',data:{labels:['Speed','Power','Defense','Magic'],datasets:[{label:'Hero',data:[80,60,70,90]}]}}",
+    "url": "test_images/chart_radar.png",
     "category": "chart",
-    "expected_keywords": ["radar", "chart", "skill"],
+    "expected_keywords": [
+      "radar",
+      "chart",
+      "skill"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "photo_random_5",
-    "url": "https://picsum.photos/seed/vision5/400/300",
+    "url": "test_images/photo_random_5.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "diagram_class",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_class.png",
     "category": "diagram",
-    "expected_keywords": ["class", "object", "attribute"],
+    "expected_keywords": [
+      "class",
+      "object",
+      "attribute"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_doughnut",
-    "url": "https://quickchart.io/chart?c={type:'doughnut',data:{labels:['Desktop','Mobile','Tablet'],datasets:[{data:[60,30,10]}]}}",
+    "url": "test_images/chart_doughnut.png",
     "category": "chart",
-    "expected_keywords": ["doughnut", "chart", "device"],
+    "expected_keywords": [
+      "doughnut",
+      "chart",
+      "device"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "photo_random_6",
-    "url": "https://picsum.photos/seed/vision6/400/300",
+    "url": "test_images/photo_random_6.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "screenshot_error",
-    "url": "https://http.cat/404.jpg",
+    "url": "test_images/screenshot_error.png",
     "category": "screenshot",
-    "expected_keywords": ["404", "error", "cat"],
+    "expected_keywords": [
+      "404",
+      "error",
+      "cat"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": true
+    }
   },
   {
     "id": "diagram_network",
-    "url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
+    "url": "test_images/diagram_network.png",
     "category": "diagram",
-    "expected_keywords": ["network", "node", "connection"],
+    "expected_keywords": [
+      "network",
+      "node",
+      "connection"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_7",
-    "url": "https://picsum.photos/seed/vision7/400/300",
+    "url": "test_images/photo_random_7.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "chart_stacked_bar",
-    "url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['2022','2023','2024'],datasets:[{label:'Cloud',data:[100,150,200]},{label:'On-prem',data:[200,180,160]}]},options:{scales:{x:{stacked:true},y:{stacked:true}}}}",
+    "url": "test_images/chart_stacked_bar.png",
     "category": "chart",
-    "expected_keywords": ["stacked", "bar", "chart"],
+    "expected_keywords": [
+      "stacked",
+      "bar",
+      "chart"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
+    "expected_structure": {
+      "min_length": 50,
+      "min_sentences": 2,
+      "has_numbers": true
+    }
   },
   {
     "id": "screenshot_dashboard",
-    "url": "https://github.githubassets.com/images/modules/site/features-code-search.png",
+    "url": "test_images/screenshot_dashboard.png",
     "category": "screenshot",
-    "expected_keywords": ["search", "code", "feature"],
+    "expected_keywords": [
+      "search",
+      "code",
+      "feature"
+    ],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   },
   {
     "id": "photo_random_8",
-    "url": "https://picsum.photos/seed/vision8/400/300",
+    "url": "test_images/photo_random_8.png",
     "category": "photo",
     "expected_keywords": [],
     "ground_truth_ocr": "",
-    "expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
+    "expected_structure": {
+      "min_length": 30,
+      "min_sentences": 1,
+      "has_numbers": false
+    }
   }
 ]

benchmarks/vision_benchmark.py

@@ -11,17 +11,19 @@ Usage:
 
     # Single image test
     python benchmarks/vision_benchmark.py --url https://example.com/image.png
+    python benchmarks/vision_benchmark.py --url benchmarks/test_images/photo_random_1.png
 
     # Generate test report
     python benchmarks/vision_benchmark.py --images benchmarks/test_images.json --output benchmarks/vision_results.json
 
-Test image dataset: benchmarks/test_images.json (50-100 diverse images)
+Test image dataset: benchmarks/test_images.json (committed local fixtures under benchmarks/test_images/)
 """
 
 import argparse
 import asyncio
 import base64
 import json
+import mimetypes
 import os
 import statistics
 import sys
@@ -67,6 +69,28 @@ EVAL_PROMPTS = {
 # ---------------------------------------------------------------------------
 
 
+def _is_remote_image_source(image_source: str) -> bool:
+    return image_source.startswith(("http://", "https://", "data:", "file://"))
+
+
+def _image_source_to_payload_url(image_source: str) -> str:
+    """Convert local image paths into data URLs; keep remote URLs unchanged."""
+    if image_source.startswith(("http://", "https://", "data:")):
+        return image_source
+
+    resolved = image_source[len("file://"):] if image_source.startswith("file://") else image_source
+    local_path = Path(os.path.expanduser(resolved)).resolve()
+    if not local_path.is_file():
+        return image_source
+
+    mime_type, _ = mimetypes.guess_type(str(local_path))
+    if not mime_type:
+        mime_type = "application/octet-stream"
+
+    encoded = base64.b64encode(local_path.read_bytes()).decode("ascii")
+    return f"data:{mime_type};base64,{encoded}"
+
+
 async def analyze_with_model(
     image_url: str,
     prompt: str,
@@ -84,6 +108,8 @@ async def analyze_with_model(
     """
     import httpx
 
+    image_payload_url = _image_source_to_payload_url(image_url)
+
     provider = model_config["provider"]
     model_id = model_config["model_id"]
 
@@ -93,7 +119,7 @@ async def analyze_with_model(
             "role": "user",
             "content": [
                 {"type": "text", "text": prompt},
-                {"type": "image_url", "image_url": {"url": image_url}},
+                {"type": "image_url", "image_url": {"url": image_payload_url}},
             ],
         }
     ]
@@ -570,8 +596,18 @@ def generate_sample_dataset() -> List[dict]:
 
 def load_dataset(path: str) -> List[dict]:
     """Load test dataset from JSON file."""
-    with open(path) as f:
-        return json.load(f)
+    dataset_path = Path(path).resolve()
+    with open(dataset_path) as f:
+        dataset = json.load(f)
+
+    base_dir = dataset_path.parent
+    for image in dataset:
+        image_url = image.get("url")
+        if not image_url or _is_remote_image_source(image_url):
+            continue
+        image["url"] = str((base_dir / image_url).resolve())
+
+    return dataset
 
 
 # ---------------------------------------------------------------------------
@@ -582,7 +618,7 @@ def load_dataset(path: str) -> List[dict]:
 async def main():
     parser = argparse.ArgumentParser(description="Vision Benchmark Suite (Issue #817)")
     parser.add_argument("--images", help="Path to test images JSON file")
-    parser.add_argument("--url", help="Single image URL to test")
+    parser.add_argument("--url", help="Single image URL or local file path to test")
     parser.add_argument("--category", default="photo", help="Category for single URL")
     parser.add_argument("--output", default=None, help="Output JSON file")
     parser.add_argument("--runs", type=int, default=1, help="Runs per model per image")

tests/test_vision_benchmark.py

@@ -11,12 +11,14 @@ import pytest
 sys.path.insert(0, str(Path(__file__).parent.parent / "benchmarks"))
 
 from vision_benchmark import (
+    analyze_with_model,
     compute_ocr_accuracy,
     compute_description_completeness,
     compute_structural_accuracy,
     aggregate_results,
     to_markdown,
     generate_sample_dataset,
+    load_dataset,
     MODELS,
     EVAL_PROMPTS,
 )
@@ -197,6 +199,71 @@ class TestMarkdown:
 
 
 class TestDataset:
+    def test_repo_dataset_uses_local_image_paths(self):
+        dataset_path = Path(__file__).parent.parent / "benchmarks" / "test_images.json"
+        dataset = json.loads(dataset_path.read_text())
+
+        assert dataset, "benchmark dataset should not be empty"
+        assert all(not entry["url"].startswith(("http://", "https://")) for entry in dataset)
+
+    def test_load_dataset_resolves_relative_local_paths(self, tmp_path):
+        images_dir = tmp_path / "images"
+        images_dir.mkdir()
+        image_path = images_dir / "sample.png"
+        image_path.write_bytes(b"png-bytes")
+
+        dataset_path = tmp_path / "dataset.json"
+        dataset_path.write_text(json.dumps([
+            {
+                "id": "sample",
+                "url": "images/sample.png",
+                "category": "photo",
+                "expected_keywords": [],
+                "expected_structure": {"min_length": 30, "min_sentences": 1},
+            }
+        ]))
+
+        loaded = load_dataset(str(dataset_path))
+
+        assert loaded[0]["url"] == str(image_path.resolve())
+
+    @pytest.mark.asyncio
+    async def test_analyze_with_model_encodes_local_file_as_data_url(self, tmp_path, monkeypatch):
+        image_path = tmp_path / "tiny.png"
+        image_path.write_bytes(
+            bytes.fromhex(
+                "89504E470D0A1A0A"
+                "0000000D49484452000000010000000108060000001F15C489"
+                "0000000D49444154789C6360000002000154A24F5D00000000"
+                "49454E44AE426082"
+            )
+        )
+
+        fake_response = MagicMock()
+        fake_response.raise_for_status.return_value = None
+        fake_response.json.return_value = {
+            "choices": [{"message": {"content": "Looks like a tiny image."}}],
+            "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
+        }
+
+        fake_client = MagicMock()
+        fake_client.post = AsyncMock(return_value=fake_response)
+        fake_ctx = MagicMock()
+        fake_ctx.__aenter__ = AsyncMock(return_value=fake_client)
+        fake_ctx.__aexit__ = AsyncMock(return_value=None)
+
+        monkeypatch.setenv("OPENROUTER_API_KEY", "test-key")
+        with patch("httpx.AsyncClient", return_value=fake_ctx):
+            result = await analyze_with_model(
+                str(image_path),
+                "Describe this image",
+                {"provider": "openrouter", "model_id": "fake/model"},
+            )
+
+        assert result["success"] is True
+        sent_url = fake_client.post.await_args.kwargs["json"]["messages"][0]["content"][1]["image_url"]["url"]
+        assert sent_url.startswith("data:image/png;base64,")
+
     def test_sample_dataset_has_entries(self):
         dataset = generate_sample_dataset()
         assert len(dataset) >= 4

tests/tools/test_fuzzy_match.py

@@ -148,184 +148,3 @@ class TestStrategyNameSurfaced:
         assert count == 0
         assert strategy is None
         assert err is not None
-
-
-class TestEscapeDriftGuard:
-    """Tests for the escape-drift guard that catches bash/JSON serialization
-    artifacts where an apostrophe gets prefixed with a spurious backslash
-    in tool-call transport.
-    """
-
-    def test_drift_blocked_apostrophe(self):
-        """File has ', old_string and new_string both have \\' — classic
-        tool-call drift. Guard must block with a helpful error instead of
-        writing \\' literals into source code."""
-        content = "x = \"hello there\"\n"
-        # Simulate transport-corrupted old_string and new_string where an
-        # apostrophe-like context got prefixed with a backslash. The content
-        # itself has no apostrophe, but both strings do — matching via
-        # whitespace/anchor strategies would otherwise succeed.
-        old_string = "x = \"hello there\" # don\\'t edit\n"
-        new_string = "x = \"hi there\" # don\\'t edit\n"
-        # This particular pair won't match anything, so it exits via
-        # no-match path. Build a case where a non-exact strategy DOES match.
-        content = "line\n    x = 1\nline"
-        old_string = "line\n  x = \\'a\\'\nline"
-        new_string = "line\n  x = \\'b\\'\nline"
-        new, count, strategy, err = fuzzy_find_and_replace(content, old_string, new_string)
-        assert count == 0
-        assert err is not None and "Escape-drift" in err
-        assert "backslash" in err.lower()
-        assert new == content  # file untouched
-
-    def test_drift_blocked_double_quote(self):
-        """Same idea but with \\" drift instead of \\'."""
-        content = 'line\n    x = 1\nline'
-        old_string = 'line\n  x = \\"a\\"\nline'
-        new_string = 'line\n  x = \\"b\\"\nline'
-        new, count, strategy, err = fuzzy_find_and_replace(content, old_string, new_string)
-        assert count == 0
-        assert err is not None and "Escape-drift" in err
-
-    def test_drift_allowed_when_file_genuinely_has_backslash_escapes(self):
-        """If the file already contains \\' (e.g. inside an existing escaped
-        string), the model is legitimately preserving it. Guard must NOT
-        fire."""
-        content = "line\n  x = \\'a\\'\nline"
-        old_string = "line\n  x = \\'a\\'\nline"
-        new_string = "line\n  x = \\'b\\'\nline"
-        new, count, strategy, err = fuzzy_find_and_replace(content, old_string, new_string)
-        assert err is None
-        assert count == 1
-        assert "\\'b\\'" in new
-
-    def test_drift_allowed_on_exact_match(self):
-        """Exact matches bypass the drift guard entirely — if the file
-        really contains the exact bytes old_string specified, it's not
-        drift."""
-        content = "hello \\'world\\'"
-        new, count, strategy, err = fuzzy_find_and_replace(
-            content, "hello \\'world\\'", "hello \\'there\\'"
-        )
-        assert err is None
-        assert count == 1
-        assert strategy == "exact"
-
-    def test_drift_allowed_when_adding_escaped_strings(self):
-        """Model is adding new content with \\' that wasn't in the original.
-        old_string has no \\', so guard doesn't fire."""
-        content = "line1\nline2\nline3"
-        old_string = "line1\nline2\nline3"
-        new_string = "line1\nprint(\\'added\\')\nline2\nline3"
-        new, count, strategy, err = fuzzy_find_and_replace(content, old_string, new_string)
-        assert err is None
-        assert count == 1
-        assert "\\'added\\'" in new
-
-    def test_no_drift_check_when_new_string_lacks_suspect_chars(self):
-        """Fast-path: if new_string has no \\' or \\", guard must not
-        fire even on fuzzy match."""
-        content = "def foo():\n    pass"  # extra space ignored by line_trimmed
-        old_string = "def foo():\n  pass"
-        new_string = "def bar():\n  return 1"
-        new, count, strategy, err = fuzzy_find_and_replace(content, old_string, new_string)
-        assert err is None
-        assert count == 1
-
-
-class TestFindClosestLines:
-    def setup_method(self):
-        from tools.fuzzy_match import find_closest_lines
-        self.find_closest_lines = find_closest_lines
-
-    def test_finds_similar_line(self):
-        content = "def foo():\n    pass\ndef bar():\n    return 1\n"
-        result = self.find_closest_lines("def baz():", content)
-        assert "def foo" in result or "def bar" in result
-
-    def test_returns_empty_for_no_match(self):
-        content = "completely different content here"
-        result = self.find_closest_lines("xyzzy_no_match_possible_!!!", content)
-        assert result == ""
-
-    def test_returns_empty_for_empty_inputs(self):
-        assert self.find_closest_lines("", "some content") == ""
-        assert self.find_closest_lines("old string", "") == ""
-
-    def test_includes_context_lines(self):
-        content = "line1\nline2\ndef target():\n    pass\nline5\n"
-        result = self.find_closest_lines("def target():", content)
-        assert "target" in result
-
-    def test_includes_line_numbers(self):
-        content = "line1\nline2\ndef foo():\n    pass\n"
-        result = self.find_closest_lines("def foo():", content)
-        # Should include line numbers in format "N| content"
-        assert "|" in result
-
-
-class TestFormatNoMatchHint:
-    """Gating tests for format_no_match_hint — the shared helper that decides
-    whether a 'Did you mean?' snippet should be appended to an error.
-    """
-
-    def setup_method(self):
-        from tools.fuzzy_match import format_no_match_hint
-        self.fmt = format_no_match_hint
-
-    def test_fires_on_could_not_find_with_match(self):
-        """Classic no-match: similar content exists → hint fires."""
-        content = "def foo():\n    pass\ndef bar():\n    pass\n"
-        result = self.fmt(
-            "Could not find a match for old_string in the file",
-            0, "def baz():", content,
-        )
-        assert "Did you mean" in result
-        assert "foo" in result or "bar" in result
-
-    def test_silent_on_ambiguous_match_error(self):
-        """'Found N matches' is not a missing-match failure — no hint."""
-        content = "aaa bbb aaa\n"
-        result = self.fmt(
-            "Found 2 matches for old_string. Provide more context to make it unique, or use replace_all=True.",
-            0, "aaa", content,
-        )
-        assert result == ""
-
-    def test_silent_on_escape_drift_error(self):
-        """Escape-drift errors are intentional blocks — hint would mislead."""
-        content = "x = 1\n"
-        result = self.fmt(
-            "Escape-drift detected: old_string and new_string contain the literal sequence '\\\\''...",
-            0, "x = \\'1\\'", content,
-        )
-        assert result == ""
-
-    def test_silent_on_identical_strings(self):
-        """old_string == new_string — hint irrelevant."""
-        result = self.fmt(
-            "old_string and new_string are identical",
-            0, "foo", "foo bar\n",
-        )
-        assert result == ""
-
-    def test_silent_when_match_count_nonzero(self):
-        """If match succeeded, we shouldn't be in the error path — defense in depth."""
-        result = self.fmt(
-            "Could not find a match for old_string in the file",
-            1, "foo", "foo bar\n",
-        )
-        assert result == ""
-
-    def test_silent_on_none_error(self):
-        """No error at all — no hint."""
-        result = self.fmt(None, 0, "foo", "bar\n")
-        assert result == ""
-
-    def test_silent_when_no_similar_content(self):
-        """Even for a valid no-match error, skip hint when nothing similar exists."""
-        result = self.fmt(
-            "Could not find a match for old_string in the file",
-            0, "totally_unique_xyzzy_qux", "abc\nxyz\n",
-        )
-        assert result == ""

tests/tools/test_patch_did_you_mean.py

@@ -1,114 +0,0 @@
-import json
-import os
-import textwrap
-from pathlib import Path
-
-import tools.skill_manager_tool as skill_manager_tool
-from tools.file_tools import patch_tool
-from tools.skill_manager_tool import _create_skill, _patch_skill
-
-
-def _disable_patch_tool_guards(monkeypatch):
-    monkeypatch.setattr("tools.file_tools._check_sensitive_path", lambda _path: None)
-    monkeypatch.setattr("tools.file_tools._check_file_staleness", lambda _path, _task_id: None)
-    monkeypatch.setattr("tools.file_tools._log_and_check_conflict", lambda _path, _task_id, _action: None)
-
-
-def test_patch_tool_replace_no_match_shows_rich_hint_without_legacy_hint(tmp_path, monkeypatch):
-    _disable_patch_tool_guards(monkeypatch)
-    sample = tmp_path / "sample.py"
-    sample.write_text("def foo():\n    return 1\n\ndef bar():\n    return 2\n", encoding="utf-8")
-
-    raw = patch_tool(
-        mode="replace",
-        path=str(sample),
-        old_string="def barycentric():",
-        new_string="def barycentric_new():",
-        task_id="qa960-replace-rich-hint",
-    )
-
-    result = json.loads(raw)
-    assert result["success"] is False
-    assert "Could not find a match" in result["error"]
-    assert "Did you mean one of these sections?" in result["error"]
-    assert "def bar():" in result["error"] or "def foo():" in result["error"]
-    assert "[Hint:" not in raw
-
-
-def test_patch_tool_replace_ambiguous_error_does_not_show_did_you_mean(tmp_path, monkeypatch):
-    _disable_patch_tool_guards(monkeypatch)
-    sample = tmp_path / "sample.py"
-    sample.write_text("aaa\nbbb\naaa\n", encoding="utf-8")
-
-    raw = patch_tool(
-        mode="replace",
-        path=str(sample),
-        old_string="aaa",
-        new_string="ccc",
-        task_id="qa960-replace-ambiguous",
-    )
-
-    result = json.loads(raw)
-    assert result["success"] is False
-    assert "Found 2 matches" in result["error"]
-    assert "Did you mean one of these sections?" not in result["error"]
-    assert "[Hint:" not in raw
-
-
-def test_patch_tool_v4a_no_match_shows_rich_hint(tmp_path, monkeypatch):
-    _disable_patch_tool_guards(monkeypatch)
-    sample = tmp_path / "sample.py"
-    sample.write_text("def foo():\n    return 1\n", encoding="utf-8")
-
-    patch = textwrap.dedent(
-        f"""\
-        *** Begin Patch
-        *** Update File: {sample}
-        @@
-        -def barycentric():
-        +def barycentric_new():
-        *** End Patch
-        """
-    )
-
-    raw = patch_tool(mode="patch", patch=patch, task_id="qa960-v4a-rich-hint")
-    result = json.loads(raw)
-    assert result["success"] is False
-    assert "Patch validation failed" in result["error"]
-    assert "Did you mean one of these sections?" in result["error"]
-    assert "def foo():" in result["error"]
-
-
-def test_skill_patch_no_match_shows_rich_hint(tmp_path, monkeypatch):
-    monkeypatch.setenv("HERMES_HOME", str(tmp_path))
-    skills_dir = tmp_path / "skills"
-    skills_dir.mkdir(parents=True, exist_ok=True)
-    monkeypatch.setattr(skill_manager_tool, "SKILLS_DIR", skills_dir)
-    monkeypatch.setattr(skill_manager_tool, "_security_scan_skill", lambda _skill_dir: None)
-
-    _create_skill(
-        "qa-skill",
-        textwrap.dedent(
-            """\
-            ---
-            name: qa-skill
-            description: test
-            ---
-
-            Step 1: Do the thing.
-            Step 2: Verify the thing.
-            """
-        ),
-    )
-
-    result = _patch_skill(
-        "qa-skill",
-        "Step 1: Do the production rollout.",
-        "Step 1: Updated.",
-    )
-
-    assert result["success"] is False
-    assert "Could not find a match" in result["error"]
-    assert "Did you mean one of these sections?" in result["error"]
-    assert "Step 1: Do the thing." in result["error"]
-    assert "file_preview" in result

tools/file_operations.py

@@ -757,14 +757,12 @@ class ShellFileOperations(FileOperations):
             content, old_string, new_string, replace_all
         )
         
-        if error or match_count == 0:
-            err_msg = error or f"Could not find match for old_string in {path}"
-            try:
-                from tools.fuzzy_match import format_no_match_hint
-                err_msg += format_no_match_hint(err_msg, match_count, old_string, content)
-            except Exception:
-                pass
-            return PatchResult(error=err_msg)
+        if error:
+            return PatchResult(error=error)
+        
+        if match_count == 0:
+            return PatchResult(error=f"Could not find match for old_string in {path}")
+        
         # Write back
         write_result = self.write_file(path, new_content)
         if write_result.error:

tools/file_tools.py

@@ -8,7 +8,6 @@ import os
 import threading
 import time
 from pathlib import Path
-from typing import Any, Dict, Optional
 from tools.binary_extensions import has_binary_extension
 from tools.file_operations import ShellFileOperations
 from agent.redact import redact_sensitive_text
@@ -691,11 +690,8 @@ def patch_tool(mode: str = "replace", path: str = None, old_string: str = None,
         result_json = json.dumps(result_dict, ensure_ascii=False)
         # Hint when old_string not found — saves iterations where the agent
         # retries with stale content instead of re-reading the file.
-        # Suppressed when patch_replace already attached a rich "Did you mean?"
-        # snippet (which is strictly more useful than the generic hint).
         if result_dict.get("error") and "Could not find" in str(result_dict["error"]):
-            if "Did you mean one of these sections?" not in str(result_dict["error"]):
-                result_json += "\n\n[Hint: old_string not found. Use read_file to verify the current content, or search_files to locate the text.]"
+            result_json += "\n\n[Hint: old_string not found. Use read_file to verify the current content, or search_files to locate the text.]"
         return result_json
     except Exception as e:
         return tool_error(str(e))

tools/fuzzy_match.py

@@ -93,21 +93,6 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
                     f"Provide more context to make it unique, or use replace_all=True."
                 )
 
-            # Escape-drift guard: when the matched strategy is NOT `exact`,
-            # we matched via some form of normalization. If new_string
-            # contains shell/JSON-style escape sequences (\\' or \\") that
-            # would be written literally into the file but the matched
-            # region of the file has no such sequences, this is almost
-            # certainly tool-call serialization drift — the model typed
-            # an apostrophe/quote and the transport added a stray
-            # backslash. Writing new_string as-is would corrupt the file.
-            # Block with a helpful error so the model re-reads and retries
-            # instead of the caller silently persisting garbage (or not).
-            if strategy_name != "exact":
-                drift_err = _detect_escape_drift(content, matches, old_string, new_string)
-                if drift_err:
-                    return content, 0, None, drift_err
-
             # Perform replacement
             new_content = _apply_replacements(content, matches, new_string)
             return new_content, len(matches), strategy_name, None
@@ -116,46 +101,6 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
     return content, 0, None, "Could not find a match for old_string in the file"
 
 
-def _detect_escape_drift(content: str, matches: List[Tuple[int, int]],
-                         old_string: str, new_string: str) -> Optional[str]:
-    """Detect tool-call escape-drift artifacts in new_string.
-
-    Looks for ``\\'`` or ``\\"`` sequences that are present in both
-    old_string and new_string (i.e. the model copy-pasted them as "context"
-    it intended to preserve) but don't exist in the matched region of the
-    file. That pattern indicates the transport layer inserted spurious
-    shell-style escapes around apostrophes or quotes — writing new_string
-    verbatim would literally insert ``\\'`` into source code.
-
-    Returns an error string if drift is detected, None otherwise.
-    """
-    # Cheap pre-check: bail out unless new_string actually contains a
-    # suspect escape sequence. This keeps the guard free for all the
-    # common, correct cases.
-    if "\\'" not in new_string and '\\"' not in new_string:
-        return None
-
-    # Aggregate matched regions of the file — that's what new_string will
-    # replace. If the suspect escapes are present there already, the
-    # model is genuinely preserving them (valid for some languages /
-    # escaped strings); accept the patch.
-    matched_regions = "".join(content[start:end] for start, end in matches)
-
-    for suspect in ("\\'", '\\"'):
-        if suspect in new_string and suspect in old_string and suspect not in matched_regions:
-            plain = suspect[1]  # "'" or '"'
-            return (
-                f"Escape-drift detected: old_string and new_string contain "
-                f"the literal sequence {suspect!r} but the matched region of "
-                f"the file does not. This is almost always a tool-call "
-                f"serialization artifact where an apostrophe or quote got "
-                f"prefixed with a spurious backslash. Re-read the file with "
-                f"read_file and pass old_string/new_string without "
-                f"backslash-escaping {plain!r} characters."
-            )
-    return None
-
-
 def _apply_replacements(content: str, matches: List[Tuple[int, int]], new_string: str) -> str:
     """
     Apply replacements at the given positions.
@@ -619,86 +564,3 @@ def _map_normalized_positions(original: str, normalized: str,
         original_matches.append((orig_start, min(orig_end, len(original))))
     
     return original_matches
-
-
-def find_closest_lines(old_string: str, content: str, context_lines: int = 2, max_results: int = 3) -> str:
-    """Find lines in content most similar to old_string for "did you mean?" feedback.
-
-    Returns a formatted string showing the closest matching lines with context,
-    or empty string if no useful match is found.
-    """
-    if not old_string or not content:
-        return ""
-
-    old_lines = old_string.splitlines()
-    content_lines = content.splitlines()
-
-    if not old_lines or not content_lines:
-        return ""
-
-    # Use first line of old_string as anchor for search
-    anchor = old_lines[0].strip()
-    if not anchor:
-        # Try second line if first is blank
-        candidates = [l.strip() for l in old_lines if l.strip()]
-        if not candidates:
-            return ""
-        anchor = candidates[0]
-
-    # Score each line in content by similarity to anchor
-    scored = []
-    for i, line in enumerate(content_lines):
-        stripped = line.strip()
-        if not stripped:
-            continue
-        ratio = SequenceMatcher(None, anchor, stripped).ratio()
-        if ratio > 0.3:
-            scored.append((ratio, i))
-
-    if not scored:
-        return ""
-
-    # Take top matches
-    scored.sort(key=lambda x: -x[0])
-    top = scored[:max_results]
-
-    parts = []
-    seen_ranges = set()
-    for _, line_idx in top:
-        start = max(0, line_idx - context_lines)
-        end = min(len(content_lines), line_idx + len(old_lines) + context_lines)
-        key = (start, end)
-        if key in seen_ranges:
-            continue
-        seen_ranges.add(key)
-        snippet = "\n".join(
-            f"{start + j + 1:4d}| {content_lines[start + j]}"
-            for j in range(end - start)
-        )
-        parts.append(snippet)
-
-    if not parts:
-        return ""
-
-    return "\n---\n".join(parts)
-
-
-def format_no_match_hint(error: Optional[str], match_count: int,
-                         old_string: str, content: str) -> str:
-    """Return a '\\n\\nDid you mean...' snippet for plain no-match errors.
-
-    Gated so the hint only fires for actual "old_string not found" failures.
-    Ambiguous-match ("Found N matches"), escape-drift, and identical-strings
-    errors all have ``match_count == 0`` but a "did you mean?" snippet would
-    be misleading — those failed for unrelated reasons.
-
-    Returns an empty string when there's nothing useful to append.
-    """
-    if match_count != 0:
-        return ""
-    if not error or not error.startswith("Could not find"):
-        return ""
-    hint = find_closest_lines(old_string, content)
-    if not hint:
-        return ""
-    return "\n\nDid you mean one of these sections?\n" + hint

tools/patch_parser.py

@@ -290,16 +290,10 @@ def _validate_operations(
                 )
                 if count == 0:
                     label = f"'{hunk.context_hint}'" if hunk.context_hint else "(no hint)"
-                    msg = (
+                    errors.append(
                         f"{op.file_path}: hunk {label} not found"
                         + (f" — {match_error}" if match_error else "")
                     )
-                    try:
-                        from tools.fuzzy_match import format_no_match_hint
-                        msg += format_no_match_hint(match_error, count, search_pattern, simulated)
-                    except Exception:
-                        pass
-                    errors.append(msg)
                 else:
                     # Advance simulation so subsequent hunks validate correctly.
                     # Reuse the result from the call above — no second fuzzy run.
@@ -543,13 +537,7 @@ def _apply_update(op: PatchOperation, file_ops: Any) -> Tuple[bool, str]:
                             error = None
                 
                 if error:
-                    err_msg = f"Could not apply hunk: {error}"
-                    try:
-                        from tools.fuzzy_match import format_no_match_hint
-                        err_msg += format_no_match_hint(error, 0, search_pattern, new_content)
-                    except Exception:
-                        pass
-                    return False, err_msg
+                    return False, f"Could not apply hunk: {error}"
         else:
             # Addition-only hunk (no context or removed lines).
             # Insert at the location indicated by the context hint, or at end of file.

tools/skill_manager_tool.py

@@ -575,15 +575,9 @@ def _patch_skill(
     if match_error:
         # Show a short preview of the file so the model can self-correct
         preview = content[:500] + ("..." if len(content) > 500 else "")
-        err_msg = match_error
-        try:
-            from tools.fuzzy_match import format_no_match_hint
-            err_msg += format_no_match_hint(match_error, match_count, old_string, content)
-        except Exception:
-            pass
         return {
             "success": False,
-            "error": err_msg,
+            "error": match_error,
             "file_preview": preview,
         }