Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5c821bda3c | ||
|
|
b87c1a5b74 |
@@ -1,354 +1,194 @@
|
||||
[
|
||||
{
|
||||
"id": "screenshot_github_home",
|
||||
"url": "test_images/screenshot_github_home.png",
|
||||
"url": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png",
|
||||
"category": "screenshot",
|
||||
"expected_keywords": [
|
||||
"github",
|
||||
"logo",
|
||||
"mark"
|
||||
],
|
||||
"expected_keywords": ["github", "logo", "mark"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "diagram_mermaid_flow",
|
||||
"url": "test_images/diagram_mermaid_flow.png",
|
||||
"url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6siSZXVhjQTlgl1nigHg5fRBOzSfebopROCu_cytObSfgLSE1ANOeZWkO2IH5upZxYot8m1hqAdpD_63WRl0xdUG1jdl9kPiOb_EWk2JBtPaiKkF4eVIYgO0EtkW-RSgC4gJ6HJYRG1UNdN0HNVd0Bftjj7X8P92qPj-F8l8T3w",
|
||||
"category": "diagram",
|
||||
"expected_keywords": [
|
||||
"flow",
|
||||
"diagram",
|
||||
"process"
|
||||
],
|
||||
"expected_keywords": ["flow", "diagram", "process"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "photo_random_1",
|
||||
"url": "test_images/photo_random_1.png",
|
||||
"url": "https://picsum.photos/seed/vision1/400/300",
|
||||
"category": "photo",
|
||||
"expected_keywords": [],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "photo_random_2",
|
||||
"url": "test_images/photo_random_2.png",
|
||||
"url": "https://picsum.photos/seed/vision2/400/300",
|
||||
"category": "photo",
|
||||
"expected_keywords": [],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "chart_simple_bar",
|
||||
"url": "test_images/chart_simple_bar.png",
|
||||
"url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['Q1','Q2','Q3','Q4'],datasets:[{label:'Revenue',data:[100,150,200,250]}]}}",
|
||||
"category": "chart",
|
||||
"expected_keywords": [
|
||||
"bar",
|
||||
"chart",
|
||||
"revenue"
|
||||
],
|
||||
"expected_keywords": ["bar", "chart", "revenue"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": true
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
||||
},
|
||||
{
|
||||
"id": "chart_pie",
|
||||
"url": "test_images/chart_pie.png",
|
||||
"url": "https://quickchart.io/chart?c={type:'pie',data:{labels:['A','B','C'],datasets:[{data:[30,50,20]}]}}",
|
||||
"category": "chart",
|
||||
"expected_keywords": [
|
||||
"pie",
|
||||
"chart",
|
||||
"percentage"
|
||||
],
|
||||
"expected_keywords": ["pie", "chart", "percentage"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": true
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
||||
},
|
||||
{
|
||||
"id": "diagram_org_chart",
|
||||
"url": "test_images/diagram_org_chart.png",
|
||||
"url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
|
||||
"category": "diagram",
|
||||
"expected_keywords": [
|
||||
"organization",
|
||||
"hierarchy",
|
||||
"chart"
|
||||
],
|
||||
"expected_keywords": ["organization", "hierarchy", "chart"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "screenshot_terminal",
|
||||
"url": "test_images/screenshot_terminal.png",
|
||||
"url": "https://raw.githubusercontent.com/nicehash/nicehash-quick-start/main/images/nicehash-terminal.png",
|
||||
"category": "screenshot",
|
||||
"expected_keywords": [
|
||||
"terminal",
|
||||
"command",
|
||||
"output"
|
||||
],
|
||||
"expected_keywords": ["terminal", "command", "output"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "photo_random_3",
|
||||
"url": "test_images/photo_random_3.png",
|
||||
"url": "https://picsum.photos/seed/vision3/400/300",
|
||||
"category": "photo",
|
||||
"expected_keywords": [],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "chart_line",
|
||||
"url": "test_images/chart_line.png",
|
||||
"url": "https://quickchart.io/chart?c={type:'line',data:{labels:['Jan','Feb','Mar','Apr'],datasets:[{label:'Temperature',data:[5,8,12,18]}]}}",
|
||||
"category": "chart",
|
||||
"expected_keywords": [
|
||||
"line",
|
||||
"chart",
|
||||
"temperature"
|
||||
],
|
||||
"expected_keywords": ["line", "chart", "temperature"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": true
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
||||
},
|
||||
{
|
||||
"id": "diagram_sequence",
|
||||
"url": "test_images/diagram_sequence.png",
|
||||
"url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
|
||||
"category": "diagram",
|
||||
"expected_keywords": [
|
||||
"sequence",
|
||||
"interaction",
|
||||
"message"
|
||||
],
|
||||
"expected_keywords": ["sequence", "interaction", "message"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "photo_random_4",
|
||||
"url": "test_images/photo_random_4.png",
|
||||
"url": "https://picsum.photos/seed/vision4/400/300",
|
||||
"category": "photo",
|
||||
"expected_keywords": [],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "screenshot_webpage",
|
||||
"url": "test_images/screenshot_webpage.png",
|
||||
"url": "https://github.githubassets.com/images/modules/site/social-cards.png",
|
||||
"category": "screenshot",
|
||||
"expected_keywords": [
|
||||
"github",
|
||||
"page",
|
||||
"web"
|
||||
],
|
||||
"expected_keywords": ["github", "page", "web"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "chart_radar",
|
||||
"url": "test_images/chart_radar.png",
|
||||
"url": "https://quickchart.io/chart?c={type:'radar',data:{labels:['Speed','Power','Defense','Magic'],datasets:[{label:'Hero',data:[80,60,70,90]}]}}",
|
||||
"category": "chart",
|
||||
"expected_keywords": [
|
||||
"radar",
|
||||
"chart",
|
||||
"skill"
|
||||
],
|
||||
"expected_keywords": ["radar", "chart", "skill"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": true
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
||||
},
|
||||
{
|
||||
"id": "photo_random_5",
|
||||
"url": "test_images/photo_random_5.png",
|
||||
"url": "https://picsum.photos/seed/vision5/400/300",
|
||||
"category": "photo",
|
||||
"expected_keywords": [],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "diagram_class",
|
||||
"url": "test_images/diagram_class.png",
|
||||
"url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
|
||||
"category": "diagram",
|
||||
"expected_keywords": [
|
||||
"class",
|
||||
"object",
|
||||
"attribute"
|
||||
],
|
||||
"expected_keywords": ["class", "object", "attribute"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "chart_doughnut",
|
||||
"url": "test_images/chart_doughnut.png",
|
||||
"url": "https://quickchart.io/chart?c={type:'doughnut',data:{labels:['Desktop','Mobile','Tablet'],datasets:[{data:[60,30,10]}]}}",
|
||||
"category": "chart",
|
||||
"expected_keywords": [
|
||||
"doughnut",
|
||||
"chart",
|
||||
"device"
|
||||
],
|
||||
"expected_keywords": ["doughnut", "chart", "device"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": true
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
||||
},
|
||||
{
|
||||
"id": "photo_random_6",
|
||||
"url": "test_images/photo_random_6.png",
|
||||
"url": "https://picsum.photos/seed/vision6/400/300",
|
||||
"category": "photo",
|
||||
"expected_keywords": [],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "screenshot_error",
|
||||
"url": "test_images/screenshot_error.png",
|
||||
"url": "https://http.cat/404.jpg",
|
||||
"category": "screenshot",
|
||||
"expected_keywords": [
|
||||
"404",
|
||||
"error",
|
||||
"cat"
|
||||
],
|
||||
"expected_keywords": ["404", "error", "cat"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": true
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": true}
|
||||
},
|
||||
{
|
||||
"id": "diagram_network",
|
||||
"url": "test_images/diagram_network.png",
|
||||
"url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
|
||||
"category": "diagram",
|
||||
"expected_keywords": [
|
||||
"network",
|
||||
"node",
|
||||
"connection"
|
||||
],
|
||||
"expected_keywords": ["network", "node", "connection"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "photo_random_7",
|
||||
"url": "test_images/photo_random_7.png",
|
||||
"url": "https://picsum.photos/seed/vision7/400/300",
|
||||
"category": "photo",
|
||||
"expected_keywords": [],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "chart_stacked_bar",
|
||||
"url": "test_images/chart_stacked_bar.png",
|
||||
"url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['2022','2023','2024'],datasets:[{label:'Cloud',data:[100,150,200]},{label:'On-prem',data:[200,180,160]}]},options:{scales:{x:{stacked:true},y:{stacked:true}}}}",
|
||||
"category": "chart",
|
||||
"expected_keywords": [
|
||||
"stacked",
|
||||
"bar",
|
||||
"chart"
|
||||
],
|
||||
"expected_keywords": ["stacked", "bar", "chart"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 50,
|
||||
"min_sentences": 2,
|
||||
"has_numbers": true
|
||||
}
|
||||
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
||||
},
|
||||
{
|
||||
"id": "screenshot_dashboard",
|
||||
"url": "test_images/screenshot_dashboard.png",
|
||||
"url": "https://github.githubassets.com/images/modules/site/features-code-search.png",
|
||||
"category": "screenshot",
|
||||
"expected_keywords": [
|
||||
"search",
|
||||
"code",
|
||||
"feature"
|
||||
],
|
||||
"expected_keywords": ["search", "code", "feature"],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
},
|
||||
{
|
||||
"id": "photo_random_8",
|
||||
"url": "test_images/photo_random_8.png",
|
||||
"url": "https://picsum.photos/seed/vision8/400/300",
|
||||
"category": "photo",
|
||||
"expected_keywords": [],
|
||||
"ground_truth_ocr": "",
|
||||
"expected_structure": {
|
||||
"min_length": 30,
|
||||
"min_sentences": 1,
|
||||
"has_numbers": false
|
||||
}
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
||||
}
|
||||
]
|
||||
|
||||
|
Before Width: | Height: | Size: 4.4 KiB |
|
Before Width: | Height: | Size: 4.1 KiB |
|
Before Width: | Height: | Size: 4.0 KiB |
|
Before Width: | Height: | Size: 3.5 KiB |
|
Before Width: | Height: | Size: 4.2 KiB |
|
Before Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 4.6 KiB |
|
Before Width: | Height: | Size: 4.8 KiB |
|
Before Width: | Height: | Size: 5.0 KiB |
|
Before Width: | Height: | Size: 5.1 KiB |
|
Before Width: | Height: | Size: 5.2 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 2.9 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 3.0 KiB |
|
Before Width: | Height: | Size: 7.1 KiB |
|
Before Width: | Height: | Size: 6.2 KiB |
|
Before Width: | Height: | Size: 7.1 KiB |
|
Before Width: | Height: | Size: 7.1 KiB |
|
Before Width: | Height: | Size: 7.2 KiB |
@@ -11,19 +11,17 @@ Usage:
|
||||
|
||||
# Single image test
|
||||
python benchmarks/vision_benchmark.py --url https://example.com/image.png
|
||||
python benchmarks/vision_benchmark.py --url benchmarks/test_images/photo_random_1.png
|
||||
|
||||
# Generate test report
|
||||
python benchmarks/vision_benchmark.py --images benchmarks/test_images.json --output benchmarks/vision_results.json
|
||||
|
||||
Test image dataset: benchmarks/test_images.json (committed local fixtures under benchmarks/test_images/)
|
||||
Test image dataset: benchmarks/test_images.json (50-100 diverse images)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import statistics
|
||||
import sys
|
||||
@@ -69,28 +67,6 @@ EVAL_PROMPTS = {
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _is_remote_image_source(image_source: str) -> bool:
|
||||
return image_source.startswith(("http://", "https://", "data:", "file://"))
|
||||
|
||||
|
||||
def _image_source_to_payload_url(image_source: str) -> str:
|
||||
"""Convert local image paths into data URLs; keep remote URLs unchanged."""
|
||||
if image_source.startswith(("http://", "https://", "data:")):
|
||||
return image_source
|
||||
|
||||
resolved = image_source[len("file://"):] if image_source.startswith("file://") else image_source
|
||||
local_path = Path(os.path.expanduser(resolved)).resolve()
|
||||
if not local_path.is_file():
|
||||
return image_source
|
||||
|
||||
mime_type, _ = mimetypes.guess_type(str(local_path))
|
||||
if not mime_type:
|
||||
mime_type = "application/octet-stream"
|
||||
|
||||
encoded = base64.b64encode(local_path.read_bytes()).decode("ascii")
|
||||
return f"data:{mime_type};base64,{encoded}"
|
||||
|
||||
|
||||
async def analyze_with_model(
|
||||
image_url: str,
|
||||
prompt: str,
|
||||
@@ -108,8 +84,6 @@ async def analyze_with_model(
|
||||
"""
|
||||
import httpx
|
||||
|
||||
image_payload_url = _image_source_to_payload_url(image_url)
|
||||
|
||||
provider = model_config["provider"]
|
||||
model_id = model_config["model_id"]
|
||||
|
||||
@@ -119,7 +93,7 @@ async def analyze_with_model(
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image_url", "image_url": {"url": image_payload_url}},
|
||||
{"type": "image_url", "image_url": {"url": image_url}},
|
||||
],
|
||||
}
|
||||
]
|
||||
@@ -596,18 +570,8 @@ def generate_sample_dataset() -> List[dict]:
|
||||
|
||||
def load_dataset(path: str) -> List[dict]:
|
||||
"""Load test dataset from JSON file."""
|
||||
dataset_path = Path(path).resolve()
|
||||
with open(dataset_path) as f:
|
||||
dataset = json.load(f)
|
||||
|
||||
base_dir = dataset_path.parent
|
||||
for image in dataset:
|
||||
image_url = image.get("url")
|
||||
if not image_url or _is_remote_image_source(image_url):
|
||||
continue
|
||||
image["url"] = str((base_dir / image_url).resolve())
|
||||
|
||||
return dataset
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -618,7 +582,7 @@ def load_dataset(path: str) -> List[dict]:
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(description="Vision Benchmark Suite (Issue #817)")
|
||||
parser.add_argument("--images", help="Path to test images JSON file")
|
||||
parser.add_argument("--url", help="Single image URL or local file path to test")
|
||||
parser.add_argument("--url", help="Single image URL to test")
|
||||
parser.add_argument("--category", default="photo", help="Category for single URL")
|
||||
parser.add_argument("--output", default=None, help="Output JSON file")
|
||||
parser.add_argument("--runs", type=int, default=1, help="Runs per model per image")
|
||||
|
||||
100
docs/issue-954-verification.md
Normal file
@@ -0,0 +1,100 @@
|
||||
# Issue #954 Verification — maps skill guest_house / camp_site / bakery
|
||||
|
||||
Status: PASS
|
||||
|
||||
## Drift noted
|
||||
|
||||
Issue #954 asked for validation on `upstream/main` (commit `c5a814b23`).
|
||||
Fresh `forge/main` did not contain `skills/productivity/maps/`, so the forge branch was behind upstream for this feature cluster.
|
||||
This branch ports the upstream maps skill files into the forge checkout and adds regression coverage.
|
||||
|
||||
## Automated verification
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
pytest -q tests/skills/test_maps_client.py
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
- 5 passed
|
||||
|
||||
Coverage added:
|
||||
|
||||
- maps skill files exist in the repo
|
||||
- `guest_house` category maps to `tourism=guest_house`
|
||||
- `camp_site` category maps to `tourism=camp_site`
|
||||
- `bakery` expands to both `shop=bakery` and `amenity=bakery`
|
||||
- dual-key bakery results dedupe correctly
|
||||
- skill documentation lists the new categories and supersedes `find-nearby`
|
||||
|
||||
## Manual evidence
|
||||
|
||||
### 1) guest_house lookup
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
python3 skills/productivity/maps/scripts/maps_client.py nearby --near "Bath, United Kingdom" --category guest_house --limit 3
|
||||
```
|
||||
|
||||
Observed results:
|
||||
|
||||
- Henrietta House — 390.3 m
|
||||
- The Windsor — 437.2 m
|
||||
- The Old Rectory Bed & Breakfast — 495.7 m
|
||||
|
||||
All returned `tourism=guest_house` in the raw tags.
|
||||
|
||||
### 2) camp_site lookup
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
python3 skills/productivity/maps/scripts/maps_client.py nearby --near "Yosemite Valley, California" --category camp_site --limit 5
|
||||
```
|
||||
|
||||
Observed result:
|
||||
|
||||
- Yellow Pine Administrative Campground — 90.3 m
|
||||
|
||||
Returned `tourism=camp_site` in the raw tags.
|
||||
|
||||
### 3) bakery lookup via `shop=bakery`
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
python3 skills/productivity/maps/scripts/maps_client.py nearby --near "Lawrenceville, New Jersey" --category bakery --radius 5000 --limit 10
|
||||
```
|
||||
|
||||
Observed results:
|
||||
|
||||
- The Gingered Peach — 713.8 m
|
||||
- WildFlour Bakery — 741.9 m
|
||||
|
||||
Both returned `shop=bakery` in the raw tags.
|
||||
|
||||
### 4) bakery lookup via `amenity=bakery`
|
||||
|
||||
Command:
|
||||
|
||||
```bash
|
||||
python3 skills/productivity/maps/scripts/maps_client.py nearby --near "20735 Stevens Creek Boulevard, Cupertino, CA" --category bakery --radius 600 --limit 5
|
||||
```
|
||||
|
||||
Observed result:
|
||||
|
||||
- Paris Baguette — 28.6 m
|
||||
|
||||
Returned `amenity=bakery` in the raw tags (and also includes `shop=bakery`), proving the dual-key union query reaches amenity-tagged bakeries too.
|
||||
|
||||
## Conclusion
|
||||
|
||||
PASS.
|
||||
|
||||
- `guest_house` resolves correctly
|
||||
- `camp_site` resolves correctly
|
||||
- `bakery` resolves through both supported keys
|
||||
- forge/main drift from upstream/main was real and is addressed on this branch
|
||||
199
skills/productivity/maps/SKILL.md
Normal file
@@ -0,0 +1,199 @@
|
||||
---
|
||||
name: maps
|
||||
description: >
|
||||
Location intelligence — geocode a place, reverse-geocode coordinates,
|
||||
find nearby places (46 POI categories), driving/walking/cycling
|
||||
distance + time, turn-by-turn directions, timezone lookup, bounding
|
||||
box + area for a named place, and POI search within a rectangle.
|
||||
Uses OpenStreetMap + Overpass + OSRM. Free, no API key.
|
||||
version: 1.2.0
|
||||
author: Mibayy
|
||||
license: MIT
|
||||
metadata:
|
||||
hermes:
|
||||
tags: [maps, geocoding, places, routing, distance, directions, nearby, location, openstreetmap, nominatim, overpass, osrm]
|
||||
category: productivity
|
||||
requires_toolsets: [terminal]
|
||||
supersedes: [find-nearby]
|
||||
---
|
||||
|
||||
# Maps Skill
|
||||
|
||||
Location intelligence using free, open data sources. 8 commands, 44 POI
|
||||
categories, zero dependencies (Python stdlib only), no API key required.
|
||||
|
||||
Data sources: OpenStreetMap/Nominatim, Overpass API, OSRM, TimeAPI.io.
|
||||
|
||||
This skill supersedes the old `find-nearby` skill — all of find-nearby's
|
||||
functionality is covered by the `nearby` command below, with the same
|
||||
`--near "<place>"` shortcut and multi-category support.
|
||||
|
||||
## When to Use
|
||||
|
||||
- User sends a Telegram location pin (latitude/longitude in the message) → `nearby`
|
||||
- User wants coordinates for a place name → `search`
|
||||
- User has coordinates and wants the address → `reverse`
|
||||
- User asks for nearby restaurants, hospitals, pharmacies, hotels, etc. → `nearby`
|
||||
- User wants driving/walking/cycling distance or travel time → `distance`
|
||||
- User wants turn-by-turn directions between two places → `directions`
|
||||
- User wants timezone information for a location → `timezone`
|
||||
- User wants to search for POIs within a geographic area → `area` + `bbox`
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Python 3.8+ (stdlib only — no pip installs needed).
|
||||
|
||||
Script path: `~/.hermes/skills/maps/scripts/maps_client.py`
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
MAPS=~/.hermes/skills/maps/scripts/maps_client.py
|
||||
```
|
||||
|
||||
### search — Geocode a place name
|
||||
|
||||
```bash
|
||||
python3 $MAPS search "Eiffel Tower"
|
||||
python3 $MAPS search "1600 Pennsylvania Ave, Washington DC"
|
||||
```
|
||||
|
||||
Returns: lat, lon, display name, type, bounding box, importance score.
|
||||
|
||||
### reverse — Coordinates to address
|
||||
|
||||
```bash
|
||||
python3 $MAPS reverse 48.8584 2.2945
|
||||
```
|
||||
|
||||
Returns: full address breakdown (street, city, state, country, postcode).
|
||||
|
||||
### nearby — Find places by category
|
||||
|
||||
```bash
|
||||
# By coordinates (from a Telegram location pin, for example)
|
||||
python3 $MAPS nearby 48.8584 2.2945 restaurant --limit 10
|
||||
python3 $MAPS nearby 40.7128 -74.0060 hospital --radius 2000
|
||||
|
||||
# By address / city / zip / landmark — --near auto-geocodes
|
||||
python3 $MAPS nearby --near "Times Square, New York" --category cafe
|
||||
python3 $MAPS nearby --near "90210" --category pharmacy
|
||||
|
||||
# Multiple categories merged into one query
|
||||
python3 $MAPS nearby --near "downtown austin" --category restaurant --category bar --limit 10
|
||||
```
|
||||
|
||||
46 categories: restaurant, cafe, bar, hospital, pharmacy, hotel, guest_house,
|
||||
camp_site, supermarket, atm, gas_station, parking, museum, park, school,
|
||||
university, bank, police, fire_station, library, airport, train_station,
|
||||
bus_stop, church, mosque, synagogue, dentist, doctor, cinema, theatre, gym,
|
||||
swimming_pool, post_office, convenience_store, bakery, bookshop, laundry,
|
||||
car_wash, car_rental, bicycle_rental, taxi, veterinary, zoo, playground,
|
||||
stadium, nightclub.
|
||||
|
||||
Each result includes: `name`, `address`, `lat`/`lon`, `distance_m`,
|
||||
`maps_url` (clickable Google Maps link), `directions_url` (Google Maps
|
||||
directions from the search point), and promoted tags when available —
|
||||
`cuisine`, `hours` (opening_hours), `phone`, `website`.
|
||||
|
||||
### distance — Travel distance and time
|
||||
|
||||
```bash
|
||||
python3 $MAPS distance "Paris" --to "Lyon"
|
||||
python3 $MAPS distance "New York" --to "Boston" --mode driving
|
||||
python3 $MAPS distance "Big Ben" --to "Tower Bridge" --mode walking
|
||||
```
|
||||
|
||||
Modes: driving (default), walking, cycling. Returns road distance, duration,
|
||||
and straight-line distance for comparison.
|
||||
|
||||
### directions — Turn-by-turn navigation
|
||||
|
||||
```bash
|
||||
python3 $MAPS directions "Eiffel Tower" --to "Louvre Museum" --mode walking
|
||||
python3 $MAPS directions "JFK Airport" --to "Times Square" --mode driving
|
||||
```
|
||||
|
||||
Returns numbered steps with instruction, distance, duration, road name, and
|
||||
maneuver type (turn, depart, arrive, etc.).
|
||||
|
||||
### timezone — Timezone for coordinates
|
||||
|
||||
```bash
|
||||
python3 $MAPS timezone 48.8584 2.2945
|
||||
python3 $MAPS timezone 35.6762 139.6503
|
||||
```
|
||||
|
||||
Returns timezone name, UTC offset, and current local time.
|
||||
|
||||
### area — Bounding box and area for a place
|
||||
|
||||
```bash
|
||||
python3 $MAPS area "Manhattan, New York"
|
||||
python3 $MAPS area "London"
|
||||
```
|
||||
|
||||
Returns bounding box coordinates, width/height in km, and approximate area.
|
||||
Useful as input for the bbox command.
|
||||
|
||||
### bbox — Search within a bounding box
|
||||
|
||||
```bash
|
||||
python3 $MAPS bbox 40.75 -74.00 40.77 -73.98 restaurant --limit 20
|
||||
```
|
||||
|
||||
Finds POIs within a geographic rectangle. Use `area` first to get the
|
||||
bounding box coordinates for a named place.
|
||||
|
||||
## Working With Telegram Location Pins
|
||||
|
||||
When a user sends a location pin, the message contains `latitude:` and
|
||||
`longitude:` fields. Extract those and pass them straight to `nearby`:
|
||||
|
||||
```bash
|
||||
# User sent a pin at 36.17, -115.14 and asked "find cafes nearby"
|
||||
python3 $MAPS nearby 36.17 -115.14 cafe --radius 1500
|
||||
```
|
||||
|
||||
Present results as a numbered list with names, distances, and the
|
||||
`maps_url` field so the user gets a tap-to-open link in chat. For "open
|
||||
now?" questions, check the `hours` field; if missing or unclear, verify
|
||||
with `web_search` since OSM hours are community-maintained and not always
|
||||
current.
|
||||
|
||||
## Workflow Examples
|
||||
|
||||
**"Find Italian restaurants near the Colosseum":**
|
||||
1. `nearby --near "Colosseum Rome" --category restaurant --radius 500`
|
||||
— one command, auto-geocoded
|
||||
|
||||
**"What's near this location pin they sent?":**
|
||||
1. Extract lat/lon from the Telegram message
|
||||
2. `nearby LAT LON cafe --radius 1500`
|
||||
|
||||
**"How do I walk from hotel to conference center?":**
|
||||
1. `directions "Hotel Name" --to "Conference Center" --mode walking`
|
||||
|
||||
**"What restaurants are in downtown Seattle?":**
|
||||
1. `area "Downtown Seattle"` → get bounding box
|
||||
2. `bbox S W N E restaurant --limit 30`
|
||||
|
||||
## Pitfalls
|
||||
|
||||
- Nominatim ToS: max 1 req/s (handled automatically by the script)
|
||||
- `nearby` requires lat/lon OR `--near "<address>"` — one of the two is needed
|
||||
- OSRM routing coverage is best for Europe and North America
|
||||
- Overpass API can be slow during peak hours; the script automatically
|
||||
falls back between mirrors (overpass-api.de → overpass.kumi.systems)
|
||||
- `distance` and `directions` use `--to` flag for the destination (not positional)
|
||||
- If a zip code alone gives ambiguous results globally, include country/state
|
||||
|
||||
## Verification
|
||||
|
||||
```bash
|
||||
python3 ~/.hermes/skills/maps/scripts/maps_client.py search "Statue of Liberty"
|
||||
# Should return lat ~40.689, lon ~-74.044
|
||||
|
||||
python3 ~/.hermes/skills/maps/scripts/maps_client.py nearby --near "Times Square" --category restaurant --limit 3
|
||||
# Should return a list of restaurants within ~500m of Times Square
|
||||
```
|
||||
1293
skills/productivity/maps/scripts/maps_client.py
Normal file
135
tests/skills/test_maps_client.py
Normal file
@@ -0,0 +1,135 @@
|
||||
"""Regression tests for the bundled maps skill."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
SCRIPT_PATH = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "skills/productivity/maps/scripts/maps_client.py"
|
||||
)
|
||||
SKILL_PATH = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "skills/productivity/maps/SKILL.md"
|
||||
)
|
||||
|
||||
|
||||
def load_module():
|
||||
assert SCRIPT_PATH.exists(), f"missing maps client script: {SCRIPT_PATH}"
|
||||
spec = importlib.util.spec_from_file_location("maps_client_test", SCRIPT_PATH)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader is not None
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_maps_skill_files_exist():
|
||||
assert SCRIPT_PATH.exists()
|
||||
assert SKILL_PATH.exists()
|
||||
|
||||
|
||||
def test_category_tags_cover_guest_house_camp_site_and_dual_key_bakery():
|
||||
module = load_module()
|
||||
|
||||
assert module.CATEGORY_TAGS["guest_house"] == ("tourism", "guest_house")
|
||||
assert module.CATEGORY_TAGS["camp_site"] == ("tourism", "camp_site")
|
||||
assert module.CATEGORY_TAGS["bakery"] == [
|
||||
("shop", "bakery"),
|
||||
("amenity", "bakery"),
|
||||
]
|
||||
assert module._tags_for("bakery") == [
|
||||
("shop", "bakery"),
|
||||
("amenity", "bakery"),
|
||||
]
|
||||
|
||||
|
||||
def test_build_overpass_queries_include_all_supported_tags():
|
||||
module = load_module()
|
||||
|
||||
bakery_query = module.build_overpass_nearby(
|
||||
None,
|
||||
None,
|
||||
40.0,
|
||||
-74.0,
|
||||
500,
|
||||
10,
|
||||
tag_pairs=module._tags_for("bakery"),
|
||||
)
|
||||
assert 'node["shop"="bakery"]' in bakery_query
|
||||
assert 'way["shop"="bakery"]' in bakery_query
|
||||
assert 'node["amenity"="bakery"]' in bakery_query
|
||||
assert 'way["amenity"="bakery"]' in bakery_query
|
||||
|
||||
guest_house_query = module.build_overpass_nearby(
|
||||
None,
|
||||
None,
|
||||
40.0,
|
||||
-74.0,
|
||||
500,
|
||||
10,
|
||||
tag_pairs=module._tags_for("guest_house"),
|
||||
)
|
||||
assert 'node["tourism"="guest_house"]' in guest_house_query
|
||||
assert 'way["tourism"="guest_house"]' in guest_house_query
|
||||
|
||||
camp_site_bbox = module.build_overpass_bbox(
|
||||
None,
|
||||
None,
|
||||
39.0,
|
||||
-75.0,
|
||||
41.0,
|
||||
-73.0,
|
||||
10,
|
||||
tag_pairs=module._tags_for("camp_site"),
|
||||
)
|
||||
assert 'node["tourism"="camp_site"]' in camp_site_bbox
|
||||
assert 'way["tourism"="camp_site"]' in camp_site_bbox
|
||||
|
||||
|
||||
def test_cmd_nearby_dedupes_dual_tag_bakery_results(monkeypatch, capsys):
|
||||
module = load_module()
|
||||
|
||||
duplicate_bakery = {
|
||||
"elements": [
|
||||
{
|
||||
"type": "node",
|
||||
"id": 101,
|
||||
"lat": 40.0,
|
||||
"lon": -74.0,
|
||||
"tags": {"name": "Wild Flour", "shop": "bakery"},
|
||||
},
|
||||
{
|
||||
"type": "node",
|
||||
"id": 101,
|
||||
"lat": 40.0,
|
||||
"lon": -74.0,
|
||||
"tags": {"name": "Wild Flour", "amenity": "bakery"},
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
monkeypatch.setattr(module, "overpass_query", lambda query: duplicate_bakery)
|
||||
args = SimpleNamespace(
|
||||
lat="40.0",
|
||||
lon="-74.0",
|
||||
near=None,
|
||||
category="bakery",
|
||||
category_list=[],
|
||||
radius=500,
|
||||
limit=10,
|
||||
)
|
||||
|
||||
module.cmd_nearby(args)
|
||||
out = capsys.readouterr().out
|
||||
assert '"count": 1' in out
|
||||
assert '"Wild Flour"' in out
|
||||
|
||||
|
||||
def test_skill_doc_lists_new_categories_and_supersession():
|
||||
text = SKILL_PATH.read_text(encoding="utf-8")
|
||||
assert "guest_house" in text
|
||||
assert "camp_site" in text
|
||||
assert "bakery" in text
|
||||
assert "supersedes: [find-nearby]" in text
|
||||
@@ -11,14 +11,12 @@ import pytest
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "benchmarks"))
|
||||
|
||||
from vision_benchmark import (
|
||||
analyze_with_model,
|
||||
compute_ocr_accuracy,
|
||||
compute_description_completeness,
|
||||
compute_structural_accuracy,
|
||||
aggregate_results,
|
||||
to_markdown,
|
||||
generate_sample_dataset,
|
||||
load_dataset,
|
||||
MODELS,
|
||||
EVAL_PROMPTS,
|
||||
)
|
||||
@@ -199,71 +197,6 @@ class TestMarkdown:
|
||||
|
||||
|
||||
class TestDataset:
|
||||
def test_repo_dataset_uses_local_image_paths(self):
|
||||
dataset_path = Path(__file__).parent.parent / "benchmarks" / "test_images.json"
|
||||
dataset = json.loads(dataset_path.read_text())
|
||||
|
||||
assert dataset, "benchmark dataset should not be empty"
|
||||
assert all(not entry["url"].startswith(("http://", "https://")) for entry in dataset)
|
||||
|
||||
def test_load_dataset_resolves_relative_local_paths(self, tmp_path):
|
||||
images_dir = tmp_path / "images"
|
||||
images_dir.mkdir()
|
||||
image_path = images_dir / "sample.png"
|
||||
image_path.write_bytes(b"png-bytes")
|
||||
|
||||
dataset_path = tmp_path / "dataset.json"
|
||||
dataset_path.write_text(json.dumps([
|
||||
{
|
||||
"id": "sample",
|
||||
"url": "images/sample.png",
|
||||
"category": "photo",
|
||||
"expected_keywords": [],
|
||||
"expected_structure": {"min_length": 30, "min_sentences": 1},
|
||||
}
|
||||
]))
|
||||
|
||||
loaded = load_dataset(str(dataset_path))
|
||||
|
||||
assert loaded[0]["url"] == str(image_path.resolve())
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_analyze_with_model_encodes_local_file_as_data_url(self, tmp_path, monkeypatch):
|
||||
image_path = tmp_path / "tiny.png"
|
||||
image_path.write_bytes(
|
||||
bytes.fromhex(
|
||||
"89504E470D0A1A0A"
|
||||
"0000000D49484452000000010000000108060000001F15C489"
|
||||
"0000000D49444154789C6360000002000154A24F5D00000000"
|
||||
"49454E44AE426082"
|
||||
)
|
||||
)
|
||||
|
||||
fake_response = MagicMock()
|
||||
fake_response.raise_for_status.return_value = None
|
||||
fake_response.json.return_value = {
|
||||
"choices": [{"message": {"content": "Looks like a tiny image."}}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
|
||||
}
|
||||
|
||||
fake_client = MagicMock()
|
||||
fake_client.post = AsyncMock(return_value=fake_response)
|
||||
fake_ctx = MagicMock()
|
||||
fake_ctx.__aenter__ = AsyncMock(return_value=fake_client)
|
||||
fake_ctx.__aexit__ = AsyncMock(return_value=None)
|
||||
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "test-key")
|
||||
with patch("httpx.AsyncClient", return_value=fake_ctx):
|
||||
result = await analyze_with_model(
|
||||
str(image_path),
|
||||
"Describe this image",
|
||||
{"provider": "openrouter", "model_id": "fake/model"},
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
sent_url = fake_client.post.await_args.kwargs["json"]["messages"][0]["content"][1]["image_url"]["url"]
|
||||
assert sent_url.startswith("data:image/png;base64,")
|
||||
|
||||
def test_sample_dataset_has_entries(self):
|
||||
dataset = generate_sample_dataset()
|
||||
assert len(dataset) >= 4
|
||||
|
||||