Timmy
eed87e454e
Some checks failed
Contributor Attribution Check / check-attribution (pull_request) Successful in 26s
Docker Build and Publish / build-and-push (pull_request) Has been skipped
Supply Chain Audit / Scan PR for supply chain risks (pull_request) Successful in 26s
Tests / e2e (pull_request) Successful in 2m38s
Tests / test (pull_request) Failing after 47m49s
Vision benchmark suite comparing Gemma 4 (google/gemma-4-27b-it) vs current Gemini 3 Flash Preview (google/gemini-3-flash-preview). Metrics: - OCR accuracy (character + word overlap) - Description completeness (keyword coverage) - Structural quality (length, sentences, numbers) - Latency (ms per image) - Token usage - Consistency across runs Features: - 24 diverse test images (screenshots, diagrams, photos, charts) - Category-specific evaluation prompts - Automated verdict with composite scoring - JSON + markdown report output - 28 unit tests passing Usage: python benchmarks/vision_benchmark.py --images benchmarks/test_images.json python benchmarks/vision_benchmark.py --url https://example.com/img.png python benchmarks/vision_benchmark.py --generate-dataset Closes #817.
195 lines
8.6 KiB
JSON
195 lines
8.6 KiB
JSON
[
|
|
{
|
|
"id": "screenshot_github_home",
|
|
"url": "https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png",
|
|
"category": "screenshot",
|
|
"expected_keywords": ["github", "logo", "mark"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "diagram_mermaid_flow",
|
|
"url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6siSZXVhjQTlgl1nigHg5fRBOzSfebopROCu_cytObSfgLSE1ANOeZWkO2IH5upZxYot8m1hqAdpD_63WRl0xdUG1jdl9kPiOb_EWk2JBtPaiKkF4eVIYgO0EtkW-RSgC4gJ6HJYRG1UNdN0HNVd0Bftjj7X8P92qPj-F8l8T3w",
|
|
"category": "diagram",
|
|
"expected_keywords": ["flow", "diagram", "process"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "photo_random_1",
|
|
"url": "https://picsum.photos/seed/vision1/400/300",
|
|
"category": "photo",
|
|
"expected_keywords": [],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "photo_random_2",
|
|
"url": "https://picsum.photos/seed/vision2/400/300",
|
|
"category": "photo",
|
|
"expected_keywords": [],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "chart_simple_bar",
|
|
"url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['Q1','Q2','Q3','Q4'],datasets:[{label:'Revenue',data:[100,150,200,250]}]}}",
|
|
"category": "chart",
|
|
"expected_keywords": ["bar", "chart", "revenue"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
|
},
|
|
{
|
|
"id": "chart_pie",
|
|
"url": "https://quickchart.io/chart?c={type:'pie',data:{labels:['A','B','C'],datasets:[{data:[30,50,20]}]}}",
|
|
"category": "chart",
|
|
"expected_keywords": ["pie", "chart", "percentage"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
|
},
|
|
{
|
|
"id": "diagram_org_chart",
|
|
"url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
|
|
"category": "diagram",
|
|
"expected_keywords": ["organization", "hierarchy", "chart"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "screenshot_terminal",
|
|
"url": "https://raw.githubusercontent.com/nicehash/nicehash-quick-start/main/images/nicehash-terminal.png",
|
|
"category": "screenshot",
|
|
"expected_keywords": ["terminal", "command", "output"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "photo_random_3",
|
|
"url": "https://picsum.photos/seed/vision3/400/300",
|
|
"category": "photo",
|
|
"expected_keywords": [],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "chart_line",
|
|
"url": "https://quickchart.io/chart?c={type:'line',data:{labels:['Jan','Feb','Mar','Apr'],datasets:[{label:'Temperature',data:[5,8,12,18]}]}}",
|
|
"category": "chart",
|
|
"expected_keywords": ["line", "chart", "temperature"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
|
},
|
|
{
|
|
"id": "diagram_sequence",
|
|
"url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
|
|
"category": "diagram",
|
|
"expected_keywords": ["sequence", "interaction", "message"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "photo_random_4",
|
|
"url": "https://picsum.photos/seed/vision4/400/300",
|
|
"category": "photo",
|
|
"expected_keywords": [],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "screenshot_webpage",
|
|
"url": "https://github.githubassets.com/images/modules/site/social-cards.png",
|
|
"category": "screenshot",
|
|
"expected_keywords": ["github", "page", "web"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "chart_radar",
|
|
"url": "https://quickchart.io/chart?c={type:'radar',data:{labels:['Speed','Power','Defense','Magic'],datasets:[{label:'Hero',data:[80,60,70,90]}]}}",
|
|
"category": "chart",
|
|
"expected_keywords": ["radar", "chart", "skill"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
|
},
|
|
{
|
|
"id": "photo_random_5",
|
|
"url": "https://picsum.photos/seed/vision5/400/300",
|
|
"category": "photo",
|
|
"expected_keywords": [],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "diagram_class",
|
|
"url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
|
|
"category": "diagram",
|
|
"expected_keywords": ["class", "object", "attribute"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "chart_doughnut",
|
|
"url": "https://quickchart.io/chart?c={type:'doughnut',data:{labels:['Desktop','Mobile','Tablet'],datasets:[{data:[60,30,10]}]}}",
|
|
"category": "chart",
|
|
"expected_keywords": ["doughnut", "chart", "device"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
|
},
|
|
{
|
|
"id": "photo_random_6",
|
|
"url": "https://picsum.photos/seed/vision6/400/300",
|
|
"category": "photo",
|
|
"expected_keywords": [],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "screenshot_error",
|
|
"url": "https://http.cat/404.jpg",
|
|
"category": "screenshot",
|
|
"expected_keywords": ["404", "error", "cat"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": true}
|
|
},
|
|
{
|
|
"id": "diagram_network",
|
|
"url": "https://mermaid.ink/img/pako:eNpdkE9PwzAMxb-K5VOl7gc7sAOIIDuAw9gptnRaSJLSJttQStmXs9LCH-ymBOI1ef_42U6cUSae4IkDxbAAWtB6iuyIWyrLgXLALrPEAfFy-iCcmk-83RSjcFZ-51ac2k7AW0JqAKY9y9IcsAPzdS3jxBb5NrHUAraH_lutjbpi6oJqG7P7IPEd3-ItJsWCaO1FVYLw8qQwANsJbIt8i1AExAX0OCwjNqoa6LoPaq7oCvbHHmv5f7pVfX4K5b8mvg",
|
|
"category": "diagram",
|
|
"expected_keywords": ["network", "node", "connection"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "photo_random_7",
|
|
"url": "https://picsum.photos/seed/vision7/400/300",
|
|
"category": "photo",
|
|
"expected_keywords": [],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "chart_stacked_bar",
|
|
"url": "https://quickchart.io/chart?c={type:'bar',data:{labels:['2022','2023','2024'],datasets:[{label:'Cloud',data:[100,150,200]},{label:'On-prem',data:[200,180,160]}]},options:{scales:{x:{stacked:true},y:{stacked:true}}}}",
|
|
"category": "chart",
|
|
"expected_keywords": ["stacked", "bar", "chart"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 50, "min_sentences": 2, "has_numbers": true}
|
|
},
|
|
{
|
|
"id": "screenshot_dashboard",
|
|
"url": "https://github.githubassets.com/images/modules/site/features-code-search.png",
|
|
"category": "screenshot",
|
|
"expected_keywords": ["search", "code", "feature"],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
},
|
|
{
|
|
"id": "photo_random_8",
|
|
"url": "https://picsum.photos/seed/vision8/400/300",
|
|
"category": "photo",
|
|
"expected_keywords": [],
|
|
"ground_truth_ocr": "",
|
|
"expected_structure": {"min_length": 30, "min_sentences": 1, "has_numbers": false}
|
|
}
|
|
]
|