wip: add regression tests for vision benchmark artifacts
This commit is contained in:
21
tests/test_vision_benchmark_artifacts.py
Normal file
21
tests/test_vision_benchmark_artifacts.py
Normal file
@@ -0,0 +1,21 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DATASET = Path("benchmarks/test_images.json")
|
||||
REPORT = Path("metrics/vision-benchmark-smoke-2026-04-22.md")
|
||||
|
||||
|
||||
def test_benchmark_dataset_is_issue_sized_and_category_complete() -> None:
|
||||
items = json.loads(DATASET.read_text(encoding="utf-8"))
|
||||
assert len(items) >= 50
|
||||
categories = {item["category"] for item in items}
|
||||
assert {"screenshot", "diagram", "photo", "ocr", "chart", "document"}.issubset(categories)
|
||||
|
||||
|
||||
def test_metrics_report_exists_with_recommendation() -> None:
|
||||
assert REPORT.exists(), "missing benchmark report under metrics/"
|
||||
text = REPORT.read_text(encoding="utf-8")
|
||||
assert "Recommendation" in text
|
||||
assert "Gemma 4" in text
|
||||
assert "Gemini" in text
|
||||
Reference in New Issue
Block a user