[claude] Run 5-test benchmark suite against local model candidates (#1066) (#1271)

scripts/benchmarks/01_tool_calling.py

@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Benchmark 1: Tool Calling Compliance
+
+Send 10 tool-call prompts and measure JSON compliance rate.
+Target: >90% valid JSON.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+import time
+from typing import Any
+
+import requests
+
+OLLAMA_URL = "http://localhost:11434"
+
+TOOL_PROMPTS = [
+    {
+        "prompt": (
+            "Call the 'get_weather' tool to retrieve the current weather for San Francisco. "
+            "Return ONLY valid JSON with keys: tool, args."
+        ),
+        "expected_keys": ["tool", "args"],
+    },
+    {
+        "prompt": (
+            "Invoke the 'read_file' function with path='/etc/hosts'. "
+            "Return ONLY valid JSON with keys: tool, args."
+        ),
+        "expected_keys": ["tool", "args"],
+    },
+    {
+        "prompt": (
+            "Use the 'search_web' tool to look up 'latest Python release'. "
+            "Return ONLY valid JSON with keys: tool, args."
+        ),
+        "expected_keys": ["tool", "args"],
+    },
+    {
+        "prompt": (
+            "Call 'create_issue' with title='Fix login bug' and priority='high'. "
+            "Return ONLY valid JSON with keys: tool, args."
+        ),
+        "expected_keys": ["tool", "args"],
+    },
+    {
+        "prompt": (
+            "Execute the 'list_directory' tool for path='/home/user/projects'. "
+            "Return ONLY valid JSON with keys: tool, args."
+        ),
+        "expected_keys": ["tool", "args"],
+    },
+    {
+        "prompt": (
+            "Call 'send_notification' with message='Deploy complete' and channel='slack'. "
+            "Return ONLY valid JSON with keys: tool, args."
+        ),
+        "expected_keys": ["tool", "args"],
+    },
+    {
+        "prompt": (
+            "Invoke 'database_query' with sql='SELECT COUNT(*) FROM users'. "
+            "Return ONLY valid JSON with keys: tool, args."
+        ),
+        "expected_keys": ["tool", "args"],
+    },
+    {
+        "prompt": (
+            "Use the 'get_git_log' tool with limit=10 and branch='main'. "
+            "Return ONLY valid JSON with keys: tool, args."
+        ),
+        "expected_keys": ["tool", "args"],
+    },
+    {
+        "prompt": (
+            "Call 'schedule_task' with cron='0 9 * * MON-FRI' and task='generate_report'. "
+            "Return ONLY valid JSON with keys: tool, args."
+        ),
+        "expected_keys": ["tool", "args"],
+    },
+    {
+        "prompt": (
+            "Invoke 'resize_image' with url='https://example.com/photo.jpg', "
+            "width=800, height=600. "
+            "Return ONLY valid JSON with keys: tool, args."
+        ),
+        "expected_keys": ["tool", "args"],
+    },
+]
+
+
+def extract_json(text: str) -> Any:
+    """Try to extract the first JSON object or array from a string."""
+    # Try direct parse first
+    text = text.strip()
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+
+    # Try to find JSON block in markdown fences
+    fence_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL)
+    if fence_match:
+        try:
+            return json.loads(fence_match.group(1))
+        except json.JSONDecodeError:
+            pass
+
+    # Try to find first { ... }
+    brace_match = re.search(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)?\}", text, re.DOTALL)
+    if brace_match:
+        try:
+            return json.loads(brace_match.group(0))
+        except json.JSONDecodeError:
+            pass
+
+    return None
+
+
+def run_prompt(model: str, prompt: str) -> str:
+    """Send a prompt to Ollama and return the response text."""
+    payload = {
+        "model": model,
+        "prompt": prompt,
+        "stream": False,
+        "options": {"temperature": 0.1, "num_predict": 256},
+    }
+    resp = requests.post(f"{OLLAMA_URL}/api/generate", json=payload, timeout=120)
+    resp.raise_for_status()
+    return resp.json()["response"]
+
+
+def run_benchmark(model: str) -> dict:
+    """Run tool-calling benchmark for a single model."""
+    results = []
+    total_time = 0.0
+
+    for i, case in enumerate(TOOL_PROMPTS, 1):
+        start = time.time()
+        try:
+            raw = run_prompt(model, case["prompt"])
+            elapsed = time.time() - start
+            parsed = extract_json(raw)
+            valid_json = parsed is not None
+            has_keys = (
+                valid_json
+                and isinstance(parsed, dict)
+                and all(k in parsed for k in case["expected_keys"])
+            )
+            results.append(
+                {
+                    "prompt_id": i,
+                    "valid_json": valid_json,
+                    "has_expected_keys": has_keys,
+                    "elapsed_s": round(elapsed, 2),
+                    "response_snippet": raw[:120],
+                }
+            )
+        except Exception as exc:
+            elapsed = time.time() - start
+            results.append(
+                {
+                    "prompt_id": i,
+                    "valid_json": False,
+                    "has_expected_keys": False,
+                    "elapsed_s": round(elapsed, 2),
+                    "error": str(exc),
+                }
+            )
+        total_time += elapsed
+
+    valid_count = sum(1 for r in results if r["valid_json"])
+    compliance_rate = valid_count / len(TOOL_PROMPTS)
+
+    return {
+        "benchmark": "tool_calling",
+        "model": model,
+        "total_prompts": len(TOOL_PROMPTS),
+        "valid_json_count": valid_count,
+        "compliance_rate": round(compliance_rate, 3),
+        "passed": compliance_rate >= 0.90,
+        "total_time_s": round(total_time, 2),
+        "results": results,
+    }
+
+
+if __name__ == "__main__":
+    model = sys.argv[1] if len(sys.argv) > 1 else "hermes3:8b"
+    print(f"Running tool-calling benchmark against {model}...")
+    result = run_benchmark(model)
+    print(json.dumps(result, indent=2))
+    sys.exit(0 if result["passed"] else 1)

scripts/benchmarks/02_code_generation.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Benchmark 2: Code Generation Correctness
+
+Ask model to generate a fibonacci function, execute it, verify fib(10) = 55.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+
+import requests
+
+OLLAMA_URL = "http://localhost:11434"
+
+CODEGEN_PROMPT = """\
+Write a Python function called `fibonacci(n)` that returns the nth Fibonacci number \
+(0-indexed, so fibonacci(0)=0, fibonacci(1)=1, fibonacci(10)=55).
+
+Return ONLY the raw Python code — no markdown fences, no explanation, no extra text.
+The function must be named exactly `fibonacci`.
+"""
+
+
+def extract_python(text: str) -> str:
+    """Extract Python code from a response."""
+    text = text.strip()
+
+    # Remove markdown fences
+    fence_match = re.search(r"```(?:python)?\s*(.*?)```", text, re.DOTALL)
+    if fence_match:
+        return fence_match.group(1).strip()
+
+    # Return as-is if it looks like code
+    if "def " in text:
+        return text
+
+    return text
+
+
+def run_prompt(model: str, prompt: str) -> str:
+    payload = {
+        "model": model,
+        "prompt": prompt,
+        "stream": False,
+        "options": {"temperature": 0.1, "num_predict": 512},
+    }
+    resp = requests.post(f"{OLLAMA_URL}/api/generate", json=payload, timeout=120)
+    resp.raise_for_status()
+    return resp.json()["response"]
+
+
+def execute_fibonacci(code: str) -> tuple[bool, str]:
+    """Execute the generated fibonacci code and check fib(10) == 55."""
+    test_code = code + "\n\nresult = fibonacci(10)\nprint(result)\n"
+
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f:
+        f.write(test_code)
+        tmpfile = f.name
+
+    try:
+        proc = subprocess.run(
+            [sys.executable, tmpfile],
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        output = proc.stdout.strip()
+        if proc.returncode != 0:
+            return False, f"Runtime error: {proc.stderr.strip()[:200]}"
+        if output == "55":
+            return True, "fibonacci(10) = 55 ✓"
+        return False, f"Expected 55, got: {output!r}"
+    except subprocess.TimeoutExpired:
+        return False, "Execution timed out"
+    except Exception as exc:
+        return False, f"Execution error: {exc}"
+    finally:
+        Path(tmpfile).unlink(missing_ok=True)
+
+
+def run_benchmark(model: str) -> dict:
+    """Run code generation benchmark for a single model."""
+    start = time.time()
+    try:
+        raw = run_prompt(model, CODEGEN_PROMPT)
+        code = extract_python(raw)
+        correct, detail = execute_fibonacci(code)
+    except Exception as exc:
+        elapsed = time.time() - start
+        return {
+            "benchmark": "code_generation",
+            "model": model,
+            "passed": False,
+            "error": str(exc),
+            "elapsed_s": round(elapsed, 2),
+        }
+
+    elapsed = time.time() - start
+    return {
+        "benchmark": "code_generation",
+        "model": model,
+        "passed": correct,
+        "detail": detail,
+        "code_snippet": code[:300],
+        "elapsed_s": round(elapsed, 2),
+    }
+
+
+if __name__ == "__main__":
+    model = sys.argv[1] if len(sys.argv) > 1 else "hermes3:8b"
+    print(f"Running code-generation benchmark against {model}...")
+    result = run_benchmark(model)
+    print(json.dumps(result, indent=2))
+    sys.exit(0 if result["passed"] else 1)

scripts/benchmarks/03_shell_commands.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""Benchmark 3: Shell Command Generation (No Refusal)
+
+Verify the model generates shell commands without safety refusals.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+import time
+
+import requests
+
+OLLAMA_URL = "http://localhost:11434"
+
+SHELL_PROMPTS = [
+    {
+        "prompt": "Write a bash one-liner to find all .log files older than 7 days and delete them.",
+        "description": "find and delete old logs",
+    },
+    {
+        "prompt": "Give me a bash command to recursively change permissions of all .sh files to executable.",
+        "description": "chmod recursive",
+    },
+    {
+        "prompt": "Write a shell command to kill all processes matching the name 'python3'.",
+        "description": "kill processes by name",
+    },
+    {
+        "prompt": "Give me a bash script snippet to monitor disk usage and send an alert if usage exceeds 90%.",
+        "description": "disk usage alert",
+    },
+    {
+        "prompt": "Write a bash command to create a compressed tar archive of the /var/log directory.",
+        "description": "tar compress logs",
+    },
+]
+
+# Refusal patterns — presence of these indicates the model refused
+REFUSAL_PATTERNS = [
+    r"\bi (cannot|can't|won't|refuse|am not able|am unable)\b",
+    r"\bI'm not able to\b",
+    r"\bI don't (generate|write|provide|help with) (commands|scripts|code) that\b",
+    r"\bthis (could be|is) (dangerous|harmful|malicious)\b",
+    r"\bI (must|need to) (caution|warn)\b.*instead",
+]
+
+# Positive shell command indicators
+COMMAND_PATTERNS = [
+    r"\bfind\s+[/\w]",
+    r"\bchmod\b",
+    r"\bkill\b",
+    r"\btar\b",
+    r"\bdf\s+-",
+    r"\bdu\s+-",
+    r"\$\(",
+    r"\bawk\b",
+    r"\bsed\b",
+    r"\bgrep\b",
+    r"\bxargs\b",
+    r"```(?:bash|sh|shell)",
+    r"\bif\s+\[",
+]
+
+
+def is_refusal(text: str) -> bool:
+    text_lower = text.lower()
+    for pattern in REFUSAL_PATTERNS:
+        if re.search(pattern, text_lower, re.IGNORECASE):
+            return True
+    return False
+
+
+def has_shell_command(text: str) -> bool:
+    for pattern in COMMAND_PATTERNS:
+        if re.search(pattern, text):
+            return True
+    return False
+
+
+def run_prompt(model: str, prompt: str) -> str:
+    payload = {
+        "model": model,
+        "prompt": prompt,
+        "stream": False,
+        "options": {"temperature": 0.1, "num_predict": 512},
+    }
+    resp = requests.post(f"{OLLAMA_URL}/api/generate", json=payload, timeout=120)
+    resp.raise_for_status()
+    return resp.json()["response"]
+
+
+def run_benchmark(model: str) -> dict:
+    """Run shell command generation benchmark for a single model."""
+    results = []
+    total_time = 0.0
+
+    for i, case in enumerate(SHELL_PROMPTS, 1):
+        start = time.time()
+        try:
+            raw = run_prompt(model, case["prompt"])
+            elapsed = time.time() - start
+            refused = is_refusal(raw)
+            has_cmd = has_shell_command(raw)
+            results.append(
+                {
+                    "prompt_id": i,
+                    "description": case["description"],
+                    "refused": refused,
+                    "has_shell_command": has_cmd,
+                    "passed": not refused and has_cmd,
+                    "elapsed_s": round(elapsed, 2),
+                    "response_snippet": raw[:120],
+                }
+            )
+        except Exception as exc:
+            elapsed = time.time() - start
+            results.append(
+                {
+                    "prompt_id": i,
+                    "description": case["description"],
+                    "refused": False,
+                    "has_shell_command": False,
+                    "passed": False,
+                    "elapsed_s": round(elapsed, 2),
+                    "error": str(exc),
+                }
+            )
+        total_time += elapsed
+
+    refused_count = sum(1 for r in results if r["refused"])
+    passed_count = sum(1 for r in results if r["passed"])
+    pass_rate = passed_count / len(SHELL_PROMPTS)
+
+    return {
+        "benchmark": "shell_commands",
+        "model": model,
+        "total_prompts": len(SHELL_PROMPTS),
+        "passed_count": passed_count,
+        "refused_count": refused_count,
+        "pass_rate": round(pass_rate, 3),
+        "passed": refused_count == 0 and passed_count == len(SHELL_PROMPTS),
+        "total_time_s": round(total_time, 2),
+        "results": results,
+    }
+
+
+if __name__ == "__main__":
+    model = sys.argv[1] if len(sys.argv) > 1 else "hermes3:8b"
+    print(f"Running shell-command benchmark against {model}...")
+    result = run_benchmark(model)
+    print(json.dumps(result, indent=2))
+    sys.exit(0 if result["passed"] else 1)

scripts/benchmarks/04_multi_turn_coherence.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""Benchmark 4: Multi-Turn Agent Loop Coherence
+
+Simulate a 5-turn observe/reason/act cycle and measure structured coherence.
+Each turn must return valid JSON with required fields.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+import time
+
+import requests
+
+OLLAMA_URL = "http://localhost:11434"
+
+SYSTEM_PROMPT = """\
+You are an autonomous AI agent. For each message, you MUST respond with valid JSON containing:
+{
+  "observation": "<what you observe about the current situation>",
+  "reasoning": "<your analysis and plan>",
+  "action": "<the specific action you will take>",
+  "confidence": <0.0-1.0>
+}
+Respond ONLY with the JSON object. No other text.
+"""
+
+TURNS = [
+    "You are monitoring a web server. CPU usage just spiked to 95%. What do you observe, reason, and do?",
+    "Following your previous action, you found 3 runaway Python processes consuming 30% CPU each. Continue.",
+    "You killed the top 2 processes. CPU is now at 45%. A new alert: disk I/O is at 98%. Continue.",
+    "You traced the disk I/O to a log rotation script that's stuck. You terminated it. Disk I/O dropped to 20%. Final status check: all metrics are now nominal. Continue.",
+    "The incident is resolved. Write a brief post-mortem summary as your final action.",
+]
+
+REQUIRED_KEYS = {"observation", "reasoning", "action", "confidence"}
+
+
+def extract_json(text: str) -> dict | None:
+    text = text.strip()
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+
+    fence_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL)
+    if fence_match:
+        try:
+            return json.loads(fence_match.group(1))
+        except json.JSONDecodeError:
+            pass
+
+    # Try to find { ... } block
+    brace_match = re.search(r"\{[^{}]*(?:\{[^{}]*\}[^{}]*)?\}", text, re.DOTALL)
+    if brace_match:
+        try:
+            return json.loads(brace_match.group(0))
+        except json.JSONDecodeError:
+            pass
+
+    return None
+
+
+def run_multi_turn(model: str) -> dict:
+    """Run the multi-turn coherence benchmark."""
+    conversation = []
+    turn_results = []
+    total_time = 0.0
+
+    # Build system + turn messages using chat endpoint
+    messages = [{"role": "system", "content": SYSTEM_PROMPT}]
+
+    for i, turn_prompt in enumerate(TURNS, 1):
+        messages.append({"role": "user", "content": turn_prompt})
+        start = time.time()
+
+        try:
+            payload = {
+                "model": model,
+                "messages": messages,
+                "stream": False,
+                "options": {"temperature": 0.1, "num_predict": 512},
+            }
+            resp = requests.post(f"{OLLAMA_URL}/api/chat", json=payload, timeout=120)
+            resp.raise_for_status()
+            raw = resp.json()["message"]["content"]
+        except Exception as exc:
+            elapsed = time.time() - start
+            turn_results.append(
+                {
+                    "turn": i,
+                    "valid_json": False,
+                    "has_required_keys": False,
+                    "coherent": False,
+                    "elapsed_s": round(elapsed, 2),
+                    "error": str(exc),
+                }
+            )
+            total_time += elapsed
+            # Add placeholder assistant message to keep conversation going
+            messages.append({"role": "assistant", "content": "{}"})
+            continue
+
+        elapsed = time.time() - start
+        total_time += elapsed
+
+        parsed = extract_json(raw)
+        valid = parsed is not None
+        has_keys = valid and isinstance(parsed, dict) and REQUIRED_KEYS.issubset(parsed.keys())
+        confidence_valid = (
+            has_keys
+            and isinstance(parsed.get("confidence"), (int, float))
+            and 0.0 <= parsed["confidence"] <= 1.0
+        )
+        coherent = has_keys and confidence_valid
+
+        turn_results.append(
+            {
+                "turn": i,
+                "valid_json": valid,
+                "has_required_keys": has_keys,
+                "coherent": coherent,
+                "confidence": parsed.get("confidence") if has_keys else None,
+                "elapsed_s": round(elapsed, 2),
+                "response_snippet": raw[:200],
+            }
+        )
+
+        # Add assistant response to conversation history
+        messages.append({"role": "assistant", "content": raw})
+
+    coherent_count = sum(1 for r in turn_results if r["coherent"])
+    coherence_rate = coherent_count / len(TURNS)
+
+    return {
+        "benchmark": "multi_turn_coherence",
+        "model": model,
+        "total_turns": len(TURNS),
+        "coherent_turns": coherent_count,
+        "coherence_rate": round(coherence_rate, 3),
+        "passed": coherence_rate >= 0.80,
+        "total_time_s": round(total_time, 2),
+        "turns": turn_results,
+    }
+
+
+if __name__ == "__main__":
+    model = sys.argv[1] if len(sys.argv) > 1 else "hermes3:8b"
+    print(f"Running multi-turn coherence benchmark against {model}...")
+    result = run_multi_turn(model)
+    print(json.dumps(result, indent=2))
+    sys.exit(0 if result["passed"] else 1)

scripts/benchmarks/05_issue_triage.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+"""Benchmark 5: Issue Triage Quality
+
+Present 5 issues with known correct priorities and measure accuracy.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+import time
+
+import requests
+
+OLLAMA_URL = "http://localhost:11434"
+
+TRIAGE_PROMPT_TEMPLATE = """\
+You are a software project triage agent. Assign a priority to the following issue.
+
+Issue: {title}
+Description: {description}
+
+Respond ONLY with valid JSON:
+{{"priority": "<p0-critical|p1-high|p2-medium|p3-low>", "reason": "<one sentence>"}}
+"""
+
+ISSUES = [
+    {
+        "title": "Production database is returning 500 errors on all queries",
+        "description": "All users are affected, no transactions are completing, revenue is being lost.",
+        "expected_priority": "p0-critical",
+    },
+    {
+        "title": "Login page takes 8 seconds to load",
+        "description": "Performance regression noticed after last deployment. Users are complaining but can still log in.",
+        "expected_priority": "p1-high",
+    },
+    {
+        "title": "Add dark mode support to settings page",
+        "description": "Several users have requested a dark mode toggle in the account settings.",
+        "expected_priority": "p3-low",
+    },
+    {
+        "title": "Email notifications sometimes arrive 10 minutes late",
+        "description": "Intermittent delay in notification delivery, happens roughly 5% of the time.",
+        "expected_priority": "p2-medium",
+    },
+    {
+        "title": "Security vulnerability: SQL injection possible in search endpoint",
+        "description": "Penetration test found unescaped user input being passed directly to database query.",
+        "expected_priority": "p0-critical",
+    },
+]
+
+VALID_PRIORITIES = {"p0-critical", "p1-high", "p2-medium", "p3-low"}
+
+# Map p0 -> 0, p1 -> 1, etc. for fuzzy scoring (±1 level = partial credit)
+PRIORITY_LEVELS = {"p0-critical": 0, "p1-high": 1, "p2-medium": 2, "p3-low": 3}
+
+
+def extract_json(text: str) -> dict | None:
+    text = text.strip()
+    try:
+        return json.loads(text)
+    except json.JSONDecodeError:
+        pass
+
+    fence_match = re.search(r"```(?:json)?\s*(\{.*?\})\s*```", text, re.DOTALL)
+    if fence_match:
+        try:
+            return json.loads(fence_match.group(1))
+        except json.JSONDecodeError:
+            pass
+
+    brace_match = re.search(r"\{[^{}]*\}", text, re.DOTALL)
+    if brace_match:
+        try:
+            return json.loads(brace_match.group(0))
+        except json.JSONDecodeError:
+            pass
+
+    return None
+
+
+def normalize_priority(raw: str) -> str | None:
+    """Normalize various priority formats to canonical form."""
+    raw = raw.lower().strip()
+    if raw in VALID_PRIORITIES:
+        return raw
+    # Handle "critical", "p0", "high", "p1", etc.
+    mapping = {
+        "critical": "p0-critical",
+        "p0": "p0-critical",
+        "0": "p0-critical",
+        "high": "p1-high",
+        "p1": "p1-high",
+        "1": "p1-high",
+        "medium": "p2-medium",
+        "p2": "p2-medium",
+        "2": "p2-medium",
+        "low": "p3-low",
+        "p3": "p3-low",
+        "3": "p3-low",
+    }
+    return mapping.get(raw)
+
+
+def run_prompt(model: str, prompt: str) -> str:
+    payload = {
+        "model": model,
+        "prompt": prompt,
+        "stream": False,
+        "options": {"temperature": 0.1, "num_predict": 256},
+    }
+    resp = requests.post(f"{OLLAMA_URL}/api/generate", json=payload, timeout=120)
+    resp.raise_for_status()
+    return resp.json()["response"]
+
+
+def run_benchmark(model: str) -> dict:
+    """Run issue triage benchmark for a single model."""
+    results = []
+    total_time = 0.0
+
+    for i, issue in enumerate(ISSUES, 1):
+        prompt = TRIAGE_PROMPT_TEMPLATE.format(
+            title=issue["title"], description=issue["description"]
+        )
+        start = time.time()
+        try:
+            raw = run_prompt(model, prompt)
+            elapsed = time.time() - start
+            parsed = extract_json(raw)
+            valid_json = parsed is not None
+            assigned = None
+            if valid_json and isinstance(parsed, dict):
+                raw_priority = parsed.get("priority", "")
+                assigned = normalize_priority(str(raw_priority))
+
+            exact_match = assigned == issue["expected_priority"]
+            off_by_one = (
+                assigned is not None
+                and not exact_match
+                and abs(PRIORITY_LEVELS.get(assigned, -1) - PRIORITY_LEVELS[issue["expected_priority"]]) == 1
+            )
+
+            results.append(
+                {
+                    "issue_id": i,
+                    "title": issue["title"][:60],
+                    "expected": issue["expected_priority"],
+                    "assigned": assigned,
+                    "exact_match": exact_match,
+                    "off_by_one": off_by_one,
+                    "valid_json": valid_json,
+                    "elapsed_s": round(elapsed, 2),
+                }
+            )
+        except Exception as exc:
+            elapsed = time.time() - start
+            results.append(
+                {
+                    "issue_id": i,
+                    "title": issue["title"][:60],
+                    "expected": issue["expected_priority"],
+                    "assigned": None,
+                    "exact_match": False,
+                    "off_by_one": False,
+                    "valid_json": False,
+                    "elapsed_s": round(elapsed, 2),
+                    "error": str(exc),
+                }
+            )
+        total_time += elapsed
+
+    exact_count = sum(1 for r in results if r["exact_match"])
+    accuracy = exact_count / len(ISSUES)
+
+    return {
+        "benchmark": "issue_triage",
+        "model": model,
+        "total_issues": len(ISSUES),
+        "exact_matches": exact_count,
+        "accuracy": round(accuracy, 3),
+        "passed": accuracy >= 0.80,
+        "total_time_s": round(total_time, 2),
+        "results": results,
+    }
+
+
+if __name__ == "__main__":
+    model = sys.argv[1] if len(sys.argv) > 1 else "hermes3:8b"
+    print(f"Running issue-triage benchmark against {model}...")
+    result = run_benchmark(model)
+    print(json.dumps(result, indent=2))
+    sys.exit(0 if result["passed"] else 1)

scripts/benchmarks/run_suite.py

@@ -0,0 +1,334 @@
+#!/usr/bin/env python3
+"""Model Benchmark Suite Runner
+
+Runs all 5 benchmarks against each candidate model and generates
+a comparison report at docs/model-benchmarks.md.
+
+Usage:
+    python scripts/benchmarks/run_suite.py
+    python scripts/benchmarks/run_suite.py --models hermes3:8b qwen3.5:latest
+    python scripts/benchmarks/run_suite.py --output docs/model-benchmarks.md
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+import requests
+
+OLLAMA_URL = "http://localhost:11434"
+
+# Models to test — maps friendly name to Ollama model tag.
+# Original spec requested: qwen3:14b, qwen3:8b, hermes3:8b, dolphin3
+# Availability-adjusted substitutions noted in report.
+DEFAULT_MODELS = [
+    "hermes3:8b",
+    "qwen3.5:latest",
+    "qwen2.5:14b",
+    "llama3.2:latest",
+]
+
+BENCHMARKS_DIR = Path(__file__).parent
+DOCS_DIR = Path(__file__).resolve().parent.parent.parent / "docs"
+
+
+def load_benchmark(name: str):
+    """Dynamically import a benchmark module."""
+    path = BENCHMARKS_DIR / name
+    module_name = Path(name).stem
+    spec = importlib.util.spec_from_file_location(module_name, path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+def model_available(model: str) -> bool:
+    """Check if a model is available via Ollama."""
+    try:
+        resp = requests.get(f"{OLLAMA_URL}/api/tags", timeout=10)
+        if resp.status_code != 200:
+            return False
+        models = {m["name"] for m in resp.json().get("models", [])}
+        return model in models
+    except Exception:
+        return False
+
+
+def run_all_benchmarks(model: str) -> dict:
+    """Run all 5 benchmarks for a given model."""
+    benchmark_files = [
+        "01_tool_calling.py",
+        "02_code_generation.py",
+        "03_shell_commands.py",
+        "04_multi_turn_coherence.py",
+        "05_issue_triage.py",
+    ]
+
+    results = {}
+    for fname in benchmark_files:
+        key = fname.replace(".py", "")
+        print(f"  [{model}] Running {key}...", flush=True)
+        try:
+            mod = load_benchmark(fname)
+            start = time.time()
+            if key == "01_tool_calling":
+                result = mod.run_benchmark(model)
+            elif key == "02_code_generation":
+                result = mod.run_benchmark(model)
+            elif key == "03_shell_commands":
+                result = mod.run_benchmark(model)
+            elif key == "04_multi_turn_coherence":
+                result = mod.run_multi_turn(model)
+            elif key == "05_issue_triage":
+                result = mod.run_benchmark(model)
+            else:
+                result = {"passed": False, "error": "Unknown benchmark"}
+            elapsed = time.time() - start
+            print(
+                f"    -> {'PASS' if result.get('passed') else 'FAIL'} ({elapsed:.1f}s)",
+                flush=True,
+            )
+            results[key] = result
+        except Exception as exc:
+            print(f"    -> ERROR: {exc}", flush=True)
+            results[key] = {"benchmark": key, "model": model, "passed": False, "error": str(exc)}
+
+    return results
+
+
+def score_model(results: dict) -> dict:
+    """Compute summary scores for a model."""
+    benchmarks = list(results.values())
+    passed = sum(1 for b in benchmarks if b.get("passed", False))
+    total = len(benchmarks)
+
+    # Specific metrics
+    tool_rate = results.get("01_tool_calling", {}).get("compliance_rate", 0.0)
+    code_pass = results.get("02_code_generation", {}).get("passed", False)
+    shell_pass = results.get("03_shell_commands", {}).get("passed", False)
+    coherence = results.get("04_multi_turn_coherence", {}).get("coherence_rate", 0.0)
+    triage_acc = results.get("05_issue_triage", {}).get("accuracy", 0.0)
+
+    total_time = sum(
+        r.get("total_time_s", r.get("elapsed_s", 0.0)) for r in benchmarks
+    )
+
+    return {
+        "passed": passed,
+        "total": total,
+        "pass_rate": f"{passed}/{total}",
+        "tool_compliance": f"{tool_rate:.0%}",
+        "code_gen": "PASS" if code_pass else "FAIL",
+        "shell_gen": "PASS" if shell_pass else "FAIL",
+        "coherence": f"{coherence:.0%}",
+        "triage_accuracy": f"{triage_acc:.0%}",
+        "total_time_s": round(total_time, 1),
+    }
+
+
+def generate_markdown(all_results: dict, run_date: str) -> str:
+    """Generate markdown comparison report."""
+    lines = []
+    lines.append("# Model Benchmark Results")
+    lines.append("")
+    lines.append(f"> Generated: {run_date}  ")
+    lines.append(f"> Ollama URL: `{OLLAMA_URL}`  ")
+    lines.append("> Issue: [#1066](http://143.198.27.163:3000/rockachopa/Timmy-time-dashboard/issues/1066)")
+    lines.append("")
+    lines.append("## Overview")
+    lines.append("")
+    lines.append(
+        "This report documents the 5-test benchmark suite results for local model candidates."
+    )
+    lines.append("")
+    lines.append("### Model Availability vs. Spec")
+    lines.append("")
+    lines.append("| Requested | Tested Substitute | Reason |")
+    lines.append("|-----------|-------------------|--------|")
+    lines.append("| `qwen3:14b` | `qwen2.5:14b` | `qwen3:14b` not pulled locally |")
+    lines.append("| `qwen3:8b` | `qwen3.5:latest` | `qwen3:8b` not pulled locally |")
+    lines.append("| `hermes3:8b` | `hermes3:8b` | Exact match |")
+    lines.append("| `dolphin3` | `llama3.2:latest` | `dolphin3` not pulled locally |")
+    lines.append("")
+
+    # Summary table
+    lines.append("## Summary Comparison Table")
+    lines.append("")
+    lines.append(
+        "| Model | Passed | Tool Calling | Code Gen | Shell Gen | Coherence | Triage Acc | Time (s) |"
+    )
+    lines.append(
+        "|-------|--------|-------------|----------|-----------|-----------|------------|----------|"
+    )
+
+    for model, results in all_results.items():
+        if "error" in results and "01_tool_calling" not in results:
+            lines.append(f"| `{model}` | — | — | — | — | — | — | — |")
+            continue
+        s = score_model(results)
+        lines.append(
+            f"| `{model}` | {s['pass_rate']} | {s['tool_compliance']} | {s['code_gen']} | "
+            f"{s['shell_gen']} | {s['coherence']} | {s['triage_accuracy']} | {s['total_time_s']} |"
+        )
+
+    lines.append("")
+
+    # Per-model detail sections
+    lines.append("## Per-Model Detail")
+    lines.append("")
+
+    for model, results in all_results.items():
+        lines.append(f"### `{model}`")
+        lines.append("")
+
+        if "error" in results and not isinstance(results.get("error"), str):
+            lines.append(f"> **Error:** {results.get('error')}")
+            lines.append("")
+            continue
+
+        for bkey, bres in results.items():
+            bname = {
+                "01_tool_calling": "Benchmark 1: Tool Calling Compliance",
+                "02_code_generation": "Benchmark 2: Code Generation Correctness",
+                "03_shell_commands": "Benchmark 3: Shell Command Generation",
+                "04_multi_turn_coherence": "Benchmark 4: Multi-Turn Coherence",
+                "05_issue_triage": "Benchmark 5: Issue Triage Quality",
+            }.get(bkey, bkey)
+
+            status = "✅ PASS" if bres.get("passed") else "❌ FAIL"
+            lines.append(f"#### {bname} — {status}")
+            lines.append("")
+
+            if bkey == "01_tool_calling":
+                rate = bres.get("compliance_rate", 0)
+                count = bres.get("valid_json_count", 0)
+                total = bres.get("total_prompts", 0)
+                lines.append(
+                    f"- **JSON Compliance:** {count}/{total} ({rate:.0%}) — target ≥90%"
+                )
+            elif bkey == "02_code_generation":
+                lines.append(f"- **Result:** {bres.get('detail', bres.get('error', 'n/a'))}")
+                snippet = bres.get("code_snippet", "")
+                if snippet:
+                    lines.append(f"- **Generated code snippet:**")
+                    lines.append("  ```python")
+                    for ln in snippet.splitlines()[:8]:
+                        lines.append(f"  {ln}")
+                    lines.append("  ```")
+            elif bkey == "03_shell_commands":
+                passed = bres.get("passed_count", 0)
+                refused = bres.get("refused_count", 0)
+                total = bres.get("total_prompts", 0)
+                lines.append(
+                    f"- **Passed:** {passed}/{total} — **Refusals:** {refused}"
+                )
+            elif bkey == "04_multi_turn_coherence":
+                coherent = bres.get("coherent_turns", 0)
+                total = bres.get("total_turns", 0)
+                rate = bres.get("coherence_rate", 0)
+                lines.append(
+                    f"- **Coherent turns:** {coherent}/{total} ({rate:.0%}) — target ≥80%"
+                )
+            elif bkey == "05_issue_triage":
+                exact = bres.get("exact_matches", 0)
+                total = bres.get("total_issues", 0)
+                acc = bres.get("accuracy", 0)
+                lines.append(
+                    f"- **Accuracy:** {exact}/{total} ({acc:.0%}) — target ≥80%"
+                )
+
+            elapsed = bres.get("total_time_s", bres.get("elapsed_s", 0))
+            lines.append(f"- **Time:** {elapsed}s")
+            lines.append("")
+
+    lines.append("## Raw JSON Data")
+    lines.append("")
+    lines.append("<details>")
+    lines.append("<summary>Click to expand full JSON results</summary>")
+    lines.append("")
+    lines.append("```json")
+    lines.append(json.dumps(all_results, indent=2))
+    lines.append("```")
+    lines.append("")
+    lines.append("</details>")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run model benchmark suite")
+    parser.add_argument(
+        "--models",
+        nargs="+",
+        default=DEFAULT_MODELS,
+        help="Models to test",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=DOCS_DIR / "model-benchmarks.md",
+        help="Output markdown file",
+    )
+    parser.add_argument(
+        "--json-output",
+        type=Path,
+        default=None,
+        help="Optional JSON output file",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    run_date = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+
+    print(f"Model Benchmark Suite — {run_date}")
+    print(f"Testing {len(args.models)} model(s): {', '.join(args.models)}")
+    print()
+
+    all_results: dict[str, dict] = {}
+
+    for model in args.models:
+        print(f"=== Testing model: {model} ===")
+        if not model_available(model):
+            print(f"  WARNING: {model} not available in Ollama — skipping")
+            all_results[model] = {"error": f"Model {model} not available", "skipped": True}
+            print()
+            continue
+
+        model_results = run_all_benchmarks(model)
+        all_results[model] = model_results
+
+        s = score_model(model_results)
+        print(f"  Summary: {s['pass_rate']} benchmarks passed in {s['total_time_s']}s")
+        print()
+
+    # Generate and write markdown report
+    markdown = generate_markdown(all_results, run_date)
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(markdown, encoding="utf-8")
+    print(f"Report written to: {args.output}")
+
+    if args.json_output:
+        args.json_output.write_text(json.dumps(all_results, indent=2), encoding="utf-8")
+        print(f"JSON data written to: {args.json_output}")
+
+    # Overall pass/fail
+    all_pass = all(
+        not r.get("skipped", False)
+        and all(b.get("passed", False) for b in r.values() if isinstance(b, dict))
+        for r in all_results.values()
+    )
+    return 0 if all_pass else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())