Compare commits
2 Commits
step35/94-
...
step35/96-
| Author | SHA1 | Date | |
|---|---|---|---|
|
d7cfc1db2c | ||
|
|
2fca513e26 |
@@ -29,6 +29,10 @@ jobs:
|
||||
run: |
|
||||
if grep -rE 'sk-or-|sk-ant-|ghp_|AKIA' . --include='*.yml' --include='*.py' --include='*.sh' 2>/dev/null | grep -v .gitea | grep -v llama-cpp-fork; then exit 1; fi
|
||||
echo "PASS: No secrets"
|
||||
- name: Tool call regression suite (issue #96)
|
||||
run: |
|
||||
python3 -m pip install -q pytest pyyaml requests
|
||||
pytest tests/tool_call_regression.py -v --tb=short
|
||||
- name: Markdown link check
|
||||
run: |
|
||||
python3 check_markdown_links.py
|
||||
|
||||
@@ -1,56 +0,0 @@
|
||||
# TurboQuant M1 Mac Benchmark — 2026-04-15
|
||||
|
||||
**Status:** Template — run `benchmarks/m1_mac_benchmark.py` on M1 Mac to populate.
|
||||
**Issue:** #94
|
||||
|
||||
## Hardware
|
||||
|
||||
| Spec | Value |
|
||||
|------|-------|
|
||||
| Chip | Apple M1 (or M1 Pro/Max/Ultra) |
|
||||
| Memory | 8/16/32/64 GB unified |
|
||||
| P-cores | 4/6/8 |
|
||||
| E-cores | 2 |
|
||||
| GPU cores | 7/8/14/16/24/32 |
|
||||
| macOS | 14.x |
|
||||
|
||||
## Results
|
||||
|
||||
| Preset | KV Type | Bits/ch | Compression | Avg tok/s | Peak Memory | GSM8K | Tool Call |
|
||||
|--------|---------|---------|-------------|-----------|-------------|-------|-----------|
|
||||
| turboquant_k8v4 | turbo4 | 3.5 | 4.2x | TBD | TBD | TBD | TBD |
|
||||
| turboquant_4bit_nc | q4_0 | 4.0 | 3.5x | TBD | TBD | TBD | TBD |
|
||||
| turboquant_3bit_nc | q3_k | 3.0 | 5.0x | TBD | TBD | TBD | TBD |
|
||||
|
||||
## How to Run
|
||||
|
||||
```bash
|
||||
# 1. Start llama-server with each preset
|
||||
# turboquant_k8v4
|
||||
llama-server -m ~/models/gemma-4-q4_k_m.gguf --port 8081 -ctk turbo4 -ctv turbo4 -c 4096
|
||||
|
||||
# 2. Run benchmark
|
||||
cd turboquant
|
||||
python3 benchmarks/m1_mac_benchmark.py \
|
||||
--url http://localhost:8081 \
|
||||
--model gemma-4 \
|
||||
--eval gsm8k \
|
||||
--output benchmarks/m1-mac-$(date +%Y-%m-%d).md
|
||||
|
||||
# 3. Repeat for other presets (change -ctk/-ctv)
|
||||
# turboquant_4bit_nc: -ctk q4_0 -ctv q4_0
|
||||
# turboquant_3bit_nc: -ctk q3_k -ctv q3_k
|
||||
|
||||
# 4. Or use vLLM
|
||||
vllm serve google/gemma-4-31b-it --kv-cache-dtype turboquant_k8v4
|
||||
python3 benchmarks/m1_mac_benchmark.py --backend vllm --eval gsm8k
|
||||
```
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Default:** TBD after benchmarks complete.
|
||||
|
||||
Decision criteria:
|
||||
- If turboquant_k8v4 GSM8K ≥ turboquant_4bit_nc GSM8K: use k8v4 (better compression, same quality)
|
||||
- If 3bit GSM8K drops >10%: don't use as default
|
||||
- Memory headroom: must fit model + KV within 70% of unified memory
|
||||
@@ -1,652 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
m1_mac_benchmark.py — Benchmark TurboQuant presets on Apple Silicon.
|
||||
|
||||
Runs all three TurboQuant presets through standardized benchmarks,
|
||||
measuring tokens/sec, peak memory, and quality. Produces a markdown
|
||||
results table for issue #94.
|
||||
|
||||
Presets:
|
||||
- turboquant_k8v4: PolarQuant WHT + 8-bit codebook + 4-bit QJL residual
|
||||
- turboquant_4bit_nc: 4-bit KV cache, no correction
|
||||
- turboquant_3bit_nc: 3-bit KV cache, no correction
|
||||
|
||||
Usage:
|
||||
# Full benchmark (requires llama-server running per preset)
|
||||
python3 benchmarks/m1_mac_benchmark.py
|
||||
|
||||
# Single preset
|
||||
python3 benchmarks/m1_mac_benchmark.py --preset turboquant_k8v4
|
||||
|
||||
# Custom server URL
|
||||
python3 benchmarks/m1_mac_benchmark.py --url http://localhost:8081
|
||||
|
||||
# With quality eval (GSM8K subset)
|
||||
python3 benchmarks/m1_mac_benchmark.py --eval gsm8k
|
||||
|
||||
# JSON output
|
||||
python3 benchmarks/m1_mac_benchmark.py --json
|
||||
|
||||
# Dry-run (validate framework without inference)
|
||||
python3 benchmarks/m1_mac_benchmark.py --dry-run
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
try:
|
||||
import requests
|
||||
except ImportError:
|
||||
requests = None
|
||||
|
||||
# ── TurboQuant Presets ────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class Preset:
|
||||
"""A TurboQuant KV cache preset."""
|
||||
name: str
|
||||
kv_type: str # -ctk/-ctv value for llama-server
|
||||
bits_per_channel: float
|
||||
compression_ratio: float
|
||||
description: str
|
||||
# vLLM equivalent (for vllm serve --kv-cache-dtype)
|
||||
vllm_dtype: str = ""
|
||||
|
||||
|
||||
PRESETS = {
|
||||
"turboquant_k8v4": Preset(
|
||||
name="turboquant_k8v4",
|
||||
kv_type="turbo4",
|
||||
bits_per_channel=3.5,
|
||||
compression_ratio=4.2,
|
||||
description="PolarQuant WHT + 8-bit codebook + 4-bit QJL residual. Best quality/compression ratio.",
|
||||
vllm_dtype="turboquant_k8v4",
|
||||
),
|
||||
"turboquant_4bit_nc": Preset(
|
||||
name="turboquant_4bit_nc",
|
||||
kv_type="q4_0",
|
||||
bits_per_channel=4.0,
|
||||
compression_ratio=3.5,
|
||||
description="4-bit KV cache, no correction. Standard baseline.",
|
||||
vllm_dtype="turboquant_4bit_nc",
|
||||
),
|
||||
"turboquant_3bit_nc": Preset(
|
||||
name="turboquant_3bit_nc",
|
||||
kv_type="q3_k",
|
||||
bits_per_channel=3.0,
|
||||
compression_ratio=5.0,
|
||||
description="3-bit KV cache, no correction. Aggressive compression, lower quality.",
|
||||
vllm_dtype="turboquant_3bit_nc",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ── Hardware Detection ────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class AppleSiliconInfo:
|
||||
"""Detected Apple Silicon hardware."""
|
||||
chip_name: str = ""
|
||||
total_memory_gb: float = 0.0
|
||||
performance_cores: int = 0
|
||||
efficiency_cores: int = 0
|
||||
gpu_cores: int = 0
|
||||
os_version: str = ""
|
||||
|
||||
|
||||
def detect_apple_silicon() -> AppleSiliconInfo:
|
||||
"""Detect Apple Silicon hardware details."""
|
||||
info = AppleSiliconInfo()
|
||||
|
||||
if platform.system() != "Darwin":
|
||||
return info
|
||||
|
||||
try:
|
||||
# Chip name
|
||||
result = subprocess.run(
|
||||
["sysctl", "-n", "machdep.cpu.brand_string"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if result.returncode == 0:
|
||||
info.chip_name = result.stdout.strip()
|
||||
|
||||
# Memory
|
||||
result = subprocess.run(
|
||||
["sysctl", "-n", "hw.memsize"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if result.returncode == 0:
|
||||
info.total_memory_gb = int(result.stdout.strip()) / (1024**3)
|
||||
|
||||
# CPU cores (performance vs efficiency)
|
||||
result = subprocess.run(
|
||||
["sysctl", "-n", "hw.perflevel0.physicalcpu"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if result.returncode == 0:
|
||||
info.performance_cores = int(result.stdout.strip())
|
||||
|
||||
result = subprocess.run(
|
||||
["sysctl", "-n", "hw.perflevel1.physicalcpu"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if result.returncode == 0:
|
||||
info.efficiency_cores = int(result.stdout.strip())
|
||||
|
||||
# OS version
|
||||
result = subprocess.run(
|
||||
["sw_vers", "-productVersion"],
|
||||
capture_output=True, text=True, timeout=5
|
||||
)
|
||||
if result.returncode == 0:
|
||||
info.os_version = result.stdout.strip()
|
||||
|
||||
# Try to get GPU core count from system_profiler (slow, optional)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["system_profiler", "SPDisplaysDataType"],
|
||||
capture_output=True, text=True, timeout=10
|
||||
)
|
||||
if result.returncode == 0:
|
||||
gpu_match = re.search(r"(\d+)\s*(?:core|Core)", result.stdout)
|
||||
if gpu_match:
|
||||
info.gpu_cores = int(gpu_match.group(1))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
except Exception as e:
|
||||
print(f"Warning: Apple Silicon detection failed: {e}", file=sys.stderr)
|
||||
|
||||
return info
|
||||
|
||||
|
||||
# ── Benchmark Prompts ─────────────────────────────────────────────────────────
|
||||
|
||||
BENCHMARK_PROMPTS = {
|
||||
"summarization": "Summarize the following text in 3 bullet points: 'The Timmy Foundation is a decentralized initiative focused on building sovereign AI. Its core principles are outlined in SOUL.md, which is inscribed on the Bitcoin blockchain. The project includes several repositories: the-nexus for 3D world-building, the-door for crisis intervention, and turboquant for local inference optimization.'",
|
||||
"code_generation": "Write a Python function that takes a list of integers and returns the two numbers that add up to a target sum. Include type hints and a docstring.",
|
||||
"reasoning": "If a TurboQuant KV cache uses 3.5 bits per channel and the uncompressed baseline uses 16 bits, what is the compression ratio? Show your calculation.",
|
||||
"creative": "Write a haiku about a blockchain inscription that can never be erased.",
|
||||
"tool_use": "Call the get_weather function with location='San Francisco' and unit='celsius'.",
|
||||
}
|
||||
|
||||
GSM8K_PROBLEMS = [
|
||||
{
|
||||
"question": "Janet's ducks lay 16 eggs per day. She eats three for breakfast every morning and bakes muffins for her friends every day with four. She sells the remainder at the farmers' market daily for $2 per egg. How much does she make every day?",
|
||||
"answer": "18",
|
||||
},
|
||||
{
|
||||
"question": "A robe takes 2 bolts of blue fiber and half that much white fiber. How many bolts in total does it take?",
|
||||
"answer": "3",
|
||||
},
|
||||
{
|
||||
"question": "Josh decides to try flipping a house. He buys a house for $80,000 and puts $50,000 in repairs. This increased the value of the house by 150%. How much profit did he make?",
|
||||
"answer": "70000",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
# ── Inference Backends ────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class BenchmarkResult:
|
||||
"""Result of a single benchmark run."""
|
||||
preset: str
|
||||
prompt_id: str
|
||||
tokens_per_sec: float = 0.0
|
||||
time_to_first_token_ms: float = 0.0
|
||||
total_tokens: int = 0
|
||||
elapsed_seconds: float = 0.0
|
||||
peak_memory_mb: float = 0.0
|
||||
output_text: str = ""
|
||||
error: str = ""
|
||||
|
||||
|
||||
def run_llama_server(prompt: str, url: str, model: str = "",
|
||||
kv_type: str = "f16", max_tokens: int = 256,
|
||||
timeout: int = 120) -> dict:
|
||||
"""Run a prompt against llama-server (OpenAI-compatible API)."""
|
||||
if requests is None:
|
||||
return {"error": "requests not installed"}
|
||||
|
||||
api_url = f"{url.rstrip('/')}/v1/chat/completions"
|
||||
start = time.time()
|
||||
ttft = None
|
||||
tokens = 0
|
||||
|
||||
try:
|
||||
resp = requests.post(api_url, json={
|
||||
"model": model or "local",
|
||||
"messages": [{"role": "user", "content": prompt}],
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0.7,
|
||||
"stream": True,
|
||||
}, stream=True, timeout=timeout)
|
||||
|
||||
output_parts = []
|
||||
for line in resp.iter_lines():
|
||||
if not line:
|
||||
continue
|
||||
line = line.decode("utf-8", errors="replace")
|
||||
if line.startswith("data: "):
|
||||
data_str = line[6:]
|
||||
if data_str.strip() == "[DONE]":
|
||||
break
|
||||
try:
|
||||
chunk = json.loads(data_str)
|
||||
delta = chunk.get("choices", [{}])[0].get("delta", {})
|
||||
content = delta.get("content", "")
|
||||
if content:
|
||||
if ttft is None:
|
||||
ttft = (time.time() - start) * 1000
|
||||
tokens += 1
|
||||
output_parts.append(content)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
elapsed = time.time() - start
|
||||
tps = tokens / elapsed if elapsed > 0 else 0.0
|
||||
|
||||
return {
|
||||
"tokens_per_sec": round(tps, 2),
|
||||
"time_to_first_token_ms": round(ttft, 1) if ttft else 0,
|
||||
"total_tokens": tokens,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"output_text": "".join(output_parts),
|
||||
}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
def run_ollama(prompt: str, url: str = "http://localhost:11434",
|
||||
model: str = "gemma4:latest", timeout: int = 120) -> dict:
|
||||
"""Run a prompt against Ollama /api/generate."""
|
||||
if requests is None:
|
||||
return {"error": "requests not installed"}
|
||||
|
||||
api_url = f"{url.rstrip('/')}/api/generate"
|
||||
start = time.time()
|
||||
ttft = None
|
||||
tokens = 0
|
||||
|
||||
try:
|
||||
resp = requests.post(api_url, json={
|
||||
"model": model,
|
||||
"prompt": prompt,
|
||||
"stream": True,
|
||||
"options": {"num_predict": 256},
|
||||
}, stream=True, timeout=timeout)
|
||||
|
||||
output_parts = []
|
||||
for line in resp.iter_lines():
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
chunk = json.loads(line)
|
||||
text = chunk.get("response", "")
|
||||
if text:
|
||||
if ttft is None:
|
||||
ttft = (time.time() - start) * 1000
|
||||
tokens += 1
|
||||
output_parts.append(text)
|
||||
if chunk.get("done", False):
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
elapsed = time.time() - start
|
||||
tps = tokens / elapsed if elapsed > 0 else 0.0
|
||||
|
||||
return {
|
||||
"tokens_per_sec": round(tps, 2),
|
||||
"time_to_first_token_ms": round(ttft, 1) if ttft else 0,
|
||||
"total_tokens": tokens,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"output_text": "".join(output_parts),
|
||||
}
|
||||
except Exception as e:
|
||||
return {"error": str(e)}
|
||||
|
||||
|
||||
def run_vllm(prompt: str, model: str = "google/gemma-4-31b-it",
|
||||
kv_dtype: str = "turboquant_k8v4", timeout: int = 120) -> dict:
|
||||
"""Run via vLLM serve (OpenAI-compatible on localhost:8000)."""
|
||||
return run_llama_server(prompt, url="http://localhost:8000",
|
||||
model=model, kv_type=kv_dtype, timeout=timeout)
|
||||
|
||||
|
||||
# ── Quality Evaluation ────────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class QualityResult:
|
||||
"""Quality evaluation result."""
|
||||
gsm8k_correct: int = 0
|
||||
gsm8k_total: int = 0
|
||||
gsm8k_accuracy: float = 0.0
|
||||
tool_call_detected: bool = False
|
||||
details: list = field(default_factory=list)
|
||||
|
||||
|
||||
def evaluate_gsm8k(output: str, expected: str) -> bool:
|
||||
"""Check if GSM8K answer is in the output."""
|
||||
# Extract the numeric answer from output
|
||||
numbers = re.findall(r'\b(\d[\d,]*)\b', output)
|
||||
if not numbers:
|
||||
return False
|
||||
# Check last number (most likely to be the answer)
|
||||
for num in reversed(numbers):
|
||||
clean = num.replace(",", "")
|
||||
if clean == expected:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def evaluate_tool_call(output: str) -> bool:
|
||||
"""Check if output contains a function/tool call."""
|
||||
indicators = [
|
||||
"get_weather", "function_call", "tool_use",
|
||||
"tool_call", '"name":', '"arguments":',
|
||||
"```json", "calling", "invoke",
|
||||
]
|
||||
return any(ind.lower() in output.lower() for ind in indicators)
|
||||
|
||||
|
||||
# ── Main Benchmark Runner ─────────────────────────────────────────────────────
|
||||
|
||||
@dataclass
|
||||
class PresetResult:
|
||||
"""Aggregate results for one preset."""
|
||||
preset: str
|
||||
kv_type: str
|
||||
bits_per_channel: float
|
||||
compression_ratio: float
|
||||
description: str
|
||||
benchmarks: list = field(default_factory=list)
|
||||
quality: Optional[QualityResult] = None
|
||||
avg_tokens_per_sec: float = 0.0
|
||||
peak_memory_mb: float = 0.0
|
||||
gsm8k_score: str = ""
|
||||
tool_call_accuracy: str = ""
|
||||
|
||||
|
||||
def run_preset_benchmark(
|
||||
preset_name: str,
|
||||
url: str = "http://localhost:8081",
|
||||
model: str = "",
|
||||
backend: str = "llama-server",
|
||||
eval_mode: str = "",
|
||||
timeout: int = 120,
|
||||
dry_run: bool = False,
|
||||
) -> PresetResult:
|
||||
"""Run all benchmarks for a single preset."""
|
||||
preset = PRESETS[preset_name]
|
||||
|
||||
result = PresetResult(
|
||||
preset=preset.name,
|
||||
kv_type=preset.kv_type,
|
||||
bits_per_channel=preset.bits_per_channel,
|
||||
compression_ratio=preset.compression_ratio,
|
||||
description=preset.description,
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
result.avg_tokens_per_sec = 42.5
|
||||
result.peak_memory_mb = 8192.0
|
||||
result.gsm8k_score = "3/3 (100%)"
|
||||
result.tool_call_accuracy = "Yes"
|
||||
return result
|
||||
|
||||
# Run each benchmark prompt
|
||||
tps_values = []
|
||||
for prompt_id, prompt in BENCHMARK_PROMPTS.items():
|
||||
print(f" Running: {prompt_id}...", end=" ", flush=True)
|
||||
|
||||
if backend == "ollama":
|
||||
bench_result = run_ollama(prompt, url=url,
|
||||
model=model or "gemma4:latest",
|
||||
timeout=timeout)
|
||||
else:
|
||||
bench_result = run_llama_server(prompt, url=url,
|
||||
model=model, kv_type=preset.kv_type,
|
||||
timeout=timeout)
|
||||
|
||||
br = BenchmarkResult(
|
||||
preset=preset_name,
|
||||
prompt_id=prompt_id,
|
||||
**{k: v for k, v in bench_result.items() if k in BenchmarkResult.__dataclass_fields__}
|
||||
)
|
||||
result.benchmarks.append(br)
|
||||
|
||||
if br.tokens_per_sec > 0:
|
||||
tps_values.append(br.tokens_per_sec)
|
||||
print(f"{br.tokens_per_sec:.1f} tok/s")
|
||||
else:
|
||||
print(f"ERROR: {br.error}")
|
||||
|
||||
# Average tokens/sec
|
||||
result.avg_tokens_per_sec = round(
|
||||
sum(tps_values) / len(tps_values), 2
|
||||
) if tps_values else 0.0
|
||||
|
||||
# Peak memory (from system, not per-request)
|
||||
try:
|
||||
if sys.platform == "darwin":
|
||||
mem_result = subprocess.run(
|
||||
["ps", "-o", "rss=", "-p", str(os.getpid())],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
if mem_result.returncode == 0:
|
||||
result.peak_memory_mb = int(mem_result.stdout.strip()) / 1024
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Quality evaluation
|
||||
if eval_mode == "gsm8k":
|
||||
quality = QualityResult()
|
||||
for problem in GSM8K_PROBLEMS:
|
||||
if backend == "ollama":
|
||||
eval_result = run_ollama(problem["question"], url=url,
|
||||
model=model or "gemma4:latest",
|
||||
timeout=timeout)
|
||||
else:
|
||||
eval_result = run_llama_server(problem["question"], url=url,
|
||||
model=model, kv_type=preset.kv_type,
|
||||
timeout=timeout)
|
||||
|
||||
output = eval_result.get("output_text", "")
|
||||
correct = evaluate_gsm8k(output, problem["answer"])
|
||||
if correct:
|
||||
quality.gsm8k_correct += 1
|
||||
quality.gsm8k_total += 1
|
||||
quality.details.append({
|
||||
"question": problem["question"][:50] + "...",
|
||||
"expected": problem["answer"],
|
||||
"correct": correct,
|
||||
})
|
||||
|
||||
quality.gsm8k_accuracy = quality.gsm8k_correct / quality.gsm8k_total if quality.gsm8k_total else 0
|
||||
result.gsm8k_score = f"{quality.gsm8k_correct}/{quality.gsm8k_total} ({quality.gsm8k_accuracy:.0%})"
|
||||
|
||||
# Tool calling test
|
||||
tool_result = run_llama_server(BENCHMARK_PROMPTS["tool_use"],
|
||||
url=url, model=model,
|
||||
kv_type=preset.kv_type, timeout=timeout)
|
||||
tool_output = tool_result.get("output_text", "")
|
||||
quality.tool_call_detected = evaluate_tool_call(tool_output)
|
||||
result.tool_call_accuracy = "Yes" if quality.tool_call_detected else "No"
|
||||
result.quality = quality
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ── Report Generation ─────────────────────────────────────────────────────────
|
||||
|
||||
def generate_markdown_report(
|
||||
hw: AppleSiliconInfo,
|
||||
results: list[PresetResult],
|
||||
model: str,
|
||||
context_length: int,
|
||||
) -> str:
|
||||
"""Generate markdown benchmark report."""
|
||||
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
lines = [
|
||||
f"# TurboQuant M1 Mac Benchmark — {date}",
|
||||
"",
|
||||
f"**Date:** {ts}",
|
||||
f"**Model:** {model}",
|
||||
f"**Context length:** {context_length}",
|
||||
"",
|
||||
"## Hardware",
|
||||
"",
|
||||
f"| Spec | Value |",
|
||||
f"|------|-------|",
|
||||
f"| Chip | {hw.chip_name or 'Unknown'} |",
|
||||
f"| Memory | {hw.total_memory_gb:.0f} GB unified |",
|
||||
f"| P-cores | {hw.performance_cores} |",
|
||||
f"| E-cores | {hw.efficiency_cores} |",
|
||||
f"| GPU cores | {hw.gpu_cores or 'N/A'} |",
|
||||
f"| macOS | {hw.os_version or 'Unknown'} |",
|
||||
"",
|
||||
"## Results",
|
||||
"",
|
||||
"| Preset | KV Type | Bits/ch | Compression | Avg tok/s | Peak Memory | GSM8K | Tool Call |",
|
||||
"|--------|---------|---------|-------------|-----------|-------------|-------|-----------|",
|
||||
]
|
||||
|
||||
for r in results:
|
||||
lines.append(
|
||||
f"| {r.preset} | {r.kv_type} | {r.bits_per_channel} | "
|
||||
f"{r.compression_ratio}x | {r.avg_tokens_per_sec:.1f} | "
|
||||
f"{r.peak_memory_mb:.0f} MB | {r.gsm8k_score or 'N/A'} | "
|
||||
f"{r.tool_call_accuracy or 'N/A'} |"
|
||||
)
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"## Per-Prompt Breakdown",
|
||||
"",
|
||||
])
|
||||
|
||||
for r in results:
|
||||
lines.append(f"### {r.preset}")
|
||||
lines.append(f"_{r.description}_")
|
||||
lines.append("")
|
||||
lines.append("| Prompt | tok/s | TTFT (ms) | Tokens | Elapsed (s) |")
|
||||
lines.append("|--------|-------|-----------|--------|-------------|")
|
||||
for b in r.benchmarks:
|
||||
lines.append(
|
||||
f"| {b.prompt_id} | {b.tokens_per_sec:.1f} | "
|
||||
f"{b.time_to_first_token_ms:.0f} | {b.total_tokens} | "
|
||||
f"{b.elapsed_seconds:.2f} |"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
# Recommendation
|
||||
if results:
|
||||
best_quality = max(results, key=lambda r: r.avg_tokens_per_sec if r.bits_per_channel >= 3.5 else 0)
|
||||
lines.extend([
|
||||
"## Recommendation",
|
||||
"",
|
||||
f"**Default for M1 Mac:** `{best_quality.preset}` ({best_quality.kv_type})",
|
||||
"",
|
||||
f"Rationale: {best_quality.description}",
|
||||
"",
|
||||
])
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ── CLI ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Benchmark TurboQuant presets on Apple Silicon"
|
||||
)
|
||||
parser.add_argument("--preset", choices=list(PRESETS.keys()),
|
||||
help="Run single preset (default: all)")
|
||||
parser.add_argument("--url", default="http://localhost:8081",
|
||||
help="Server URL (default: http://localhost:8081)")
|
||||
parser.add_argument("--model", default="",
|
||||
help="Model name (auto-detected if empty)")
|
||||
parser.add_argument("--backend", choices=["llama-server", "ollama", "vllm"],
|
||||
default="llama-server")
|
||||
parser.add_argument("--eval", choices=["", "gsm8k"], default="",
|
||||
help="Quality evaluation mode")
|
||||
parser.add_argument("--context", type=int, default=4096,
|
||||
help="Context length tested (for report)")
|
||||
parser.add_argument("--timeout", type=int, default=120)
|
||||
parser.add_argument("--json", action="store_true", help="JSON output")
|
||||
parser.add_argument("--output", help="Save markdown report to file")
|
||||
parser.add_argument("--dry-run", action="store_true",
|
||||
help="Validate framework without inference")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Detect hardware
|
||||
hw = detect_apple_silicon()
|
||||
if hw.chip_name:
|
||||
print(f"Hardware: {hw.chip_name}, {hw.total_memory_gb:.0f}GB, "
|
||||
f"{hw.performance_cores}P+{hw.efficiency_cores}E cores")
|
||||
else:
|
||||
print("Hardware: Non-Apple Silicon (running in simulation mode)")
|
||||
|
||||
# Determine presets to run
|
||||
preset_names = [args.preset] if args.preset else list(PRESETS.keys())
|
||||
|
||||
results = []
|
||||
for name in preset_names:
|
||||
print(f"\n--- {name} ---")
|
||||
preset_result = run_preset_benchmark(
|
||||
name, url=args.url, model=args.model,
|
||||
backend=args.backend, eval_mode=args.eval,
|
||||
timeout=args.timeout, dry_run=args.dry_run,
|
||||
)
|
||||
results.append(preset_result)
|
||||
|
||||
# Output
|
||||
if args.json:
|
||||
output = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"hardware": {
|
||||
"chip": hw.chip_name,
|
||||
"memory_gb": hw.total_memory_gb,
|
||||
"p_cores": hw.performance_cores,
|
||||
"e_cores": hw.efficiency_cores,
|
||||
"gpu_cores": hw.gpu_cores,
|
||||
"macos": hw.os_version,
|
||||
},
|
||||
"model": args.model or "auto",
|
||||
"context_length": args.context,
|
||||
"results": [asdict(r) for r in results],
|
||||
}
|
||||
print(json.dumps(output, indent=2, default=str))
|
||||
else:
|
||||
report = generate_markdown_report(hw, results, args.model, args.context)
|
||||
print("\n" + report)
|
||||
|
||||
# Save report
|
||||
output_path = args.output
|
||||
if not output_path:
|
||||
date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||
output_path = f"benchmarks/m1-mac-{date}.md"
|
||||
|
||||
report = generate_markdown_report(hw, results, args.model, args.context)
|
||||
# Save locally for reference (actual commit happens via API)
|
||||
print(f"\nReport saved to {output_path}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
2
benchmarks/tool-call-regression.md
Normal file
2
benchmarks/tool-call-regression.md
Normal file
@@ -0,0 +1,2 @@
|
||||
| Timestamp | Model | Preset | Accuracy | read_file | web_search | terminal | execute_code | delegate_task | Parallel |
|
||||
|-----------|-------|--------|----------|-----------|------------|----------|--------------|---------------|----------|
|
||||
@@ -1,152 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Tests for m1_mac_benchmark.py"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import pytest
|
||||
from unittest.mock import patch, MagicMock
|
||||
from datetime import datetime, timezone
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
|
||||
from benchmarks.m1_mac_benchmark import (
|
||||
Preset,
|
||||
AppleSiliconInfo,
|
||||
BenchmarkResult,
|
||||
PresetResult,
|
||||
QualityResult,
|
||||
PRESETS,
|
||||
detect_apple_silicon,
|
||||
evaluate_gsm8k,
|
||||
evaluate_tool_call,
|
||||
generate_markdown_report,
|
||||
run_preset_benchmark,
|
||||
)
|
||||
|
||||
|
||||
class TestPresets:
|
||||
def test_all_presets_defined(self):
|
||||
assert "turboquant_k8v4" in PRESETS
|
||||
assert "turboquant_4bit_nc" in PRESETS
|
||||
assert "turboquant_3bit_nc" in PRESETS
|
||||
|
||||
def test_preset_fields(self):
|
||||
for name, preset in PRESETS.items():
|
||||
assert preset.name == name
|
||||
assert preset.bits_per_channel > 0
|
||||
assert preset.compression_ratio > 1
|
||||
assert preset.kv_type
|
||||
assert preset.description
|
||||
|
||||
def test_presets_ordered_by_bits(self):
|
||||
"""k8v4 should be ~3.5b, 4bit should be 4.0, 3bit should be 3.0."""
|
||||
assert PRESETS["turboquant_4bit_nc"].bits_per_channel > PRESETS["turboquant_k8v4"].bits_per_channel
|
||||
assert PRESETS["turboquant_k8v4"].bits_per_channel > PRESETS["turboquant_3bit_nc"].bits_per_channel
|
||||
|
||||
|
||||
class TestGSM8KEval:
|
||||
def test_correct_answer(self):
|
||||
output = "Janet makes 9 + 9 = 18 dollars per day."
|
||||
assert evaluate_gsm8k(output, "18") is True
|
||||
|
||||
def test_correct_with_commas(self):
|
||||
output = "The profit is $70,000."
|
||||
assert evaluate_gsm8k(output, "70000") is True
|
||||
|
||||
def test_wrong_answer(self):
|
||||
output = "The answer is 42 dollars."
|
||||
assert evaluate_gsm8k(output, "18") is False
|
||||
|
||||
def test_no_number(self):
|
||||
output = "I'm not sure about this problem."
|
||||
assert evaluate_gsm8k(output, "18") is False
|
||||
|
||||
def test_correct_answer_not_last(self):
|
||||
"""If the answer appears in the reasoning, not just at the end."""
|
||||
output = "There are 16 eggs. She eats 3, uses 4. That leaves 9. She sells for $2 each = 18 dollars."
|
||||
assert evaluate_gsm8k(output, "18") is True
|
||||
|
||||
|
||||
class TestToolCallEval:
|
||||
def test_function_name(self):
|
||||
output = "I'll call get_weather with the parameters."
|
||||
assert evaluate_tool_call(output) is True
|
||||
|
||||
def test_json_format(self):
|
||||
output = '```json\n{"name": "get_weather", "arguments": {}}\n```'
|
||||
assert evaluate_tool_call(output) is True
|
||||
|
||||
def test_no_tool(self):
|
||||
output = "The weather in San Francisco is sunny."
|
||||
assert evaluate_tool_call(output) is False
|
||||
|
||||
|
||||
class TestMarkdownReport:
|
||||
def test_generates_report(self):
|
||||
hw = AppleSiliconInfo(
|
||||
chip_name="Apple M1 Max",
|
||||
total_memory_gb=32,
|
||||
performance_cores=8,
|
||||
efficiency_cores=2,
|
||||
gpu_cores=24,
|
||||
os_version="14.2",
|
||||
)
|
||||
results = [
|
||||
PresetResult(
|
||||
preset="turboquant_k8v4",
|
||||
kv_type="turbo4",
|
||||
bits_per_channel=3.5,
|
||||
compression_ratio=4.2,
|
||||
description="Best quality",
|
||||
avg_tokens_per_sec=45.2,
|
||||
peak_memory_mb=8192,
|
||||
gsm8k_score="2/3 (67%)",
|
||||
tool_call_accuracy="Yes",
|
||||
benchmarks=[BenchmarkResult(
|
||||
preset="turboquant_k8v4",
|
||||
prompt_id="summarization",
|
||||
tokens_per_sec=45.2,
|
||||
time_to_first_token_ms=150,
|
||||
total_tokens=128,
|
||||
elapsed_seconds=2.83,
|
||||
)],
|
||||
),
|
||||
]
|
||||
report = generate_markdown_report(hw, results, "gemma-4", 4096)
|
||||
|
||||
assert "TurboQuant M1 Mac Benchmark" in report
|
||||
assert "Apple M1 Max" in report
|
||||
assert "turboquant_k8v4" in report
|
||||
assert "45.2" in report
|
||||
assert "Recommendation" in report
|
||||
|
||||
def test_empty_results(self):
|
||||
hw = AppleSiliconInfo()
|
||||
report = generate_markdown_report(hw, [], "test", 4096)
|
||||
assert "TurboQuant M1 Mac Benchmark" in report
|
||||
|
||||
|
||||
class TestDryRun:
|
||||
def test_dry_run_returns_results(self):
|
||||
result = run_preset_benchmark("turboquant_k8v4", dry_run=True)
|
||||
assert result.preset == "turboquant_k8v4"
|
||||
assert result.avg_tokens_per_sec > 0
|
||||
assert result.peak_memory_mb > 0
|
||||
|
||||
def test_dry_run_all_presets(self):
|
||||
for name in PRESETS:
|
||||
result = run_preset_benchmark(name, dry_run=True)
|
||||
assert result.preset == name
|
||||
assert result.avg_tokens_per_sec > 0
|
||||
|
||||
|
||||
class TestHardwareDetection:
|
||||
@patch("benchmarks.m1_mac_benchmark.platform.system", return_value="Linux")
|
||||
def test_non_apple(self, mock_system):
|
||||
hw = detect_apple_silicon()
|
||||
assert hw.chip_name == ""
|
||||
|
||||
def test_returns_info_structure(self):
|
||||
hw = detect_apple_silicon()
|
||||
assert isinstance(hw, AppleSiliconInfo)
|
||||
assert isinstance(hw.total_memory_gb, float)
|
||||
225
tests/test_tool_call_regression.py
Normal file
225
tests/test_tool_call_regression.py
Normal file
@@ -0,0 +1,225 @@
|
||||
"""
|
||||
TurboQuant Compressed Model Tool Call Regression Suite — Issue #96
|
||||
|
||||
Run: pytest tests/tool_call_regression.py -v
|
||||
Generate matrix: pytest tests/tool_call_regression.py --generate-matrix
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import time
|
||||
import unittest
|
||||
from typing import Dict
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
BENCHMARKS_DIR = ROOT / "benchmarks"
|
||||
RESULTS_MATRIX = BENCHMARKS_DIR / "tool-call-regression.md"
|
||||
|
||||
CORE_TOOLS = [
|
||||
{"name": "read_file", "description": "Read a text file", "args": {"path": "/tmp/test.txt"}},
|
||||
{"name": "web_search", "description": "Search the web", "args": {"query": "turboquant"}},
|
||||
{"name": "terminal", "description": "Run a shell command", "args": {"command": "echo ok"}},
|
||||
{"name": "execute_code", "description": "Run Python code", "args": {"code": "print(1)"}},
|
||||
{"name": "delegate_task", "description": "Delegate to subagent", "args": {"goal": "test"}},
|
||||
]
|
||||
|
||||
PARALLEL_TOOLS = [
|
||||
{"name": "read_file", "args": {"path": "/tmp/a.txt"}},
|
||||
{"name": "web_search", "args": {"query": "python"}},
|
||||
{"name": "execute_code", "args": {"code": "x=1"}},
|
||||
]
|
||||
|
||||
PASS_THRESHOLD = 0.95
|
||||
|
||||
|
||||
class TestToolSchemaContract(unittest.TestCase):
|
||||
def test_core_tool_schemas_are_valid_functions(self):
|
||||
for tool in CORE_TOOLS:
|
||||
schema = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": tool["name"],
|
||||
"description": tool["description"],
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"required": list(tool["args"].keys()),
|
||||
},
|
||||
},
|
||||
}
|
||||
parsed = json.loads(json.dumps(schema))
|
||||
assert parsed["type"] == "function"
|
||||
fn = parsed["function"]
|
||||
assert fn["name"] == tool["name"]
|
||||
assert fn["description"]
|
||||
assert "parameters" in fn
|
||||
|
||||
def test_parallel_tool_set_is_unique(self):
|
||||
names = [t["name"] for t in PARALLEL_TOOLS]
|
||||
assert len(names) == len(set(names))
|
||||
|
||||
def test_tool_call_response_format(self):
|
||||
tc = {"id": "call_abc", "type": "function",
|
||||
"function": {"name": "read_file", "arguments": json.dumps({"path": "/tmp/test.txt"})}}
|
||||
assert tc["type"] == "function"
|
||||
args = json.loads(tc["function"]["arguments"])
|
||||
assert "path" in args
|
||||
|
||||
def test_parallel_response_contains_multiple_calls(self):
|
||||
calls = [
|
||||
{"id": "c1", "type": "function", "function": {"name": "read_file", "arguments": "{}"}},
|
||||
{"id": "c2", "type": "function", "function": {"name": "web_search", "arguments": "{}"}},
|
||||
{"id": "c3", "type": "function", "function": {"name": "execute_code","arguments": "{}"}},
|
||||
]
|
||||
assert len(calls) >= 3
|
||||
call_names = {c["function"]["name"] for c in calls}
|
||||
assert len(call_names) >= 2
|
||||
|
||||
|
||||
class TestProfileConfig(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
import yaml
|
||||
cls.profile = yaml.safe_load((ROOT / "profiles" / "hermes-profile-gemma4-turboquant.yaml").read_text())
|
||||
|
||||
def test_primary_provider_has_all_required_fields(self):
|
||||
"""Provider must have model, endpoint, and turboquant config."""
|
||||
p = self.profile["providers"]["primary"]
|
||||
assert "model" in p
|
||||
assert "endpoint" in p
|
||||
assert "turboquant" in p
|
||||
def test_turboquant_enabled(self):
|
||||
tq = self.profile["providers"]["primary"].get("turboquant", {})
|
||||
assert tq.get("enabled") is True
|
||||
assert tq.get("kv_type") in ("turbo2", "turbo3", "turbo4")
|
||||
|
||||
def test_server_command_has_turboquant_flags(self):
|
||||
cmd = self.profile["providers"]["primary"].get("server_command", "")
|
||||
assert "-ctk" in cmd and "-ctv" in cmd
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not os.environ.get("TURBOQUANT_SERVER_URL"),
|
||||
reason="Set TURBOQUANT_SERVER_URL to run live regression"
|
||||
)
|
||||
class TestLiveRegression:
|
||||
RESULTS: Dict[str, bool] = {}
|
||||
|
||||
def _call_model(self, tools, prompt, timeout=120):
|
||||
import requests
|
||||
url = os.environ["TURBOQUANT_SERVER_URL"]
|
||||
resp = requests.post(
|
||||
f"{url}/v1/chat/completions",
|
||||
json={"model": "gemma-4", "messages": [{"role": "user", "content": prompt}],
|
||||
"tools": tools, "tool_choice": "auto"},
|
||||
timeout=timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def _has_valid_tool_call(self, data, expected_name):
|
||||
msg = data["choices"][0]["message"]
|
||||
for tc in msg.get("tool_calls", []):
|
||||
if tc["function"]["name"] == expected_name:
|
||||
json.loads(tc["function"]["arguments"])
|
||||
return True
|
||||
return False
|
||||
|
||||
def test_read_file(self):
|
||||
tools = [{"type":"function","function":{"name":"read_file","description":"Read file",
|
||||
"parameters":{"type":"object","properties":{"path":{"type":"string"}},"required":["path"]}}}]
|
||||
data = self._call_model(tools, "Read /tmp/test.txt")
|
||||
self.__class__.RESULTS["read_file"] = self._has_valid_tool_call(data, "read_file")
|
||||
|
||||
def test_web_search(self):
|
||||
tools = [{"type":"function","function":{"name":"web_search","description":"Search",
|
||||
"parameters":{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}}}]
|
||||
data = self._call_model(tools, "Search for Python")
|
||||
self.__class__.RESULTS["web_search"] = self._has_valid_tool_call(data, "web_search")
|
||||
|
||||
def test_terminal(self):
|
||||
tools = [{"type":"function","function":{"name":"terminal","description":"Shell",
|
||||
"parameters":{"type":"object","properties":{"command":{"type":"string"}},"required":["command"]}}}]
|
||||
data = self._call_model(tools, "List files")
|
||||
self.__class__.RESULTS["terminal"] = self._has_valid_tool_call(data, "terminal")
|
||||
|
||||
def test_execute_code(self):
|
||||
tools = [{"type":"function","function":{"name":"execute_code","description":"Code",
|
||||
"parameters":{"type":"object","properties":{"code":{"type":"string"}},"required":["code"]}}}]
|
||||
data = self._call_model(tools, "Run: print('test')")
|
||||
self.__class__.RESULTS["execute_code"] = self._has_valid_tool_call(data, "execute_code")
|
||||
|
||||
def test_delegate_task(self):
|
||||
tools = [{"type":"function","function":{"name":"delegate_task","description":"Delegate",
|
||||
"parameters":{"type":"object","properties":{"goal":{"type":"string"}},"required":["goal"]}}}]
|
||||
data = self._call_model(tools, "Delegate task: test")
|
||||
self.__class__.RESULTS["delegate_task"] = self._has_valid_tool_call(data, "delegate_task")
|
||||
|
||||
def test_parallel_tool_calling(self):
|
||||
tools = [
|
||||
{"type":"function","function":{"name":"read_file","description":"Read",
|
||||
"parameters":{"type":"object","properties":{"path":{"type":"string"}},"required":["path"]}},},
|
||||
{"type":"function","function":{"name":"web_search","description":"Search",
|
||||
"parameters":{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}},},
|
||||
{"type":"function","function":{"name":"execute_code","description":"Code",
|
||||
"parameters":{"type":"object","properties":{"code":{"type":"string"}},"required":["code"]}},},
|
||||
]
|
||||
data = self._call_model(tools, "Read a.txt, search python, run code")
|
||||
msg = data["choices"][0]["message"]
|
||||
calls = msg.get("tool_calls", [])
|
||||
names = {c["function"]["name"] for c in calls}
|
||||
self.__class__.RESULTS["parallel"] = len(names) >= 2
|
||||
|
||||
@classmethod
|
||||
def _accuracy(cls) -> float:
|
||||
if not cls.RESULTS:
|
||||
return 1.0
|
||||
return sum(1 for v in cls.RESULTS.values() if v) / len(cls.RESULTS)
|
||||
|
||||
@classmethod
|
||||
def teardown_class(cls):
|
||||
acc = cls._accuracy()
|
||||
print(f"\nTool Call Regression Accuracy: {acc*100:.1f}% (threshold {PASS_THRESHOLD*100:.0f}%)")
|
||||
for name, passed in cls.RESULTS.items():
|
||||
print(f" {name}: {'PASS' if passed else 'FAIL'}")
|
||||
assert acc >= PASS_THRESHOLD, f"Accuracy {acc*100:.1f}% below {PASS_THRESHOLD*100:.0f}% gate"
|
||||
if os.environ.get("GENERATE_MATRIX"):
|
||||
_append_matrix(acc, cls.RESULTS)
|
||||
|
||||
|
||||
def _append_matrix(accuracy: float, results: Dict[str, bool]):
|
||||
timestamp = time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime())
|
||||
tool_names = [t["name"] for t in CORE_TOOLS]
|
||||
tool_checks = ["✓" if results.get(n, False) else "✗" for n in tool_names]
|
||||
parallel_check = "✓" if results.get("parallel") else "✗"
|
||||
row = f"| {timestamp} | gemma-4 | turbo4 | {accuracy*100:.1f}% | " + " | ".join(tool_checks) + f" | {parallel_check} |\n"
|
||||
header = (
|
||||
"| Timestamp | Model | Preset | Accuracy | "
|
||||
+ " | ".join(tool_names)
|
||||
+ " | Parallel |\n"
|
||||
"|-----------|-------|--------|----------|"
|
||||
+ "---|" * (len(tool_names) + 1) + "\n"
|
||||
)
|
||||
if not RESULTS_MATRIX.exists():
|
||||
RESULTS_MATRIX.write_text(header + row)
|
||||
else:
|
||||
content = RESULTS_MATRIX.read_text()
|
||||
if header not in content:
|
||||
content = header + row + content
|
||||
else:
|
||||
content = header + row + content.split(header, 1)[1]
|
||||
RESULTS_MATRIX.write_text(content)
|
||||
print(f"Matrix updated: {RESULTS_MATRIX}")
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
parser.addoption("--generate-matrix", action="store_true",
|
||||
help="Update benchmarks/tool-call-regression.md with live results")
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
if config.getoption("--generate-matrix"):
|
||||
os.environ["GENERATE_MATRIX"] = "1"
|
||||
Reference in New Issue
Block a user