064ab602dd
All checks were successful
Smoke Test / smoke (pull_request) Successful in 16s
Implements Issue #80: benchmark turboquant vs llama.cpp baseline on M1. New files: - benchmarks/run_m1_benchmark.py — comprehensive benchmark runner - benchmarks/run_benchmark_m1.sh — shell wrapper for easy execution - tests/test_m1_benchmark.py — unit tests for benchmark functions Measures: - Tokens/sec throughput (f16 vs turbo4, 3-run average) - Memory usage (RSS monitoring during inference) - Quality via perplexity (llama-perplexity on wikitext-2) Generates: - benchmarks/m1_benchmark_results.json — raw results - benchmarks/m1_benchmark_report.md — markdown comparison table Closes #80
137 lines
4.3 KiB
Python
137 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Tests for run_m1_benchmark.py (Issue #80)
|
|
|
|
Validates core benchmark functions without requiring a live server.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
# Add parent dir to path
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from benchmarks.run_m1_benchmark import (
|
|
MemoryMonitor,
|
|
get_system_info,
|
|
generate_report,
|
|
)
|
|
|
|
|
|
class TestMemoryMonitor(unittest.TestCase):
|
|
def test_init(self):
|
|
mon = MemoryMonitor(pid=1, interval=0.1)
|
|
self.assertEqual(mon.pid, 1)
|
|
self.assertEqual(mon.samples, [])
|
|
|
|
def test_get_stats_empty(self):
|
|
mon = MemoryMonitor(pid=1)
|
|
stats = mon.get_stats()
|
|
self.assertEqual(stats["avg_mb"], 0)
|
|
self.assertEqual(stats["peak_mb"], 0)
|
|
self.assertEqual(stats["samples"], 0)
|
|
|
|
def test_get_stats_with_samples(self):
|
|
mon = MemoryMonitor(pid=1)
|
|
mon.samples = [100.0, 150.0, 200.0, 120.0]
|
|
stats = mon.get_stats()
|
|
self.assertEqual(stats["peak_mb"], 200.0)
|
|
self.assertEqual(stats["min_mb"], 100.0)
|
|
self.assertEqual(stats["avg_mb"], 142.5)
|
|
self.assertEqual(stats["samples"], 4)
|
|
|
|
|
|
class TestSystemInfo(unittest.TestCase):
|
|
def test_returns_dict(self):
|
|
info = get_system_info()
|
|
self.assertIsInstance(info, dict)
|
|
self.assertIn("platform", info)
|
|
self.assertIn("python", info)
|
|
|
|
|
|
class TestReportGeneration(unittest.TestCase):
|
|
def test_basic_report(self):
|
|
results = {
|
|
"timestamp": "2026-04-15T12:00:00Z",
|
|
"system": {"chip": "Apple M1", "memory_gb": 16, "cpu_cores": 8},
|
|
"model": "test-model",
|
|
"throughput": {
|
|
"f16": {
|
|
"avg_tok_per_sec": 100.0,
|
|
"avg_latency": 2.5,
|
|
"avg_ttft": 0.3,
|
|
"results": [
|
|
{"tokens_per_sec": 100, "latency_s": 2.5, "status": "success"},
|
|
],
|
|
},
|
|
"turbo4": {
|
|
"avg_tok_per_sec": 90.0,
|
|
"avg_latency": 2.8,
|
|
"avg_ttft": 0.35,
|
|
"results": [
|
|
{"tokens_per_sec": 90, "latency_s": 2.8, "status": "success"},
|
|
],
|
|
},
|
|
},
|
|
"memory": {
|
|
"f16": {"peak_mb": 1000, "avg_mb": 900},
|
|
"turbo4": {"peak_mb": 300, "avg_mb": 250},
|
|
},
|
|
"perplexity": {
|
|
"f16": {"perplexity": 12.5, "tokens": 5000, "elapsed_seconds": 120},
|
|
"turbo4": {"perplexity": 12.8, "tokens": 5000, "elapsed_seconds": 130},
|
|
"delta": 0.3,
|
|
"pass": True,
|
|
"threshold": 0.5,
|
|
},
|
|
"issues_discovered": [],
|
|
}
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f:
|
|
report_path = f.name
|
|
|
|
try:
|
|
report = generate_report(results, report_path)
|
|
self.assertIn("TurboQuant M1 Benchmark Report", report)
|
|
self.assertIn("f16", report)
|
|
self.assertIn("turbo4", report)
|
|
self.assertIn("PASS", report)
|
|
|
|
# Verify file was written
|
|
with open(report_path) as f:
|
|
written = f.read()
|
|
self.assertEqual(written, report)
|
|
finally:
|
|
os.unlink(report_path)
|
|
|
|
def test_report_with_issues(self):
|
|
results = {
|
|
"timestamp": "2026-04-15T12:00:00Z",
|
|
"system": {"chip": "M1", "memory_gb": 16, "cpu_cores": 8},
|
|
"model": "test",
|
|
"throughput": {"f16": {"results": []}, "turbo4": {"results": []}},
|
|
"memory": {"f16": {}, "turbo4": {}},
|
|
"perplexity": {},
|
|
"issues_discovered": [
|
|
{"title": "Test issue", "description": "Something went wrong"}
|
|
],
|
|
}
|
|
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False) as f:
|
|
report_path = f.name
|
|
|
|
try:
|
|
report = generate_report(results, report_path)
|
|
self.assertIn("Issues Discovered", report)
|
|
self.assertIn("Test issue", report)
|
|
finally:
|
|
os.unlink(report_path)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|