118 lines
4.2 KiB
Python
118 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Tests for benchmarks/quality_gate.py — Perplexity Quality Gate (#63)."""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import textwrap
|
|
from pathlib import Path
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "benchmarks"))
|
|
from quality_gate import (
|
|
PerplexityResult,
|
|
QualityGateResult,
|
|
measure_perplexity_ollama_proxy,
|
|
run_quality_gate,
|
|
)
|
|
|
|
|
|
class TestPerplexityResult:
|
|
def test_to_dict(self):
|
|
r = PerplexityResult(
|
|
backend="llama-server", kv_type="f16",
|
|
perplexity=12.5, is_proxy=False, tokens=1000,
|
|
elapsed_seconds=10.0, method="llama-perplexity", exit_code=0,
|
|
)
|
|
d = r.to_dict()
|
|
assert d["backend"] == "llama-server"
|
|
assert d["perplexity"] == 12.5
|
|
assert d["is_proxy"] is False
|
|
|
|
def test_proxy_flag(self):
|
|
r = PerplexityResult(
|
|
backend="ollama-proxy", kv_type="f16",
|
|
perplexity=3.2, is_proxy=True, method="proxy heuristic",
|
|
)
|
|
assert r.is_proxy is True
|
|
|
|
|
|
class TestQualityGateResult:
|
|
def test_pass(self):
|
|
f16 = PerplexityResult("llama-server", "f16", 10.0, False)
|
|
turbo4 = PerplexityResult("llama-server", "turbo4", 10.3, False)
|
|
gate = QualityGateResult(f16=f16, turbo4=turbo4, delta=0.3, threshold=0.5, passed=True, is_proxy=False)
|
|
assert gate.passed is True
|
|
assert gate.delta == 0.3
|
|
|
|
def test_fail(self):
|
|
f16 = PerplexityResult("llama-server", "f16", 10.0, False)
|
|
turbo4 = PerplexityResult("llama-server", "turbo4", 11.0, False)
|
|
gate = QualityGateResult(f16=f16, turbo4=turbo4, delta=1.0, threshold=0.5, passed=False, is_proxy=False)
|
|
assert gate.passed is False
|
|
|
|
def test_proxy_warning(self):
|
|
f16 = PerplexityResult("ollama-proxy", "f16", 5.0, True)
|
|
gate = QualityGateResult(f16=f16, turbo4=None, delta=None, threshold=0.5, passed=False, is_proxy=True, warning="Only F16 measured")
|
|
assert gate.is_proxy is True
|
|
summary = gate.summary()
|
|
assert "PROXY" in summary or "Proxy" in summary
|
|
|
|
def test_to_dict(self):
|
|
f16 = PerplexityResult("llama-server", "f16", 10.0, False)
|
|
gate = QualityGateResult(f16=f16, turbo4=None, delta=None, threshold=0.5, passed=False, is_proxy=False)
|
|
d = gate.to_dict()
|
|
assert d["f16"]["perplexity"] == 10.0
|
|
assert d["turbo4"] is None
|
|
assert d["delta"] is None
|
|
|
|
def test_summary_format(self):
|
|
f16 = PerplexityResult("llama-server", "f16", 10.0, False)
|
|
turbo4 = PerplexityResult("llama-server", "turbo4", 10.2, False)
|
|
gate = QualityGateResult(f16=f16, turbo4=turbo4, delta=0.2, threshold=0.5, passed=True, is_proxy=False)
|
|
summary = gate.summary()
|
|
assert "F16" in summary
|
|
assert "Turbo4" in summary
|
|
assert "PASS" in summary
|
|
assert "0.2000" in summary
|
|
|
|
|
|
class TestOllamaProxy:
|
|
def test_with_corpus_file(self):
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
|
|
f.write("The quick brown fox jumps over the lazy dog.\n" * 100)
|
|
f.flush()
|
|
result = measure_perplexity_ollama_proxy("test-model", f.name)
|
|
os.unlink(f.name)
|
|
# Result should be proxy
|
|
assert result.is_proxy is True
|
|
assert result.backend == "ollama-proxy"
|
|
|
|
def test_with_missing_corpus(self):
|
|
result = measure_perplexity_ollama_proxy("test-model", "/nonexistent/corpus.txt")
|
|
assert result.is_proxy is True
|
|
|
|
|
|
class TestRunQualityGate:
|
|
def test_unknown_backend(self):
|
|
result = run_quality_gate(backend="unknown", model="test")
|
|
assert result.passed is False
|
|
assert "Unknown backend" in result.warning
|
|
|
|
def test_llama_server_missing_binary(self):
|
|
result = run_quality_gate(
|
|
backend="llama-server",
|
|
model="test.gguf",
|
|
corpus="/tmp/nonexistent_corpus.txt",
|
|
llama_bin="/nonexistent/llama-perplexity",
|
|
)
|
|
assert result.f16 is not None
|
|
assert result.f16.error is not None
|
|
assert "not found" in result.f16.error.lower()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import unittest
|
|
unittest.main()
|