diff --git a/tests/test_quality_gate.py b/tests/test_quality_gate.py new file mode 100644 index 00000000..75a8aae5 --- /dev/null +++ b/tests/test_quality_gate.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Tests for benchmarks/quality_gate.py — Perplexity Quality Gate (#63).""" + +import json +import os +import sys +import tempfile +import textwrap +from pathlib import Path +from unittest.mock import patch, MagicMock + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "benchmarks")) +from quality_gate import ( + PerplexityResult, + QualityGateResult, + measure_perplexity_ollama_proxy, + run_quality_gate, +) + + +class TestPerplexityResult: + def test_to_dict(self): + r = PerplexityResult( + backend="llama-server", kv_type="f16", + perplexity=12.5, is_proxy=False, tokens=1000, + elapsed_seconds=10.0, method="llama-perplexity", exit_code=0, + ) + d = r.to_dict() + assert d["backend"] == "llama-server" + assert d["perplexity"] == 12.5 + assert d["is_proxy"] is False + + def test_proxy_flag(self): + r = PerplexityResult( + backend="ollama-proxy", kv_type="f16", + perplexity=3.2, is_proxy=True, method="proxy heuristic", + ) + assert r.is_proxy is True + + +class TestQualityGateResult: + def test_pass(self): + f16 = PerplexityResult("llama-server", "f16", 10.0, False) + turbo4 = PerplexityResult("llama-server", "turbo4", 10.3, False) + gate = QualityGateResult(f16=f16, turbo4=turbo4, delta=0.3, threshold=0.5, passed=True, is_proxy=False) + assert gate.passed is True + assert gate.delta == 0.3 + + def test_fail(self): + f16 = PerplexityResult("llama-server", "f16", 10.0, False) + turbo4 = PerplexityResult("llama-server", "turbo4", 11.0, False) + gate = QualityGateResult(f16=f16, turbo4=turbo4, delta=1.0, threshold=0.5, passed=False, is_proxy=False) + assert gate.passed is False + + def test_proxy_warning(self): + f16 = PerplexityResult("ollama-proxy", "f16", 5.0, True) + gate = QualityGateResult(f16=f16, turbo4=None, delta=None, threshold=0.5, passed=False, is_proxy=True, warning="Only F16 measured") + assert gate.is_proxy is True + summary = gate.summary() + assert "PROXY" in summary or "Proxy" in summary + + def test_to_dict(self): + f16 = PerplexityResult("llama-server", "f16", 10.0, False) + gate = QualityGateResult(f16=f16, turbo4=None, delta=None, threshold=0.5, passed=False, is_proxy=False) + d = gate.to_dict() + assert d["f16"]["perplexity"] == 10.0 + assert d["turbo4"] is None + assert d["delta"] is None + + def test_summary_format(self): + f16 = PerplexityResult("llama-server", "f16", 10.0, False) + turbo4 = PerplexityResult("llama-server", "turbo4", 10.2, False) + gate = QualityGateResult(f16=f16, turbo4=turbo4, delta=0.2, threshold=0.5, passed=True, is_proxy=False) + summary = gate.summary() + assert "F16" in summary + assert "Turbo4" in summary + assert "PASS" in summary + assert "0.2000" in summary + + +class TestOllamaProxy: + def test_with_corpus_file(self): + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + f.write("The quick brown fox jumps over the lazy dog.\n" * 100) + f.flush() + result = measure_perplexity_ollama_proxy("test-model", f.name) + os.unlink(f.name) + # Result should be proxy + assert result.is_proxy is True + assert result.backend == "ollama-proxy" + + def test_with_missing_corpus(self): + result = measure_perplexity_ollama_proxy("test-model", "/nonexistent/corpus.txt") + assert result.is_proxy is True + + +class TestRunQualityGate: + def test_unknown_backend(self): + result = run_quality_gate(backend="unknown", model="test") + assert result.passed is False + assert "Unknown backend" in result.warning + + def test_llama_server_missing_binary(self): + result = run_quality_gate( + backend="llama-server", + model="test.gguf", + corpus="/tmp/nonexistent_corpus.txt", + llama_bin="/nonexistent/llama-perplexity", + ) + assert result.f16 is not None + assert result.f16.error is not None + assert "not found" in result.f16.error.lower() + + +if __name__ == "__main__": + import unittest + unittest.main()