388 lines
15 KiB
Python
388 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
McDonald Wizard Test Suite
|
|
|
|
Tests for the McDonald chatbot wizard harness and Hermes shim.
|
|
|
|
Usage:
|
|
pytest tests/test_mcdonald_wizard.py -v
|
|
RUN_LIVE_TESTS=1 pytest tests/test_mcdonald_wizard.py -v # real API calls
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from nexus.mcdonald_wizard import (
|
|
DEFAULT_ENDPOINT,
|
|
DEFAULT_RETRIES,
|
|
DEFAULT_TIMEOUT,
|
|
WIZARD_ID,
|
|
McdonaldWizard,
|
|
WizardResponse,
|
|
mcdonald_wizard,
|
|
)
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# FIXTURES
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
@pytest.fixture
|
|
def wizard():
|
|
"""Wizard with a fake API key so no real calls are made."""
|
|
return McdonaldWizard(api_key="fake-key-for-testing")
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_ok_response():
|
|
"""Mock requests.post returning a successful API response."""
|
|
mock = MagicMock()
|
|
mock.status_code = 200
|
|
mock.json.return_value = {
|
|
"choices": [{"message": {"content": "Behold, the golden arches!"}}],
|
|
"model": "mc-wizard-v1",
|
|
}
|
|
return mock
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_rate_limit_response():
|
|
"""Mock requests.post returning a 429 rate-limit error."""
|
|
mock = MagicMock()
|
|
mock.status_code = 429
|
|
mock.text = "Rate limit exceeded"
|
|
return mock
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_server_error_response():
|
|
"""Mock requests.post returning a 500 server error."""
|
|
mock = MagicMock()
|
|
mock.status_code = 500
|
|
mock.text = "Internal server error"
|
|
return mock
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# WizardResponse dataclass
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestWizardResponse:
|
|
def test_default_creation(self):
|
|
resp = WizardResponse()
|
|
assert resp.text == ""
|
|
assert resp.model == ""
|
|
assert resp.latency_ms == 0.0
|
|
assert resp.attempt == 1
|
|
assert resp.error is None
|
|
assert resp.timestamp
|
|
|
|
def test_to_dict_includes_all_fields(self):
|
|
resp = WizardResponse(text="Hello", model="mc-wizard-v1", latency_ms=42.5, attempt=2)
|
|
d = resp.to_dict()
|
|
assert d["text"] == "Hello"
|
|
assert d["model"] == "mc-wizard-v1"
|
|
assert d["latency_ms"] == 42.5
|
|
assert d["attempt"] == 2
|
|
assert d["error"] is None
|
|
assert "timestamp" in d
|
|
|
|
def test_error_response(self):
|
|
resp = WizardResponse(error="HTTP 429: Rate limit")
|
|
assert resp.error == "HTTP 429: Rate limit"
|
|
assert resp.text == ""
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# McdonaldWizard — initialization
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestMcdonaldWizardInit:
|
|
def test_default_endpoint(self, wizard):
|
|
assert wizard.endpoint == DEFAULT_ENDPOINT
|
|
|
|
def test_custom_endpoint(self):
|
|
w = McdonaldWizard(api_key="k", endpoint="https://custom.example.com/chat")
|
|
assert w.endpoint == "https://custom.example.com/chat"
|
|
|
|
def test_default_timeout(self, wizard):
|
|
assert wizard.timeout == DEFAULT_TIMEOUT
|
|
|
|
def test_default_retries(self, wizard):
|
|
assert wizard.max_retries == DEFAULT_RETRIES
|
|
|
|
def test_no_api_key_warning(self, caplog):
|
|
import logging
|
|
|
|
with caplog.at_level(logging.WARNING, logger="mcdonald_wizard"):
|
|
McdonaldWizard(api_key="")
|
|
assert "MCDONALDS_API_KEY" in caplog.text
|
|
|
|
def test_api_key_from_env(self, monkeypatch):
|
|
monkeypatch.setenv("MCDONALDS_API_KEY", "env-key-123")
|
|
w = McdonaldWizard()
|
|
assert w.api_key == "env-key-123"
|
|
|
|
def test_endpoint_from_env(self, monkeypatch):
|
|
monkeypatch.setenv("MCDONALDS_ENDPOINT", "https://env.example.com/chat")
|
|
w = McdonaldWizard(api_key="k")
|
|
assert w.endpoint == "https://env.example.com/chat"
|
|
|
|
def test_initial_stats_zero(self, wizard):
|
|
assert wizard.request_count == 0
|
|
assert wizard.total_latency_ms == 0.0
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# McdonaldWizard — ask (mocked HTTP)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestAsk:
|
|
def test_ask_no_api_key_returns_error(self):
|
|
w = McdonaldWizard(api_key="")
|
|
resp = w.ask("Hello wizard")
|
|
assert resp.error is not None
|
|
assert "MCDONALDS_API_KEY" in resp.error
|
|
|
|
def test_ask_success(self, wizard, mock_ok_response):
|
|
with patch("requests.post", return_value=mock_ok_response):
|
|
resp = wizard.ask("What is your wisdom?")
|
|
|
|
assert resp.error is None
|
|
assert resp.text == "Behold, the golden arches!"
|
|
assert resp.model == "mc-wizard-v1"
|
|
assert resp.latency_ms >= 0.0
|
|
assert resp.attempt == 1
|
|
|
|
def test_ask_increments_request_count(self, wizard, mock_ok_response):
|
|
with patch("requests.post", return_value=mock_ok_response):
|
|
wizard.ask("q1")
|
|
wizard.ask("q2")
|
|
|
|
assert wizard.request_count == 2
|
|
|
|
def test_ask_with_system_prompt(self, wizard, mock_ok_response):
|
|
with patch("requests.post", return_value=mock_ok_response) as mock_post:
|
|
wizard.ask("Hello", system="You are a wise McDonald wizard")
|
|
|
|
payload = mock_post.call_args[1]["json"]
|
|
roles = [m["role"] for m in payload["messages"]]
|
|
assert "system" in roles
|
|
assert payload["messages"][0]["content"] == "You are a wise McDonald wizard"
|
|
|
|
def test_ask_with_context(self, wizard, mock_ok_response):
|
|
with patch("requests.post", return_value=mock_ok_response) as mock_post:
|
|
wizard.ask("Continue please", context="Prior context here")
|
|
|
|
payload = mock_post.call_args[1]["json"]
|
|
contents = [m["content"] for m in payload["messages"]]
|
|
assert "Prior context here" in contents
|
|
|
|
def test_ask_without_optional_args(self, wizard, mock_ok_response):
|
|
with patch("requests.post", return_value=mock_ok_response) as mock_post:
|
|
wizard.ask("Simple prompt")
|
|
|
|
payload = mock_post.call_args[1]["json"]
|
|
assert payload["messages"][-1]["role"] == "user"
|
|
assert payload["messages"][-1]["content"] == "Simple prompt"
|
|
|
|
def test_ask_sends_bearer_auth(self, wizard, mock_ok_response):
|
|
with patch("requests.post", return_value=mock_ok_response) as mock_post:
|
|
wizard.ask("Hello")
|
|
|
|
headers = mock_post.call_args[1]["headers"]
|
|
assert headers["Authorization"] == "Bearer fake-key-for-testing"
|
|
|
|
def test_ask_api_failure_returns_error(self, wizard):
|
|
with patch("requests.post", side_effect=Exception("Connection refused")):
|
|
resp = wizard.ask("Hello")
|
|
|
|
assert resp.error is not None
|
|
assert "failed" in resp.error.lower()
|
|
assert wizard.request_count == 1
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# McdonaldWizard — retry behaviour
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestRetry:
|
|
def test_retries_on_429(self, wizard, mock_ok_response, mock_rate_limit_response):
|
|
call_count = [0]
|
|
|
|
def side_effect(*args, **kwargs):
|
|
call_count[0] += 1
|
|
if call_count[0] < 2:
|
|
return mock_rate_limit_response
|
|
return mock_ok_response
|
|
|
|
with patch("requests.post", side_effect=side_effect):
|
|
with patch("time.sleep"): # suppress actual sleep
|
|
resp = wizard.ask("Hello")
|
|
|
|
assert resp.error is None
|
|
assert resp.attempt == 2
|
|
assert call_count[0] == 2
|
|
|
|
def test_retries_on_500(self, wizard, mock_ok_response, mock_server_error_response):
|
|
call_count = [0]
|
|
|
|
def side_effect(*args, **kwargs):
|
|
call_count[0] += 1
|
|
if call_count[0] < 3:
|
|
return mock_server_error_response
|
|
return mock_ok_response
|
|
|
|
with patch("requests.post", side_effect=side_effect):
|
|
with patch("time.sleep"):
|
|
resp = wizard.ask("Hello")
|
|
|
|
assert resp.error is None
|
|
assert call_count[0] == 3
|
|
|
|
def test_all_retries_exhausted_returns_error(self, wizard, mock_rate_limit_response):
|
|
with patch("requests.post", return_value=mock_rate_limit_response):
|
|
with patch("time.sleep"):
|
|
resp = wizard.ask("Hello")
|
|
|
|
assert resp.error is not None
|
|
assert wizard.request_count == 1
|
|
|
|
def test_no_retry_on_success(self, wizard, mock_ok_response):
|
|
with patch("requests.post", return_value=mock_ok_response) as mock_post:
|
|
resp = wizard.ask("Hello")
|
|
|
|
assert mock_post.call_count == 1
|
|
assert resp.attempt == 1
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# McdonaldWizard — session stats
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestSessionStats:
|
|
def test_initial_stats(self, wizard):
|
|
stats = wizard.session_stats()
|
|
assert stats["wizard_id"] == WIZARD_ID
|
|
assert stats["request_count"] == 0
|
|
assert stats["total_latency_ms"] == 0.0
|
|
assert stats["avg_latency_ms"] == 0.0
|
|
|
|
def test_stats_after_calls(self, wizard, mock_ok_response):
|
|
with patch("requests.post", return_value=mock_ok_response):
|
|
wizard.ask("a")
|
|
wizard.ask("b")
|
|
|
|
stats = wizard.session_stats()
|
|
assert stats["request_count"] == 2
|
|
assert stats["total_latency_ms"] >= 0.0
|
|
assert stats["avg_latency_ms"] >= 0.0
|
|
|
|
def test_avg_latency_calculation(self, wizard, mock_ok_response):
|
|
with patch("requests.post", return_value=mock_ok_response):
|
|
wizard.ask("x")
|
|
|
|
stats = wizard.session_stats()
|
|
assert stats["avg_latency_ms"] == stats["total_latency_ms"]
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Hermes tool function
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
class TestHermesTool:
|
|
def test_mcdonald_wizard_tool_returns_dict(self, monkeypatch):
|
|
mock_resp = WizardResponse(text="I am the wizard", model="mc-v1")
|
|
mock_wizard = MagicMock()
|
|
mock_wizard.ask.return_value = mock_resp
|
|
|
|
import nexus.mcdonald_wizard as _mod
|
|
|
|
monkeypatch.setattr(_mod, "_wizard_instance", mock_wizard)
|
|
|
|
result = mcdonald_wizard("What do you know?")
|
|
|
|
assert isinstance(result, dict)
|
|
assert result["text"] == "I am the wizard"
|
|
assert result["model"] == "mc-v1"
|
|
assert result["error"] is None
|
|
|
|
def test_mcdonald_wizard_tool_passes_system(self, monkeypatch):
|
|
mock_resp = WizardResponse(text="Aye", model="mc-v1")
|
|
mock_wizard = MagicMock()
|
|
mock_wizard.ask.return_value = mock_resp
|
|
|
|
import nexus.mcdonald_wizard as _mod
|
|
|
|
monkeypatch.setattr(_mod, "_wizard_instance", mock_wizard)
|
|
mcdonald_wizard("Hello", system="Be brief")
|
|
|
|
mock_wizard.ask.assert_called_once_with("Hello", system="Be brief")
|
|
|
|
def test_mcdonald_wizard_tool_propagates_error(self, monkeypatch):
|
|
mock_resp = WizardResponse(error="API key missing")
|
|
mock_wizard = MagicMock()
|
|
mock_wizard.ask.return_value = mock_resp
|
|
|
|
import nexus.mcdonald_wizard as _mod
|
|
|
|
monkeypatch.setattr(_mod, "_wizard_instance", mock_wizard)
|
|
|
|
result = mcdonald_wizard("Hello")
|
|
assert result["error"] == "API key missing"
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# Live API tests (skipped unless RUN_LIVE_TESTS=1 and MCDONALDS_API_KEY set)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
|
|
|
|
def _live_tests_enabled():
|
|
return (
|
|
os.environ.get("RUN_LIVE_TESTS") == "1"
|
|
and bool(os.environ.get("MCDONALDS_API_KEY"))
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
not _live_tests_enabled(),
|
|
reason="Live tests require RUN_LIVE_TESTS=1 and MCDONALDS_API_KEY",
|
|
)
|
|
@pytest.mark.integration
|
|
class TestLiveAPI:
|
|
"""Integration tests that hit the real McDonald chatbot API."""
|
|
|
|
@pytest.fixture
|
|
def live_wizard(self):
|
|
return McdonaldWizard()
|
|
|
|
def test_live_ask(self, live_wizard):
|
|
resp = live_wizard.ask("Say 'McReady' and nothing else.")
|
|
assert resp.error is None
|
|
assert resp.text.strip()
|
|
assert resp.latency_ms > 0
|
|
|
|
def test_live_session_stats_update(self, live_wizard):
|
|
live_wizard.ask("Ping")
|
|
stats = live_wizard.session_stats()
|
|
assert stats["request_count"] == 1
|
|
assert stats["total_latency_ms"] > 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"])
|