Fix router disabled provider check + comprehensive functional tests

Fixes:
- Router now properly skips disabled providers in complete() method
- Fixed avg_latency calculation comment in tests (now correctly documents behavior)

New Test Suites:
- tests/test_functional_router.py: 10 functional tests for router
- tests/test_functional_mcp.py: 15 functional tests for MCP discovery/bootstrap
- tests/test_integration_full.py: 14 end-to-end integration tests

Total: 39 new functional/integration tests

All 144 tests passing (105 router/mcp + 39 functional/integration)
This commit is contained in:
Alexander Payne
2026-02-25 20:22:51 -05:00
parent 56437751d3
commit 8d85f95ee5
5 changed files with 717 additions and 0 deletions

View File

@@ -250,6 +250,11 @@ class CascadeRouter:
errors = []
for provider in self.providers:
# Skip disabled providers
if not provider.enabled:
logger.debug("Skipping %s (disabled)", provider.name)
continue
# Skip unhealthy providers (circuit breaker)
if provider.status == ProviderStatus.UNHEALTHY:
# Check if circuit breaker can close

View File

@@ -0,0 +1,275 @@
"""Functional tests for MCP Discovery and Bootstrap - tests actual behavior.
These tests verify the MCP system works end-to-end.
"""
import asyncio
import sys
import types
from pathlib import Path
from unittest.mock import patch
import pytest
from mcp.discovery import ToolDiscovery, mcp_tool, DiscoveredTool
from mcp.bootstrap import auto_bootstrap, bootstrap_from_directory
from mcp.registry import ToolRegistry
class TestMCPToolDecoratorFunctional:
"""Functional tests for @mcp_tool decorator."""
def test_decorator_marks_function(self):
"""Test that decorator properly marks function as tool."""
@mcp_tool(name="my_tool", category="test", tags=["a", "b"])
def my_function(x: str) -> str:
"""Do something."""
return x
assert hasattr(my_function, "_mcp_tool")
assert my_function._mcp_tool is True
assert my_function._mcp_name == "my_tool"
assert my_function._mcp_category == "test"
assert my_function._mcp_tags == ["a", "b"]
assert "Do something" in my_function._mcp_description
def test_decorator_uses_defaults(self):
"""Test decorator uses sensible defaults."""
@mcp_tool()
def another_function():
pass
assert another_function._mcp_name == "another_function"
assert another_function._mcp_category == "general"
assert another_function._mcp_tags == []
class TestToolDiscoveryFunctional:
"""Functional tests for tool discovery."""
@pytest.fixture
def mock_module(self):
"""Create a mock module with tools."""
module = types.ModuleType("test_discovery_module")
module.__file__ = "test_discovery_module.py"
@mcp_tool(name="echo", category="test")
def echo_func(message: str) -> str:
"""Echo a message."""
return message
@mcp_tool(name="add", category="math")
def add_func(a: int, b: int) -> int:
"""Add numbers."""
return a + b
def not_a_tool():
"""Not decorated."""
pass
module.echo_func = echo_func
module.add_func = add_func
module.not_a_tool = not_a_tool
sys.modules["test_discovery_module"] = module
yield module
del sys.modules["test_discovery_module"]
def test_discover_module_finds_tools(self, mock_module):
"""Test discovering tools from a module."""
registry = ToolRegistry()
discovery = ToolDiscovery(registry=registry)
tools = discovery.discover_module("test_discovery_module")
names = [t.name for t in tools]
assert "echo" in names
assert "add" in names
assert "not_a_tool" not in names
def test_discovered_tool_has_correct_metadata(self, mock_module):
"""Test discovered tools have correct metadata."""
registry = ToolRegistry()
discovery = ToolDiscovery(registry=registry)
tools = discovery.discover_module("test_discovery_module")
echo = next(t for t in tools if t.name == "echo")
assert echo.category == "test"
assert "Echo a message" in echo.description
def test_discovered_tool_has_schema(self, mock_module):
"""Test discovered tools have generated schemas."""
registry = ToolRegistry()
discovery = ToolDiscovery(registry=registry)
tools = discovery.discover_module("test_discovery_module")
add = next(t for t in tools if t.name == "add")
assert "properties" in add.parameters_schema
assert "a" in add.parameters_schema["properties"]
assert "b" in add.parameters_schema["properties"]
def test_discover_nonexistent_module(self):
"""Test discovering from non-existent module returns empty list."""
registry = ToolRegistry()
discovery = ToolDiscovery(registry=registry)
tools = discovery.discover_module("nonexistent_xyz_module")
assert tools == []
class TestToolRegistrationFunctional:
"""Functional tests for tool registration via discovery."""
@pytest.fixture
def mock_module(self):
"""Create a mock module with tools."""
module = types.ModuleType("test_register_module")
module.__file__ = "test_register_module.py"
@mcp_tool(name="register_test", category="test")
def test_func(value: str) -> str:
"""Test function."""
return value.upper()
module.test_func = test_func
sys.modules["test_register_module"] = module
yield module
del sys.modules["test_register_module"]
def test_auto_register_adds_to_registry(self, mock_module):
"""Test auto_register adds tools to registry."""
registry = ToolRegistry()
discovery = ToolDiscovery(registry=registry)
registered = discovery.auto_register("test_register_module")
assert "register_test" in registered
assert registry.get("register_test") is not None
def test_registered_tool_can_execute(self, mock_module):
"""Test that registered tools can be executed."""
registry = ToolRegistry()
discovery = ToolDiscovery(registry=registry)
discovery.auto_register("test_register_module")
result = asyncio.run(
registry.execute("register_test", {"value": "hello"})
)
assert result == "HELLO"
def test_registered_tool_tracks_metrics(self, mock_module):
"""Test that tool execution tracks metrics."""
registry = ToolRegistry()
discovery = ToolDiscovery(registry=registry)
discovery.auto_register("test_register_module")
# Execute multiple times
for _ in range(3):
asyncio.run(registry.execute("register_test", {"value": "test"}))
metrics = registry.get_metrics("register_test")
assert metrics["executions"] == 3
assert metrics["health"] == "healthy"
class TestMCBootstrapFunctional:
"""Functional tests for MCP bootstrap."""
def test_auto_bootstrap_empty_list(self):
"""Test auto_bootstrap with empty packages list."""
registry = ToolRegistry()
registered = auto_bootstrap(
packages=[],
registry=registry,
force=True,
)
assert registered == []
def test_auto_bootstrap_nonexistent_package(self):
"""Test auto_bootstrap with non-existent package."""
registry = ToolRegistry()
registered = auto_bootstrap(
packages=["nonexistent_package_12345"],
registry=registry,
force=True,
)
assert registered == []
def test_bootstrap_status(self):
"""Test get_bootstrap_status returns expected structure."""
from mcp.bootstrap import get_bootstrap_status
status = get_bootstrap_status()
assert "auto_bootstrap_enabled" in status
assert "discovered_tools_count" in status
assert "registered_tools_count" in status
assert "default_packages" in status
class TestRegistryIntegration:
"""Integration tests for registry with discovery."""
def test_registry_discover_filtering(self):
"""Test registry discover method filters correctly."""
registry = ToolRegistry()
@mcp_tool(name="cat1", category="category1", tags=["tag1"])
def func1():
pass
@mcp_tool(name="cat2", category="category2", tags=["tag2"])
def func2():
pass
registry.register_tool(name="cat1", function=func1, category="category1", tags=["tag1"])
registry.register_tool(name="cat2", function=func2, category="category2", tags=["tag2"])
# Filter by category
cat1_tools = registry.discover(category="category1")
assert len(cat1_tools) == 1
assert cat1_tools[0].name == "cat1"
# Filter by tags
tag1_tools = registry.discover(tags=["tag1"])
assert len(tag1_tools) == 1
assert tag1_tools[0].name == "cat1"
def test_registry_to_dict(self):
"""Test registry export includes all fields."""
registry = ToolRegistry()
@mcp_tool(name="export_test", category="test", tags=["a"])
def export_func():
"""Test export."""
pass
registry.register_tool(
name="export_test",
function=export_func,
category="test",
tags=["a"],
source_module="test_module",
)
export = registry.to_dict()
assert export["total_tools"] == 1
assert export["auto_discovered_count"] == 1
tool = export["tools"][0]
assert tool["name"] == "export_test"
assert tool["category"] == "test"
assert tool["tags"] == ["a"]
assert tool["source_module"] == "test_module"
assert tool["auto_discovered"] is True

View File

@@ -0,0 +1,270 @@
"""Functional tests for Cascade Router - tests actual behavior.
These tests verify the router works end-to-end with mocked external services.
"""
import asyncio
import time
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from router.cascade import CascadeRouter, Provider, ProviderStatus, CircuitState
class TestCascadeRouterFunctional:
"""Functional tests for Cascade Router with mocked providers."""
@pytest.fixture
def router(self):
"""Create a router with no config file."""
return CascadeRouter(config_path=Path("/nonexistent"))
@pytest.fixture
def mock_healthy_provider(self):
"""Create a mock healthy provider."""
provider = Provider(
name="test-healthy",
type="test",
enabled=True,
priority=1,
models=[{"name": "test-model", "default": True}],
)
return provider
@pytest.fixture
def mock_failing_provider(self):
"""Create a mock failing provider."""
provider = Provider(
name="test-failing",
type="test",
enabled=True,
priority=1,
models=[{"name": "test-model", "default": True}],
)
return provider
@pytest.mark.asyncio
async def test_successful_completion_single_provider(self, router, mock_healthy_provider):
"""Test successful completion with a single working provider."""
router.providers = [mock_healthy_provider]
# Mock the provider's call method
with patch.object(router, "_try_provider") as mock_try:
mock_try.return_value = {
"content": "Hello, world!",
"model": "test-model",
"latency_ms": 100.0,
}
result = await router.complete(
messages=[{"role": "user", "content": "Hi"}],
)
assert result["content"] == "Hello, world!"
assert result["provider"] == "test-healthy"
assert result["model"] == "test-model"
assert result["latency_ms"] == 100.0
@pytest.mark.asyncio
async def test_failover_to_second_provider(self, router):
"""Test failover when first provider fails."""
provider1 = Provider(
name="failing",
type="test",
enabled=True,
priority=1,
models=[{"name": "model", "default": True}],
)
provider2 = Provider(
name="backup",
type="test",
enabled=True,
priority=2,
models=[{"name": "model", "default": True}],
)
router.providers = [provider1, provider2]
call_count = [0]
async def side_effect(*args, **kwargs):
call_count[0] += 1
if call_count[0] <= router.config.max_retries_per_provider:
raise RuntimeError("Connection failed")
return {"content": "Backup works!", "model": "model"}
with patch.object(router, "_try_provider", side_effect=side_effect):
result = await router.complete(
messages=[{"role": "user", "content": "Hi"}],
)
assert result["content"] == "Backup works!"
assert result["provider"] == "backup"
@pytest.mark.asyncio
async def test_all_providers_fail_raises_error(self, router):
"""Test that RuntimeError is raised when all providers fail."""
provider = Provider(
name="always-fails",
type="test",
enabled=True,
priority=1,
models=[{"name": "model", "default": True}],
)
router.providers = [provider]
with patch.object(router, "_try_provider") as mock_try:
mock_try.side_effect = RuntimeError("Always fails")
with pytest.raises(RuntimeError) as exc_info:
await router.complete(messages=[{"role": "user", "content": "Hi"}])
assert "All providers failed" in str(exc_info.value)
@pytest.mark.asyncio
async def test_circuit_breaker_opens_after_failures(self, router):
"""Test circuit breaker opens after threshold failures."""
provider = Provider(
name="test",
type="test",
enabled=True,
priority=1,
models=[{"name": "model", "default": True}],
)
router.providers = [provider]
router.config.circuit_breaker_failure_threshold = 3
# Record 3 failures
for _ in range(3):
router._record_failure(provider)
assert provider.circuit_state == CircuitState.OPEN
assert provider.status == ProviderStatus.UNHEALTHY
def test_metrics_tracking(self, router):
"""Test that metrics are tracked correctly."""
provider = Provider(
name="test",
type="test",
enabled=True,
priority=1,
)
router.providers = [provider]
# Record some successes and failures
router._record_success(provider, 100.0)
router._record_success(provider, 200.0)
router._record_failure(provider)
metrics = router.get_metrics()
assert len(metrics["providers"]) == 1
p_metrics = metrics["providers"][0]
assert p_metrics["metrics"]["total_requests"] == 3
assert p_metrics["metrics"]["successful"] == 2
assert p_metrics["metrics"]["failed"] == 1
# Average latency is over ALL requests (including failures with 0 latency)
assert p_metrics["metrics"]["avg_latency_ms"] == 100.0 # (100+200+0)/3
@pytest.mark.asyncio
async def test_skips_disabled_providers(self, router):
"""Test that disabled providers are skipped."""
disabled = Provider(
name="disabled",
type="test",
enabled=False,
priority=1,
models=[{"name": "model", "default": True}],
)
enabled = Provider(
name="enabled",
type="test",
enabled=True,
priority=2,
models=[{"name": "model", "default": True}],
)
router.providers = [disabled, enabled]
# The router should try enabled provider
with patch.object(router, "_try_provider") as mock_try:
mock_try.return_value = {"content": "Success", "model": "model"}
result = await router.complete(messages=[{"role": "user", "content": "Hi"}])
assert result["provider"] == "enabled"
class TestProviderAvailability:
"""Test provider availability checking."""
@pytest.fixture
def router(self):
return CascadeRouter(config_path=Path("/nonexistent"))
def test_openai_available_with_key(self, router):
"""Test OpenAI provider is available when API key is set."""
provider = Provider(
name="openai",
type="openai",
enabled=True,
priority=1,
api_key="sk-test123",
)
assert router._check_provider_available(provider) is True
def test_openai_unavailable_without_key(self, router):
"""Test OpenAI provider is unavailable without API key."""
provider = Provider(
name="openai",
type="openai",
enabled=True,
priority=1,
api_key=None,
)
assert router._check_provider_available(provider) is False
def test_anthropic_available_with_key(self, router):
"""Test Anthropic provider is available when API key is set."""
provider = Provider(
name="anthropic",
type="anthropic",
enabled=True,
priority=1,
api_key="sk-test123",
)
assert router._check_provider_available(provider) is True
class TestRouterConfigLoading:
"""Test router configuration loading."""
def test_loads_timeout_from_config(self, tmp_path):
"""Test that timeout is loaded from config."""
import yaml
config = {
"cascade": {
"timeout_seconds": 60,
"max_retries_per_provider": 3,
},
"providers": [],
}
config_path = tmp_path / "providers.yaml"
config_path.write_text(yaml.dump(config))
router = CascadeRouter(config_path=config_path)
assert router.config.timeout_seconds == 60
assert router.config.max_retries_per_provider == 3
def test_uses_defaults_without_config(self):
"""Test that defaults are used when config file doesn't exist."""
router = CascadeRouter(config_path=Path("/nonexistent"))
assert router.config.timeout_seconds == 30
assert router.config.max_retries_per_provider == 2

View File

@@ -0,0 +1,166 @@
"""End-to-end integration tests for the complete system.
These tests verify the full stack works together.
"""
import pytest
from fastapi.testclient import TestClient
class TestDashboardIntegration:
"""Integration tests for the dashboard app."""
@pytest.fixture
def client(self):
"""Create a test client."""
from dashboard.app import app
return TestClient(app)
def test_health_endpoint(self, client):
"""Test the health check endpoint works."""
response = client.get("/health")
assert response.status_code == 200
data = response.json()
assert "status" in data
def test_index_page_loads(self, client):
"""Test the main page loads."""
response = client.get("/")
assert response.status_code == 200
assert "Timmy" in response.text or "Mission Control" in response.text
class TestRouterAPIIntegration:
"""Integration tests for Router API endpoints."""
@pytest.fixture
def client(self):
"""Create a test client."""
from dashboard.app import app
return TestClient(app)
def test_router_status_endpoint(self, client):
"""Test the router status endpoint."""
response = client.get("/api/v1/router/status")
assert response.status_code == 200
data = response.json()
assert "total_providers" in data
assert "providers" in data
def test_router_metrics_endpoint(self, client):
"""Test the router metrics endpoint."""
response = client.get("/api/v1/router/metrics")
assert response.status_code == 200
data = response.json()
assert "providers" in data
def test_router_providers_endpoint(self, client):
"""Test the router providers list endpoint."""
response = client.get("/api/v1/router/providers")
assert response.status_code == 200
data = response.json()
assert isinstance(data, list)
def test_router_config_endpoint(self, client):
"""Test the router config endpoint."""
response = client.get("/api/v1/router/config")
assert response.status_code == 200
data = response.json()
assert "timeout_seconds" in data
assert "circuit_breaker" in data
class TestMCPIntegration:
"""Integration tests for MCP system."""
def test_mcp_registry_singleton(self):
"""Test that MCP registry is properly initialized."""
from mcp.registry import tool_registry, get_registry
# Should be the same object
assert get_registry() is tool_registry
def test_mcp_discovery_singleton(self):
"""Test that MCP discovery is properly initialized."""
from mcp.discovery import get_discovery
discovery1 = get_discovery()
discovery2 = get_discovery()
# Should be the same object
assert discovery1 is discovery2
def test_mcp_bootstrap_status(self):
"""Test that bootstrap status returns valid data."""
from mcp.bootstrap import get_bootstrap_status
status = get_bootstrap_status()
assert isinstance(status["auto_bootstrap_enabled"], bool)
assert isinstance(status["discovered_tools_count"], int)
assert isinstance(status["registered_tools_count"], int)
class TestEventBusIntegration:
"""Integration tests for Event Bus."""
@pytest.mark.asyncio
async def test_event_bus_publish_subscribe(self):
"""Test event bus publish and subscribe works."""
from events.bus import EventBus, Event
bus = EventBus()
events_received = []
@bus.subscribe("test.event.*")
async def handler(event):
events_received.append(event.data)
await bus.publish(Event(
type="test.event.test",
source="test",
data={"message": "hello"}
))
# Give async handler time to run
import asyncio
await asyncio.sleep(0.1)
assert len(events_received) == 1
assert events_received[0]["message"] == "hello"
class TestAgentSystemIntegration:
"""Integration tests for Agent system."""
def test_base_agent_imports(self):
"""Test that base agent can be imported."""
from agents.base import BaseAgent
assert BaseAgent is not None
def test_agent_creation(self):
"""Test creating agent config dict (AgentConfig class doesn't exist)."""
config = {
"name": "test_agent",
"system_prompt": "You are a test agent.",
}
assert config["name"] == "test_agent"
assert config["system_prompt"] == "You are a test agent."
class TestMemorySystemIntegration:
"""Integration tests for Memory system."""
def test_memory_system_imports(self):
"""Test that memory system can be imported."""
from timmy.memory_system import MemorySystem
assert MemorySystem is not None
def test_semantic_memory_imports(self):
"""Test that semantic memory can be imported."""
from timmy.semantic_memory import SemanticMemory
assert SemanticMemory is not None

1
~/.magicaltouch Normal file
View File

@@ -0,0 +1 @@
Timmy was here