test: add fallback chain integration tests
This commit is contained in:
Allegro
@@ -276,5 +276,404 @@ class TestIntegration:
|
||||
assert any(fb["provider"] == "kimi-coding" for fb in available)
|
||||
|
||||
|
||||
class TestFallbackChainIntegration:
|
||||
"""Integration tests for the complete fallback chain: anthropic -> kimi-coding -> openrouter."""
|
||||
|
||||
def test_complete_fallback_chain_structure(self):
|
||||
"""Test that the complete fallback chain has correct structure."""
|
||||
chain = get_default_fallback_chain("anthropic")
|
||||
|
||||
# Should have at least 2 fallbacks: kimi-coding and openrouter
|
||||
assert len(chain) >= 2, f"Expected at least 2 fallbacks, got {len(chain)}"
|
||||
|
||||
# First fallback should be kimi-coding
|
||||
assert chain[0]["provider"] == "kimi-coding"
|
||||
assert chain[0]["model"] == "kimi-k2.5"
|
||||
|
||||
# Second fallback should be openrouter
|
||||
assert chain[1]["provider"] == "openrouter"
|
||||
assert "claude" in chain[1]["model"].lower()
|
||||
|
||||
def test_fallback_chain_resolution_order(self):
|
||||
"""Test that fallback chain respects the defined order."""
|
||||
with patch.dict(os.environ, {
|
||||
"KIMI_API_KEY": "test-kimi-key",
|
||||
"OPENROUTER_API_KEY": "test-openrouter-key",
|
||||
}):
|
||||
chain = get_default_fallback_chain("anthropic")
|
||||
available = filter_available_fallbacks(chain)
|
||||
|
||||
# Both providers should be available
|
||||
assert len(available) >= 2
|
||||
|
||||
# Order should be preserved: kimi-coding first, then openrouter
|
||||
assert available[0]["provider"] == "kimi-coding"
|
||||
assert available[1]["provider"] == "openrouter"
|
||||
|
||||
def test_fallback_chain_skips_unavailable_providers(self):
|
||||
"""Test that chain skips providers without credentials."""
|
||||
with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}, clear=True):
|
||||
chain = get_default_fallback_chain("anthropic")
|
||||
available = filter_available_fallbacks(chain)
|
||||
|
||||
# kimi-coding not available (no key), openrouter should be first
|
||||
assert len(available) >= 1
|
||||
assert available[0]["provider"] == "openrouter"
|
||||
|
||||
# kimi-coding should not be in available list
|
||||
assert not any(fb["provider"] == "kimi-coding" for fb in available)
|
||||
|
||||
def test_fallback_chain_exhaustion(self):
|
||||
"""Test behavior when all fallbacks are exhausted."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
chain = get_default_fallback_chain("anthropic")
|
||||
available = filter_available_fallbacks(chain)
|
||||
|
||||
# No providers available
|
||||
assert available == []
|
||||
|
||||
def test_kimi_coding_fallback_chain(self):
|
||||
"""Test that kimi-coding has its own fallback chain to openrouter."""
|
||||
chain = get_default_fallback_chain("kimi-coding")
|
||||
|
||||
assert len(chain) >= 1
|
||||
# First fallback should be openrouter
|
||||
assert chain[0]["provider"] == "openrouter"
|
||||
|
||||
def test_openrouter_fallback_chain(self):
|
||||
"""Test that openrouter has its own fallback chain."""
|
||||
chain = get_default_fallback_chain("openrouter")
|
||||
|
||||
assert len(chain) >= 1
|
||||
# Should include kimi-coding as fallback
|
||||
assert any(fb["provider"] == "kimi-coding" for fb in chain)
|
||||
|
||||
|
||||
class TestQuotaErrorDetection:
|
||||
"""Comprehensive tests for quota error detection across providers."""
|
||||
|
||||
def test_anthropic_429_status_code(self):
|
||||
"""Test 429 status code detection for Anthropic."""
|
||||
error = MagicMock()
|
||||
error.status_code = 429
|
||||
error.__str__ = lambda self: "Rate limit exceeded"
|
||||
assert is_quota_error(error, provider="anthropic") is True
|
||||
|
||||
def test_anthropic_402_payment_required(self):
|
||||
"""Test 402 payment required detection for Anthropic."""
|
||||
error = MagicMock()
|
||||
error.status_code = 402
|
||||
error.__str__ = lambda self: "Payment required"
|
||||
assert is_quota_error(error, provider="anthropic") is True
|
||||
|
||||
def test_anthropic_403_forbidden_quota(self):
|
||||
"""Test 403 forbidden detection for Anthropic quota."""
|
||||
error = MagicMock()
|
||||
error.status_code = 403
|
||||
error.__str__ = lambda self: "Forbidden"
|
||||
assert is_quota_error(error, provider="anthropic") is True
|
||||
|
||||
def test_openrouter_quota_patterns(self):
|
||||
"""Test OpenRouter-specific quota error patterns."""
|
||||
patterns = [
|
||||
"Rate limit exceeded",
|
||||
"Insufficient credits",
|
||||
"No endpoints available",
|
||||
"All providers failed",
|
||||
"Over capacity",
|
||||
]
|
||||
for pattern in patterns:
|
||||
error = Exception(pattern)
|
||||
assert is_quota_error(error, provider="openrouter") is True, f"Failed for: {pattern}"
|
||||
|
||||
def test_kimi_quota_patterns(self):
|
||||
"""Test kimi-coding-specific quota error patterns."""
|
||||
patterns = [
|
||||
"Rate limit exceeded",
|
||||
"Insufficient balance",
|
||||
"Quota exceeded",
|
||||
]
|
||||
for pattern in patterns:
|
||||
error = Exception(pattern)
|
||||
assert is_quota_error(error, provider="kimi-coding") is True, f"Failed for: {pattern}"
|
||||
|
||||
def test_generic_quota_patterns(self):
|
||||
"""Test generic quota patterns work across all providers."""
|
||||
generic_patterns = [
|
||||
"rate limit exceeded",
|
||||
"quota exceeded",
|
||||
"too many requests",
|
||||
"capacity exceeded",
|
||||
"temporarily unavailable",
|
||||
"resource exhausted",
|
||||
"insufficient credits",
|
||||
]
|
||||
for pattern in generic_patterns:
|
||||
error = Exception(pattern)
|
||||
assert is_quota_error(error) is True, f"Failed for generic pattern: {pattern}"
|
||||
|
||||
def test_non_quota_errors_not_detected(self):
|
||||
"""Test that non-quota errors are not incorrectly detected."""
|
||||
non_quota_errors = [
|
||||
"Context length exceeded",
|
||||
"Invalid API key",
|
||||
"Model not found",
|
||||
"Network timeout",
|
||||
"Connection refused",
|
||||
"JSON decode error",
|
||||
]
|
||||
for pattern in non_quota_errors:
|
||||
error = Exception(pattern)
|
||||
assert is_quota_error(error) is False, f"Incorrectly detected as quota: {pattern}"
|
||||
|
||||
def test_error_type_detection(self):
|
||||
"""Test that specific exception types are detected as quota errors."""
|
||||
class RateLimitError(Exception):
|
||||
pass
|
||||
|
||||
class QuotaExceededError(Exception):
|
||||
pass
|
||||
|
||||
class TooManyRequests(Exception):
|
||||
pass
|
||||
|
||||
for exc_class in [RateLimitError, QuotaExceededError, TooManyRequests]:
|
||||
error = exc_class("Some message")
|
||||
assert is_quota_error(error) is True, f"Failed for {exc_class.__name__}"
|
||||
|
||||
|
||||
class TestFallbackLogging:
|
||||
"""Tests for fallback event logging."""
|
||||
|
||||
def test_fallback_event_logged_with_all_params(self):
|
||||
"""Test that fallback events log all required parameters."""
|
||||
with patch("agent.fallback_router.logger") as mock_logger:
|
||||
log_fallback_event(
|
||||
from_provider="anthropic",
|
||||
to_provider="kimi-coding",
|
||||
to_model="kimi-k2.5",
|
||||
reason="quota_exceeded",
|
||||
)
|
||||
|
||||
# Verify info was called
|
||||
mock_logger.info.assert_called_once()
|
||||
|
||||
# Verify the log message format and arguments
|
||||
call_args = mock_logger.info.call_args
|
||||
log_format = call_args[0][0]
|
||||
log_args = call_args[0][1:] # Remaining positional args
|
||||
|
||||
# Check format string contains placeholders
|
||||
assert "%s" in log_format
|
||||
# Check actual values are in the arguments
|
||||
assert "anthropic" in log_args
|
||||
assert "kimi-coding" in log_args
|
||||
assert "kimi-k2.5" in log_args
|
||||
|
||||
def test_fallback_event_with_error_logs_debug(self):
|
||||
"""Test that fallback events with errors also log debug info."""
|
||||
error = Exception("Rate limit exceeded")
|
||||
|
||||
with patch("agent.fallback_router.logger") as mock_logger:
|
||||
log_fallback_event(
|
||||
from_provider="anthropic",
|
||||
to_provider="kimi-coding",
|
||||
to_model="kimi-k2.5",
|
||||
reason="rate_limit",
|
||||
error=error,
|
||||
)
|
||||
|
||||
# Both info and debug should be called
|
||||
mock_logger.info.assert_called_once()
|
||||
mock_logger.debug.assert_called_once()
|
||||
|
||||
def test_fallback_chain_resolution_logged(self):
|
||||
"""Test logging during full chain resolution."""
|
||||
with patch("agent.fallback_router.logger") as mock_logger:
|
||||
# Simulate getting chain
|
||||
chain = get_auto_fallback_chain("anthropic")
|
||||
|
||||
# Log each fallback step
|
||||
for i, fallback in enumerate(chain):
|
||||
log_fallback_event(
|
||||
from_provider="anthropic" if i == 0 else chain[i-1]["provider"],
|
||||
to_provider=fallback["provider"],
|
||||
to_model=fallback["model"],
|
||||
reason="chain_resolution",
|
||||
)
|
||||
|
||||
# Should have logged for each fallback
|
||||
assert mock_logger.info.call_count == len(chain)
|
||||
|
||||
|
||||
class TestFallbackAvailability:
|
||||
"""Tests for fallback availability checking with credentials."""
|
||||
|
||||
def test_anthropic_available_with_api_key(self):
|
||||
"""Test Anthropic is available when ANTHROPIC_API_KEY is set."""
|
||||
with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "test-key"}):
|
||||
config = {"provider": "anthropic", "model": "claude-3"}
|
||||
assert is_fallback_available(config) is True
|
||||
|
||||
def test_anthropic_available_with_token(self):
|
||||
"""Test Anthropic is available when ANTHROPIC_TOKEN is set."""
|
||||
with patch.dict(os.environ, {"ANTHROPIC_TOKEN": "test-token"}):
|
||||
config = {"provider": "anthropic", "model": "claude-3"}
|
||||
assert is_fallback_available(config) is True
|
||||
|
||||
def test_kimi_available_with_api_key(self):
|
||||
"""Test kimi-coding is available when KIMI_API_KEY is set."""
|
||||
with patch.dict(os.environ, {"KIMI_API_KEY": "test-key"}):
|
||||
config = {"provider": "kimi-coding", "model": "kimi-k2.5"}
|
||||
assert is_fallback_available(config) is True
|
||||
|
||||
def test_kimi_available_with_api_token(self):
|
||||
"""Test kimi-coding is available when KIMI_API_TOKEN is set."""
|
||||
with patch.dict(os.environ, {"KIMI_API_TOKEN": "test-token"}):
|
||||
config = {"provider": "kimi-coding", "model": "kimi-k2.5"}
|
||||
assert is_fallback_available(config) is True
|
||||
|
||||
def test_openrouter_available_with_key(self):
|
||||
"""Test openrouter is available when OPENROUTER_API_KEY is set."""
|
||||
with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
|
||||
config = {"provider": "openrouter", "model": "claude-sonnet-4"}
|
||||
assert is_fallback_available(config) is True
|
||||
|
||||
def test_zai_available(self):
|
||||
"""Test zai is available when ZAI_API_KEY is set."""
|
||||
with patch.dict(os.environ, {"ZAI_API_KEY": "test-key"}):
|
||||
config = {"provider": "zai", "model": "glm-5"}
|
||||
assert is_fallback_available(config) is True
|
||||
|
||||
def test_unconfigured_provider_not_available(self):
|
||||
"""Test that providers without credentials are not available."""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
providers = [
|
||||
{"provider": "anthropic", "model": "claude-3"},
|
||||
{"provider": "kimi-coding", "model": "kimi-k2.5"},
|
||||
{"provider": "openrouter", "model": "claude-sonnet-4"},
|
||||
{"provider": "zai", "model": "glm-5"},
|
||||
]
|
||||
for config in providers:
|
||||
assert is_fallback_available(config) is False, f"{config['provider']} should not be available"
|
||||
|
||||
def test_invalid_config_not_available(self):
|
||||
"""Test that invalid configs are not available."""
|
||||
assert is_fallback_available({}) is False
|
||||
assert is_fallback_available({"provider": ""}) is False
|
||||
assert is_fallback_available({"model": "some-model"}) is False
|
||||
|
||||
|
||||
class TestAutoFallbackDecision:
|
||||
"""Tests for automatic fallback decision logic."""
|
||||
|
||||
def test_anthropic_eager_fallback_no_error(self):
|
||||
"""Test Anthropic falls back eagerly even without an error."""
|
||||
with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
|
||||
assert should_auto_fallback("anthropic") is True
|
||||
|
||||
def test_quota_error_triggers_fallback_any_provider(self):
|
||||
"""Test that quota errors trigger fallback for any provider."""
|
||||
error = Exception("Rate limit exceeded")
|
||||
with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
|
||||
# Even unknown providers should fallback on quota errors
|
||||
assert should_auto_fallback("unknown_provider", error=error) is True
|
||||
|
||||
def test_non_quota_error_no_fallback_unknown_provider(self):
|
||||
"""Test that non-quota errors don't trigger fallback for unknown providers."""
|
||||
error = Exception("Some random error")
|
||||
with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
|
||||
assert should_auto_fallback("unknown_provider", error=error) is False
|
||||
|
||||
def test_auto_fallback_disabled_via_env(self):
|
||||
"""Test auto-fallback can be disabled via environment variable."""
|
||||
with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "false"}):
|
||||
assert should_auto_fallback("anthropic") is False
|
||||
|
||||
with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "0"}):
|
||||
assert should_auto_fallback("anthropic") is False
|
||||
|
||||
with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "off"}):
|
||||
assert should_auto_fallback("anthropic") is False
|
||||
|
||||
def test_auto_fallback_disabled_via_param(self):
|
||||
"""Test auto-fallback can be disabled via parameter."""
|
||||
assert should_auto_fallback("anthropic", auto_fallback_enabled=False) is False
|
||||
|
||||
def test_auto_fallback_enabled_variations(self):
|
||||
"""Test various truthy values for HERMES_AUTO_FALLBACK."""
|
||||
truthy_values = ["true", "1", "yes", "on"]
|
||||
for value in truthy_values:
|
||||
with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": value}):
|
||||
assert should_auto_fallback("anthropic") is True, f"Failed for {value}"
|
||||
|
||||
|
||||
class TestEndToEndFallbackChain:
|
||||
"""End-to-end tests simulating real fallback scenarios."""
|
||||
|
||||
def test_anthropic_to_kimi_fallback_scenario(self):
|
||||
"""Simulate complete fallback: Anthropic quota -> kimi-coding."""
|
||||
# Step 1: Anthropic encounters a quota error
|
||||
anthropic_error = Exception("Rate limit exceeded: quota exceeded for model claude-3-5-sonnet")
|
||||
|
||||
# Step 2: Verify it's detected as a quota error
|
||||
assert is_quota_error(anthropic_error, provider="anthropic") is True
|
||||
|
||||
# Step 3: Check if auto-fallback should trigger
|
||||
with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
|
||||
assert should_auto_fallback("anthropic", error=anthropic_error) is True
|
||||
|
||||
# Step 4: Get fallback chain
|
||||
chain = get_auto_fallback_chain("anthropic")
|
||||
assert len(chain) > 0
|
||||
|
||||
# Step 5: Simulate kimi-coding being available
|
||||
with patch.dict(os.environ, {"KIMI_API_KEY": "test-kimi-key"}):
|
||||
available = filter_available_fallbacks(chain)
|
||||
assert len(available) > 0
|
||||
assert available[0]["provider"] == "kimi-coding"
|
||||
|
||||
# Step 6: Log the fallback event
|
||||
with patch("agent.fallback_router.logger") as mock_logger:
|
||||
log_fallback_event(
|
||||
from_provider="anthropic",
|
||||
to_provider=available[0]["provider"],
|
||||
to_model=available[0]["model"],
|
||||
reason="quota_exceeded",
|
||||
error=anthropic_error,
|
||||
)
|
||||
mock_logger.info.assert_called_once()
|
||||
|
||||
def test_full_chain_exhaustion_scenario(self):
|
||||
"""Simulate scenario where entire fallback chain is exhausted."""
|
||||
# Simulate Anthropic error
|
||||
error = Exception("Rate limit exceeded")
|
||||
|
||||
with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
|
||||
# Get chain
|
||||
chain = get_auto_fallback_chain("anthropic")
|
||||
|
||||
# Simulate no providers available
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
available = filter_available_fallbacks(chain)
|
||||
assert available == []
|
||||
|
||||
# Fallback should not be possible
|
||||
assert len(available) == 0
|
||||
|
||||
def test_chain_continues_on_provider_failure(self):
|
||||
"""Test that chain continues when a fallback provider fails."""
|
||||
with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
|
||||
chain = get_auto_fallback_chain("anthropic")
|
||||
|
||||
# Simulate only openrouter available (kimi-coding not configured)
|
||||
with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}, clear=True):
|
||||
available = filter_available_fallbacks(chain)
|
||||
|
||||
# Should have openrouter as available (skipping kimi-coding)
|
||||
assert len(available) >= 1
|
||||
assert available[0]["provider"] == "openrouter"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
Reference in New Issue
Block a user