test: add fallback chain integration tests

tests/test_fallback_router.py

@@ -276,5 +276,404 @@ class TestIntegration:
             assert any(fb["provider"] == "kimi-coding" for fb in available)
 
 
+class TestFallbackChainIntegration:
+    """Integration tests for the complete fallback chain: anthropic -> kimi-coding -> openrouter."""
+
+    def test_complete_fallback_chain_structure(self):
+        """Test that the complete fallback chain has correct structure."""
+        chain = get_default_fallback_chain("anthropic")
+        
+        # Should have at least 2 fallbacks: kimi-coding and openrouter
+        assert len(chain) >= 2, f"Expected at least 2 fallbacks, got {len(chain)}"
+        
+        # First fallback should be kimi-coding
+        assert chain[0]["provider"] == "kimi-coding"
+        assert chain[0]["model"] == "kimi-k2.5"
+        
+        # Second fallback should be openrouter
+        assert chain[1]["provider"] == "openrouter"
+        assert "claude" in chain[1]["model"].lower()
+
+    def test_fallback_chain_resolution_order(self):
+        """Test that fallback chain respects the defined order."""
+        with patch.dict(os.environ, {
+            "KIMI_API_KEY": "test-kimi-key",
+            "OPENROUTER_API_KEY": "test-openrouter-key",
+        }):
+            chain = get_default_fallback_chain("anthropic")
+            available = filter_available_fallbacks(chain)
+            
+            # Both providers should be available
+            assert len(available) >= 2
+            
+            # Order should be preserved: kimi-coding first, then openrouter
+            assert available[0]["provider"] == "kimi-coding"
+            assert available[1]["provider"] == "openrouter"
+
+    def test_fallback_chain_skips_unavailable_providers(self):
+        """Test that chain skips providers without credentials."""
+        with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}, clear=True):
+            chain = get_default_fallback_chain("anthropic")
+            available = filter_available_fallbacks(chain)
+            
+            # kimi-coding not available (no key), openrouter should be first
+            assert len(available) >= 1
+            assert available[0]["provider"] == "openrouter"
+            
+            # kimi-coding should not be in available list
+            assert not any(fb["provider"] == "kimi-coding" for fb in available)
+
+    def test_fallback_chain_exhaustion(self):
+        """Test behavior when all fallbacks are exhausted."""
+        with patch.dict(os.environ, {}, clear=True):
+            chain = get_default_fallback_chain("anthropic")
+            available = filter_available_fallbacks(chain)
+            
+            # No providers available
+            assert available == []
+
+    def test_kimi_coding_fallback_chain(self):
+        """Test that kimi-coding has its own fallback chain to openrouter."""
+        chain = get_default_fallback_chain("kimi-coding")
+        
+        assert len(chain) >= 1
+        # First fallback should be openrouter
+        assert chain[0]["provider"] == "openrouter"
+        
+    def test_openrouter_fallback_chain(self):
+        """Test that openrouter has its own fallback chain."""
+        chain = get_default_fallback_chain("openrouter")
+        
+        assert len(chain) >= 1
+        # Should include kimi-coding as fallback
+        assert any(fb["provider"] == "kimi-coding" for fb in chain)
+
+
+class TestQuotaErrorDetection:
+    """Comprehensive tests for quota error detection across providers."""
+
+    def test_anthropic_429_status_code(self):
+        """Test 429 status code detection for Anthropic."""
+        error = MagicMock()
+        error.status_code = 429
+        error.__str__ = lambda self: "Rate limit exceeded"
+        assert is_quota_error(error, provider="anthropic") is True
+
+    def test_anthropic_402_payment_required(self):
+        """Test 402 payment required detection for Anthropic."""
+        error = MagicMock()
+        error.status_code = 402
+        error.__str__ = lambda self: "Payment required"
+        assert is_quota_error(error, provider="anthropic") is True
+
+    def test_anthropic_403_forbidden_quota(self):
+        """Test 403 forbidden detection for Anthropic quota."""
+        error = MagicMock()
+        error.status_code = 403
+        error.__str__ = lambda self: "Forbidden"
+        assert is_quota_error(error, provider="anthropic") is True
+
+    def test_openrouter_quota_patterns(self):
+        """Test OpenRouter-specific quota error patterns."""
+        patterns = [
+            "Rate limit exceeded",
+            "Insufficient credits",
+            "No endpoints available",
+            "All providers failed",
+            "Over capacity",
+        ]
+        for pattern in patterns:
+            error = Exception(pattern)
+            assert is_quota_error(error, provider="openrouter") is True, f"Failed for: {pattern}"
+
+    def test_kimi_quota_patterns(self):
+        """Test kimi-coding-specific quota error patterns."""
+        patterns = [
+            "Rate limit exceeded",
+            "Insufficient balance",
+            "Quota exceeded",
+        ]
+        for pattern in patterns:
+            error = Exception(pattern)
+            assert is_quota_error(error, provider="kimi-coding") is True, f"Failed for: {pattern}"
+
+    def test_generic_quota_patterns(self):
+        """Test generic quota patterns work across all providers."""
+        generic_patterns = [
+            "rate limit exceeded",
+            "quota exceeded",
+            "too many requests",
+            "capacity exceeded",
+            "temporarily unavailable",
+            "resource exhausted",
+            "insufficient credits",
+        ]
+        for pattern in generic_patterns:
+            error = Exception(pattern)
+            assert is_quota_error(error) is True, f"Failed for generic pattern: {pattern}"
+
+    def test_non_quota_errors_not_detected(self):
+        """Test that non-quota errors are not incorrectly detected."""
+        non_quota_errors = [
+            "Context length exceeded",
+            "Invalid API key",
+            "Model not found",
+            "Network timeout",
+            "Connection refused",
+            "JSON decode error",
+        ]
+        for pattern in non_quota_errors:
+            error = Exception(pattern)
+            assert is_quota_error(error) is False, f"Incorrectly detected as quota: {pattern}"
+
+    def test_error_type_detection(self):
+        """Test that specific exception types are detected as quota errors."""
+        class RateLimitError(Exception):
+            pass
+        
+        class QuotaExceededError(Exception):
+            pass
+        
+        class TooManyRequests(Exception):
+            pass
+        
+        for exc_class in [RateLimitError, QuotaExceededError, TooManyRequests]:
+            error = exc_class("Some message")
+            assert is_quota_error(error) is True, f"Failed for {exc_class.__name__}"
+
+
+class TestFallbackLogging:
+    """Tests for fallback event logging."""
+
+    def test_fallback_event_logged_with_all_params(self):
+        """Test that fallback events log all required parameters."""
+        with patch("agent.fallback_router.logger") as mock_logger:
+            log_fallback_event(
+                from_provider="anthropic",
+                to_provider="kimi-coding",
+                to_model="kimi-k2.5",
+                reason="quota_exceeded",
+            )
+            
+            # Verify info was called
+            mock_logger.info.assert_called_once()
+            
+            # Verify the log message format and arguments
+            call_args = mock_logger.info.call_args
+            log_format = call_args[0][0]
+            log_args = call_args[0][1:]  # Remaining positional args
+            
+            # Check format string contains placeholders
+            assert "%s" in log_format
+            # Check actual values are in the arguments
+            assert "anthropic" in log_args
+            assert "kimi-coding" in log_args
+            assert "kimi-k2.5" in log_args
+
+    def test_fallback_event_with_error_logs_debug(self):
+        """Test that fallback events with errors also log debug info."""
+        error = Exception("Rate limit exceeded")
+        
+        with patch("agent.fallback_router.logger") as mock_logger:
+            log_fallback_event(
+                from_provider="anthropic",
+                to_provider="kimi-coding",
+                to_model="kimi-k2.5",
+                reason="rate_limit",
+                error=error,
+            )
+            
+            # Both info and debug should be called
+            mock_logger.info.assert_called_once()
+            mock_logger.debug.assert_called_once()
+
+    def test_fallback_chain_resolution_logged(self):
+        """Test logging during full chain resolution."""
+        with patch("agent.fallback_router.logger") as mock_logger:
+            # Simulate getting chain
+            chain = get_auto_fallback_chain("anthropic")
+            
+            # Log each fallback step
+            for i, fallback in enumerate(chain):
+                log_fallback_event(
+                    from_provider="anthropic" if i == 0 else chain[i-1]["provider"],
+                    to_provider=fallback["provider"],
+                    to_model=fallback["model"],
+                    reason="chain_resolution",
+                )
+            
+            # Should have logged for each fallback
+            assert mock_logger.info.call_count == len(chain)
+
+
+class TestFallbackAvailability:
+    """Tests for fallback availability checking with credentials."""
+
+    def test_anthropic_available_with_api_key(self):
+        """Test Anthropic is available when ANTHROPIC_API_KEY is set."""
+        with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "test-key"}):
+            config = {"provider": "anthropic", "model": "claude-3"}
+            assert is_fallback_available(config) is True
+
+    def test_anthropic_available_with_token(self):
+        """Test Anthropic is available when ANTHROPIC_TOKEN is set."""
+        with patch.dict(os.environ, {"ANTHROPIC_TOKEN": "test-token"}):
+            config = {"provider": "anthropic", "model": "claude-3"}
+            assert is_fallback_available(config) is True
+
+    def test_kimi_available_with_api_key(self):
+        """Test kimi-coding is available when KIMI_API_KEY is set."""
+        with patch.dict(os.environ, {"KIMI_API_KEY": "test-key"}):
+            config = {"provider": "kimi-coding", "model": "kimi-k2.5"}
+            assert is_fallback_available(config) is True
+
+    def test_kimi_available_with_api_token(self):
+        """Test kimi-coding is available when KIMI_API_TOKEN is set."""
+        with patch.dict(os.environ, {"KIMI_API_TOKEN": "test-token"}):
+            config = {"provider": "kimi-coding", "model": "kimi-k2.5"}
+            assert is_fallback_available(config) is True
+
+    def test_openrouter_available_with_key(self):
+        """Test openrouter is available when OPENROUTER_API_KEY is set."""
+        with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}):
+            config = {"provider": "openrouter", "model": "claude-sonnet-4"}
+            assert is_fallback_available(config) is True
+
+    def test_zai_available(self):
+        """Test zai is available when ZAI_API_KEY is set."""
+        with patch.dict(os.environ, {"ZAI_API_KEY": "test-key"}):
+            config = {"provider": "zai", "model": "glm-5"}
+            assert is_fallback_available(config) is True
+
+    def test_unconfigured_provider_not_available(self):
+        """Test that providers without credentials are not available."""
+        with patch.dict(os.environ, {}, clear=True):
+            providers = [
+                {"provider": "anthropic", "model": "claude-3"},
+                {"provider": "kimi-coding", "model": "kimi-k2.5"},
+                {"provider": "openrouter", "model": "claude-sonnet-4"},
+                {"provider": "zai", "model": "glm-5"},
+            ]
+            for config in providers:
+                assert is_fallback_available(config) is False, f"{config['provider']} should not be available"
+
+    def test_invalid_config_not_available(self):
+        """Test that invalid configs are not available."""
+        assert is_fallback_available({}) is False
+        assert is_fallback_available({"provider": ""}) is False
+        assert is_fallback_available({"model": "some-model"}) is False
+
+
+class TestAutoFallbackDecision:
+    """Tests for automatic fallback decision logic."""
+
+    def test_anthropic_eager_fallback_no_error(self):
+        """Test Anthropic falls back eagerly even without an error."""
+        with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
+            assert should_auto_fallback("anthropic") is True
+
+    def test_quota_error_triggers_fallback_any_provider(self):
+        """Test that quota errors trigger fallback for any provider."""
+        error = Exception("Rate limit exceeded")
+        with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
+            # Even unknown providers should fallback on quota errors
+            assert should_auto_fallback("unknown_provider", error=error) is True
+
+    def test_non_quota_error_no_fallback_unknown_provider(self):
+        """Test that non-quota errors don't trigger fallback for unknown providers."""
+        error = Exception("Some random error")
+        with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
+            assert should_auto_fallback("unknown_provider", error=error) is False
+
+    def test_auto_fallback_disabled_via_env(self):
+        """Test auto-fallback can be disabled via environment variable."""
+        with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "false"}):
+            assert should_auto_fallback("anthropic") is False
+            
+        with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "0"}):
+            assert should_auto_fallback("anthropic") is False
+            
+        with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "off"}):
+            assert should_auto_fallback("anthropic") is False
+
+    def test_auto_fallback_disabled_via_param(self):
+        """Test auto-fallback can be disabled via parameter."""
+        assert should_auto_fallback("anthropic", auto_fallback_enabled=False) is False
+
+    def test_auto_fallback_enabled_variations(self):
+        """Test various truthy values for HERMES_AUTO_FALLBACK."""
+        truthy_values = ["true", "1", "yes", "on"]
+        for value in truthy_values:
+            with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": value}):
+                assert should_auto_fallback("anthropic") is True, f"Failed for {value}"
+
+
+class TestEndToEndFallbackChain:
+    """End-to-end tests simulating real fallback scenarios."""
+
+    def test_anthropic_to_kimi_fallback_scenario(self):
+        """Simulate complete fallback: Anthropic quota -> kimi-coding."""
+        # Step 1: Anthropic encounters a quota error
+        anthropic_error = Exception("Rate limit exceeded: quota exceeded for model claude-3-5-sonnet")
+        
+        # Step 2: Verify it's detected as a quota error
+        assert is_quota_error(anthropic_error, provider="anthropic") is True
+        
+        # Step 3: Check if auto-fallback should trigger
+        with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
+            assert should_auto_fallback("anthropic", error=anthropic_error) is True
+            
+            # Step 4: Get fallback chain
+            chain = get_auto_fallback_chain("anthropic")
+            assert len(chain) > 0
+            
+            # Step 5: Simulate kimi-coding being available
+            with patch.dict(os.environ, {"KIMI_API_KEY": "test-kimi-key"}):
+                available = filter_available_fallbacks(chain)
+                assert len(available) > 0
+                assert available[0]["provider"] == "kimi-coding"
+                
+                # Step 6: Log the fallback event
+                with patch("agent.fallback_router.logger") as mock_logger:
+                    log_fallback_event(
+                        from_provider="anthropic",
+                        to_provider=available[0]["provider"],
+                        to_model=available[0]["model"],
+                        reason="quota_exceeded",
+                        error=anthropic_error,
+                    )
+                    mock_logger.info.assert_called_once()
+
+    def test_full_chain_exhaustion_scenario(self):
+        """Simulate scenario where entire fallback chain is exhausted."""
+        # Simulate Anthropic error
+        error = Exception("Rate limit exceeded")
+        
+        with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
+            # Get chain
+            chain = get_auto_fallback_chain("anthropic")
+            
+            # Simulate no providers available
+            with patch.dict(os.environ, {}, clear=True):
+                available = filter_available_fallbacks(chain)
+                assert available == []
+                
+                # Fallback should not be possible
+                assert len(available) == 0
+
+    def test_chain_continues_on_provider_failure(self):
+        """Test that chain continues when a fallback provider fails."""
+        with patch.dict(os.environ, {"HERMES_AUTO_FALLBACK": "true"}):
+            chain = get_auto_fallback_chain("anthropic")
+            
+            # Simulate only openrouter available (kimi-coding not configured)
+            with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}, clear=True):
+                available = filter_available_fallbacks(chain)
+                
+                # Should have openrouter as available (skipping kimi-coding)
+                assert len(available) >= 1
+                assert available[0]["provider"] == "openrouter"
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v"])