992498463e
Some checks failed
Forge CI / smoke-and-build (pull_request) Failing after 1m32s
- Expand validate_config_structure() to catch: - fallback_providers format errors (non-list, missing provider/model) - session_reset.idle_minutes <= 0 (causes immediate resets) - session_reset.at_hour out of 0-23 range - API_SERVER enabled without API_SERVER_KEY - Unknown root-level keys that look like misplaced custom_providers fields - Add _validate_fallback_providers() in gateway/config.py to validate fallback chain at gateway startup (logs warnings for malformed entries) - Add API_SERVER_KEY check in gateway config loader (warns on unauthenticated endpoint) - Expand _KNOWN_ROOT_KEYS to include all valid top-level config sections (session_reset, browser, checkpoints, voice, stt, tts, etc.) - Add 13 new tests for fallback_providers and session_reset validation - All existing tests pass (47/47) Closes #328
283 lines
11 KiB
Python
283 lines
11 KiB
Python
"""Tests for config.yaml structure validation (validate_config_structure)."""
|
|
|
|
import pytest
|
|
|
|
from hermes_cli.config import validate_config_structure, ConfigIssue
|
|
|
|
|
|
class TestCustomProvidersValidation:
|
|
"""custom_providers must be a YAML list, not a dict."""
|
|
|
|
def test_dict_instead_of_list(self):
|
|
"""The exact Discord user scenario — custom_providers as flat dict."""
|
|
issues = validate_config_structure({
|
|
"custom_providers": {
|
|
"name": "Generativelanguage.googleapis.com",
|
|
"base_url": "https://generativelanguage.googleapis.com/v1beta/openai",
|
|
"api_key": "xxx",
|
|
"model": "models/gemini-2.5-flash",
|
|
"rate_limit_delay": 2.0,
|
|
"fallback_model": {
|
|
"provider": "openrouter",
|
|
"model": "qwen/qwen3.6-plus:free",
|
|
},
|
|
},
|
|
"fallback_providers": [],
|
|
})
|
|
errors = [i for i in issues if i.severity == "error"]
|
|
assert any("dict" in i.message and "list" in i.message for i in errors), (
|
|
"Should detect custom_providers as dict instead of list"
|
|
)
|
|
|
|
def test_dict_detects_misplaced_fields(self):
|
|
"""When custom_providers is a dict, detect fields that look misplaced."""
|
|
issues = validate_config_structure({
|
|
"custom_providers": {
|
|
"name": "test",
|
|
"base_url": "https://example.com",
|
|
"api_key": "xxx",
|
|
},
|
|
})
|
|
warnings = [i for i in issues if i.severity == "warning"]
|
|
# Should flag base_url, api_key as looking like custom_providers entry fields
|
|
misplaced = [i for i in warnings if "custom_providers entry fields" in i.message]
|
|
assert len(misplaced) == 1
|
|
|
|
def test_dict_detects_nested_fallback(self):
|
|
"""When fallback_model gets swallowed into custom_providers dict."""
|
|
issues = validate_config_structure({
|
|
"custom_providers": {
|
|
"name": "test",
|
|
"fallback_model": {"provider": "openrouter", "model": "test"},
|
|
},
|
|
})
|
|
errors = [i for i in issues if i.severity == "error"]
|
|
assert any("fallback_model" in i.message and "inside" in i.message for i in errors)
|
|
|
|
def test_valid_list_no_issues(self):
|
|
"""Properly formatted custom_providers should produce no issues."""
|
|
issues = validate_config_structure({
|
|
"custom_providers": [
|
|
{"name": "gemini", "base_url": "https://example.com/v1"},
|
|
],
|
|
"model": {"provider": "custom", "default": "test"},
|
|
})
|
|
assert len(issues) == 0
|
|
|
|
def test_list_entry_missing_name(self):
|
|
"""List entry without name should warn."""
|
|
issues = validate_config_structure({
|
|
"custom_providers": [{"base_url": "https://example.com/v1"}],
|
|
"model": {"provider": "custom"},
|
|
})
|
|
assert any("missing 'name'" in i.message for i in issues)
|
|
|
|
def test_list_entry_missing_base_url(self):
|
|
"""List entry without base_url should warn."""
|
|
issues = validate_config_structure({
|
|
"custom_providers": [{"name": "test"}],
|
|
"model": {"provider": "custom"},
|
|
})
|
|
assert any("missing 'base_url'" in i.message for i in issues)
|
|
|
|
def test_list_entry_not_dict(self):
|
|
"""Non-dict list entries should warn."""
|
|
issues = validate_config_structure({
|
|
"custom_providers": ["not-a-dict"],
|
|
"model": {"provider": "custom"},
|
|
})
|
|
assert any("not a dict" in i.message for i in issues)
|
|
|
|
def test_none_custom_providers_no_issues(self):
|
|
"""No custom_providers at all should be fine."""
|
|
issues = validate_config_structure({
|
|
"model": {"provider": "openrouter"},
|
|
})
|
|
assert len(issues) == 0
|
|
|
|
|
|
class TestFallbackModelValidation:
|
|
"""fallback_model should be a top-level dict with provider + model."""
|
|
|
|
def test_missing_provider(self):
|
|
issues = validate_config_structure({
|
|
"fallback_model": {"model": "anthropic/claude-sonnet-4"},
|
|
})
|
|
assert any("missing 'provider'" in i.message for i in issues)
|
|
|
|
def test_missing_model(self):
|
|
issues = validate_config_structure({
|
|
"fallback_model": {"provider": "openrouter"},
|
|
})
|
|
assert any("missing 'model'" in i.message for i in issues)
|
|
|
|
def test_valid_fallback(self):
|
|
issues = validate_config_structure({
|
|
"fallback_model": {
|
|
"provider": "openrouter",
|
|
"model": "anthropic/claude-sonnet-4",
|
|
},
|
|
})
|
|
# Only fallback-related issues should be absent
|
|
fb_issues = [i for i in issues if "fallback" in i.message.lower()]
|
|
assert len(fb_issues) == 0
|
|
|
|
def test_non_dict_fallback(self):
|
|
issues = validate_config_structure({
|
|
"fallback_model": "openrouter:anthropic/claude-sonnet-4",
|
|
})
|
|
assert any("should be a dict" in i.message for i in issues)
|
|
|
|
def test_empty_fallback_dict_no_issues(self):
|
|
"""Empty fallback_model dict means disabled — no warnings needed."""
|
|
issues = validate_config_structure({
|
|
"fallback_model": {},
|
|
})
|
|
fb_issues = [i for i in issues if "fallback" in i.message.lower()]
|
|
assert len(fb_issues) == 0
|
|
|
|
|
|
class TestMissingModelSection:
|
|
"""Warn when custom_providers exists but model section is missing."""
|
|
|
|
def test_custom_providers_without_model(self):
|
|
issues = validate_config_structure({
|
|
"custom_providers": [
|
|
{"name": "test", "base_url": "https://example.com/v1"},
|
|
],
|
|
})
|
|
assert any("no 'model' section" in i.message for i in issues)
|
|
|
|
def test_custom_providers_with_model(self):
|
|
issues = validate_config_structure({
|
|
"custom_providers": [
|
|
{"name": "test", "base_url": "https://example.com/v1"},
|
|
],
|
|
"model": {"provider": "custom", "default": "test-model"},
|
|
})
|
|
# Should not warn about missing model section
|
|
assert not any("no 'model' section" in i.message for i in issues)
|
|
|
|
|
|
class TestConfigIssueDataclass:
|
|
"""ConfigIssue should be a proper dataclass."""
|
|
|
|
def test_fields(self):
|
|
issue = ConfigIssue(severity="error", message="test msg", hint="test hint")
|
|
assert issue.severity == "error"
|
|
assert issue.message == "test msg"
|
|
assert issue.hint == "test hint"
|
|
|
|
def test_equality(self):
|
|
a = ConfigIssue("error", "msg", "hint")
|
|
b = ConfigIssue("error", "msg", "hint")
|
|
assert a == b
|
|
|
|
|
|
class TestFallbackProvidersValidation:
|
|
"""fallback_providers must be a list of dicts with provider + model."""
|
|
|
|
def test_non_list(self):
|
|
"""fallback_providers as string should error."""
|
|
issues = validate_config_structure({
|
|
"fallback_providers": "openrouter:google/gemini-3-flash-preview",
|
|
})
|
|
errors = [i for i in issues if i.severity == "error"]
|
|
assert any("fallback_providers" in i.message and "list" in i.message for i in errors)
|
|
|
|
def test_dict_instead_of_list(self):
|
|
"""fallback_providers as dict should error."""
|
|
issues = validate_config_structure({
|
|
"fallback_providers": {"provider": "openrouter", "model": "test"},
|
|
})
|
|
errors = [i for i in issues if i.severity == "error"]
|
|
assert any("fallback_providers" in i.message and "dict" in i.message for i in errors)
|
|
|
|
def test_entry_missing_provider(self):
|
|
"""Entry without provider should warn."""
|
|
issues = validate_config_structure({
|
|
"fallback_providers": [{"model": "google/gemini-3-flash-preview"}],
|
|
})
|
|
assert any("missing 'provider'" in i.message for i in issues)
|
|
|
|
def test_entry_missing_model(self):
|
|
"""Entry without model should warn."""
|
|
issues = validate_config_structure({
|
|
"fallback_providers": [{"provider": "openrouter"}],
|
|
})
|
|
assert any("missing 'model'" in i.message for i in issues)
|
|
|
|
def test_entry_not_dict(self):
|
|
"""Non-dict entries should warn."""
|
|
issues = validate_config_structure({
|
|
"fallback_providers": ["not-a-dict"],
|
|
})
|
|
assert any("not a dict" in i.message for i in issues)
|
|
|
|
def test_valid_entries(self):
|
|
"""Valid fallback_providers should produce no fallback-related issues."""
|
|
issues = validate_config_structure({
|
|
"fallback_providers": [
|
|
{"provider": "openrouter", "model": "google/gemini-3-flash-preview"},
|
|
{"provider": "gemini", "model": "gemini-2.5-flash"},
|
|
],
|
|
})
|
|
fb_issues = [i for i in issues if "fallback_providers" in i.message]
|
|
assert len(fb_issues) == 0
|
|
|
|
def test_empty_list_no_issues(self):
|
|
"""Empty list is valid (fallback disabled)."""
|
|
issues = validate_config_structure({
|
|
"fallback_providers": [],
|
|
})
|
|
fb_issues = [i for i in issues if "fallback_providers" in i.message]
|
|
assert len(fb_issues) == 0
|
|
|
|
|
|
class TestSessionResetValidation:
|
|
"""session_reset.idle_minutes must be positive."""
|
|
|
|
def test_zero_idle_minutes(self):
|
|
"""idle_minutes=0 should warn."""
|
|
issues = validate_config_structure({
|
|
"session_reset": {"idle_minutes": 0},
|
|
})
|
|
assert any("idle_minutes=0" in i.message for i in issues)
|
|
|
|
def test_negative_idle_minutes(self):
|
|
"""idle_minutes=-5 should warn."""
|
|
issues = validate_config_structure({
|
|
"session_reset": {"idle_minutes": -5},
|
|
})
|
|
assert any("idle_minutes=-5" in i.message for i in issues)
|
|
|
|
def test_string_idle_minutes(self):
|
|
"""idle_minutes as string should warn."""
|
|
issues = validate_config_structure({
|
|
"session_reset": {"idle_minutes": "abc"},
|
|
})
|
|
assert any("idle_minutes=" in i.message for i in issues)
|
|
|
|
def test_valid_idle_minutes(self):
|
|
"""Valid idle_minutes should not warn."""
|
|
issues = validate_config_structure({
|
|
"session_reset": {"idle_minutes": 1440},
|
|
})
|
|
idle_issues = [i for i in issues if "idle_minutes" in i.message]
|
|
assert len(idle_issues) == 0
|
|
|
|
def test_invalid_at_hour(self):
|
|
"""at_hour=25 should warn."""
|
|
issues = validate_config_structure({
|
|
"session_reset": {"at_hour": 25},
|
|
})
|
|
assert any("at_hour=25" in i.message for i in issues)
|
|
|
|
def test_valid_at_hour(self):
|
|
"""Valid at_hour should not warn."""
|
|
issues = validate_config_structure({
|
|
"session_reset": {"at_hour": 4},
|
|
})
|
|
hour_issues = [i for i in issues if "at_hour" in i.message]
|
|
assert len(hour_issues) == 0
|