f3006ebef9
* refactor: re-architect tests to mirror the codebase
* Update tests.yml
* fix: add missing tool_error imports after registry refactor
* fix(tests): replace patch.dict with monkeypatch to prevent env var leaks under xdist
patch.dict(os.environ) can leak TERMINAL_ENV across xdist workers,
causing test_code_execution tests to hit the Modal remote path.
* fix(tests): fix update_check and telegram xdist failures
- test_update_check: replace patch("hermes_cli.banner.os.getenv") with
monkeypatch.setenv("HERMES_HOME") — banner.py no longer imports os
directly, it uses get_hermes_home() from hermes_constants.
- test_telegram_conflict/approval_buttons: provide real exception classes
for telegram.error mock (NetworkError, TimedOut, BadRequest) so the
except clause in connect() doesn't fail with "catching classes that do
not inherit from BaseException" when xdist pollutes sys.modules.
* fix(tests): accept unavailable_models kwarg in _prompt_model_selection mock
260 lines
11 KiB
Python
260 lines
11 KiB
Python
"""Tests that provider selection via `hermes model` always persists correctly.
|
|
|
|
Regression tests for the bug where _save_model_choice could save config.model
|
|
as a plain string, causing subsequent provider writes (which check
|
|
isinstance(model, dict)) to silently fail — leaving the provider unset and
|
|
falling back to auto-detection.
|
|
"""
|
|
|
|
import os
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture
|
|
def config_home(tmp_path, monkeypatch):
|
|
"""Isolated HERMES_HOME with a minimal string-format config."""
|
|
home = tmp_path / "hermes"
|
|
home.mkdir()
|
|
config_yaml = home / "config.yaml"
|
|
# Start with model as a plain string — the format that triggered the bug
|
|
config_yaml.write_text("model: some-old-model\n")
|
|
env_file = home / ".env"
|
|
env_file.write_text("")
|
|
monkeypatch.setenv("HERMES_HOME", str(home))
|
|
# Clear env vars that could interfere
|
|
monkeypatch.delenv("HERMES_MODEL", raising=False)
|
|
monkeypatch.delenv("LLM_MODEL", raising=False)
|
|
monkeypatch.delenv("HERMES_INFERENCE_PROVIDER", raising=False)
|
|
monkeypatch.delenv("GITHUB_TOKEN", raising=False)
|
|
monkeypatch.delenv("GH_TOKEN", raising=False)
|
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
|
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
|
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
|
return home
|
|
|
|
|
|
class TestSaveModelChoiceAlwaysDict:
|
|
def test_string_model_becomes_dict(self, config_home):
|
|
"""When config.model is a plain string, _save_model_choice must
|
|
convert it to a dict so provider can be set afterwards."""
|
|
from hermes_cli.auth import _save_model_choice
|
|
|
|
_save_model_choice("kimi-k2.5")
|
|
|
|
import yaml
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict), (
|
|
f"Expected model to be a dict after save, got {type(model)}: {model}"
|
|
)
|
|
assert model["default"] == "kimi-k2.5"
|
|
|
|
def test_dict_model_stays_dict(self, config_home):
|
|
"""When config.model is already a dict, _save_model_choice preserves it."""
|
|
import yaml
|
|
(config_home / "config.yaml").write_text(
|
|
"model:\n default: old-model\n provider: openrouter\n"
|
|
)
|
|
from hermes_cli.auth import _save_model_choice
|
|
|
|
_save_model_choice("new-model")
|
|
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict)
|
|
assert model["default"] == "new-model"
|
|
assert model["provider"] == "openrouter" # preserved
|
|
|
|
|
|
class TestProviderPersistsAfterModelSave:
|
|
def test_api_key_provider_saved_when_model_was_string(self, config_home, monkeypatch):
|
|
"""_model_flow_api_key_provider must persist the provider even when
|
|
config.model started as a plain string."""
|
|
from hermes_cli.auth import PROVIDER_REGISTRY
|
|
|
|
pconfig = PROVIDER_REGISTRY.get("kimi-coding")
|
|
if not pconfig:
|
|
pytest.skip("kimi-coding not in PROVIDER_REGISTRY")
|
|
|
|
# Simulate: user has a Kimi API key, model was a string
|
|
monkeypatch.setenv("KIMI_API_KEY", "sk-kimi-test-key")
|
|
|
|
from hermes_cli.main import _model_flow_api_key_provider
|
|
from hermes_cli.config import load_config
|
|
|
|
# Mock the model selection prompt to return "kimi-k2.5"
|
|
# Also mock input() for the base URL prompt and builtins.input
|
|
with patch("hermes_cli.auth._prompt_model_selection", return_value="kimi-k2.5"), \
|
|
patch("hermes_cli.auth.deactivate_provider"), \
|
|
patch("builtins.input", return_value=""):
|
|
_model_flow_api_key_provider(load_config(), "kimi-coding", "old-model")
|
|
|
|
import yaml
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict), f"model should be dict, got {type(model)}"
|
|
assert model.get("provider") == "kimi-coding", (
|
|
f"provider should be 'kimi-coding', got {model.get('provider')}"
|
|
)
|
|
assert model.get("default") == "kimi-k2.5"
|
|
|
|
def test_copilot_provider_saved_when_selected(self, config_home):
|
|
"""_model_flow_copilot should persist provider/base_url/model together."""
|
|
from hermes_cli.main import _model_flow_copilot
|
|
from hermes_cli.config import load_config
|
|
|
|
with patch(
|
|
"hermes_cli.auth.resolve_api_key_provider_credentials",
|
|
return_value={
|
|
"provider": "copilot",
|
|
"api_key": "gh-cli-token",
|
|
"base_url": "https://api.githubcopilot.com",
|
|
"source": "gh auth token",
|
|
},
|
|
), patch(
|
|
"hermes_cli.models.fetch_github_model_catalog",
|
|
return_value=[
|
|
{
|
|
"id": "gpt-4.1",
|
|
"capabilities": {"type": "chat", "supports": {}},
|
|
"supported_endpoints": ["/chat/completions"],
|
|
},
|
|
{
|
|
"id": "gpt-5.4",
|
|
"capabilities": {"type": "chat", "supports": {"reasoning_effort": ["low", "medium", "high"]}},
|
|
"supported_endpoints": ["/responses"],
|
|
},
|
|
],
|
|
), patch(
|
|
"hermes_cli.auth._prompt_model_selection",
|
|
return_value="gpt-5.4",
|
|
), patch(
|
|
"hermes_cli.main._prompt_reasoning_effort_selection",
|
|
return_value="high",
|
|
), patch(
|
|
"hermes_cli.auth.deactivate_provider",
|
|
):
|
|
_model_flow_copilot(load_config(), "old-model")
|
|
|
|
import yaml
|
|
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict), f"model should be dict, got {type(model)}"
|
|
assert model.get("provider") == "copilot"
|
|
assert model.get("base_url") == "https://api.githubcopilot.com"
|
|
assert model.get("default") == "gpt-5.4"
|
|
assert model.get("api_mode") == "codex_responses"
|
|
assert config["agent"]["reasoning_effort"] == "high"
|
|
|
|
def test_copilot_acp_provider_saved_when_selected(self, config_home):
|
|
"""_model_flow_copilot_acp should persist provider/base_url/model together."""
|
|
from hermes_cli.main import _model_flow_copilot_acp
|
|
from hermes_cli.config import load_config
|
|
|
|
with patch(
|
|
"hermes_cli.auth.get_external_process_provider_status",
|
|
return_value={
|
|
"resolved_command": "/usr/local/bin/copilot",
|
|
"command": "copilot",
|
|
"base_url": "acp://copilot",
|
|
},
|
|
), patch(
|
|
"hermes_cli.auth.resolve_external_process_provider_credentials",
|
|
return_value={
|
|
"provider": "copilot-acp",
|
|
"api_key": "copilot-acp",
|
|
"base_url": "acp://copilot",
|
|
"command": "/usr/local/bin/copilot",
|
|
"args": ["--acp", "--stdio"],
|
|
"source": "process",
|
|
},
|
|
), patch(
|
|
"hermes_cli.auth.resolve_api_key_provider_credentials",
|
|
return_value={
|
|
"provider": "copilot",
|
|
"api_key": "gh-cli-token",
|
|
"base_url": "https://api.githubcopilot.com",
|
|
"source": "gh auth token",
|
|
},
|
|
), patch(
|
|
"hermes_cli.models.fetch_github_model_catalog",
|
|
return_value=[
|
|
{
|
|
"id": "gpt-4.1",
|
|
"capabilities": {"type": "chat", "supports": {}},
|
|
"supported_endpoints": ["/chat/completions"],
|
|
},
|
|
{
|
|
"id": "gpt-5.4",
|
|
"capabilities": {"type": "chat", "supports": {"reasoning_effort": ["low", "medium", "high"]}},
|
|
"supported_endpoints": ["/responses"],
|
|
},
|
|
],
|
|
), patch(
|
|
"hermes_cli.auth._prompt_model_selection",
|
|
return_value="gpt-5.4",
|
|
), patch(
|
|
"hermes_cli.auth.deactivate_provider",
|
|
):
|
|
_model_flow_copilot_acp(load_config(), "old-model")
|
|
|
|
import yaml
|
|
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict), f"model should be dict, got {type(model)}"
|
|
assert model.get("provider") == "copilot-acp"
|
|
assert model.get("base_url") == "acp://copilot"
|
|
assert model.get("default") == "gpt-5.4"
|
|
assert model.get("api_mode") == "chat_completions"
|
|
|
|
def test_opencode_go_models_are_selectable_and_persist_normalized(self, config_home, monkeypatch):
|
|
from hermes_cli.main import _model_flow_api_key_provider
|
|
from hermes_cli.config import load_config
|
|
|
|
monkeypatch.setenv("OPENCODE_GO_API_KEY", "test-key")
|
|
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["opencode-go/kimi-k2.5", "opencode-go/minimax-m2.7"]), \
|
|
patch("hermes_cli.auth._prompt_model_selection", return_value="kimi-k2.5"), \
|
|
patch("hermes_cli.auth.deactivate_provider"), \
|
|
patch("builtins.input", return_value=""):
|
|
_model_flow_api_key_provider(load_config(), "opencode-go", "opencode-go/kimi-k2.5")
|
|
|
|
import yaml
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict)
|
|
assert model.get("provider") == "opencode-go"
|
|
assert model.get("default") == "kimi-k2.5"
|
|
assert model.get("api_mode") == "chat_completions"
|
|
|
|
def test_opencode_go_same_provider_switch_recomputes_api_mode(self, config_home, monkeypatch):
|
|
from hermes_cli.main import _model_flow_api_key_provider
|
|
from hermes_cli.config import load_config
|
|
|
|
monkeypatch.setenv("OPENCODE_GO_API_KEY", "test-key")
|
|
(config_home / "config.yaml").write_text(
|
|
"model:\n"
|
|
" default: kimi-k2.5\n"
|
|
" provider: opencode-go\n"
|
|
" base_url: https://opencode.ai/zen/go/v1\n"
|
|
" api_mode: chat_completions\n"
|
|
)
|
|
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=["opencode-go/kimi-k2.5", "opencode-go/minimax-m2.5"]), \
|
|
patch("hermes_cli.auth._prompt_model_selection", return_value="minimax-m2.5"), \
|
|
patch("hermes_cli.auth.deactivate_provider"), \
|
|
patch("builtins.input", return_value=""):
|
|
_model_flow_api_key_provider(load_config(), "opencode-go", "kimi-k2.5")
|
|
|
|
import yaml
|
|
config = yaml.safe_load((config_home / "config.yaml").read_text()) or {}
|
|
model = config.get("model")
|
|
assert isinstance(model, dict)
|
|
assert model.get("provider") == "opencode-go"
|
|
assert model.get("default") == "minimax-m2.5"
|
|
assert model.get("api_mode") == "anthropic_messages"
|