f3006ebef9
* refactor: re-architect tests to mirror the codebase
* Update tests.yml
* fix: add missing tool_error imports after registry refactor
* fix(tests): replace patch.dict with monkeypatch to prevent env var leaks under xdist
patch.dict(os.environ) can leak TERMINAL_ENV across xdist workers,
causing test_code_execution tests to hit the Modal remote path.
* fix(tests): fix update_check and telegram xdist failures
- test_update_check: replace patch("hermes_cli.banner.os.getenv") with
monkeypatch.setenv("HERMES_HOME") — banner.py no longer imports os
directly, it uses get_hermes_home() from hermes_constants.
- test_telegram_conflict/approval_buttons: provide real exception classes
for telegram.error mock (NetworkError, TimedOut, BadRequest) so the
except clause in connect() doesn't fail with "catching classes that do
not inherit from BaseException" when xdist pollutes sys.modules.
* fix(tests): accept unavailable_models kwarg in _prompt_model_selection mock
644 lines
25 KiB
Python
644 lines
25 KiB
Python
import importlib
|
|
import sys
|
|
import types
|
|
from contextlib import nullcontext
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
|
|
from hermes_cli.auth import AuthError
|
|
from hermes_cli import main as hermes_main
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Module isolation: _import_cli() wipes tools.* / cli / run_agent from
|
|
# sys.modules so it can re-import cli fresh. Without cleanup the wiped
|
|
# modules leak into subsequent tests on the same xdist worker, breaking
|
|
# mock patches that target "tools.file_tools._get_file_ops" etc.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _reset_modules(prefixes: tuple[str, ...]):
|
|
for name in list(sys.modules):
|
|
if any(name == p or name.startswith(p + ".") for p in prefixes):
|
|
sys.modules.pop(name, None)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _restore_cli_and_tool_modules():
|
|
"""Save and restore tools/cli/run_agent modules around every test."""
|
|
prefixes = ("tools", "cli", "run_agent")
|
|
original_modules = {
|
|
name: module
|
|
for name, module in sys.modules.items()
|
|
if any(name == p or name.startswith(p + ".") for p in prefixes)
|
|
}
|
|
try:
|
|
yield
|
|
finally:
|
|
_reset_modules(prefixes)
|
|
sys.modules.update(original_modules)
|
|
|
|
|
|
def _install_prompt_toolkit_stubs():
|
|
class _Dummy:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
class _Condition:
|
|
def __init__(self, func):
|
|
self.func = func
|
|
|
|
def __bool__(self):
|
|
return bool(self.func())
|
|
|
|
class _ANSI(str):
|
|
pass
|
|
|
|
root = types.ModuleType("prompt_toolkit")
|
|
history = types.ModuleType("prompt_toolkit.history")
|
|
styles = types.ModuleType("prompt_toolkit.styles")
|
|
patch_stdout = types.ModuleType("prompt_toolkit.patch_stdout")
|
|
application = types.ModuleType("prompt_toolkit.application")
|
|
layout = types.ModuleType("prompt_toolkit.layout")
|
|
processors = types.ModuleType("prompt_toolkit.layout.processors")
|
|
filters = types.ModuleType("prompt_toolkit.filters")
|
|
dimension = types.ModuleType("prompt_toolkit.layout.dimension")
|
|
menus = types.ModuleType("prompt_toolkit.layout.menus")
|
|
widgets = types.ModuleType("prompt_toolkit.widgets")
|
|
key_binding = types.ModuleType("prompt_toolkit.key_binding")
|
|
completion = types.ModuleType("prompt_toolkit.completion")
|
|
formatted_text = types.ModuleType("prompt_toolkit.formatted_text")
|
|
|
|
history.FileHistory = _Dummy
|
|
styles.Style = _Dummy
|
|
patch_stdout.patch_stdout = lambda *args, **kwargs: nullcontext()
|
|
application.Application = _Dummy
|
|
layout.Layout = _Dummy
|
|
layout.HSplit = _Dummy
|
|
layout.Window = _Dummy
|
|
layout.FormattedTextControl = _Dummy
|
|
layout.ConditionalContainer = _Dummy
|
|
processors.Processor = _Dummy
|
|
processors.Transformation = _Dummy
|
|
processors.PasswordProcessor = _Dummy
|
|
processors.ConditionalProcessor = _Dummy
|
|
filters.Condition = _Condition
|
|
dimension.Dimension = _Dummy
|
|
menus.CompletionsMenu = _Dummy
|
|
widgets.TextArea = _Dummy
|
|
key_binding.KeyBindings = _Dummy
|
|
completion.Completer = _Dummy
|
|
completion.Completion = _Dummy
|
|
formatted_text.ANSI = _ANSI
|
|
root.print_formatted_text = lambda *args, **kwargs: None
|
|
|
|
sys.modules.setdefault("prompt_toolkit", root)
|
|
sys.modules.setdefault("prompt_toolkit.history", history)
|
|
sys.modules.setdefault("prompt_toolkit.styles", styles)
|
|
sys.modules.setdefault("prompt_toolkit.patch_stdout", patch_stdout)
|
|
sys.modules.setdefault("prompt_toolkit.application", application)
|
|
sys.modules.setdefault("prompt_toolkit.layout", layout)
|
|
sys.modules.setdefault("prompt_toolkit.layout.processors", processors)
|
|
sys.modules.setdefault("prompt_toolkit.filters", filters)
|
|
sys.modules.setdefault("prompt_toolkit.layout.dimension", dimension)
|
|
sys.modules.setdefault("prompt_toolkit.layout.menus", menus)
|
|
sys.modules.setdefault("prompt_toolkit.widgets", widgets)
|
|
sys.modules.setdefault("prompt_toolkit.key_binding", key_binding)
|
|
sys.modules.setdefault("prompt_toolkit.completion", completion)
|
|
sys.modules.setdefault("prompt_toolkit.formatted_text", formatted_text)
|
|
|
|
|
|
def _import_cli():
|
|
for name in list(sys.modules):
|
|
if name == "cli" or name == "run_agent" or name == "tools" or name.startswith("tools."):
|
|
sys.modules.pop(name, None)
|
|
|
|
if "firecrawl" not in sys.modules:
|
|
sys.modules["firecrawl"] = types.SimpleNamespace(Firecrawl=object)
|
|
|
|
try:
|
|
importlib.import_module("prompt_toolkit")
|
|
except ModuleNotFoundError:
|
|
_install_prompt_toolkit_stubs()
|
|
return importlib.import_module("cli")
|
|
|
|
|
|
def test_hermes_cli_init_does_not_eagerly_resolve_runtime_provider(monkeypatch):
|
|
cli = _import_cli()
|
|
calls = {"count": 0}
|
|
|
|
def _unexpected_runtime_resolve(**kwargs):
|
|
calls["count"] += 1
|
|
raise AssertionError("resolve_runtime_provider should not be called in HermesCLI.__init__")
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _unexpected_runtime_resolve)
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.format_runtime_provider_error", lambda exc: str(exc))
|
|
|
|
shell = cli.HermesCLI(model="gpt-5", compact=True, max_turns=1)
|
|
|
|
assert shell is not None
|
|
assert calls["count"] == 0
|
|
|
|
|
|
def test_runtime_resolution_failure_is_not_sticky(monkeypatch):
|
|
cli = _import_cli()
|
|
calls = {"count": 0}
|
|
|
|
def _runtime_resolve(**kwargs):
|
|
calls["count"] += 1
|
|
if calls["count"] == 1:
|
|
raise RuntimeError("temporary auth failure")
|
|
return {
|
|
"provider": "openrouter",
|
|
"api_mode": "chat_completions",
|
|
"base_url": "https://openrouter.ai/api/v1",
|
|
"api_key": "test-key",
|
|
"source": "env/config",
|
|
}
|
|
|
|
class _DummyAgent:
|
|
def __init__(self, *args, **kwargs):
|
|
self.kwargs = kwargs
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _runtime_resolve)
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.format_runtime_provider_error", lambda exc: str(exc))
|
|
monkeypatch.setattr(cli, "AIAgent", _DummyAgent)
|
|
|
|
shell = cli.HermesCLI(model="gpt-5", compact=True, max_turns=1)
|
|
|
|
assert shell._init_agent() is False
|
|
assert shell._init_agent() is True
|
|
assert calls["count"] == 2
|
|
assert shell.agent is not None
|
|
|
|
|
|
def test_runtime_resolution_rebuilds_agent_on_routing_change(monkeypatch):
|
|
cli = _import_cli()
|
|
|
|
def _runtime_resolve(**kwargs):
|
|
return {
|
|
"provider": "openai-codex",
|
|
"api_mode": "codex_responses",
|
|
"base_url": "https://same-endpoint.example/v1",
|
|
"api_key": "same-key",
|
|
"source": "env/config",
|
|
}
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _runtime_resolve)
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.format_runtime_provider_error", lambda exc: str(exc))
|
|
|
|
shell = cli.HermesCLI(model="gpt-5", compact=True, max_turns=1)
|
|
shell.provider = "openrouter"
|
|
shell.api_mode = "chat_completions"
|
|
shell.base_url = "https://same-endpoint.example/v1"
|
|
shell.api_key = "same-key"
|
|
shell.agent = object()
|
|
|
|
assert shell._ensure_runtime_credentials() is True
|
|
assert shell.agent is None
|
|
assert shell.provider == "openai-codex"
|
|
assert shell.api_mode == "codex_responses"
|
|
|
|
|
|
def test_cli_turn_routing_uses_primary_when_disabled(monkeypatch):
|
|
cli = _import_cli()
|
|
shell = cli.HermesCLI(model="gpt-5", compact=True, max_turns=1)
|
|
shell.provider = "openrouter"
|
|
shell.api_mode = "chat_completions"
|
|
shell.base_url = "https://openrouter.ai/api/v1"
|
|
shell.api_key = "sk-primary"
|
|
shell._smart_model_routing = {"enabled": False}
|
|
|
|
result = shell._resolve_turn_agent_config("what time is it in tokyo?")
|
|
|
|
assert result["model"] == "gpt-5"
|
|
assert result["runtime"]["provider"] == "openrouter"
|
|
assert result["label"] is None
|
|
|
|
|
|
def test_cli_turn_routing_uses_cheap_model_when_simple(monkeypatch):
|
|
cli = _import_cli()
|
|
|
|
def _runtime_resolve(**kwargs):
|
|
assert kwargs["requested"] == "zai"
|
|
return {
|
|
"provider": "zai",
|
|
"api_mode": "chat_completions",
|
|
"base_url": "https://open.z.ai/api/v1",
|
|
"api_key": "cheap-key",
|
|
"source": "env/config",
|
|
}
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _runtime_resolve)
|
|
|
|
shell = cli.HermesCLI(model="anthropic/claude-sonnet-4", compact=True, max_turns=1)
|
|
shell.provider = "openrouter"
|
|
shell.api_mode = "chat_completions"
|
|
shell.base_url = "https://openrouter.ai/api/v1"
|
|
shell.api_key = "primary-key"
|
|
shell._smart_model_routing = {
|
|
"enabled": True,
|
|
"cheap_model": {"provider": "zai", "model": "glm-5-air"},
|
|
"max_simple_chars": 160,
|
|
"max_simple_words": 28,
|
|
}
|
|
|
|
result = shell._resolve_turn_agent_config("what time is it in tokyo?")
|
|
|
|
assert result["model"] == "glm-5-air"
|
|
assert result["runtime"]["provider"] == "zai"
|
|
assert result["runtime"]["api_key"] == "cheap-key"
|
|
assert result["label"] is not None
|
|
|
|
|
|
def test_cli_prefers_config_provider_over_stale_env_override(monkeypatch):
|
|
cli = _import_cli()
|
|
|
|
monkeypatch.setenv("HERMES_INFERENCE_PROVIDER", "openrouter")
|
|
config_copy = dict(cli.CLI_CONFIG)
|
|
model_copy = dict(config_copy.get("model", {}))
|
|
model_copy["provider"] = "custom"
|
|
model_copy["base_url"] = "https://api.fireworks.ai/inference/v1"
|
|
config_copy["model"] = model_copy
|
|
monkeypatch.setattr(cli, "CLI_CONFIG", config_copy)
|
|
|
|
shell = cli.HermesCLI(model="fireworks/minimax-m2p5", compact=True, max_turns=1)
|
|
|
|
assert shell.requested_provider == "custom"
|
|
|
|
|
|
def test_codex_provider_replaces_incompatible_default_model(monkeypatch):
|
|
"""When provider resolves to openai-codex and no model was explicitly
|
|
chosen, the global config default (e.g. anthropic/claude-opus-4.6) must
|
|
be replaced with a Codex-compatible model. Fixes #651."""
|
|
cli = _import_cli()
|
|
|
|
monkeypatch.delenv("LLM_MODEL", raising=False)
|
|
monkeypatch.delenv("OPENAI_MODEL", raising=False)
|
|
# Ensure local user config does not leak a model into the test
|
|
monkeypatch.setitem(cli.CLI_CONFIG, "model", {
|
|
"default": "",
|
|
"base_url": "https://openrouter.ai/api/v1",
|
|
})
|
|
|
|
def _runtime_resolve(**kwargs):
|
|
return {
|
|
"provider": "openai-codex",
|
|
"api_mode": "codex_responses",
|
|
"base_url": "https://chatgpt.com/backend-api/codex",
|
|
"api_key": "test-key",
|
|
"source": "env/config",
|
|
}
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _runtime_resolve)
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.format_runtime_provider_error", lambda exc: str(exc))
|
|
monkeypatch.setattr(
|
|
"hermes_cli.codex_models.get_codex_model_ids",
|
|
lambda access_token=None: ["gpt-5.2-codex", "gpt-5.1-codex-mini"],
|
|
)
|
|
|
|
shell = cli.HermesCLI(compact=True, max_turns=1)
|
|
|
|
assert shell._model_is_default is True
|
|
assert shell._ensure_runtime_credentials() is True
|
|
assert shell.provider == "openai-codex"
|
|
assert "anthropic" not in shell.model
|
|
assert "claude" not in shell.model
|
|
assert shell.model == "gpt-5.2-codex"
|
|
|
|
|
|
def test_model_flow_nous_prints_subscription_guidance_without_mutating_explicit_tts(monkeypatch, capsys):
|
|
monkeypatch.setenv("HERMES_ENABLE_NOUS_MANAGED_TOOLS", "1")
|
|
config = {
|
|
"model": {"provider": "nous", "default": "claude-opus-4-6"},
|
|
"tts": {"provider": "elevenlabs"},
|
|
"browser": {"cloud_provider": "browser-use"},
|
|
}
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.auth.get_provider_auth_state",
|
|
lambda provider: {"access_token": "nous-token"},
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.auth.resolve_nous_runtime_credentials",
|
|
lambda *args, **kwargs: {
|
|
"base_url": "https://inference.example.com/v1",
|
|
"api_key": "nous-key",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.auth.fetch_nous_models",
|
|
lambda *args, **kwargs: ["claude-opus-4-6"],
|
|
)
|
|
monkeypatch.setattr("hermes_cli.auth._prompt_model_selection", lambda model_ids, current_model="", pricing=None, **kw: "claude-opus-4-6")
|
|
monkeypatch.setattr("hermes_cli.auth._save_model_choice", lambda model: None)
|
|
monkeypatch.setattr("hermes_cli.auth._update_config_for_provider", lambda provider, url: None)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.nous_subscription.get_nous_subscription_explainer_lines",
|
|
lambda: ["Nous subscription enables managed web tools."],
|
|
)
|
|
|
|
hermes_main._model_flow_nous(config, current_model="claude-opus-4-6")
|
|
|
|
out = capsys.readouterr().out
|
|
assert "Nous subscription enables managed web tools." in out
|
|
assert config["tts"]["provider"] == "elevenlabs"
|
|
assert config["browser"]["cloud_provider"] == "browser-use"
|
|
|
|
|
|
def test_model_flow_nous_applies_managed_tts_default_when_unconfigured(monkeypatch, capsys):
|
|
monkeypatch.setenv("HERMES_ENABLE_NOUS_MANAGED_TOOLS", "1")
|
|
config = {
|
|
"model": {"provider": "nous", "default": "claude-opus-4-6"},
|
|
"tts": {"provider": "edge"},
|
|
}
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.auth.get_provider_auth_state",
|
|
lambda provider: {"access_token": "nous-token"},
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.auth.resolve_nous_runtime_credentials",
|
|
lambda *args, **kwargs: {
|
|
"base_url": "https://inference.example.com/v1",
|
|
"api_key": "nous-key",
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.auth.fetch_nous_models",
|
|
lambda *args, **kwargs: ["claude-opus-4-6"],
|
|
)
|
|
monkeypatch.setattr("hermes_cli.auth._prompt_model_selection", lambda model_ids, current_model="", pricing=None, **kw: "claude-opus-4-6")
|
|
monkeypatch.setattr("hermes_cli.auth._save_model_choice", lambda model: None)
|
|
monkeypatch.setattr("hermes_cli.auth._update_config_for_provider", lambda provider, url: None)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.nous_subscription.get_nous_subscription_explainer_lines",
|
|
lambda: ["Nous subscription enables managed web tools."],
|
|
)
|
|
|
|
hermes_main._model_flow_nous(config, current_model="claude-opus-4-6")
|
|
|
|
out = capsys.readouterr().out
|
|
assert "Nous subscription enables managed web tools." in out
|
|
assert "OpenAI TTS via your Nous subscription" in out
|
|
assert config["tts"]["provider"] == "openai"
|
|
|
|
|
|
def test_codex_provider_uses_config_model(monkeypatch):
|
|
"""Model comes from config.yaml, not LLM_MODEL env var.
|
|
Config.yaml is the single source of truth to avoid multi-agent conflicts."""
|
|
cli = _import_cli()
|
|
|
|
# LLM_MODEL env var should be IGNORED (even if set)
|
|
monkeypatch.setenv("LLM_MODEL", "should-be-ignored")
|
|
monkeypatch.delenv("OPENAI_MODEL", raising=False)
|
|
|
|
# Set model via config
|
|
monkeypatch.setitem(cli.CLI_CONFIG, "model", {
|
|
"default": "gpt-5.2-codex",
|
|
"provider": "openai-codex",
|
|
"base_url": "https://chatgpt.com/backend-api/codex",
|
|
})
|
|
|
|
def _runtime_resolve(**kwargs):
|
|
return {
|
|
"provider": "openai-codex",
|
|
"api_mode": "codex_responses",
|
|
"base_url": "https://chatgpt.com/backend-api/codex",
|
|
"api_key": "fake-codex-token",
|
|
"source": "env/config",
|
|
}
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _runtime_resolve)
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.format_runtime_provider_error", lambda exc: str(exc))
|
|
# Prevent live API call from overriding the config model
|
|
monkeypatch.setattr(
|
|
"hermes_cli.codex_models.get_codex_model_ids",
|
|
lambda access_token=None: ["gpt-5.2-codex"],
|
|
)
|
|
|
|
shell = cli.HermesCLI(compact=True, max_turns=1)
|
|
|
|
assert shell._ensure_runtime_credentials() is True
|
|
assert shell.provider == "openai-codex"
|
|
# Model from config (may be normalized by codex provider logic)
|
|
assert "codex" in shell.model.lower()
|
|
# LLM_MODEL env var is NOT used
|
|
assert shell.model != "should-be-ignored"
|
|
|
|
|
|
def test_codex_config_model_not_replaced_by_normalization(monkeypatch):
|
|
"""When the user sets model.default in config.yaml to a specific codex
|
|
model, _normalize_model_for_provider must NOT replace it with the latest
|
|
available model from the API. Regression test for #1887."""
|
|
cli = _import_cli()
|
|
|
|
monkeypatch.delenv("LLM_MODEL", raising=False)
|
|
monkeypatch.delenv("OPENAI_MODEL", raising=False)
|
|
|
|
# User explicitly configured gpt-5.3-codex in config.yaml
|
|
monkeypatch.setitem(cli.CLI_CONFIG, "model", {
|
|
"default": "gpt-5.3-codex",
|
|
"provider": "openai-codex",
|
|
"base_url": "https://chatgpt.com/backend-api/codex",
|
|
})
|
|
|
|
def _runtime_resolve(**kwargs):
|
|
return {
|
|
"provider": "openai-codex",
|
|
"api_mode": "codex_responses",
|
|
"base_url": "https://chatgpt.com/backend-api/codex",
|
|
"api_key": "fake-key",
|
|
"source": "env/config",
|
|
}
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _runtime_resolve)
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.format_runtime_provider_error", lambda exc: str(exc))
|
|
# API returns a DIFFERENT model than what the user configured
|
|
monkeypatch.setattr(
|
|
"hermes_cli.codex_models.get_codex_model_ids",
|
|
lambda access_token=None: ["gpt-5.4", "gpt-5.3-codex"],
|
|
)
|
|
|
|
shell = cli.HermesCLI(compact=True, max_turns=1)
|
|
|
|
# Config model is NOT the global default — user made a deliberate choice
|
|
assert shell._model_is_default is False
|
|
assert shell._ensure_runtime_credentials() is True
|
|
assert shell.provider == "openai-codex"
|
|
# Model must stay as user configured, not replaced by gpt-5.4
|
|
assert shell.model == "gpt-5.3-codex"
|
|
|
|
|
|
def test_codex_provider_preserves_explicit_codex_model(monkeypatch):
|
|
"""If the user explicitly passes a Codex-compatible model, it must be
|
|
preserved even when the provider resolves to openai-codex."""
|
|
cli = _import_cli()
|
|
|
|
monkeypatch.delenv("LLM_MODEL", raising=False)
|
|
monkeypatch.delenv("OPENAI_MODEL", raising=False)
|
|
|
|
def _runtime_resolve(**kwargs):
|
|
return {
|
|
"provider": "openai-codex",
|
|
"api_mode": "codex_responses",
|
|
"base_url": "https://chatgpt.com/backend-api/codex",
|
|
"api_key": "test-key",
|
|
"source": "env/config",
|
|
}
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _runtime_resolve)
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.format_runtime_provider_error", lambda exc: str(exc))
|
|
|
|
shell = cli.HermesCLI(model="gpt-5.1-codex-mini", compact=True, max_turns=1)
|
|
|
|
assert shell._model_is_default is False
|
|
assert shell._ensure_runtime_credentials() is True
|
|
assert shell.model == "gpt-5.1-codex-mini"
|
|
|
|
|
|
def test_codex_provider_strips_provider_prefix_from_model(monkeypatch):
|
|
"""openai/gpt-5.3-codex should become gpt-5.3-codex — the Codex
|
|
Responses API does not accept provider-prefixed model slugs."""
|
|
cli = _import_cli()
|
|
|
|
monkeypatch.delenv("LLM_MODEL", raising=False)
|
|
monkeypatch.delenv("OPENAI_MODEL", raising=False)
|
|
|
|
def _runtime_resolve(**kwargs):
|
|
return {
|
|
"provider": "openai-codex",
|
|
"api_mode": "codex_responses",
|
|
"base_url": "https://chatgpt.com/backend-api/codex",
|
|
"api_key": "test-key",
|
|
"source": "env/config",
|
|
}
|
|
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.resolve_runtime_provider", _runtime_resolve)
|
|
monkeypatch.setattr("hermes_cli.runtime_provider.format_runtime_provider_error", lambda exc: str(exc))
|
|
|
|
shell = cli.HermesCLI(model="openai/gpt-5.3-codex", compact=True, max_turns=1)
|
|
|
|
assert shell._ensure_runtime_credentials() is True
|
|
assert shell.model == "gpt-5.3-codex"
|
|
|
|
|
|
def test_cmd_model_falls_back_to_auto_on_invalid_provider(monkeypatch, capsys):
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.load_config",
|
|
lambda: {"model": {"default": "gpt-5", "provider": "invalid-provider"}},
|
|
)
|
|
monkeypatch.setattr("hermes_cli.config.save_config", lambda cfg: None)
|
|
monkeypatch.setattr("hermes_cli.config.get_env_value", lambda key: "")
|
|
monkeypatch.setattr("hermes_cli.config.save_env_value", lambda key, value: None)
|
|
|
|
def _resolve_provider(requested, **kwargs):
|
|
if requested == "invalid-provider":
|
|
raise AuthError("Unknown provider 'invalid-provider'.", code="invalid_provider")
|
|
return "openrouter"
|
|
|
|
monkeypatch.setattr("hermes_cli.auth.resolve_provider", _resolve_provider)
|
|
monkeypatch.setattr(hermes_main, "_prompt_provider_choice", lambda choices, **kwargs: len(choices) - 1)
|
|
monkeypatch.setattr("sys.stdin", type("FakeTTY", (), {"isatty": lambda self: True})())
|
|
|
|
hermes_main.cmd_model(SimpleNamespace())
|
|
output = capsys.readouterr().out
|
|
|
|
assert "Warning:" in output
|
|
assert "falling back to auto provider detection" in output.lower()
|
|
assert "No change." in output
|
|
|
|
|
|
def test_model_flow_custom_saves_verified_v1_base_url(monkeypatch, capsys):
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.get_env_value",
|
|
lambda key: "" if key in {"OPENAI_BASE_URL", "OPENAI_API_KEY"} else "",
|
|
)
|
|
saved_env = {}
|
|
monkeypatch.setattr("hermes_cli.config.save_env_value", lambda key, value: saved_env.__setitem__(key, value))
|
|
monkeypatch.setattr("hermes_cli.auth._save_model_choice", lambda model: saved_env.__setitem__("MODEL", model))
|
|
monkeypatch.setattr("hermes_cli.auth.deactivate_provider", lambda: None)
|
|
monkeypatch.setattr("hermes_cli.main._save_custom_provider", lambda *args, **kwargs: None)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.models.probe_api_models",
|
|
lambda api_key, base_url: {
|
|
"models": ["llm"],
|
|
"probed_url": "http://localhost:8000/v1/models",
|
|
"resolved_base_url": "http://localhost:8000/v1",
|
|
"suggested_base_url": "http://localhost:8000/v1",
|
|
"used_fallback": True,
|
|
},
|
|
)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.load_config",
|
|
lambda: {"model": {"default": "", "provider": "custom", "base_url": ""}},
|
|
)
|
|
monkeypatch.setattr("hermes_cli.config.save_config", lambda cfg: None)
|
|
|
|
# After the probe detects a single model ("llm"), the flow asks
|
|
# "Use this model? [Y/n]:" — confirm with Enter, then context length.
|
|
answers = iter(["http://localhost:8000", "local-key", "", ""])
|
|
monkeypatch.setattr("builtins.input", lambda _prompt="": next(answers))
|
|
monkeypatch.setattr("getpass.getpass", lambda _prompt="": next(answers))
|
|
|
|
hermes_main._model_flow_custom({})
|
|
output = capsys.readouterr().out
|
|
|
|
assert "Saving the working base URL instead" in output
|
|
assert "Detected model: llm" in output
|
|
# OPENAI_BASE_URL is no longer saved to .env — config.yaml is authoritative
|
|
assert "OPENAI_BASE_URL" not in saved_env
|
|
assert saved_env["MODEL"] == "llm"
|
|
|
|
|
|
def test_cmd_model_forwards_nous_login_tls_options(monkeypatch):
|
|
monkeypatch.setattr(hermes_main, "_require_tty", lambda *a: None)
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.load_config",
|
|
lambda: {"model": {"default": "gpt-5", "provider": "nous"}},
|
|
)
|
|
monkeypatch.setattr("hermes_cli.config.save_config", lambda cfg: None)
|
|
monkeypatch.setattr("hermes_cli.config.get_env_value", lambda key: "")
|
|
monkeypatch.setattr("hermes_cli.config.save_env_value", lambda key, value: None)
|
|
monkeypatch.setattr("hermes_cli.auth.resolve_provider", lambda requested, **kwargs: "nous")
|
|
monkeypatch.setattr("hermes_cli.auth.get_provider_auth_state", lambda provider_id: None)
|
|
monkeypatch.setattr(hermes_main, "_prompt_provider_choice", lambda choices, **kwargs: 0)
|
|
|
|
captured = {}
|
|
|
|
def _fake_login(login_args, provider_config):
|
|
captured["portal_url"] = login_args.portal_url
|
|
captured["inference_url"] = login_args.inference_url
|
|
captured["client_id"] = login_args.client_id
|
|
captured["scope"] = login_args.scope
|
|
captured["no_browser"] = login_args.no_browser
|
|
captured["timeout"] = login_args.timeout
|
|
captured["ca_bundle"] = login_args.ca_bundle
|
|
captured["insecure"] = login_args.insecure
|
|
|
|
monkeypatch.setattr("hermes_cli.auth._login_nous", _fake_login)
|
|
|
|
hermes_main.cmd_model(
|
|
SimpleNamespace(
|
|
portal_url="https://portal.nousresearch.com",
|
|
inference_url="https://inference.nousresearch.com/v1",
|
|
client_id="hermes-local",
|
|
scope="openid profile",
|
|
no_browser=True,
|
|
timeout=7.5,
|
|
ca_bundle="/tmp/local-ca.pem",
|
|
insecure=True,
|
|
)
|
|
)
|
|
|
|
assert captured == {
|
|
"portal_url": "https://portal.nousresearch.com",
|
|
"inference_url": "https://inference.nousresearch.com/v1",
|
|
"client_id": "hermes-local",
|
|
"scope": "openid profile",
|
|
"no_browser": True,
|
|
"timeout": 7.5,
|
|
"ca_bundle": "/tmp/local-ca.pem",
|
|
"insecure": True,
|
|
}
|