b641ee88f4
* feat(model): persist base_url on /model switch, auto-detect for bare /model custom Phase 2+3 of the /model command overhaul: Phase 2 — Persist base_url on model switch: - CLI: save model.base_url when switching to a non-OpenRouter endpoint; clear it when switching away from custom to prevent stale URLs leaking into the new provider's resolution - Gateway: same logic using direct YAML write Phase 3 — Better feedback and edge cases: - Bare '/model custom' now auto-detects the model from the endpoint using _auto_detect_local_model() and saves all three config values (model, provider, base_url) atomically - Shows endpoint URL in success messages when switching to/from custom providers (both CLI and gateway) - Clear error messages when no custom endpoint is configured - Updated test assertions for the additional save_config_value call Fixes #2562 (Phase 2+3) * feat(model): support custom:name:model triple syntax for named custom providers Phase 5 of the /model command overhaul. Extends parse_model_input() to handle the triple syntax: /model custom:local-server:qwen → provider='custom:local-server', model='qwen' /model custom:my-model → provider='custom', model='my-model' (unchanged) The 'custom:local-server' provider string is already supported by _get_named_custom_provider() in runtime_provider.py, which matches it against the custom_providers list in config.yaml. This just wires the parsing so users can do it from the /model slash command. Added 4 tests covering single, triple, whitespace, and empty model cases.
451 lines
18 KiB
Python
451 lines
18 KiB
Python
"""Tests for provider-aware `/model` validation in hermes_cli.models."""
|
|
|
|
from unittest.mock import patch
|
|
|
|
from hermes_cli.models import (
|
|
copilot_model_api_mode,
|
|
fetch_github_model_catalog,
|
|
curated_models_for_provider,
|
|
fetch_api_models,
|
|
github_model_reasoning_efforts,
|
|
normalize_copilot_model_id,
|
|
normalize_provider,
|
|
parse_model_input,
|
|
probe_api_models,
|
|
provider_label,
|
|
provider_model_ids,
|
|
validate_requested_model,
|
|
)
|
|
|
|
|
|
# -- helpers -----------------------------------------------------------------
|
|
|
|
FAKE_API_MODELS = [
|
|
"anthropic/claude-opus-4.6",
|
|
"anthropic/claude-sonnet-4.5",
|
|
"openai/gpt-5.4-pro",
|
|
"openai/gpt-5.4",
|
|
"google/gemini-3-pro-preview",
|
|
]
|
|
|
|
|
|
def _validate(model, provider="openrouter", api_models=FAKE_API_MODELS, **kw):
|
|
"""Shortcut: call validate_requested_model with mocked API."""
|
|
probe_payload = {
|
|
"models": api_models,
|
|
"probed_url": "http://localhost:11434/v1/models",
|
|
"resolved_base_url": kw.get("base_url", "") or "http://localhost:11434/v1",
|
|
"suggested_base_url": None,
|
|
"used_fallback": False,
|
|
}
|
|
with patch("hermes_cli.models.fetch_api_models", return_value=api_models), \
|
|
patch("hermes_cli.models.probe_api_models", return_value=probe_payload):
|
|
return validate_requested_model(model, provider, **kw)
|
|
|
|
|
|
# -- parse_model_input -------------------------------------------------------
|
|
|
|
class TestParseModelInput:
|
|
def test_plain_model_keeps_current_provider(self):
|
|
provider, model = parse_model_input("anthropic/claude-sonnet-4.5", "openrouter")
|
|
assert provider == "openrouter"
|
|
assert model == "anthropic/claude-sonnet-4.5"
|
|
|
|
def test_provider_colon_model_switches_provider(self):
|
|
provider, model = parse_model_input("openrouter:anthropic/claude-sonnet-4.5", "nous")
|
|
assert provider == "openrouter"
|
|
assert model == "anthropic/claude-sonnet-4.5"
|
|
|
|
def test_provider_alias_resolved(self):
|
|
provider, model = parse_model_input("glm:glm-5", "openrouter")
|
|
assert provider == "zai"
|
|
assert model == "glm-5"
|
|
|
|
def test_no_slash_no_colon_keeps_provider(self):
|
|
provider, model = parse_model_input("gpt-5.4", "openrouter")
|
|
assert provider == "openrouter"
|
|
assert model == "gpt-5.4"
|
|
|
|
def test_nous_provider_switch(self):
|
|
provider, model = parse_model_input("nous:hermes-3", "openrouter")
|
|
assert provider == "nous"
|
|
assert model == "hermes-3"
|
|
|
|
def test_empty_model_after_colon_keeps_current(self):
|
|
provider, model = parse_model_input("openrouter:", "nous")
|
|
assert provider == "nous"
|
|
assert model == "openrouter:"
|
|
|
|
def test_colon_at_start_keeps_current(self):
|
|
provider, model = parse_model_input(":something", "openrouter")
|
|
assert provider == "openrouter"
|
|
assert model == ":something"
|
|
|
|
def test_unknown_prefix_colon_not_treated_as_provider(self):
|
|
"""Colons are only provider delimiters if the left side is a known provider."""
|
|
provider, model = parse_model_input("anthropic/claude-3.5-sonnet:beta", "openrouter")
|
|
assert provider == "openrouter"
|
|
assert model == "anthropic/claude-3.5-sonnet:beta"
|
|
|
|
def test_http_url_not_treated_as_provider(self):
|
|
provider, model = parse_model_input("http://localhost:8080/model", "openrouter")
|
|
assert provider == "openrouter"
|
|
assert model == "http://localhost:8080/model"
|
|
|
|
def test_custom_colon_model_single(self):
|
|
"""custom:model-name → anonymous custom provider."""
|
|
provider, model = parse_model_input("custom:qwen-2.5", "openrouter")
|
|
assert provider == "custom"
|
|
assert model == "qwen-2.5"
|
|
|
|
def test_custom_triple_syntax(self):
|
|
"""custom:name:model → named custom provider."""
|
|
provider, model = parse_model_input("custom:local-server:qwen-2.5", "openrouter")
|
|
assert provider == "custom:local-server"
|
|
assert model == "qwen-2.5"
|
|
|
|
def test_custom_triple_spaces(self):
|
|
"""Triple syntax should handle whitespace."""
|
|
provider, model = parse_model_input("custom: my-server : my-model ", "openrouter")
|
|
assert provider == "custom:my-server"
|
|
assert model == "my-model"
|
|
|
|
def test_custom_triple_empty_model_falls_back(self):
|
|
"""custom:name: with no model → treated as custom:name (bare)."""
|
|
provider, model = parse_model_input("custom:name:", "openrouter")
|
|
# Empty model after second colon → no triple match, falls through
|
|
assert provider == "custom"
|
|
assert model == "name:"
|
|
|
|
|
|
# -- curated_models_for_provider ---------------------------------------------
|
|
|
|
class TestCuratedModelsForProvider:
|
|
def test_openrouter_returns_curated_list(self):
|
|
models = curated_models_for_provider("openrouter")
|
|
assert len(models) > 0
|
|
assert any("claude" in m[0] for m in models)
|
|
|
|
def test_zai_returns_glm_models(self):
|
|
models = curated_models_for_provider("zai")
|
|
assert any("glm" in m[0] for m in models)
|
|
|
|
def test_unknown_provider_returns_empty(self):
|
|
assert curated_models_for_provider("totally-unknown") == []
|
|
|
|
|
|
# -- normalize_provider ------------------------------------------------------
|
|
|
|
class TestNormalizeProvider:
|
|
def test_defaults_to_openrouter(self):
|
|
assert normalize_provider(None) == "openrouter"
|
|
assert normalize_provider("") == "openrouter"
|
|
|
|
def test_known_aliases(self):
|
|
assert normalize_provider("glm") == "zai"
|
|
assert normalize_provider("kimi") == "kimi-coding"
|
|
assert normalize_provider("moonshot") == "kimi-coding"
|
|
assert normalize_provider("github-copilot") == "copilot"
|
|
|
|
def test_case_insensitive(self):
|
|
assert normalize_provider("OpenRouter") == "openrouter"
|
|
|
|
|
|
class TestProviderLabel:
|
|
def test_known_labels_and_auto(self):
|
|
assert provider_label("anthropic") == "Anthropic"
|
|
assert provider_label("kimi") == "Kimi / Moonshot"
|
|
assert provider_label("copilot") == "GitHub Copilot"
|
|
assert provider_label("copilot-acp") == "GitHub Copilot ACP"
|
|
assert provider_label("auto") == "Auto"
|
|
|
|
def test_unknown_provider_preserves_original_name(self):
|
|
assert provider_label("my-custom-provider") == "my-custom-provider"
|
|
|
|
|
|
# -- provider_model_ids ------------------------------------------------------
|
|
|
|
class TestProviderModelIds:
|
|
def test_openrouter_returns_curated_list(self):
|
|
ids = provider_model_ids("openrouter")
|
|
assert len(ids) > 0
|
|
assert all("/" in mid for mid in ids)
|
|
|
|
def test_unknown_provider_returns_empty(self):
|
|
assert provider_model_ids("some-unknown-provider") == []
|
|
|
|
def test_zai_returns_glm_models(self):
|
|
assert "glm-5" in provider_model_ids("zai")
|
|
|
|
def test_copilot_prefers_live_catalog(self):
|
|
with patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={"api_key": "gh-token"}), \
|
|
patch("hermes_cli.models._fetch_github_models", return_value=["gpt-5.4", "claude-sonnet-4.6"]):
|
|
assert provider_model_ids("copilot") == ["gpt-5.4", "claude-sonnet-4.6"]
|
|
|
|
def test_copilot_acp_reuses_copilot_catalog(self):
|
|
with patch("hermes_cli.auth.resolve_api_key_provider_credentials", return_value={"api_key": "gh-token"}), \
|
|
patch("hermes_cli.models._fetch_github_models", return_value=["gpt-5.4", "claude-sonnet-4.6"]):
|
|
assert provider_model_ids("copilot-acp") == ["gpt-5.4", "claude-sonnet-4.6"]
|
|
|
|
def test_copilot_acp_falls_back_to_copilot_defaults(self):
|
|
with patch("hermes_cli.auth.resolve_api_key_provider_credentials", side_effect=Exception("no token")), \
|
|
patch("hermes_cli.models._fetch_github_models", return_value=None):
|
|
ids = provider_model_ids("copilot-acp")
|
|
|
|
assert "gpt-5.4" in ids
|
|
assert "copilot-acp" not in ids
|
|
|
|
|
|
# -- fetch_api_models --------------------------------------------------------
|
|
|
|
class TestFetchApiModels:
|
|
def test_returns_none_when_no_base_url(self):
|
|
assert fetch_api_models("key", None) is None
|
|
|
|
def test_returns_none_on_network_error(self):
|
|
with patch("hermes_cli.models.urllib.request.urlopen", side_effect=Exception("timeout")):
|
|
assert fetch_api_models("key", "https://example.com/v1") is None
|
|
|
|
def test_probe_api_models_tries_v1_fallback(self):
|
|
class _Resp:
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
def read(self):
|
|
return b'{"data": [{"id": "local-model"}]}'
|
|
|
|
calls = []
|
|
|
|
def _fake_urlopen(req, timeout=5.0):
|
|
calls.append(req.full_url)
|
|
if req.full_url.endswith("/v1/models"):
|
|
return _Resp()
|
|
raise Exception("404")
|
|
|
|
with patch("hermes_cli.models.urllib.request.urlopen", side_effect=_fake_urlopen):
|
|
probe = probe_api_models("key", "http://localhost:8000")
|
|
|
|
assert calls == ["http://localhost:8000/models", "http://localhost:8000/v1/models"]
|
|
assert probe["models"] == ["local-model"]
|
|
assert probe["resolved_base_url"] == "http://localhost:8000/v1"
|
|
assert probe["used_fallback"] is True
|
|
|
|
def test_probe_api_models_uses_copilot_catalog(self):
|
|
class _Resp:
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
def read(self):
|
|
return b'{"data": [{"id": "gpt-5.4", "model_picker_enabled": true, "supported_endpoints": ["/responses"], "capabilities": {"type": "chat", "supports": {"reasoning_effort": ["low", "medium", "high"]}}}, {"id": "claude-sonnet-4.6", "model_picker_enabled": true, "supported_endpoints": ["/chat/completions"], "capabilities": {"type": "chat", "supports": {"reasoning_effort": ["low", "medium", "high"]}}}, {"id": "text-embedding-3-small", "model_picker_enabled": true, "capabilities": {"type": "embedding"}}]}'
|
|
|
|
with patch("hermes_cli.models.urllib.request.urlopen", return_value=_Resp()) as mock_urlopen:
|
|
probe = probe_api_models("gh-token", "https://api.githubcopilot.com")
|
|
|
|
assert mock_urlopen.call_args[0][0].full_url == "https://api.githubcopilot.com/models"
|
|
assert probe["models"] == ["gpt-5.4", "claude-sonnet-4.6"]
|
|
assert probe["resolved_base_url"] == "https://api.githubcopilot.com"
|
|
assert probe["used_fallback"] is False
|
|
|
|
def test_fetch_github_model_catalog_filters_non_chat_models(self):
|
|
class _Resp:
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc, tb):
|
|
return False
|
|
|
|
def read(self):
|
|
return b'{"data": [{"id": "gpt-5.4", "model_picker_enabled": true, "supported_endpoints": ["/responses"], "capabilities": {"type": "chat", "supports": {"reasoning_effort": ["low", "medium", "high"]}}}, {"id": "text-embedding-3-small", "model_picker_enabled": true, "capabilities": {"type": "embedding"}}]}'
|
|
|
|
with patch("hermes_cli.models.urllib.request.urlopen", return_value=_Resp()):
|
|
catalog = fetch_github_model_catalog("gh-token")
|
|
|
|
assert catalog is not None
|
|
assert [item["id"] for item in catalog] == ["gpt-5.4"]
|
|
|
|
|
|
class TestGithubReasoningEfforts:
|
|
def test_gpt5_supports_minimal_to_high(self):
|
|
catalog = [{
|
|
"id": "gpt-5.4",
|
|
"capabilities": {"type": "chat", "supports": {"reasoning_effort": ["low", "medium", "high"]}},
|
|
"supported_endpoints": ["/responses"],
|
|
}]
|
|
assert github_model_reasoning_efforts("gpt-5.4", catalog=catalog) == [
|
|
"low",
|
|
"medium",
|
|
"high",
|
|
]
|
|
|
|
def test_legacy_catalog_reasoning_still_supported(self):
|
|
catalog = [{"id": "openai/o3", "capabilities": ["reasoning"]}]
|
|
assert github_model_reasoning_efforts("openai/o3", catalog=catalog) == [
|
|
"low",
|
|
"medium",
|
|
"high",
|
|
]
|
|
|
|
def test_non_reasoning_model_returns_empty(self):
|
|
catalog = [{"id": "gpt-4.1", "capabilities": {"type": "chat", "supports": {}}}]
|
|
assert github_model_reasoning_efforts("gpt-4.1", catalog=catalog) == []
|
|
|
|
|
|
class TestCopilotNormalization:
|
|
def test_normalize_old_github_models_slug(self):
|
|
catalog = [{"id": "gpt-4.1"}, {"id": "gpt-5.4"}]
|
|
assert normalize_copilot_model_id("openai/gpt-4.1-mini", catalog=catalog) == "gpt-4.1"
|
|
|
|
def test_copilot_api_mode_gpt5_uses_responses(self):
|
|
"""GPT-5+ models should use Responses API (matching opencode)."""
|
|
assert copilot_model_api_mode("gpt-5.4") == "codex_responses"
|
|
assert copilot_model_api_mode("gpt-5.4-mini") == "codex_responses"
|
|
assert copilot_model_api_mode("gpt-5.3-codex") == "codex_responses"
|
|
assert copilot_model_api_mode("gpt-5.2-codex") == "codex_responses"
|
|
assert copilot_model_api_mode("gpt-5.2") == "codex_responses"
|
|
|
|
def test_copilot_api_mode_gpt5_mini_uses_chat(self):
|
|
"""gpt-5-mini is the exception — uses Chat Completions."""
|
|
assert copilot_model_api_mode("gpt-5-mini") == "chat_completions"
|
|
|
|
def test_copilot_api_mode_non_gpt5_uses_chat(self):
|
|
"""Non-GPT-5 models use Chat Completions."""
|
|
assert copilot_model_api_mode("gpt-4.1") == "chat_completions"
|
|
assert copilot_model_api_mode("gpt-4o") == "chat_completions"
|
|
assert copilot_model_api_mode("gpt-4o-mini") == "chat_completions"
|
|
assert copilot_model_api_mode("claude-sonnet-4.6") == "chat_completions"
|
|
assert copilot_model_api_mode("claude-opus-4.6") == "chat_completions"
|
|
assert copilot_model_api_mode("gemini-2.5-pro") == "chat_completions"
|
|
|
|
def test_copilot_api_mode_with_catalog_both_endpoints(self):
|
|
"""When catalog shows both endpoints, model ID pattern wins."""
|
|
catalog = [{
|
|
"id": "gpt-5.4",
|
|
"supported_endpoints": ["/chat/completions", "/responses"],
|
|
}]
|
|
# GPT-5.4 should use responses even though chat/completions is listed
|
|
assert copilot_model_api_mode("gpt-5.4", catalog=catalog) == "codex_responses"
|
|
|
|
def test_copilot_api_mode_with_catalog_only_responses(self):
|
|
catalog = [{
|
|
"id": "gpt-5.4",
|
|
"supported_endpoints": ["/responses"],
|
|
"capabilities": {"type": "chat"},
|
|
}]
|
|
assert copilot_model_api_mode("gpt-5.4", catalog=catalog) == "codex_responses"
|
|
|
|
|
|
# -- validate — format checks -----------------------------------------------
|
|
|
|
class TestValidateFormatChecks:
|
|
def test_empty_model_rejected(self):
|
|
result = _validate("")
|
|
assert result["accepted"] is False
|
|
assert "empty" in result["message"]
|
|
|
|
def test_whitespace_only_rejected(self):
|
|
result = _validate(" ")
|
|
assert result["accepted"] is False
|
|
|
|
def test_model_with_spaces_rejected(self):
|
|
result = _validate("anthropic/ claude-opus")
|
|
assert result["accepted"] is False
|
|
|
|
def test_no_slash_model_still_probes_api(self):
|
|
result = _validate("gpt-5.4", api_models=["gpt-5.4", "gpt-5.4-pro"])
|
|
assert result["accepted"] is True
|
|
assert result["persist"] is True
|
|
|
|
def test_no_slash_model_rejected_if_not_in_api(self):
|
|
result = _validate("gpt-5.4", api_models=["openai/gpt-5.4"])
|
|
assert result["accepted"] is True
|
|
assert "not found" in result["message"]
|
|
|
|
|
|
# -- validate — API found ----------------------------------------------------
|
|
|
|
class TestValidateApiFound:
|
|
def test_model_found_in_api(self):
|
|
result = _validate("anthropic/claude-opus-4.6")
|
|
assert result["accepted"] is True
|
|
assert result["persist"] is True
|
|
assert result["recognized"] is True
|
|
|
|
def test_model_found_for_custom_endpoint(self):
|
|
result = _validate(
|
|
"my-model", provider="openrouter",
|
|
api_models=["my-model"], base_url="http://localhost:11434/v1",
|
|
)
|
|
assert result["accepted"] is True
|
|
assert result["persist"] is True
|
|
assert result["recognized"] is True
|
|
|
|
|
|
# -- validate — API not found ------------------------------------------------
|
|
|
|
class TestValidateApiNotFound:
|
|
def test_model_not_in_api_accepted_with_warning(self):
|
|
result = _validate("anthropic/claude-nonexistent")
|
|
assert result["accepted"] is True
|
|
assert result["persist"] is True
|
|
assert "not found" in result["message"]
|
|
|
|
def test_warning_includes_suggestions(self):
|
|
result = _validate("anthropic/claude-opus-4.5")
|
|
assert result["accepted"] is True
|
|
assert "Similar models" in result["message"]
|
|
|
|
|
|
# -- validate — API unreachable — accept and persist everything ----------------
|
|
|
|
class TestValidateApiFallback:
|
|
def test_any_model_accepted_when_api_down(self):
|
|
result = _validate("anthropic/claude-opus-4.6", api_models=None)
|
|
assert result["accepted"] is True
|
|
assert result["persist"] is True
|
|
|
|
def test_unknown_model_also_accepted_when_api_down(self):
|
|
"""No hardcoded catalog gatekeeping — accept, persist, and warn."""
|
|
result = _validate("anthropic/claude-next-gen", api_models=None)
|
|
assert result["accepted"] is True
|
|
assert result["persist"] is True
|
|
assert "could not reach" in result["message"].lower()
|
|
|
|
def test_zai_model_accepted_when_api_down(self):
|
|
result = _validate("glm-5", provider="zai", api_models=None)
|
|
assert result["accepted"] is True
|
|
assert result["persist"] is True
|
|
|
|
def test_unknown_provider_accepted_when_api_down(self):
|
|
result = _validate("some-model", provider="totally-unknown", api_models=None)
|
|
assert result["accepted"] is True
|
|
assert result["persist"] is True
|
|
|
|
def test_custom_endpoint_warns_with_probed_url_and_v1_hint(self):
|
|
with patch(
|
|
"hermes_cli.models.probe_api_models",
|
|
return_value={
|
|
"models": None,
|
|
"probed_url": "http://localhost:8000/v1/models",
|
|
"resolved_base_url": "http://localhost:8000",
|
|
"suggested_base_url": "http://localhost:8000/v1",
|
|
"used_fallback": False,
|
|
},
|
|
):
|
|
result = validate_requested_model(
|
|
"qwen3",
|
|
"custom",
|
|
api_key="local-key",
|
|
base_url="http://localhost:8000",
|
|
)
|
|
|
|
assert result["accepted"] is True
|
|
assert result["persist"] is True
|
|
assert "http://localhost:8000/v1/models" in result["message"]
|
|
assert "http://localhost:8000/v1" in result["message"]
|