Timmy_Foundation/hermes-agent
15
0
Fork 0
Code Issues 248 Pull Requests 14 Actions 1 Packages Projects Releases Wiki Activity

Refresh OpenRouter model catalog

Browse Source 
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Carlos
2026-04-09 15:11:58 -05:00
committed by Teknium
parent 38ccd9eb95
commit 7368854398
4 changed files with 180 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
hermes_cli/main.py
View File

@@ -1142,10 +1142,10 @@ def _model_flow_openrouter(config, current_model=""):
print()
from hermes_cli.models import model_ids, get_pricing_for_provider
openrouter_models = model_ids()
openrouter_models = model_ids(force_refresh=True)
# Fetch live pricing (non-blocking — returns empty dict on failure)
pricing = get_pricing_for_provider("openrouter")
pricing = get_pricing_for_provider("openrouter", force_refresh=True)
selected = _prompt_model_selection(openrouter_models, current_model=current_model, pricing=pricing)
if selected:

101
hermes_cli/models.py
View File

@@ -24,18 +24,19 @@ COPILOT_REASONING_EFFORTS_O_SERIES = ["low", "medium", "high"]
GITHUB_MODELS_BASE_URL = COPILOT_BASE_URL
GITHUB_MODELS_CATALOG_URL = COPILOT_MODELS_URL
# Fallback OpenRouter snapshot used when the live catalog is unavailable.
# (model_id, display description shown in menus)
OPENROUTER_MODELS: list[tuple[str, str]] = [
("anthropic/claude-opus-4.6", "recommended"),
("anthropic/claude-sonnet-4.6", ""),
("qwen/qwen3.6-plus:free", "free"),
("qwen/qwen3.6-plus", ""),
("anthropic/claude-sonnet-4.5", ""),
("anthropic/claude-haiku-4.5", ""),
("openai/gpt-5.4", ""),
("openai/gpt-5.4-mini", ""),
("xiaomi/mimo-v2-pro", ""),
("openai/gpt-5.3-codex", ""),
("google/gemini-3-pro-preview", ""),
("google/gemini-3-pro-image-preview", ""),
("google/gemini-3-flash-preview", ""),
("google/gemini-3.1-pro-preview", ""),
("google/gemini-3.1-flash-lite-preview", ""),
@@ -47,7 +48,7 @@ OPENROUTER_MODELS: list[tuple[str, str]] = [
("z-ai/glm-5.1", ""),
("z-ai/glm-5-turbo", ""),
("moonshotai/kimi-k2.5", ""),
("x-ai/grok-4.20-beta", ""),
("x-ai/grok-4.20", ""),
("nvidia/nemotron-3-super-120b-a12b", ""),
("nvidia/nemotron-3-super-120b-a12b:free", "free"),
("arcee-ai/trinity-large-preview:free", "free"),
@@ -56,6 +57,8 @@ OPENROUTER_MODELS: list[tuple[str, str]] = [
("openai/gpt-5.4-nano", ""),
]
_openrouter_catalog_cache: list[tuple[str, str]] | None = None
_PROVIDER_MODELS: dict[str, list[str]] = {
"nous": [
"anthropic/claude-opus-4.6",
@@ -530,15 +533,79 @@ _PROVIDER_ALIASES = {
}
def model_ids() -> list[str]:
def _openrouter_model_is_free(pricing: Any) -> bool:
"""Return True when both prompt and completion pricing are zero."""
if not isinstance(pricing, dict):
return False
try:
return float(pricing.get("prompt", "0")) == 0 and float(pricing.get("completion", "0")) == 0
except (TypeError, ValueError):
return False
def fetch_openrouter_models(
timeout: float = 8.0,
*,
force_refresh: bool = False,
) -> list[tuple[str, str]]:
"""Return the curated OpenRouter picker list, refreshed from the live catalog when possible."""
global _openrouter_catalog_cache
if _openrouter_catalog_cache is not None and not force_refresh:
return list(_openrouter_catalog_cache)
fallback = list(OPENROUTER_MODELS)
preferred_ids = [mid for mid, _ in fallback]
try:
req = urllib.request.Request(
"https://openrouter.ai/api/v1/models",
headers={"Accept": "application/json"},
)
with urllib.request.urlopen(req, timeout=timeout) as resp:
payload = json.loads(resp.read().decode())
except Exception:
return list(_openrouter_catalog_cache or fallback)
live_items = payload.get("data", [])
if not isinstance(live_items, list):
return list(_openrouter_catalog_cache or fallback)
live_by_id: dict[str, dict[str, Any]] = {}
for item in live_items:
if not isinstance(item, dict):
continue
mid = str(item.get("id") or "").strip()
if not mid:
continue
live_by_id[mid] = item
curated: list[tuple[str, str]] = []
for preferred_id in preferred_ids:
live_item = live_by_id.get(preferred_id)
if live_item is None:
continue
desc = "free" if _openrouter_model_is_free(live_item.get("pricing")) else ""
curated.append((preferred_id, desc))
if not curated:
return list(_openrouter_catalog_cache or fallback)
first_id, _ = curated[0]
curated[0] = (first_id, "recommended")
_openrouter_catalog_cache = curated
return list(curated)
def model_ids(*, force_refresh: bool = False) -> list[str]:
"""Return just the OpenRouter model-id strings."""
return [mid for mid, _ in OPENROUTER_MODELS]
return [mid for mid, _ in fetch_openrouter_models(force_refresh=force_refresh)]
def menu_labels() -> list[str]:
def menu_labels(*, force_refresh: bool = False) -> list[str]:
"""Return display labels like 'anthropic/claude-opus-4.6 (recommended)'."""
labels = []
for mid, desc in OPENROUTER_MODELS:
for mid, desc in fetch_openrouter_models(force_refresh=force_refresh):
labels.append(f"{mid} ({desc})" if desc else mid)
return labels
@@ -727,13 +794,14 @@ def _resolve_nous_pricing_credentials() -> tuple[str, str]:
return ("", "")
def get_pricing_for_provider(provider: str) -> dict[str, dict[str, str]]:
def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> dict[str, dict[str, str]]:
"""Return live pricing for providers that support it (openrouter, nous)."""
normalized = normalize_provider(provider)
if normalized == "openrouter":
return fetch_models_with_pricing(
api_key=_resolve_openrouter_api_key(),
base_url="https://openrouter.ai/api",
force_refresh=force_refresh,
)
if normalized == "nous":
api_key, base_url = _resolve_nous_pricing_credentials()
@@ -746,6 +814,7 @@ def get_pricing_for_provider(provider: str) -> dict[str, dict[str, str]]:
return fetch_models_with_pricing(
api_key=api_key,
base_url=stripped,
force_refresh=force_refresh,
)
return {}
@@ -854,7 +923,11 @@ def _get_custom_base_url() -> str:
return ""
def curated_models_for_provider(provider: Optional[str]) -> list[tuple[str, str]]:
def curated_models_for_provider(
provider: Optional[str],
*,
force_refresh: bool = False,
) -> list[tuple[str, str]]:
"""Return ``(model_id, description)`` tuples for a provider's model list.
Tries to fetch the live model list from the provider's API first,
@@ -863,7 +936,7 @@ def curated_models_for_provider(provider: Optional[str]) -> list[tuple[str, str]
"""
normalized = normalize_provider(provider)
if normalized == "openrouter":
return list(OPENROUTER_MODELS)
return fetch_openrouter_models(force_refresh=force_refresh)
# Try live API first (Codex, Nous, etc. all support /models)
live = provider_model_ids(normalized)
@@ -982,12 +1055,12 @@ def _find_openrouter_slug(model_name: str) -> Optional[str]:
return None
# Exact match (already has provider/ prefix)
for mid, _ in OPENROUTER_MODELS:
for mid in model_ids():
if name_lower == mid.lower():
return mid
# Try matching just the model part (after the /)
for mid, _ in OPENROUTER_MODELS:
for mid in model_ids():
if "/" in mid:
_, model_part = mid.split("/", 1)
if name_lower == model_part.lower():
@@ -1101,7 +1174,7 @@ def _resolve_copilot_catalog_api_key() -> str:
return ""
def provider_model_ids(provider: Optional[str]) -> list[str]:
def provider_model_ids(provider: Optional[str], *, force_refresh: bool = False) -> list[str]:
"""Return the best known model catalog for a provider.
Tries live API endpoints for providers that support them (Codex, Nous),
@@ -1109,7 +1182,7 @@ def provider_model_ids(provider: Optional[str]) -> list[str]:
"""
normalized = normalize_provider(provider)
if normalized == "openrouter":
return model_ids()
return model_ids(force_refresh=force_refresh)
if normalized == "openai-codex":
from hermes_cli.codex_models import get_codex_model_ids

18
tests/hermes_cli/test_model_validation.py
View File

@@ -124,7 +124,14 @@ class TestParseModelInput:
class TestCuratedModelsForProvider:
def test_openrouter_returns_curated_list(self):
models = curated_models_for_provider("openrouter")
with patch(
"hermes_cli.models.fetch_openrouter_models",
return_value=[
("anthropic/claude-opus-4.6", "recommended"),
("qwen/qwen3.6-plus", ""),
],
):
models = curated_models_for_provider("openrouter")
assert len(models) > 0
assert any("claude" in m[0] for m in models)
@@ -169,7 +176,14 @@ class TestProviderLabel:
class TestProviderModelIds:
def test_openrouter_returns_curated_list(self):
ids = provider_model_ids("openrouter")
with patch(
"hermes_cli.models.fetch_openrouter_models",
return_value=[
("anthropic/claude-opus-4.6", "recommended"),
("qwen/qwen3.6-plus", ""),
],
):
ids = provider_model_ids("openrouter")
assert len(ids) > 0
assert all("/" in mid for mid in ids)

97
tests/hermes_cli/test_models.py
View File

@@ -3,7 +3,7 @@
from unittest.mock import patch, MagicMock
from hermes_cli.models import (
OPENROUTER_MODELS, menu_labels, model_ids, detect_provider_for_model,
OPENROUTER_MODELS, fetch_openrouter_models, menu_labels, model_ids, detect_provider_for_model,
filter_nous_free_models, _NOUS_ALLOWED_FREE_MODELS,
is_nous_free_tier, partition_nous_models_by_tier,
check_nous_free_tier, clear_nous_free_tier_cache,
@@ -11,43 +11,57 @@ from hermes_cli.models import (
)
import hermes_cli.models as _models_mod
LIVE_OPENROUTER_MODELS = [
("anthropic/claude-opus-4.6", "recommended"),
("qwen/qwen3.6-plus", ""),
("nvidia/nemotron-3-super-120b-a12b:free", "free"),
]
class TestModelIds:
def test_returns_non_empty_list(self):
ids = model_ids()
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
ids = model_ids()
assert isinstance(ids, list)
assert len(ids) > 0
def test_ids_match_models_list(self):
ids = model_ids()
expected = [mid for mid, _ in OPENROUTER_MODELS]
def test_ids_match_fetched_catalog(self):
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
ids = model_ids()
expected = [mid for mid, _ in LIVE_OPENROUTER_MODELS]
assert ids == expected
def test_all_ids_contain_provider_slash(self):
"""Model IDs should follow the provider/model format."""
for mid in model_ids():
assert "/" in mid, f"Model ID '{mid}' missing provider/ prefix"
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
for mid in model_ids():
assert "/" in mid, f"Model ID '{mid}' missing provider/ prefix"
def test_no_duplicate_ids(self):
ids = model_ids()
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
ids = model_ids()
assert len(ids) == len(set(ids)), "Duplicate model IDs found"
class TestMenuLabels:
def test_same_length_as_model_ids(self):
assert len(menu_labels()) == len(model_ids())
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
assert len(menu_labels()) == len(model_ids())
def test_first_label_marked_recommended(self):
labels = menu_labels()
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
labels = menu_labels()
assert "recommended" in labels[0].lower()
def test_each_label_contains_its_model_id(self):
for label, mid in zip(menu_labels(), model_ids()):
assert mid in label, f"Label '{label}' doesn't contain model ID '{mid}'"
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
for label, mid in zip(menu_labels(), model_ids()):
assert mid in label, f"Label '{label}' doesn't contain model ID '{mid}'"
def test_non_recommended_labels_have_no_tag(self):
"""Only the first model should have (recommended)."""
labels = menu_labels()
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
labels = menu_labels()
for label in labels[1:]:
assert "recommended" not in label.lower(), f"Unexpected 'recommended' in '{label}'"
@@ -65,30 +79,65 @@ class TestOpenRouterModels:
assert len(OPENROUTER_MODELS) >= 5
class TestFetchOpenRouterModels:
def test_live_fetch_recomputes_free_tags(self, monkeypatch):
class _Resp:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def read(self):
return b'{"data":[{"id":"anthropic/claude-opus-4.6","pricing":{"prompt":"0.000015","completion":"0.000075"}},{"id":"qwen/qwen3.6-plus","pricing":{"prompt":"0.000000325","completion":"0.00000195"}},{"id":"nvidia/nemotron-3-super-120b-a12b:free","pricing":{"prompt":"0","completion":"0"}}]}'
monkeypatch.setattr(_models_mod, "_openrouter_catalog_cache", None)
with patch("hermes_cli.models.urllib.request.urlopen", return_value=_Resp()):
models = fetch_openrouter_models(force_refresh=True)
assert models == [
("anthropic/claude-opus-4.6", "recommended"),
("qwen/qwen3.6-plus", ""),
("nvidia/nemotron-3-super-120b-a12b:free", "free"),
]
def test_falls_back_to_static_snapshot_on_fetch_failure(self, monkeypatch):
monkeypatch.setattr(_models_mod, "_openrouter_catalog_cache", None)
with patch("hermes_cli.models.urllib.request.urlopen", side_effect=OSError("boom")):
models = fetch_openrouter_models(force_refresh=True)
assert models == OPENROUTER_MODELS
class TestFindOpenrouterSlug:
def test_exact_match(self):
from hermes_cli.models import _find_openrouter_slug
assert _find_openrouter_slug("anthropic/claude-opus-4.6") == "anthropic/claude-opus-4.6"
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
assert _find_openrouter_slug("anthropic/claude-opus-4.6") == "anthropic/claude-opus-4.6"
def test_bare_name_match(self):
from hermes_cli.models import _find_openrouter_slug
result = _find_openrouter_slug("claude-opus-4.6")
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
result = _find_openrouter_slug("claude-opus-4.6")
assert result == "anthropic/claude-opus-4.6"
def test_case_insensitive(self):
from hermes_cli.models import _find_openrouter_slug
result = _find_openrouter_slug("Anthropic/Claude-Opus-4.6")
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
result = _find_openrouter_slug("Anthropic/Claude-Opus-4.6")
assert result is not None
def test_unknown_returns_none(self):
from hermes_cli.models import _find_openrouter_slug
assert _find_openrouter_slug("totally-fake-model-xyz") is None
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
assert _find_openrouter_slug("totally-fake-model-xyz") is None
class TestDetectProviderForModel:
def test_anthropic_model_detected(self):
"""claude-opus-4-6 should resolve to anthropic provider."""
result = detect_provider_for_model("claude-opus-4-6", "openai-codex")
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
result = detect_provider_for_model("claude-opus-4-6", "openai-codex")
assert result is not None
assert result[0] == "anthropic"
@@ -105,7 +154,8 @@ class TestDetectProviderForModel:
def test_openrouter_slug_match(self):
"""Models in the OpenRouter catalog should be found."""
result = detect_provider_for_model("anthropic/claude-opus-4.6", "openai-codex")
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
result = detect_provider_for_model("anthropic/claude-opus-4.6", "openai-codex")
assert result is not None
assert result[0] == "openrouter"
assert result[1] == "anthropic/claude-opus-4.6"
@@ -119,18 +169,21 @@ class TestDetectProviderForModel:
):
monkeypatch.delenv(env_var, raising=False)
"""Bare model names should get mapped to full OpenRouter slugs."""
result = detect_provider_for_model("claude-opus-4.6", "openai-codex")
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
result = detect_provider_for_model("claude-opus-4.6", "openai-codex")
assert result is not None
# Should find it on OpenRouter with full slug
assert result[1] == "anthropic/claude-opus-4.6"
def test_unknown_model_returns_none(self):
"""Completely unknown model names should return None."""
assert detect_provider_for_model("nonexistent-model-xyz", "openai-codex") is None
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
assert detect_provider_for_model("nonexistent-model-xyz", "openai-codex") is None
def test_aggregator_not_suggested(self):
"""nous/openrouter should never be auto-suggested as target provider."""
result = detect_provider_for_model("claude-opus-4-6", "openai-codex")
with patch("hermes_cli.models.fetch_openrouter_models", return_value=LIVE_OPENROUTER_MODELS):
result = detect_provider_for_model("claude-opus-4-6", "openai-codex")
assert result is not None
assert result[0] not in ("nous",) # nous has claude models but shouldn't be suggested