Timmy_Foundation/hermes-agent
16
0
Fork 0
Code Issues 76 Pull Requests 2 Actions 54 Packages Projects Releases Wiki Activity

feat: validate /model against live API instead of hardcoded lists

Browse Source 
Replace the static catalog-based model validation with a live API probe.
The /model command now hits the provider's /models endpoint to check if
the requested model actually exists:

- Model found in API → accepted + saved to config
- Model NOT found in API → rejected with 'Error: not a valid model'
  and fuzzy-match suggestions from the live model list
- API unreachable → graceful fallback to hardcoded catalog (session-only
  for unrecognized models)
- Format errors (empty, spaces, missing '/') still caught instantly
  without a network call

The API probe takes ~0.2s for OpenRouter (346 models) and works with any
OpenAI-compatible endpoint (Ollama, vLLM, custom, etc.).

32 tests covering all paths: format checks, API found, API not found,
API unreachable fallback, CLI integration.
This commit is contained in:
teknium1
2026-03-08 05:22:15 -07:00
parent 77f47768dd
commit 245d174359
4 changed files with 226 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
tests/test_cli_model_command.py
View File

@@ -13,58 +13,17 @@ class TestModelCommand:
cli_obj.provider = "openrouter"
cli_obj.requested_provider = "openrouter"
cli_obj.base_url = "https://openrouter.ai/api/v1"
cli_obj.api_key = "test-key"
cli_obj._explicit_api_key = None
cli_obj._explicit_base_url = None
return cli_obj
def test_invalid_model_does_not_change_current_model(self, capsys):
def test_valid_model_from_api_saved_to_config(self, capsys):
cli_obj = self._make_cli()
with patch("hermes_cli.auth.resolve_provider", return_value="openrouter"), \
patch("hermes_cli.models.validate_requested_model", return_value={
"accepted": False,
"persist": False,
"recognized": False,
"message": "OpenRouter model IDs should use the `provider/model` format.",
}), \
patch("cli.save_config_value") as save_mock:
cli_obj.process_command("/model invalid-model")
output = capsys.readouterr().out
assert "Current model unchanged" in output
assert cli_obj.model == "anthropic/claude-opus-4.6"
assert cli_obj.agent is not None
save_mock.assert_not_called()
def test_unknown_model_stays_session_only(self, capsys):
cli_obj = self._make_cli()
with patch("hermes_cli.auth.resolve_provider", return_value="openrouter"), \
patch("hermes_cli.models.validate_requested_model", return_value={
"accepted": True,
"persist": False,
"recognized": False,
"message": "Using it for this session only; config unchanged.",
}), \
patch("cli.save_config_value") as save_mock:
cli_obj.process_command("/model anthropic/claude-sonnet-next")
output = capsys.readouterr().out
assert "session only" in output
assert cli_obj.model == "anthropic/claude-sonnet-next"
assert cli_obj.agent is None
save_mock.assert_not_called()
def test_known_model_is_saved_to_config(self, capsys):
cli_obj = self._make_cli()
with patch("hermes_cli.auth.resolve_provider", return_value="openrouter"), \
patch("hermes_cli.models.validate_requested_model", return_value={
"accepted": True,
"persist": True,
"recognized": True,
"message": None,
}), \
patch("hermes_cli.models.fetch_api_models",
return_value=["anthropic/claude-sonnet-4.5", "openai/gpt-5.4"]), \
patch("cli.save_config_value", return_value=True) as save_mock:
cli_obj.process_command("/model anthropic/claude-sonnet-4.5")
@@ -74,12 +33,56 @@ class TestModelCommand:
assert cli_obj.agent is None
save_mock.assert_called_once_with("model.default", "anthropic/claude-sonnet-4.5")
def test_invalid_model_from_api_is_rejected(self, capsys):
cli_obj = self._make_cli()
with patch("hermes_cli.auth.resolve_provider", return_value="openrouter"), \
patch("hermes_cli.models.fetch_api_models",
return_value=["anthropic/claude-opus-4.6"]), \
patch("cli.save_config_value") as save_mock:
cli_obj.process_command("/model anthropic/fake-model")
output = capsys.readouterr().out
assert "not a valid model" in output
assert cli_obj.model == "anthropic/claude-opus-4.6" # unchanged
assert cli_obj.agent is not None # not reset
save_mock.assert_not_called()
def test_model_when_api_unreachable_falls_back_session_only(self, capsys):
cli_obj = self._make_cli()
with patch("hermes_cli.auth.resolve_provider", return_value="openrouter"), \
patch("hermes_cli.models.fetch_api_models", return_value=None), \
patch("cli.save_config_value") as save_mock:
cli_obj.process_command("/model anthropic/claude-sonnet-next")
output = capsys.readouterr().out
assert "session only" in output
assert cli_obj.model == "anthropic/claude-sonnet-next"
assert cli_obj.agent is None
save_mock.assert_not_called()
def test_bad_format_rejected_without_api_call(self, capsys):
cli_obj = self._make_cli()
with patch("hermes_cli.auth.resolve_provider", return_value="openrouter"), \
patch("hermes_cli.models.fetch_api_models") as fetch_mock, \
patch("cli.save_config_value") as save_mock:
cli_obj.process_command("/model invalid-no-slash")
output = capsys.readouterr().out
assert "provider/model" in output
assert cli_obj.model == "anthropic/claude-opus-4.6" # unchanged
fetch_mock.assert_not_called() # no API call for format errors
save_mock.assert_not_called()
def test_validation_crash_falls_back_to_save(self, capsys):
"""If validate_requested_model throws, /model should still work (old behavior)."""
cli_obj = self._make_cli()
with patch("hermes_cli.auth.resolve_provider", return_value="openrouter"), \
patch("hermes_cli.models.validate_requested_model", side_effect=RuntimeError("boom")), \
patch("hermes_cli.models.validate_requested_model",
side_effect=RuntimeError("boom")), \
patch("cli.save_config_value", return_value=True) as save_mock:
cli_obj.process_command("/model anthropic/claude-sonnet-4.5")