from unittest.mock import MagicMock, patch def test_create_timmy_custom_db_file(): with ( patch("timmy.agent.Agent"), patch("timmy.agent.Ollama"), patch("timmy.agent.SqliteDb") as MockDb, patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): from timmy.agent import create_timmy create_timmy(db_file="custom.db") MockDb.assert_called_once_with(db_file="custom.db") def test_create_timmy_embeds_system_prompt(): with ( patch("timmy.agent.Agent") as MockAgent, patch("timmy.agent.Ollama"), patch("timmy.agent.SqliteDb"), patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): from timmy.agent import create_timmy create_timmy() kwargs = MockAgent.call_args.kwargs # Prompt should contain base system prompt (may have memory context appended) # Default model (llama3.2) uses the lite prompt assert "Timmy" in kwargs["description"] # ── Ollama host regression (container connectivity) ───────────────────────── def test_create_timmy_passes_ollama_url_to_model(): """Regression: Ollama model must receive settings.ollama_url as host. Without this, containers default to localhost:11434 which is unreachable when Ollama runs on the Docker host. """ with ( patch("timmy.agent.Agent"), patch("timmy.agent.Ollama") as MockOllama, patch("timmy.agent.SqliteDb"), patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): from timmy.agent import create_timmy create_timmy() kwargs = MockOllama.call_args.kwargs assert "host" in kwargs, "Ollama() must receive host= parameter" from config import settings assert kwargs["host"] == settings.ollama_url def test_create_timmy_respects_custom_ollama_url(): """Ollama host should follow OLLAMA_URL when overridden in config.""" custom_url = "http://host.docker.internal:11434" with ( patch("timmy.agent.Agent"), patch("timmy.agent.Ollama") as MockOllama, patch("timmy.agent.SqliteDb"), patch("timmy.agent.settings") as mock_settings, patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): mock_settings.ollama_model = "llama3.2" mock_settings.ollama_url = custom_url mock_settings.ollama_num_ctx = 4096 mock_settings.timmy_model_backend = "ollama" from timmy.agent import create_timmy create_timmy() kwargs = MockOllama.call_args.kwargs assert kwargs["host"] == custom_url # ── AirLLM path ────────────────────────────────────────────────────────────── def test_create_timmy_airllm_returns_airllm_agent(): """backend='airllm' must return a TimmyAirLLMAgent, not an Agno Agent.""" with patch("timmy.backends.is_apple_silicon", return_value=False): from timmy.agent import create_timmy from timmy.backends import TimmyAirLLMAgent result = create_timmy(backend="airllm", model_size="8b") assert isinstance(result, TimmyAirLLMAgent) def test_create_timmy_airllm_does_not_call_agno_agent(): """When using the airllm backend, Agno Agent should never be instantiated.""" with ( patch("timmy.agent.Agent") as MockAgent, patch("timmy.backends.is_apple_silicon", return_value=False), ): from timmy.agent import create_timmy create_timmy(backend="airllm", model_size="8b") MockAgent.assert_not_called() def test_create_timmy_explicit_ollama_ignores_autodetect(): """backend='ollama' must always use Ollama, even on Apple Silicon.""" with ( patch("timmy.agent.Agent") as MockAgent, patch("timmy.agent.Ollama"), patch("timmy.agent.SqliteDb"), patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): from timmy.agent import create_timmy create_timmy(backend="ollama") MockAgent.assert_called_once() # ── _resolve_backend ───────────────────────────────────────────────────────── def test_resolve_backend_explicit_takes_priority(): from timmy.agent import _resolve_backend assert _resolve_backend("airllm") == "airllm" assert _resolve_backend("ollama") == "ollama" def test_resolve_backend_defaults_to_ollama_without_config(): """Default config (timmy_model_backend='ollama') → 'ollama'.""" from timmy.agent import _resolve_backend assert _resolve_backend(None) == "ollama" def test_resolve_backend_auto_uses_airllm_on_apple_silicon(): """'auto' on Apple Silicon with airllm stubbed → 'airllm'.""" with ( patch("timmy.backends.is_apple_silicon", return_value=True), patch("timmy.agent.settings") as mock_settings, ): mock_settings.timmy_model_backend = "auto" mock_settings.ollama_model = "llama3.2" from timmy.agent import _resolve_backend assert _resolve_backend(None) == "airllm" def test_resolve_backend_auto_falls_back_on_non_apple(): """'auto' on non-Apple Silicon → 'ollama'.""" with ( patch("timmy.backends.is_apple_silicon", return_value=False), patch("timmy.agent.settings") as mock_settings, ): mock_settings.timmy_model_backend = "auto" mock_settings.ollama_model = "llama3.2" from timmy.agent import _resolve_backend assert _resolve_backend(None) == "ollama" # ── _model_supports_tools ──────────────────────────────────────────────────── def test_model_supports_tools_llama32_returns_false(): """llama3.2 (3B) is too small for reliable tool calling.""" from timmy.agent import _model_supports_tools assert _model_supports_tools("llama3.2") is False assert _model_supports_tools("llama3.2:latest") is False def test_model_supports_tools_llama31_returns_true(): """llama3.1 (8B+) can handle tool calling.""" from timmy.agent import _model_supports_tools assert _model_supports_tools("llama3.1") is True assert _model_supports_tools("llama3.3") is True def test_model_supports_tools_other_small_models(): """Other known small models should not get tools.""" from timmy.agent import _model_supports_tools assert _model_supports_tools("phi-3") is False assert _model_supports_tools("tinyllama") is False def test_model_supports_tools_unknown_model_gets_tools(): """Unknown models default to tool-capable (optimistic).""" from timmy.agent import _model_supports_tools assert _model_supports_tools("mistral") is True assert _model_supports_tools("qwen2.5:72b") is True # ── Tool gating in create_timmy ────────────────────────────────────────────── def test_create_timmy_no_tools_for_small_model(): """Small models (llama3.2) should get no tools.""" mock_toolkit = MagicMock() with ( patch("timmy.agent.Agent") as MockAgent, patch("timmy.agent.Ollama"), patch("timmy.agent.SqliteDb"), patch("timmy.agent.create_full_toolkit", return_value=mock_toolkit), patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): from timmy.agent import create_timmy create_timmy() kwargs = MockAgent.call_args.kwargs # llama3.2 is in _SMALL_MODEL_PATTERNS → tools should be None assert kwargs["tools"] is None def test_create_timmy_includes_tools_for_large_model(): """A tool-capable model (e.g. llama3.1) should attempt to include tools.""" mock_toolkit = MagicMock() with ( patch("timmy.agent.Agent") as MockAgent, patch("timmy.agent.Ollama"), patch("timmy.agent.SqliteDb"), patch("timmy.agent.create_full_toolkit", return_value=mock_toolkit), patch("timmy.agent.settings") as mock_settings, patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.1", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): mock_settings.ollama_model = "llama3.1" mock_settings.ollama_url = "http://localhost:11434" mock_settings.ollama_num_ctx = 4096 mock_settings.timmy_model_backend = "ollama" mock_settings.telemetry_enabled = False from timmy.agent import create_timmy create_timmy() kwargs = MockAgent.call_args.kwargs assert mock_toolkit in kwargs["tools"] def test_create_timmy_no_unsupported_agent_kwargs(): """Regression guard: show_tool_calls and tool_call_limit are not valid agno 2.x params. These were removed in f95c960 (Feb 26) and must not be reintroduced. """ with ( patch("timmy.agent.Agent") as MockAgent, patch("timmy.agent.Ollama"), patch("timmy.agent.SqliteDb"), patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): from timmy.agent import create_timmy create_timmy() kwargs = MockAgent.call_args.kwargs assert "show_tool_calls" not in kwargs, "show_tool_calls is not a valid Agent param" def test_create_timmy_no_extra_kwargs(): """All kwargs passed to Agent() must be from the known-valid set. agno is mocked globally in conftest, so we can't inspect the real class here. Instead, maintain an explicit allowlist of params validated against agno 2.5.5. If a new param is needed, verify it exists in agno first, then add it here. """ VALID_AGENT_KWARGS = { "name", "model", "db", "description", "add_history_to_context", "num_history_runs", "markdown", "tools", "tool_call_limit", "telemetry", } with ( patch("timmy.agent.Agent") as MockAgent, patch("timmy.agent.Ollama"), patch("timmy.agent.SqliteDb"), patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): from timmy.agent import create_timmy create_timmy() kwargs = MockAgent.call_args.kwargs invalid = set(kwargs.keys()) - VALID_AGENT_KWARGS assert not invalid, ( f"Unknown Agent kwargs {invalid} — verify they exist in agno " f"before adding to VALID_AGENT_KWARGS" ) # ── skip_mcp flag (#72) ───────────────────────────────────────────────────── def test_create_timmy_skip_mcp_omits_mcp_tools(): """create_timmy(skip_mcp=True) must not add MCP tool servers.""" with ( patch("timmy.agent.Agent"), patch("timmy.agent.Ollama"), patch("timmy.agent.SqliteDb"), patch("timmy.mcp_tools.create_gitea_mcp_tools") as mock_gitea_mcp, patch("timmy.mcp_tools.create_filesystem_mcp_tools") as mock_fs_mcp, patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): from timmy.agent import create_timmy create_timmy(skip_mcp=True) # MCP factory functions should never be called mock_gitea_mcp.assert_not_called() mock_fs_mcp.assert_not_called() def test_create_timmy_default_includes_mcp_tools(): """create_timmy() without skip_mcp should attempt MCP tool creation.""" with ( patch("timmy.agent.Agent"), patch("timmy.agent.Ollama"), patch("timmy.agent.SqliteDb"), patch("timmy.mcp_tools.create_gitea_mcp_tools", return_value=None) as mock_gitea_mcp, patch("timmy.mcp_tools.create_filesystem_mcp_tools", return_value=None) as mock_fs_mcp, patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.1", False)), patch("timmy.agent._check_model_available", return_value=True), patch("timmy.agent._warmup_model", return_value=True), ): from timmy.agent import create_timmy create_timmy(skip_mcp=False) # MCP factories should be called when skip_mcp is False mock_gitea_mcp.assert_called_once() mock_fs_mcp.assert_called_once() # ── Configurable fallback chain tests ──────────────────────────────────────── def test_settings_has_fallback_model_lists(): """settings.fallback_models and vision_fallback_models exist and are lists.""" from config import settings assert isinstance(settings.fallback_models, list) assert isinstance(settings.vision_fallback_models, list) assert len(settings.fallback_models) > 0 assert len(settings.vision_fallback_models) > 0 def test_resolve_model_uses_configurable_text_fallback(): """_resolve_model_with_fallback walks settings.fallback_models for text models.""" with patch("timmy.agent.settings") as mock_settings: mock_settings.ollama_model = "nonexistent-model" mock_settings.fallback_models = ["custom-a", "custom-b"] mock_settings.vision_fallback_models = ["vision-a"] # First model in chain is available with patch("timmy.agent._check_model_available", side_effect=lambda m: m == "custom-a"): from timmy.agent import _resolve_model_with_fallback model, is_fallback = _resolve_model_with_fallback( requested_model="nonexistent-model", require_vision=False, auto_pull=False, ) assert model == "custom-a" assert is_fallback is True def test_resolve_model_uses_configurable_vision_fallback(): """_resolve_model_with_fallback walks settings.vision_fallback_models for vision.""" with patch("timmy.agent.settings") as mock_settings: mock_settings.ollama_model = "nonexistent-model" mock_settings.fallback_models = ["text-a"] mock_settings.vision_fallback_models = ["vision-x", "vision-y"] with patch("timmy.agent._check_model_available", side_effect=lambda m: m == "vision-y"): from timmy.agent import _resolve_model_with_fallback model, is_fallback = _resolve_model_with_fallback( requested_model="nonexistent-model", require_vision=True, auto_pull=False, ) assert model == "vision-y" assert is_fallback is True def test_get_effective_ollama_model_walks_fallback_chain(): """get_effective_ollama_model uses settings.fallback_models.""" with ( patch("config.settings") as mock_settings, patch("config.check_ollama_model_available", side_effect=lambda m: m == "fb-2") as _, ): mock_settings.ollama_model = "gone-model" mock_settings.ollama_url = "http://localhost:11434" mock_settings.fallback_models = ["fb-1", "fb-2", "fb-3"] from config import get_effective_ollama_model result = get_effective_ollama_model() assert result == "fb-2" def test_no_hardcoded_fallback_constants_in_agent(): """agent.py must not define module-level DEFAULT_MODEL_FALLBACKS.""" import timmy.agent as agent_mod assert not hasattr(agent_mod, "DEFAULT_MODEL_FALLBACKS"), ( "Hardcoded DEFAULT_MODEL_FALLBACKS still exists — use settings.fallback_models" ) assert not hasattr(agent_mod, "VISION_MODEL_FALLBACKS"), ( "Hardcoded VISION_MODEL_FALLBACKS still exists — use settings.vision_fallback_models" )