d5361a0385
Remove `airllm` from timmy_model_backend Literal type and delete the airllm_model_size field plus associated comments. Replace the one settings.airllm_model_size reference in agent.py with a hardcoded default, and clean up mock assignments in tests. Fixes #473 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
453 lines
17 KiB
Python
453 lines
17 KiB
Python
from unittest.mock import MagicMock, patch
|
|
|
|
|
|
def test_create_timmy_custom_db_file():
|
|
with (
|
|
patch("timmy.agent.Agent"),
|
|
patch("timmy.agent.Ollama"),
|
|
patch("timmy.agent.SqliteDb") as MockDb,
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy(db_file="custom.db")
|
|
|
|
MockDb.assert_called_once_with(db_file="custom.db")
|
|
|
|
|
|
def test_create_timmy_embeds_system_prompt():
|
|
|
|
with (
|
|
patch("timmy.agent.Agent") as MockAgent,
|
|
patch("timmy.agent.Ollama"),
|
|
patch("timmy.agent.SqliteDb"),
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy()
|
|
|
|
kwargs = MockAgent.call_args.kwargs
|
|
# Prompt should contain base system prompt (may have memory context appended)
|
|
# Default model (llama3.2) uses the lite prompt
|
|
assert "Timmy" in kwargs["description"]
|
|
|
|
|
|
# ── Ollama host regression (container connectivity) ─────────────────────────
|
|
|
|
|
|
def test_create_timmy_passes_ollama_url_to_model():
|
|
"""Regression: Ollama model must receive settings.ollama_url as host.
|
|
|
|
Without this, containers default to localhost:11434 which is unreachable
|
|
when Ollama runs on the Docker host.
|
|
"""
|
|
with (
|
|
patch("timmy.agent.Agent"),
|
|
patch("timmy.agent.Ollama") as MockOllama,
|
|
patch("timmy.agent.SqliteDb"),
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy()
|
|
|
|
kwargs = MockOllama.call_args.kwargs
|
|
assert "host" in kwargs, "Ollama() must receive host= parameter"
|
|
from config import settings
|
|
|
|
assert kwargs["host"] == settings.ollama_url
|
|
|
|
|
|
def test_create_timmy_respects_custom_ollama_url():
|
|
"""Ollama host should follow OLLAMA_URL when overridden in config."""
|
|
custom_url = "http://host.docker.internal:11434"
|
|
with (
|
|
patch("timmy.agent.Agent"),
|
|
patch("timmy.agent.Ollama") as MockOllama,
|
|
patch("timmy.agent.SqliteDb"),
|
|
patch("timmy.agent.settings") as mock_settings,
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
mock_settings.ollama_model = "llama3.2"
|
|
mock_settings.ollama_url = custom_url
|
|
mock_settings.ollama_num_ctx = 4096
|
|
mock_settings.timmy_model_backend = "ollama"
|
|
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy()
|
|
|
|
kwargs = MockOllama.call_args.kwargs
|
|
assert kwargs["host"] == custom_url
|
|
|
|
|
|
# ── AirLLM path ──────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_create_timmy_airllm_returns_airllm_agent():
|
|
"""backend='airllm' must return a TimmyAirLLMAgent, not an Agno Agent."""
|
|
with patch("timmy.backends.is_apple_silicon", return_value=False):
|
|
from timmy.agent import create_timmy
|
|
from timmy.backends import TimmyAirLLMAgent
|
|
|
|
result = create_timmy(backend="airllm", model_size="8b")
|
|
|
|
assert isinstance(result, TimmyAirLLMAgent)
|
|
|
|
|
|
def test_create_timmy_airllm_does_not_call_agno_agent():
|
|
"""When using the airllm backend, Agno Agent should never be instantiated."""
|
|
with (
|
|
patch("timmy.agent.Agent") as MockAgent,
|
|
patch("timmy.backends.is_apple_silicon", return_value=False),
|
|
):
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy(backend="airllm", model_size="8b")
|
|
|
|
MockAgent.assert_not_called()
|
|
|
|
|
|
def test_create_timmy_explicit_ollama_ignores_autodetect():
|
|
"""backend='ollama' must always use Ollama, even on Apple Silicon."""
|
|
with (
|
|
patch("timmy.agent.Agent") as MockAgent,
|
|
patch("timmy.agent.Ollama"),
|
|
patch("timmy.agent.SqliteDb"),
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy(backend="ollama")
|
|
|
|
MockAgent.assert_called_once()
|
|
|
|
|
|
# ── _resolve_backend ─────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_resolve_backend_explicit_takes_priority():
|
|
from timmy.agent import _resolve_backend
|
|
|
|
assert _resolve_backend("airllm") == "airllm"
|
|
assert _resolve_backend("ollama") == "ollama"
|
|
|
|
|
|
def test_resolve_backend_defaults_to_ollama_without_config():
|
|
"""Default config (timmy_model_backend='ollama') → 'ollama'."""
|
|
from timmy.agent import _resolve_backend
|
|
|
|
assert _resolve_backend(None) == "ollama"
|
|
|
|
|
|
def test_resolve_backend_auto_uses_airllm_on_apple_silicon():
|
|
"""'auto' on Apple Silicon with airllm stubbed → 'airllm'."""
|
|
with (
|
|
patch("timmy.backends.is_apple_silicon", return_value=True),
|
|
patch("timmy.agent.settings") as mock_settings,
|
|
):
|
|
mock_settings.timmy_model_backend = "auto"
|
|
mock_settings.ollama_model = "llama3.2"
|
|
|
|
from timmy.agent import _resolve_backend
|
|
|
|
assert _resolve_backend(None) == "airllm"
|
|
|
|
|
|
def test_resolve_backend_auto_falls_back_on_non_apple():
|
|
"""'auto' on non-Apple Silicon → 'ollama'."""
|
|
with (
|
|
patch("timmy.backends.is_apple_silicon", return_value=False),
|
|
patch("timmy.agent.settings") as mock_settings,
|
|
):
|
|
mock_settings.timmy_model_backend = "auto"
|
|
mock_settings.ollama_model = "llama3.2"
|
|
|
|
from timmy.agent import _resolve_backend
|
|
|
|
assert _resolve_backend(None) == "ollama"
|
|
|
|
|
|
# ── _model_supports_tools ────────────────────────────────────────────────────
|
|
|
|
|
|
def test_model_supports_tools_llama32_returns_false():
|
|
"""llama3.2 (3B) is too small for reliable tool calling."""
|
|
from timmy.agent import _model_supports_tools
|
|
|
|
assert _model_supports_tools("llama3.2") is False
|
|
assert _model_supports_tools("llama3.2:latest") is False
|
|
|
|
|
|
def test_model_supports_tools_llama31_returns_true():
|
|
"""llama3.1 (8B+) can handle tool calling."""
|
|
from timmy.agent import _model_supports_tools
|
|
|
|
assert _model_supports_tools("llama3.1") is True
|
|
assert _model_supports_tools("llama3.3") is True
|
|
|
|
|
|
def test_model_supports_tools_other_small_models():
|
|
"""Other known small models should not get tools."""
|
|
from timmy.agent import _model_supports_tools
|
|
|
|
assert _model_supports_tools("phi-3") is False
|
|
assert _model_supports_tools("tinyllama") is False
|
|
|
|
|
|
def test_model_supports_tools_unknown_model_gets_tools():
|
|
"""Unknown models default to tool-capable (optimistic)."""
|
|
from timmy.agent import _model_supports_tools
|
|
|
|
assert _model_supports_tools("mistral") is True
|
|
assert _model_supports_tools("qwen2.5:72b") is True
|
|
|
|
|
|
# ── Tool gating in create_timmy ──────────────────────────────────────────────
|
|
|
|
|
|
def test_create_timmy_no_tools_for_small_model():
|
|
"""Small models (llama3.2) should get no tools."""
|
|
mock_toolkit = MagicMock()
|
|
with (
|
|
patch("timmy.agent.Agent") as MockAgent,
|
|
patch("timmy.agent.Ollama"),
|
|
patch("timmy.agent.SqliteDb"),
|
|
patch("timmy.agent.create_full_toolkit", return_value=mock_toolkit),
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy()
|
|
|
|
kwargs = MockAgent.call_args.kwargs
|
|
# llama3.2 is in _SMALL_MODEL_PATTERNS → tools should be None
|
|
assert kwargs["tools"] is None
|
|
|
|
|
|
def test_create_timmy_includes_tools_for_large_model():
|
|
"""A tool-capable model (e.g. llama3.1) should attempt to include tools."""
|
|
mock_toolkit = MagicMock()
|
|
|
|
with (
|
|
patch("timmy.agent.Agent") as MockAgent,
|
|
patch("timmy.agent.Ollama"),
|
|
patch("timmy.agent.SqliteDb"),
|
|
patch("timmy.agent.create_full_toolkit", return_value=mock_toolkit),
|
|
patch("timmy.agent.settings") as mock_settings,
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.1", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
mock_settings.ollama_model = "llama3.1"
|
|
mock_settings.ollama_url = "http://localhost:11434"
|
|
mock_settings.ollama_num_ctx = 4096
|
|
mock_settings.timmy_model_backend = "ollama"
|
|
mock_settings.telemetry_enabled = False
|
|
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy()
|
|
|
|
kwargs = MockAgent.call_args.kwargs
|
|
assert mock_toolkit in kwargs["tools"]
|
|
|
|
|
|
def test_create_timmy_no_unsupported_agent_kwargs():
|
|
"""Regression guard: show_tool_calls and tool_call_limit are not valid agno 2.x params.
|
|
|
|
These were removed in f95c960 (Feb 26) and must not be reintroduced.
|
|
"""
|
|
with (
|
|
patch("timmy.agent.Agent") as MockAgent,
|
|
patch("timmy.agent.Ollama"),
|
|
patch("timmy.agent.SqliteDb"),
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy()
|
|
|
|
kwargs = MockAgent.call_args.kwargs
|
|
assert "show_tool_calls" not in kwargs, "show_tool_calls is not a valid Agent param"
|
|
|
|
|
|
def test_create_timmy_no_extra_kwargs():
|
|
"""All kwargs passed to Agent() must be from the known-valid set.
|
|
|
|
agno is mocked globally in conftest, so we can't inspect the real class here.
|
|
Instead, maintain an explicit allowlist of params validated against agno 2.5.5.
|
|
If a new param is needed, verify it exists in agno first, then add it here.
|
|
"""
|
|
VALID_AGENT_KWARGS = {
|
|
"name",
|
|
"model",
|
|
"db",
|
|
"description",
|
|
"add_history_to_context",
|
|
"num_history_runs",
|
|
"markdown",
|
|
"tools",
|
|
"tool_call_limit",
|
|
"telemetry",
|
|
}
|
|
|
|
with (
|
|
patch("timmy.agent.Agent") as MockAgent,
|
|
patch("timmy.agent.Ollama"),
|
|
patch("timmy.agent.SqliteDb"),
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy()
|
|
|
|
kwargs = MockAgent.call_args.kwargs
|
|
invalid = set(kwargs.keys()) - VALID_AGENT_KWARGS
|
|
assert not invalid, (
|
|
f"Unknown Agent kwargs {invalid} — verify they exist in agno "
|
|
f"before adding to VALID_AGENT_KWARGS"
|
|
)
|
|
|
|
|
|
# ── skip_mcp flag (#72) ─────────────────────────────────────────────────────
|
|
|
|
|
|
def test_create_timmy_skip_mcp_omits_mcp_tools():
|
|
"""create_timmy(skip_mcp=True) must not add MCP tool servers."""
|
|
with (
|
|
patch("timmy.agent.Agent"),
|
|
patch("timmy.agent.Ollama"),
|
|
patch("timmy.agent.SqliteDb"),
|
|
patch("timmy.mcp_tools.create_gitea_mcp_tools") as mock_gitea_mcp,
|
|
patch("timmy.mcp_tools.create_filesystem_mcp_tools") as mock_fs_mcp,
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy(skip_mcp=True)
|
|
|
|
# MCP factory functions should never be called
|
|
mock_gitea_mcp.assert_not_called()
|
|
mock_fs_mcp.assert_not_called()
|
|
|
|
|
|
def test_create_timmy_default_includes_mcp_tools():
|
|
"""create_timmy() without skip_mcp should attempt MCP tool creation."""
|
|
with (
|
|
patch("timmy.agent.Agent"),
|
|
patch("timmy.agent.Ollama"),
|
|
patch("timmy.agent.SqliteDb"),
|
|
patch("timmy.mcp_tools.create_gitea_mcp_tools", return_value=None) as mock_gitea_mcp,
|
|
patch("timmy.mcp_tools.create_filesystem_mcp_tools", return_value=None) as mock_fs_mcp,
|
|
patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.1", False)),
|
|
patch("timmy.agent._check_model_available", return_value=True),
|
|
patch("timmy.agent._warmup_model", return_value=True),
|
|
):
|
|
from timmy.agent import create_timmy
|
|
|
|
create_timmy(skip_mcp=False)
|
|
|
|
# MCP factories should be called when skip_mcp is False
|
|
mock_gitea_mcp.assert_called_once()
|
|
mock_fs_mcp.assert_called_once()
|
|
|
|
|
|
# ── Configurable fallback chain tests ────────────────────────────────────────
|
|
|
|
|
|
def test_settings_has_fallback_model_lists():
|
|
"""settings.fallback_models and vision_fallback_models exist and are lists."""
|
|
from config import settings
|
|
|
|
assert isinstance(settings.fallback_models, list)
|
|
assert isinstance(settings.vision_fallback_models, list)
|
|
assert len(settings.fallback_models) > 0
|
|
assert len(settings.vision_fallback_models) > 0
|
|
|
|
|
|
def test_resolve_model_uses_configurable_text_fallback():
|
|
"""_resolve_model_with_fallback walks settings.fallback_models for text models."""
|
|
with patch("timmy.agent.settings") as mock_settings:
|
|
mock_settings.ollama_model = "nonexistent-model"
|
|
mock_settings.fallback_models = ["custom-a", "custom-b"]
|
|
mock_settings.vision_fallback_models = ["vision-a"]
|
|
|
|
# First model in chain is available
|
|
with patch("timmy.agent._check_model_available", side_effect=lambda m: m == "custom-a"):
|
|
from timmy.agent import _resolve_model_with_fallback
|
|
|
|
model, is_fallback = _resolve_model_with_fallback(
|
|
requested_model="nonexistent-model",
|
|
require_vision=False,
|
|
auto_pull=False,
|
|
)
|
|
assert model == "custom-a"
|
|
assert is_fallback is True
|
|
|
|
|
|
def test_resolve_model_uses_configurable_vision_fallback():
|
|
"""_resolve_model_with_fallback walks settings.vision_fallback_models for vision."""
|
|
with patch("timmy.agent.settings") as mock_settings:
|
|
mock_settings.ollama_model = "nonexistent-model"
|
|
mock_settings.fallback_models = ["text-a"]
|
|
mock_settings.vision_fallback_models = ["vision-x", "vision-y"]
|
|
|
|
with patch("timmy.agent._check_model_available", side_effect=lambda m: m == "vision-y"):
|
|
from timmy.agent import _resolve_model_with_fallback
|
|
|
|
model, is_fallback = _resolve_model_with_fallback(
|
|
requested_model="nonexistent-model",
|
|
require_vision=True,
|
|
auto_pull=False,
|
|
)
|
|
assert model == "vision-y"
|
|
assert is_fallback is True
|
|
|
|
|
|
def test_get_effective_ollama_model_walks_fallback_chain():
|
|
"""get_effective_ollama_model uses settings.fallback_models."""
|
|
with (
|
|
patch("config.settings") as mock_settings,
|
|
patch("config.check_ollama_model_available", side_effect=lambda m: m == "fb-2") as _,
|
|
):
|
|
mock_settings.ollama_model = "gone-model"
|
|
mock_settings.ollama_url = "http://localhost:11434"
|
|
mock_settings.fallback_models = ["fb-1", "fb-2", "fb-3"]
|
|
|
|
from config import get_effective_ollama_model
|
|
|
|
result = get_effective_ollama_model()
|
|
assert result == "fb-2"
|
|
|
|
|
|
def test_no_hardcoded_fallback_constants_in_agent():
|
|
"""agent.py must not define module-level DEFAULT_MODEL_FALLBACKS."""
|
|
import timmy.agent as agent_mod
|
|
|
|
assert not hasattr(agent_mod, "DEFAULT_MODEL_FALLBACKS"), (
|
|
"Hardcoded DEFAULT_MODEL_FALLBACKS still exists — use settings.fallback_models"
|
|
)
|
|
assert not hasattr(agent_mod, "VISION_MODEL_FALLBACKS"), (
|
|
"Hardcoded VISION_MODEL_FALLBACKS still exists — use settings.vision_fallback_models"
|
|
)
|