Timmy-time-dashboard/tests/timmy/test_agent.py

from unittest.mock import MagicMock, patch


def test_create_timmy_custom_db_file():
    with (
        patch("timmy.agent.Agent"),
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb") as MockDb,
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        from timmy.agent import create_timmy

        create_timmy(db_file="custom.db")

        MockDb.assert_called_once_with(db_file="custom.db")


def test_create_timmy_embeds_system_prompt():

    with (
        patch("timmy.agent.Agent") as MockAgent,
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb"),
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        from timmy.agent import create_timmy

        create_timmy()

        kwargs = MockAgent.call_args.kwargs
        # Prompt should contain base system prompt (may have memory context appended)
        # Default model (llama3.2) uses the lite prompt
        assert "Timmy" in kwargs["description"]


# ── Ollama host regression (container connectivity) ─────────────────────────


def test_create_timmy_passes_ollama_url_to_model():
    """Regression: Ollama model must receive settings.ollama_url as host.

    Without this, containers default to localhost:11434 which is unreachable
    when Ollama runs on the Docker host.
    """
    with (
        patch("timmy.agent.Agent"),
        patch("timmy.agent.Ollama") as MockOllama,
        patch("timmy.agent.SqliteDb"),
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        from timmy.agent import create_timmy

        create_timmy()

        kwargs = MockOllama.call_args.kwargs
        assert "host" in kwargs, "Ollama() must receive host= parameter"
        from config import settings

        assert kwargs["host"] == settings.ollama_url


def test_create_timmy_respects_custom_ollama_url():
    """Ollama host should follow OLLAMA_URL when overridden in config."""
    custom_url = "http://host.docker.internal:11434"
    with (
        patch("timmy.agent.Agent"),
        patch("timmy.agent.Ollama") as MockOllama,
        patch("timmy.agent.SqliteDb"),
        patch("timmy.agent.settings") as mock_settings,
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        mock_settings.ollama_model = "llama3.2"
        mock_settings.ollama_url = custom_url
        mock_settings.ollama_num_ctx = 4096
        mock_settings.timmy_model_backend = "ollama"
        mock_settings.airllm_model_size = "70b"

        from timmy.agent import create_timmy

        create_timmy()

        kwargs = MockOllama.call_args.kwargs
        assert kwargs["host"] == custom_url


# ── AirLLM path ──────────────────────────────────────────────────────────────


def test_create_timmy_airllm_returns_airllm_agent():
    """backend='airllm' must return a TimmyAirLLMAgent, not an Agno Agent."""
    with patch("timmy.backends.is_apple_silicon", return_value=False):
        from timmy.agent import create_timmy
        from timmy.backends import TimmyAirLLMAgent

        result = create_timmy(backend="airllm", model_size="8b")

    assert isinstance(result, TimmyAirLLMAgent)


def test_create_timmy_airllm_does_not_call_agno_agent():
    """When using the airllm backend, Agno Agent should never be instantiated."""
    with (
        patch("timmy.agent.Agent") as MockAgent,
        patch("timmy.backends.is_apple_silicon", return_value=False),
    ):
        from timmy.agent import create_timmy

        create_timmy(backend="airllm", model_size="8b")

    MockAgent.assert_not_called()


def test_create_timmy_explicit_ollama_ignores_autodetect():
    """backend='ollama' must always use Ollama, even on Apple Silicon."""
    with (
        patch("timmy.agent.Agent") as MockAgent,
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb"),
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        from timmy.agent import create_timmy

        create_timmy(backend="ollama")

    MockAgent.assert_called_once()


# ── _resolve_backend ─────────────────────────────────────────────────────────


def test_resolve_backend_explicit_takes_priority():
    from timmy.agent import _resolve_backend

    assert _resolve_backend("airllm") == "airllm"
    assert _resolve_backend("ollama") == "ollama"


def test_resolve_backend_defaults_to_ollama_without_config():
    """Default config (timmy_model_backend='ollama') → 'ollama'."""
    from timmy.agent import _resolve_backend

    assert _resolve_backend(None) == "ollama"


def test_resolve_backend_auto_uses_airllm_on_apple_silicon():
    """'auto' on Apple Silicon with airllm stubbed → 'airllm'."""
    with (
        patch("timmy.backends.is_apple_silicon", return_value=True),
        patch("timmy.agent.settings") as mock_settings,
    ):
        mock_settings.timmy_model_backend = "auto"
        mock_settings.airllm_model_size = "70b"
        mock_settings.ollama_model = "llama3.2"

        from timmy.agent import _resolve_backend

        assert _resolve_backend(None) == "airllm"


def test_resolve_backend_auto_falls_back_on_non_apple():
    """'auto' on non-Apple Silicon → 'ollama'."""
    with (
        patch("timmy.backends.is_apple_silicon", return_value=False),
        patch("timmy.agent.settings") as mock_settings,
    ):
        mock_settings.timmy_model_backend = "auto"
        mock_settings.airllm_model_size = "70b"
        mock_settings.ollama_model = "llama3.2"

        from timmy.agent import _resolve_backend

        assert _resolve_backend(None) == "ollama"


# ── _model_supports_tools ────────────────────────────────────────────────────


def test_model_supports_tools_llama32_returns_false():
    """llama3.2 (3B) is too small for reliable tool calling."""
    from timmy.agent import _model_supports_tools

    assert _model_supports_tools("llama3.2") is False
    assert _model_supports_tools("llama3.2:latest") is False


def test_model_supports_tools_llama31_returns_true():
    """llama3.1 (8B+) can handle tool calling."""
    from timmy.agent import _model_supports_tools

    assert _model_supports_tools("llama3.1") is True
    assert _model_supports_tools("llama3.3") is True


def test_model_supports_tools_other_small_models():
    """Other known small models should not get tools."""
    from timmy.agent import _model_supports_tools

    assert _model_supports_tools("phi-3") is False
    assert _model_supports_tools("tinyllama") is False


def test_model_supports_tools_unknown_model_gets_tools():
    """Unknown models default to tool-capable (optimistic)."""
    from timmy.agent import _model_supports_tools

    assert _model_supports_tools("mistral") is True
    assert _model_supports_tools("qwen2.5:72b") is True


# ── Tool gating in create_timmy ──────────────────────────────────────────────


def test_create_timmy_no_tools_for_small_model():
    """Small models (llama3.2) should get no tools."""
    mock_toolkit = MagicMock()
    with (
        patch("timmy.agent.Agent") as MockAgent,
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb"),
        patch("timmy.agent.create_full_toolkit", return_value=mock_toolkit),
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        from timmy.agent import create_timmy

        create_timmy()

        kwargs = MockAgent.call_args.kwargs
        # llama3.2 is in _SMALL_MODEL_PATTERNS → tools should be None
        assert kwargs["tools"] is None


def test_create_timmy_includes_tools_for_large_model():
    """A tool-capable model (e.g. llama3.1) should attempt to include tools."""
    mock_toolkit = MagicMock()

    with (
        patch("timmy.agent.Agent") as MockAgent,
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb"),
        patch("timmy.agent.create_full_toolkit", return_value=mock_toolkit),
        patch("timmy.agent.settings") as mock_settings,
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.1", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        mock_settings.ollama_model = "llama3.1"
        mock_settings.ollama_url = "http://localhost:11434"
        mock_settings.ollama_num_ctx = 4096
        mock_settings.timmy_model_backend = "ollama"
        mock_settings.airllm_model_size = "70b"
        mock_settings.telemetry_enabled = False

        from timmy.agent import create_timmy

        create_timmy()

        kwargs = MockAgent.call_args.kwargs
        assert mock_toolkit in kwargs["tools"]


def test_create_timmy_no_unsupported_agent_kwargs():
    """Regression guard: show_tool_calls and tool_call_limit are not valid agno 2.x params.

    These were removed in f95c960 (Feb 26) and must not be reintroduced.
    """
    with (
        patch("timmy.agent.Agent") as MockAgent,
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb"),
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        from timmy.agent import create_timmy

        create_timmy()

        kwargs = MockAgent.call_args.kwargs
        assert "show_tool_calls" not in kwargs, "show_tool_calls is not a valid Agent param"


def test_create_timmy_no_extra_kwargs():
    """All kwargs passed to Agent() must be from the known-valid set.

    agno is mocked globally in conftest, so we can't inspect the real class here.
    Instead, maintain an explicit allowlist of params validated against agno 2.5.5.
    If a new param is needed, verify it exists in agno first, then add it here.
    """
    VALID_AGENT_KWARGS = {
        "name",
        "model",
        "db",
        "description",
        "add_history_to_context",
        "num_history_runs",
        "markdown",
        "tools",
        "tool_call_limit",
        "telemetry",
    }

    with (
        patch("timmy.agent.Agent") as MockAgent,
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb"),
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        from timmy.agent import create_timmy

        create_timmy()

        kwargs = MockAgent.call_args.kwargs
        invalid = set(kwargs.keys()) - VALID_AGENT_KWARGS
        assert not invalid, (
            f"Unknown Agent kwargs {invalid} — verify they exist in agno "
            f"before adding to VALID_AGENT_KWARGS"
        )


# ── skip_mcp flag (#72) ─────────────────────────────────────────────────────


def test_create_timmy_skip_mcp_omits_mcp_tools():
    """create_timmy(skip_mcp=True) must not add MCP tool servers."""
    with (
        patch("timmy.agent.Agent"),
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb"),
        patch("timmy.mcp_tools.create_gitea_mcp_tools") as mock_gitea_mcp,
        patch("timmy.mcp_tools.create_filesystem_mcp_tools") as mock_fs_mcp,
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.2:3b", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        from timmy.agent import create_timmy

        create_timmy(skip_mcp=True)

        # MCP factory functions should never be called
        mock_gitea_mcp.assert_not_called()
        mock_fs_mcp.assert_not_called()


def test_create_timmy_default_includes_mcp_tools():
    """create_timmy() without skip_mcp should attempt MCP tool creation."""
    with (
        patch("timmy.agent.Agent"),
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb"),
        patch("timmy.mcp_tools.create_gitea_mcp_tools", return_value=None) as mock_gitea_mcp,
        patch("timmy.mcp_tools.create_filesystem_mcp_tools", return_value=None) as mock_fs_mcp,
        patch("timmy.agent._resolve_model_with_fallback", return_value=("llama3.1", False)),
        patch("timmy.agent._check_model_available", return_value=True),
        patch("timmy.agent._warmup_model", return_value=True),
    ):
        from timmy.agent import create_timmy

        create_timmy(skip_mcp=False)

        # MCP factories should be called when skip_mcp is False
        mock_gitea_mcp.assert_called_once()
        mock_fs_mcp.assert_called_once()


# ── Configurable fallback chain tests ────────────────────────────────────────


def test_settings_has_fallback_model_lists():
    """settings.fallback_models and vision_fallback_models exist and are lists."""
    from config import settings

    assert isinstance(settings.fallback_models, list)
    assert isinstance(settings.vision_fallback_models, list)
    assert len(settings.fallback_models) > 0
    assert len(settings.vision_fallback_models) > 0


def test_resolve_model_uses_configurable_text_fallback():
    """_resolve_model_with_fallback walks settings.fallback_models for text models."""
    with patch("timmy.agent.settings") as mock_settings:
        mock_settings.ollama_model = "nonexistent-model"
        mock_settings.fallback_models = ["custom-a", "custom-b"]
        mock_settings.vision_fallback_models = ["vision-a"]

        # First model in chain is available
        with patch("timmy.agent._check_model_available", side_effect=lambda m: m == "custom-a"):
            from timmy.agent import _resolve_model_with_fallback

            model, is_fallback = _resolve_model_with_fallback(
                requested_model="nonexistent-model",
                require_vision=False,
                auto_pull=False,
            )
            assert model == "custom-a"
            assert is_fallback is True


def test_resolve_model_uses_configurable_vision_fallback():
    """_resolve_model_with_fallback walks settings.vision_fallback_models for vision."""
    with patch("timmy.agent.settings") as mock_settings:
        mock_settings.ollama_model = "nonexistent-model"
        mock_settings.fallback_models = ["text-a"]
        mock_settings.vision_fallback_models = ["vision-x", "vision-y"]

        with patch("timmy.agent._check_model_available", side_effect=lambda m: m == "vision-y"):
            from timmy.agent import _resolve_model_with_fallback

            model, is_fallback = _resolve_model_with_fallback(
                requested_model="nonexistent-model",
                require_vision=True,
                auto_pull=False,
            )
            assert model == "vision-y"
            assert is_fallback is True


def test_get_effective_ollama_model_walks_fallback_chain():
    """get_effective_ollama_model uses settings.fallback_models."""
    with (
        patch("config.settings") as mock_settings,
        patch("config.check_ollama_model_available", side_effect=lambda m: m == "fb-2") as _,
    ):
        mock_settings.ollama_model = "gone-model"
        mock_settings.ollama_url = "http://localhost:11434"
        mock_settings.fallback_models = ["fb-1", "fb-2", "fb-3"]

        from config import get_effective_ollama_model

        result = get_effective_ollama_model()
        assert result == "fb-2"


def test_no_hardcoded_fallback_constants_in_agent():
    """agent.py must not define module-level DEFAULT_MODEL_FALLBACKS."""
    import timmy.agent as agent_mod

    assert not hasattr(agent_mod, "DEFAULT_MODEL_FALLBACKS"), (
        "Hardcoded DEFAULT_MODEL_FALLBACKS still exists — use settings.fallback_models"
    )
    assert not hasattr(agent_mod, "VISION_MODEL_FALLBACKS"), (
        "Hardcoded VISION_MODEL_FALLBACKS still exists — use settings.vision_fallback_models"
    )


# ── _build_tools_list helper ─────────────────────────────────────────────────


def test_build_tools_list_returns_empty_when_no_tools():
    """When use_tools=False, _build_tools_list returns an empty list."""
    from timmy.agent import _build_tools_list

    result = _build_tools_list(use_tools=False, skip_mcp=False)
    assert result == []


def test_build_tools_list_includes_toolkit():
    """When use_tools=True, _build_tools_list includes the toolkit."""
    mock_toolkit = MagicMock()
    with patch("timmy.agent.create_full_toolkit", return_value=mock_toolkit):
        from timmy.agent import _build_tools_list

        result = _build_tools_list(use_tools=True, skip_mcp=True)

    assert mock_toolkit in result


def test_build_tools_list_adds_mcp_when_not_skipped():
    """When skip_mcp=False, _build_tools_list attempts MCP tools."""
    mock_toolkit = MagicMock()
    mock_gitea = MagicMock()
    with (
        patch("timmy.agent.create_full_toolkit", return_value=mock_toolkit),
        patch("timmy.mcp_tools.create_gitea_mcp_tools", return_value=mock_gitea),
        patch("timmy.mcp_tools.create_filesystem_mcp_tools", return_value=None),
    ):
        from timmy.agent import _build_tools_list

        result = _build_tools_list(use_tools=True, skip_mcp=False)

    assert mock_toolkit in result
    assert mock_gitea in result


# ── _build_prompt helper ─────────────────────────────────────────────────────


def test_build_prompt_returns_base_when_no_memory():
    """_build_prompt returns base prompt when memory context is empty."""
    with patch("timmy.memory_system.memory_system") as mock_mem:
        mock_mem.get_system_context.return_value = ""
        from timmy.agent import _build_prompt

        result = _build_prompt(use_tools=True, session_id="test")

    assert "Timmy" in result


def test_build_prompt_appends_memory_context():
    """_build_prompt appends memory context when available."""
    with patch("timmy.memory_system.memory_system") as mock_mem:
        mock_mem.get_system_context.return_value = "User likes pizza"
        from timmy.agent import _build_prompt

        result = _build_prompt(use_tools=True, session_id="test")

    assert "User likes pizza" in result
    assert "GROUNDED CONTEXT" in result


def test_build_prompt_truncates_long_memory_for_small_models():
    """_build_prompt truncates memory for small models (use_tools=False)."""
    long_context = "x" * 5000
    with patch("timmy.memory_system.memory_system") as mock_mem:
        mock_mem.get_system_context.return_value = long_context
        from timmy.agent import _build_prompt

        result = _build_prompt(use_tools=False, session_id="test")

    # Max context is 2000 for small models + truncation marker
    assert "[truncated]" in result


# ── _create_ollama_agent helper ──────────────────────────────────────────────


def test_create_ollama_agent_passes_correct_kwargs():
    """_create_ollama_agent passes the expected kwargs to Agent()."""
    with (
        patch("timmy.agent.Agent") as MockAgent,
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb"),
    ):
        from timmy.agent import _create_ollama_agent

        _create_ollama_agent(
            model_name="test-model",
            db_file="test.db",
            tools_list=[MagicMock()],
            full_prompt="Test prompt",
            use_tools=True,
        )

        kwargs = MockAgent.call_args.kwargs
        assert kwargs["description"] == "Test prompt"
        assert kwargs["tools"] is not None


def test_create_ollama_agent_none_tools_when_empty():
    """_create_ollama_agent passes tools=None when tools_list is empty."""
    with (
        patch("timmy.agent.Agent") as MockAgent,
        patch("timmy.agent.Ollama"),
        patch("timmy.agent.SqliteDb"),
    ):
        from timmy.agent import _create_ollama_agent

        _create_ollama_agent(
            model_name="test-model",
            db_file="test.db",
            tools_list=[],
            full_prompt="Test prompt",
            use_tools=False,
        )

        kwargs = MockAgent.call_args.kwargs
        assert kwargs["tools"] is None