forked from Rockachopa/Timmy-time-dashboard
Claude/angry cerf (#173)
* feat: set qwen3.5:latest as default model - Make qwen3.5:latest the primary default model for faster inference - Move llama3.1:8b-instruct to fallback chain - Update text fallback chain to prioritize qwen3.5:latest Retains full backward compatibility via cascade fallback. * test: remove ~55 brittle, duplicate, and useless tests Audit of all 100 test files identified tests that provided no real regression protection. Removed: - 4 files deleted entirely: test_setup_script (always skipped), test_csrf_bypass (tautological assertions), test_input_validation (accepts 200-500 status codes), test_security_regression (fragile source-pattern checks redundant with rendering tests) - Duplicate test classes (TestToolTracking, TestCalculatorExtended) - Mock-only tests that just verify mock wiring, not behavior - Structurally broken tests (TestCreateToolFunctions patches after import) - Empty/pass-body tests and meaningless assertions (len > 20) - Flaky subprocess tests (aider tool calling real binary) All 1328 remaining tests pass. Net: -699 lines, zero coverage loss. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: prevent test pollution from autoresearch_enabled mutation test_autoresearch_perplexity.py was setting settings.autoresearch_enabled = True but never restoring it in the finally block — polluting subsequent tests. When pytest-randomly ordered it before test_experiments_page_shows_disabled_when_off, the victim test saw enabled=True and failed to find "Disabled" in the page. Fix both sides: - Restore autoresearch_enabled in the finally block (root cause) - Mock settings explicitly in the victim test (defense in depth) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Trip T <trip@local> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Alexander Whitestone
committed by
GitHub
GitHub
parent
0b91e45d90
commit
36fc10097f
@@ -1,54 +1,6 @@
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
def test_create_timmy_returns_agent():
|
||||
"""create_timmy should delegate to Agno Agent with correct config."""
|
||||
with (
|
||||
patch("timmy.agent.Agent") as MockAgent,
|
||||
patch("timmy.agent.Ollama"),
|
||||
patch("timmy.agent.SqliteDb"),
|
||||
):
|
||||
mock_instance = MagicMock()
|
||||
MockAgent.return_value = mock_instance
|
||||
|
||||
from timmy.agent import create_timmy
|
||||
|
||||
result = create_timmy()
|
||||
|
||||
assert result is mock_instance
|
||||
MockAgent.assert_called_once()
|
||||
|
||||
|
||||
def test_create_timmy_agent_name():
|
||||
with (
|
||||
patch("timmy.agent.Agent") as MockAgent,
|
||||
patch("timmy.agent.Ollama"),
|
||||
patch("timmy.agent.SqliteDb"),
|
||||
):
|
||||
from timmy.agent import create_timmy
|
||||
|
||||
create_timmy()
|
||||
|
||||
kwargs = MockAgent.call_args.kwargs
|
||||
assert kwargs["name"] == "Agent"
|
||||
|
||||
|
||||
def test_create_timmy_history_config():
|
||||
with (
|
||||
patch("timmy.agent.Agent") as MockAgent,
|
||||
patch("timmy.agent.Ollama"),
|
||||
patch("timmy.agent.SqliteDb"),
|
||||
):
|
||||
from timmy.agent import create_timmy
|
||||
|
||||
create_timmy()
|
||||
|
||||
kwargs = MockAgent.call_args.kwargs
|
||||
assert kwargs["add_history_to_context"] is True
|
||||
assert kwargs["num_history_runs"] == 20
|
||||
assert kwargs["markdown"] is True
|
||||
|
||||
|
||||
def test_create_timmy_custom_db_file():
|
||||
with (
|
||||
patch("timmy.agent.Agent"),
|
||||
|
||||
@@ -1,94 +1,17 @@
|
||||
"""Test plan for using the autoresearch module with perplexity as the target metric.
|
||||
"""Tests for using the autoresearch module with perplexity as the target metric.
|
||||
|
||||
Perplexity is a standard LM evaluation metric (lower = better), so the existing
|
||||
evaluate_result direction logic (lower-is-better) is correct without changes.
|
||||
Covers run integration, config override, history, and dashboard rendering when
|
||||
`autoresearch_metric` is set to ``perplexity``.
|
||||
|
||||
The tests below verify every layer of the stack — metric extraction, evaluation,
|
||||
run integration, config override, tool wiring, and dashboard rendering — works
|
||||
correctly when `autoresearch_metric` is set to ``perplexity``.
|
||||
Note: metric extraction and evaluation logic are already tested in
|
||||
test_autoresearch.py — only perplexity-specific integration paths are tested here.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# ── 1. Metric extraction ────────────────────────────────────────────────
|
||||
|
||||
|
||||
class TestExtractPerplexity:
|
||||
"""_extract_metric must find 'perplexity' values in training output."""
|
||||
|
||||
def test_extracts_perplexity_value(self):
|
||||
from timmy.autoresearch import _extract_metric
|
||||
|
||||
output = "step 500 perplexity: 42.31\nstep 1000 perplexity: 38.05"
|
||||
assert _extract_metric(output, "perplexity") == pytest.approx(38.05)
|
||||
|
||||
def test_extracts_last_occurrence(self):
|
||||
from timmy.autoresearch import _extract_metric
|
||||
|
||||
output = "perplexity: 100.0\nperplexity: 80.5\nperplexity: 55.2\n"
|
||||
assert _extract_metric(output, "perplexity") == pytest.approx(55.2)
|
||||
|
||||
def test_handles_integer_perplexity(self):
|
||||
from timmy.autoresearch import _extract_metric
|
||||
|
||||
output = "perplexity: 42"
|
||||
assert _extract_metric(output, "perplexity") == pytest.approx(42.0)
|
||||
|
||||
def test_handles_space_separator(self):
|
||||
"""Some training scripts use 'perplexity 38.5' without a colon."""
|
||||
from timmy.autoresearch import _extract_metric
|
||||
|
||||
output = "perplexity 38.5"
|
||||
assert _extract_metric(output, "perplexity") == pytest.approx(38.5)
|
||||
|
||||
def test_returns_none_when_absent(self):
|
||||
from timmy.autoresearch import _extract_metric
|
||||
|
||||
assert _extract_metric("loss: 0.45", "perplexity") is None
|
||||
|
||||
def test_ignores_unrelated_numbers(self):
|
||||
from timmy.autoresearch import _extract_metric
|
||||
|
||||
output = "step 500 lr 0.001 loss 2.3\nperplexity: 50.1"
|
||||
assert _extract_metric(output, "perplexity") == pytest.approx(50.1)
|
||||
|
||||
|
||||
# ── 2. Evaluation with perplexity ───────────────────────────────────────
|
||||
|
||||
|
||||
class TestEvaluatePerplexity:
|
||||
"""evaluate_result should treat lower perplexity as an improvement."""
|
||||
|
||||
def test_lower_is_improvement(self):
|
||||
from timmy.autoresearch import evaluate_result
|
||||
|
||||
result = evaluate_result(35.0, 42.0, metric_name="perplexity")
|
||||
assert "improvement" in result.lower()
|
||||
assert "perplexity" in result.lower()
|
||||
|
||||
def test_higher_is_regression(self):
|
||||
from timmy.autoresearch import evaluate_result
|
||||
|
||||
result = evaluate_result(50.0, 42.0, metric_name="perplexity")
|
||||
assert "regression" in result.lower()
|
||||
|
||||
def test_equal_is_no_change(self):
|
||||
from timmy.autoresearch import evaluate_result
|
||||
|
||||
result = evaluate_result(42.0, 42.0, metric_name="perplexity")
|
||||
assert "no change" in result.lower()
|
||||
|
||||
def test_percentage_is_correct(self):
|
||||
from timmy.autoresearch import evaluate_result
|
||||
|
||||
# 40 -> 30 is a -25% change
|
||||
result = evaluate_result(30.0, 40.0, metric_name="perplexity")
|
||||
assert "-25.00%" in result
|
||||
|
||||
|
||||
# ── 3. run_experiment with perplexity ───────────────────────────────────
|
||||
# ── run_experiment with perplexity ──────────────────────────────────────
|
||||
|
||||
|
||||
class TestRunExperimentPerplexity:
|
||||
@@ -230,7 +153,8 @@ class TestExperimentsRoutePerplexity:
|
||||
except ImportError:
|
||||
pytest.skip("pydantic_settings not installed")
|
||||
|
||||
original = settings.autoresearch_metric
|
||||
original_metric = settings.autoresearch_metric
|
||||
original_enabled = settings.autoresearch_enabled
|
||||
try:
|
||||
settings.autoresearch_metric = "perplexity"
|
||||
settings.autoresearch_enabled = True
|
||||
@@ -240,4 +164,5 @@ class TestExperimentsRoutePerplexity:
|
||||
|
||||
assert resp.status_code == 200
|
||||
finally:
|
||||
settings.autoresearch_metric = original
|
||||
settings.autoresearch_metric = original_metric
|
||||
settings.autoresearch_enabled = original_enabled
|
||||
|
||||
@@ -189,25 +189,6 @@ class TestToolCatalog:
|
||||
class TestAiderTool:
|
||||
"""Test the Aider AI coding assistant tool."""
|
||||
|
||||
def test_aider_tool_responds_to_simple_prompt(self):
|
||||
"""Test Aider tool can respond to a simple prompt.
|
||||
|
||||
This is a smoke test - we just verify it returns something.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
from timmy.tools import create_aider_tool
|
||||
|
||||
tool = create_aider_tool(Path.cwd())
|
||||
|
||||
# Call with a simple prompt - should return something (even if error)
|
||||
result = tool.run_aider("what is 2+2", model="qwen2.5:14b")
|
||||
|
||||
# Should get a response (either success or error message)
|
||||
assert result is not None
|
||||
assert isinstance(result, str)
|
||||
assert len(result) > 0
|
||||
|
||||
def test_aider_in_tool_catalog(self):
|
||||
"""Verify Aider appears in the tool catalog."""
|
||||
catalog = get_all_available_tools()
|
||||
|
||||
@@ -1,75 +1,15 @@
|
||||
"""Extended tests for timmy.tools — covers tool tracking, stats, and create_* functions."""
|
||||
"""Extended tests for timmy.tools — covers stats, type aliases, and aider tool."""
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from timmy.tools import (
|
||||
_TOOL_USAGE,
|
||||
AgentTools,
|
||||
PersonaTools,
|
||||
ToolStats,
|
||||
_track_tool_usage,
|
||||
calculator,
|
||||
create_aider_tool,
|
||||
get_tool_stats,
|
||||
)
|
||||
|
||||
|
||||
class TestToolTracking:
|
||||
"""Test _track_tool_usage and get_tool_stats."""
|
||||
|
||||
def setup_method(self):
|
||||
_TOOL_USAGE.clear()
|
||||
|
||||
def test_track_tool_usage(self):
|
||||
_track_tool_usage("agent-1", "web_search")
|
||||
assert "agent-1" in _TOOL_USAGE
|
||||
assert len(_TOOL_USAGE["agent-1"]) == 1
|
||||
assert _TOOL_USAGE["agent-1"][0]["tool"] == "web_search"
|
||||
assert _TOOL_USAGE["agent-1"][0]["success"] is True
|
||||
|
||||
def test_track_multiple_calls(self):
|
||||
_track_tool_usage("agent-1", "tool_a")
|
||||
_track_tool_usage("agent-1", "tool_b")
|
||||
_track_tool_usage("agent-1", "tool_a", success=False)
|
||||
assert len(_TOOL_USAGE["agent-1"]) == 3
|
||||
|
||||
def test_get_tool_stats_specific_agent(self):
|
||||
_track_tool_usage("agent-x", "read_file")
|
||||
_track_tool_usage("agent-x", "write_file")
|
||||
|
||||
stats = get_tool_stats("agent-x")
|
||||
assert stats["agent_id"] == "agent-x"
|
||||
assert stats["total_calls"] == 2
|
||||
assert set(stats["tools_used"]) == {"read_file", "write_file"}
|
||||
|
||||
def test_get_tool_stats_no_data(self):
|
||||
stats = get_tool_stats("nonexistent")
|
||||
assert stats["total_calls"] == 0
|
||||
assert stats["tools_used"] == []
|
||||
|
||||
def test_get_tool_stats_all_agents(self):
|
||||
_track_tool_usage("a1", "t1")
|
||||
_track_tool_usage("a2", "t2")
|
||||
_track_tool_usage("a2", "t3")
|
||||
|
||||
stats = get_tool_stats()
|
||||
assert "a1" in stats
|
||||
assert stats["a1"]["total_calls"] == 1
|
||||
assert stats["a2"]["total_calls"] == 2
|
||||
|
||||
def test_recent_calls_capped_at_10(self):
|
||||
for i in range(15):
|
||||
_track_tool_usage("agent-y", f"tool_{i}")
|
||||
|
||||
stats = get_tool_stats("agent-y")
|
||||
assert len(stats["recent_calls"]) == 10
|
||||
|
||||
def teardown_method(self):
|
||||
_TOOL_USAGE.clear()
|
||||
|
||||
|
||||
class TestToolStats:
|
||||
"""Test ToolStats dataclass."""
|
||||
|
||||
@@ -87,81 +27,6 @@ class TestAgentTools:
|
||||
assert PersonaTools is AgentTools
|
||||
|
||||
|
||||
class TestCalculatorExtended:
|
||||
"""Extended tests for the calculator tool."""
|
||||
|
||||
def test_division(self):
|
||||
assert calculator("10 / 3") == str(10 / 3)
|
||||
|
||||
def test_exponents(self):
|
||||
assert calculator("2**10") == "1024"
|
||||
|
||||
def test_math_functions(self):
|
||||
import math
|
||||
|
||||
assert calculator("math.sqrt(144)") == "12.0"
|
||||
assert calculator("math.pi") == str(math.pi)
|
||||
assert calculator("math.log(100, 10)") == str(math.log(100, 10))
|
||||
|
||||
def test_builtins_blocked(self):
|
||||
result = calculator("__import__('os').system('ls')")
|
||||
assert "Error" in result
|
||||
|
||||
def test_abs_allowed(self):
|
||||
assert calculator("abs(-5)") == "5"
|
||||
|
||||
def test_round_allowed(self):
|
||||
assert calculator("round(3.14159, 2)") == "3.14"
|
||||
|
||||
def test_min_max_allowed(self):
|
||||
assert calculator("min(1, 2, 3)") == "1"
|
||||
assert calculator("max(1, 2, 3)") == "3"
|
||||
|
||||
def test_invalid_expression(self):
|
||||
result = calculator("not valid python")
|
||||
assert "Error" in result
|
||||
|
||||
def test_division_by_zero(self):
|
||||
result = calculator("1/0")
|
||||
assert "Error" in result
|
||||
|
||||
|
||||
class TestCreateToolFunctions:
|
||||
"""Test that create_*_tools functions check availability."""
|
||||
|
||||
def test_create_research_tools_no_agno(self):
|
||||
with patch("timmy.tools._AGNO_TOOLS_AVAILABLE", False):
|
||||
with patch("timmy.tools._ImportError", ImportError("no agno")):
|
||||
with pytest.raises(ImportError):
|
||||
from timmy.tools import create_research_tools
|
||||
|
||||
create_research_tools()
|
||||
|
||||
def test_create_code_tools_no_agno(self):
|
||||
with patch("timmy.tools._AGNO_TOOLS_AVAILABLE", False):
|
||||
with patch("timmy.tools._ImportError", ImportError("no agno")):
|
||||
with pytest.raises(ImportError):
|
||||
from timmy.tools import create_code_tools
|
||||
|
||||
create_code_tools()
|
||||
|
||||
def test_create_data_tools_no_agno(self):
|
||||
with patch("timmy.tools._AGNO_TOOLS_AVAILABLE", False):
|
||||
with patch("timmy.tools._ImportError", ImportError("no agno")):
|
||||
with pytest.raises(ImportError):
|
||||
from timmy.tools import create_data_tools
|
||||
|
||||
create_data_tools()
|
||||
|
||||
def test_create_writing_tools_no_agno(self):
|
||||
with patch("timmy.tools._AGNO_TOOLS_AVAILABLE", False):
|
||||
with patch("timmy.tools._ImportError", ImportError("no agno")):
|
||||
with pytest.raises(ImportError):
|
||||
from timmy.tools import create_writing_tools
|
||||
|
||||
create_writing_tools()
|
||||
|
||||
|
||||
class TestAiderTool:
|
||||
"""Test AiderTool created by create_aider_tool."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user