forked from Rockachopa/Timmy-time-dashboard
feat: code quality audit + autoresearch integration + infra hardening (#150)
This commit is contained in:
Alexander Whitestone
committed by
GitHub
GitHub
parent
fd0ede0d51
commit
ae3bb1cc21
@@ -4,20 +4,17 @@ Tests cover planning, execution, max_steps enforcement, failure
|
||||
adaptation, progress callbacks, and response cleaning.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import MagicMock, patch, AsyncMock
|
||||
from timmy.agentic_loop import (
|
||||
run_agentic_loop,
|
||||
_parse_steps,
|
||||
AgenticResult,
|
||||
AgenticStep,
|
||||
)
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from timmy.agentic_loop import AgenticResult, AgenticStep, _parse_steps, run_agentic_loop
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _mock_run(content: str):
|
||||
"""Create a mock return value for agent.run()."""
|
||||
m = MagicMock()
|
||||
@@ -29,6 +26,7 @@ def _mock_run(content: str):
|
||||
# _parse_steps
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseSteps:
|
||||
def test_numbered_with_dot(self):
|
||||
text = "1. Search for data\n2. Write to file\n3. Verify"
|
||||
@@ -50,20 +48,24 @@ class TestParseSteps:
|
||||
# run_agentic_loop
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_planning_phase_produces_steps():
|
||||
"""Planning prompt returns numbered step list."""
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.run = MagicMock(side_effect=[
|
||||
_mock_run("1. Search AI news\n2. Write to file\n3. Verify"),
|
||||
_mock_run("Found 5 articles about AI."),
|
||||
_mock_run("Wrote summary to /tmp/ai_news.md"),
|
||||
_mock_run("File verified, 15 lines."),
|
||||
_mock_run("Searched, wrote, verified."),
|
||||
])
|
||||
mock_agent.run = MagicMock(
|
||||
side_effect=[
|
||||
_mock_run("1. Search AI news\n2. Write to file\n3. Verify"),
|
||||
_mock_run("Found 5 articles about AI."),
|
||||
_mock_run("Wrote summary to /tmp/ai_news.md"),
|
||||
_mock_run("File verified, 15 lines."),
|
||||
_mock_run("Searched, wrote, verified."),
|
||||
]
|
||||
)
|
||||
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), \
|
||||
patch("timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock):
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), patch(
|
||||
"timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock
|
||||
):
|
||||
result = await run_agentic_loop("Search AI news and write summary")
|
||||
|
||||
assert result.status == "completed"
|
||||
@@ -74,15 +76,18 @@ async def test_planning_phase_produces_steps():
|
||||
async def test_loop_executes_all_steps():
|
||||
"""Loop calls agent.run() for plan + each step + summary."""
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.run = MagicMock(side_effect=[
|
||||
_mock_run("1. Do A\n2. Do B"),
|
||||
_mock_run("A done"),
|
||||
_mock_run("B done"),
|
||||
_mock_run("All done"),
|
||||
])
|
||||
mock_agent.run = MagicMock(
|
||||
side_effect=[
|
||||
_mock_run("1. Do A\n2. Do B"),
|
||||
_mock_run("A done"),
|
||||
_mock_run("B done"),
|
||||
_mock_run("All done"),
|
||||
]
|
||||
)
|
||||
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), \
|
||||
patch("timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock):
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), patch(
|
||||
"timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock
|
||||
):
|
||||
result = await run_agentic_loop("Do A and B")
|
||||
|
||||
# plan + 2 steps + summary = 4 calls
|
||||
@@ -94,15 +99,18 @@ async def test_loop_executes_all_steps():
|
||||
async def test_loop_respects_max_steps():
|
||||
"""Loop stops at max_steps and returns status='partial'."""
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.run = MagicMock(side_effect=[
|
||||
_mock_run("1. A\n2. B\n3. C\n4. D\n5. E"),
|
||||
_mock_run("A done"),
|
||||
_mock_run("B done"),
|
||||
_mock_run("Completed 2 of 5 steps."),
|
||||
])
|
||||
mock_agent.run = MagicMock(
|
||||
side_effect=[
|
||||
_mock_run("1. A\n2. B\n3. C\n4. D\n5. E"),
|
||||
_mock_run("A done"),
|
||||
_mock_run("B done"),
|
||||
_mock_run("Completed 2 of 5 steps."),
|
||||
]
|
||||
)
|
||||
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), \
|
||||
patch("timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock):
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), patch(
|
||||
"timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock
|
||||
):
|
||||
result = await run_agentic_loop("Do 5 things", max_steps=2)
|
||||
|
||||
assert len(result.steps) == 2
|
||||
@@ -113,17 +121,20 @@ async def test_loop_respects_max_steps():
|
||||
async def test_failure_triggers_adaptation():
|
||||
"""Failed step feeds error back to model, step marked as adapted."""
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.run = MagicMock(side_effect=[
|
||||
_mock_run("1. Read config\n2. Update setting\n3. Verify"),
|
||||
_mock_run("Config: timeout=30"),
|
||||
Exception("Permission denied"),
|
||||
_mock_run("Adapted: wrote to ~/config.yaml instead"),
|
||||
_mock_run("Verified: timeout=60"),
|
||||
_mock_run("Updated config via alternative path."),
|
||||
])
|
||||
mock_agent.run = MagicMock(
|
||||
side_effect=[
|
||||
_mock_run("1. Read config\n2. Update setting\n3. Verify"),
|
||||
_mock_run("Config: timeout=30"),
|
||||
Exception("Permission denied"),
|
||||
_mock_run("Adapted: wrote to ~/config.yaml instead"),
|
||||
_mock_run("Verified: timeout=60"),
|
||||
_mock_run("Updated config via alternative path."),
|
||||
]
|
||||
)
|
||||
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), \
|
||||
patch("timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock):
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), patch(
|
||||
"timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock
|
||||
):
|
||||
result = await run_agentic_loop("Update config timeout to 60")
|
||||
|
||||
assert result.status == "completed"
|
||||
@@ -139,15 +150,18 @@ async def test_progress_callback_fires():
|
||||
events.append((step, total))
|
||||
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.run = MagicMock(side_effect=[
|
||||
_mock_run("1. Do A\n2. Do B"),
|
||||
_mock_run("A done"),
|
||||
_mock_run("B done"),
|
||||
_mock_run("All done"),
|
||||
])
|
||||
mock_agent.run = MagicMock(
|
||||
side_effect=[
|
||||
_mock_run("1. Do A\n2. Do B"),
|
||||
_mock_run("A done"),
|
||||
_mock_run("B done"),
|
||||
_mock_run("All done"),
|
||||
]
|
||||
)
|
||||
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), \
|
||||
patch("timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock):
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), patch(
|
||||
"timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock
|
||||
):
|
||||
await run_agentic_loop("Do A and B", on_progress=on_progress)
|
||||
|
||||
assert len(events) == 2
|
||||
@@ -159,15 +173,18 @@ async def test_progress_callback_fires():
|
||||
async def test_result_contains_step_metadata():
|
||||
"""AgenticResult.steps has status and duration per step."""
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.run = MagicMock(side_effect=[
|
||||
_mock_run("1. Search\n2. Write"),
|
||||
_mock_run("Found results"),
|
||||
_mock_run("Written to file"),
|
||||
_mock_run("Done"),
|
||||
])
|
||||
mock_agent.run = MagicMock(
|
||||
side_effect=[
|
||||
_mock_run("1. Search\n2. Write"),
|
||||
_mock_run("Found results"),
|
||||
_mock_run("Written to file"),
|
||||
_mock_run("Done"),
|
||||
]
|
||||
)
|
||||
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), \
|
||||
patch("timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock):
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), patch(
|
||||
"timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock
|
||||
):
|
||||
result = await run_agentic_loop("Search and write")
|
||||
|
||||
for step in result.steps:
|
||||
@@ -191,8 +208,9 @@ async def test_config_default_used():
|
||||
|
||||
mock_agent.run = MagicMock(side_effect=side_effects)
|
||||
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), \
|
||||
patch("timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock):
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), patch(
|
||||
"timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock
|
||||
):
|
||||
result = await run_agentic_loop("Do 14 things", max_steps=0)
|
||||
|
||||
# Should be capped at 10 (config default)
|
||||
@@ -205,8 +223,9 @@ async def test_planning_failure_returns_failed():
|
||||
mock_agent = MagicMock()
|
||||
mock_agent.run = MagicMock(side_effect=Exception("Model offline"))
|
||||
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), \
|
||||
patch("timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock):
|
||||
with patch("timmy.agentic_loop._get_loop_agent", return_value=mock_agent), patch(
|
||||
"timmy.agentic_loop._broadcast_progress", new_callable=AsyncMock
|
||||
):
|
||||
result = await run_agentic_loop("Do something")
|
||||
|
||||
assert result.status == "failed"
|
||||
|
||||
Reference in New Issue
Block a user