forked from Rockachopa/Timmy-time-dashboard
Claude (Opus 4.6)
f2a277f7b5
Co-authored-by: Claude (Opus 4.6) <claude@hermes.local> Co-committed-by: Claude (Opus 4.6) <claude@hermes.local>
354 lines
13 KiB
Python
354 lines
13 KiB
Python
"""Tests for research triage — action item extraction and Gitea issue filing."""
|
|
|
|
import json
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from timmy.research_triage import (
|
|
ActionItem,
|
|
_parse_llm_response,
|
|
_validate_action_item,
|
|
create_gitea_issue,
|
|
extract_action_items,
|
|
triage_research_report,
|
|
)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ActionItem
|
|
# ---------------------------------------------------------------------------
|
|
|
|
SAMPLE_REPORT = """
|
|
## Research: MCP Abstraction Layer
|
|
|
|
### Finding 1: FastMCP overhead is negligible
|
|
FastMCP averages 26.45ms per tool call. Total overhead <3% of budget.
|
|
|
|
### Finding 2: Agno tool calling is broken
|
|
Agno issues #2231, #2625 document persistent breakage with Ollama.
|
|
Fix: Use Ollama's `format` parameter with Pydantic JSON schemas.
|
|
|
|
### Recommendation
|
|
Implement three-tier router for structured output.
|
|
"""
|
|
|
|
SAMPLE_LLM_RESPONSE = json.dumps(
|
|
[
|
|
{
|
|
"title": "[Router] Implement three-tier structured output router",
|
|
"body": (
|
|
"**What:** Build a three-tier router that uses Ollama's "
|
|
"`format` parameter for structured output.\n"
|
|
"**Why:** Agno's native tool calling is broken (#2231, #2625). "
|
|
"Pydantic JSON schemas with `format` bypass the issue.\n"
|
|
"**Suggested approach:** Add format parameter support to "
|
|
"CascadeRouter.\n"
|
|
"**Acceptance criteria:** Tool calls return valid JSON matching "
|
|
"the Pydantic schema."
|
|
),
|
|
"labels": ["actionable", "feature", "kimi-ready"],
|
|
"priority": "high",
|
|
"source_urls": ["https://github.com/agno-agi/agno/issues/2231"],
|
|
},
|
|
]
|
|
)
|
|
|
|
|
|
class TestActionItem:
|
|
def test_to_issue_body_basic(self):
|
|
item = ActionItem(title="Test", body="Test body")
|
|
body = item.to_issue_body()
|
|
assert "Test body" in body
|
|
assert "Auto-triaged" in body
|
|
|
|
def test_to_issue_body_with_source_issue(self):
|
|
item = ActionItem(title="Test", body="Test body")
|
|
body = item.to_issue_body(source_issue=946)
|
|
assert "#946" in body
|
|
assert "Origin" in body
|
|
|
|
def test_to_issue_body_with_source_urls(self):
|
|
item = ActionItem(
|
|
title="Test",
|
|
body="Body",
|
|
source_urls=["https://example.com/finding"],
|
|
)
|
|
body = item.to_issue_body()
|
|
assert "https://example.com/finding" in body
|
|
assert "Source Evidence" in body
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _parse_llm_response
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestParseLlmResponse:
|
|
def test_plain_json(self):
|
|
items = _parse_llm_response('[{"title": "foo"}]')
|
|
assert len(items) == 1
|
|
assert items[0]["title"] == "foo"
|
|
|
|
def test_fenced_json(self):
|
|
raw = '```json\n[{"title": "bar"}]\n```'
|
|
items = _parse_llm_response(raw)
|
|
assert len(items) == 1
|
|
assert items[0]["title"] == "bar"
|
|
|
|
def test_empty_array(self):
|
|
assert _parse_llm_response("[]") == []
|
|
|
|
def test_non_array_returns_empty(self):
|
|
assert _parse_llm_response('{"title": "not an array"}') == []
|
|
|
|
def test_invalid_json_raises(self):
|
|
with pytest.raises(json.JSONDecodeError):
|
|
_parse_llm_response("not json at all")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# _validate_action_item
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidateActionItem:
|
|
def test_valid_item(self):
|
|
raw = {
|
|
"title": "[Area] A specific clear title",
|
|
"body": "Detailed body with enough content to be useful.",
|
|
"labels": ["actionable", "bug"],
|
|
"priority": "high",
|
|
}
|
|
item = _validate_action_item(raw)
|
|
assert item is not None
|
|
assert item.title == "[Area] A specific clear title"
|
|
assert item.priority == "high"
|
|
assert "actionable" in item.labels
|
|
|
|
def test_short_title_rejected(self):
|
|
raw = {"title": "Short", "body": "Detailed body with enough content here."}
|
|
assert _validate_action_item(raw) is None
|
|
|
|
def test_short_body_rejected(self):
|
|
raw = {"title": "A perfectly fine title here", "body": "Too short"}
|
|
assert _validate_action_item(raw) is None
|
|
|
|
def test_missing_title_rejected(self):
|
|
raw = {"body": "Detailed body with enough content to be useful."}
|
|
assert _validate_action_item(raw) is None
|
|
|
|
def test_non_dict_rejected(self):
|
|
assert _validate_action_item("not a dict") is None
|
|
|
|
def test_actionable_label_auto_added(self):
|
|
raw = {
|
|
"title": "A perfectly fine title here",
|
|
"body": "Detailed body with enough content to be useful.",
|
|
"labels": ["bug"],
|
|
}
|
|
item = _validate_action_item(raw)
|
|
assert item is not None
|
|
assert "actionable" in item.labels
|
|
|
|
def test_labels_as_csv_string(self):
|
|
raw = {
|
|
"title": "A perfectly fine title here",
|
|
"body": "Detailed body with enough content to be useful.",
|
|
"labels": "bug, feature",
|
|
}
|
|
item = _validate_action_item(raw)
|
|
assert item is not None
|
|
assert "bug" in item.labels
|
|
assert "feature" in item.labels
|
|
|
|
def test_invalid_priority_defaults_medium(self):
|
|
raw = {
|
|
"title": "A perfectly fine title here",
|
|
"body": "Detailed body with enough content to be useful.",
|
|
"priority": "urgent",
|
|
}
|
|
item = _validate_action_item(raw)
|
|
assert item is not None
|
|
assert item.priority == "medium"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# extract_action_items
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestExtractActionItems:
|
|
@pytest.mark.asyncio
|
|
async def test_extracts_items_from_report(self):
|
|
mock_llm = AsyncMock(return_value=SAMPLE_LLM_RESPONSE)
|
|
items = await extract_action_items(SAMPLE_REPORT, llm_caller=mock_llm)
|
|
assert len(items) == 1
|
|
assert "three-tier" in items[0].title.lower()
|
|
assert items[0].priority == "high"
|
|
mock_llm.assert_called_once()
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_report_returns_empty(self):
|
|
items = await extract_action_items("")
|
|
assert items == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_failure_returns_empty(self):
|
|
mock_llm = AsyncMock(side_effect=RuntimeError("LLM down"))
|
|
items = await extract_action_items(SAMPLE_REPORT, llm_caller=mock_llm)
|
|
assert items == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_returns_empty_string(self):
|
|
mock_llm = AsyncMock(return_value="")
|
|
items = await extract_action_items(SAMPLE_REPORT, llm_caller=mock_llm)
|
|
assert items == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_llm_returns_invalid_json(self):
|
|
mock_llm = AsyncMock(return_value="not valid json")
|
|
items = await extract_action_items(SAMPLE_REPORT, llm_caller=mock_llm)
|
|
assert items == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_caps_at_five_items(self):
|
|
many_items = [
|
|
{
|
|
"title": f"[Area] Action item number {i} is specific",
|
|
"body": f"Detailed body for action item {i} with enough words.",
|
|
"labels": ["actionable"],
|
|
"priority": "medium",
|
|
}
|
|
for i in range(10)
|
|
]
|
|
mock_llm = AsyncMock(return_value=json.dumps(many_items))
|
|
items = await extract_action_items(SAMPLE_REPORT, llm_caller=mock_llm)
|
|
assert len(items) <= 5
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# create_gitea_issue
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCreateGiteaIssue:
|
|
@pytest.mark.asyncio
|
|
async def test_creates_issue_via_api(self):
|
|
item = ActionItem(
|
|
title="[Test] Create a test issue",
|
|
body="This is a test issue body with details.",
|
|
labels=["actionable"],
|
|
)
|
|
issue_resp = MagicMock()
|
|
issue_resp.status_code = 201
|
|
issue_resp.json.return_value = {"number": 42, "title": item.title}
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.post.return_value = issue_resp
|
|
|
|
with (
|
|
patch("timmy.research_triage.settings") as mock_settings,
|
|
patch(
|
|
"timmy.research_triage._resolve_label_ids", new_callable=AsyncMock, return_value=[1]
|
|
),
|
|
patch("timmy.research_triage.httpx.AsyncClient") as mock_cls,
|
|
):
|
|
mock_settings.gitea_enabled = True
|
|
mock_settings.gitea_token = "test-token"
|
|
mock_settings.gitea_repo = "owner/repo"
|
|
mock_settings.gitea_url = "http://localhost:3000"
|
|
mock_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_cls.return_value.__aexit__ = AsyncMock(return_value=False)
|
|
result = await create_gitea_issue(item, source_issue=946)
|
|
|
|
assert result is not None
|
|
assert result["number"] == 42
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_returns_none_when_disabled(self):
|
|
item = ActionItem(title="[Test] Disabled test", body="Body content here.")
|
|
with patch("timmy.research_triage.settings") as mock_settings:
|
|
mock_settings.gitea_enabled = False
|
|
mock_settings.gitea_token = ""
|
|
result = await create_gitea_issue(item)
|
|
assert result is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_handles_connection_error(self):
|
|
item = ActionItem(
|
|
title="[Test] Connection fail",
|
|
body="Body content for connection test.",
|
|
)
|
|
mock_client = AsyncMock()
|
|
mock_client.post.side_effect = httpx.ConnectError("refused")
|
|
|
|
with (
|
|
patch("timmy.research_triage.settings") as mock_settings,
|
|
patch(
|
|
"timmy.research_triage._resolve_label_ids", new_callable=AsyncMock, return_value=[]
|
|
),
|
|
patch("timmy.research_triage.httpx.AsyncClient") as mock_cls,
|
|
):
|
|
mock_settings.gitea_enabled = True
|
|
mock_settings.gitea_token = "test-token"
|
|
mock_settings.gitea_repo = "owner/repo"
|
|
mock_settings.gitea_url = "http://localhost:3000"
|
|
mock_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_cls.return_value.__aexit__ = AsyncMock(return_value=False)
|
|
result = await create_gitea_issue(item)
|
|
assert result is None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# triage_research_report (integration)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestTriageResearchReport:
|
|
@pytest.mark.asyncio
|
|
async def test_dry_run_extracts_without_filing(self):
|
|
mock_llm = AsyncMock(return_value=SAMPLE_LLM_RESPONSE)
|
|
results = await triage_research_report(
|
|
SAMPLE_REPORT, source_issue=946, llm_caller=mock_llm, dry_run=True
|
|
)
|
|
assert len(results) == 1
|
|
assert results[0]["action_item"] is not None
|
|
assert results[0]["gitea_issue"] is None
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_empty_report_returns_empty(self):
|
|
results = await triage_research_report("", llm_caller=AsyncMock(return_value="[]"))
|
|
assert results == []
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_end_to_end_with_mock_gitea(self):
|
|
mock_llm = AsyncMock(return_value=SAMPLE_LLM_RESPONSE)
|
|
|
|
issue_resp = MagicMock()
|
|
issue_resp.status_code = 201
|
|
issue_resp.json.return_value = {"number": 99, "title": "test"}
|
|
|
|
mock_client = AsyncMock()
|
|
mock_client.post.return_value = issue_resp
|
|
|
|
with (
|
|
patch("timmy.research_triage.settings") as mock_settings,
|
|
patch(
|
|
"timmy.research_triage._resolve_label_ids", new_callable=AsyncMock, return_value=[]
|
|
),
|
|
patch("timmy.research_triage.httpx.AsyncClient") as mock_cls,
|
|
):
|
|
mock_settings.gitea_enabled = True
|
|
mock_settings.gitea_token = "test-token"
|
|
mock_settings.gitea_repo = "owner/repo"
|
|
mock_settings.gitea_url = "http://localhost:3000"
|
|
mock_cls.return_value.__aenter__ = AsyncMock(return_value=mock_client)
|
|
mock_cls.return_value.__aexit__ = AsyncMock(return_value=False)
|
|
results = await triage_research_report(
|
|
SAMPLE_REPORT, source_issue=946, llm_caller=mock_llm
|
|
)
|
|
|
|
assert len(results) == 1
|
|
assert results[0]["gitea_issue"]["number"] == 99
|