2e64b160b5
- Add Kaizen Retro to cron/jobs.json with explicit local model/provider - Add Telegram message chunking for reports approaching the 4096-char limit - Fix classify_issue_type false positives on short substrings (ci in cleanup) - Add 28 unit tests covering classification, max-attempts detection, suggestion generation, report formatting, and Telegram chunking
284 lines
11 KiB
Python
284 lines
11 KiB
Python
"""Tests for the Kaizen Retro burn-cycle retrospective script."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import json
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
REPO_ROOT = Path(__file__).parent.parent
|
|
|
|
# Load kaizen_retro.py as a module (it lives in scripts/, not a package)
|
|
spec = importlib.util.spec_from_file_location("kaizen_retro", REPO_ROOT / "scripts" / "kaizen_retro.py")
|
|
kr = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(kr)
|
|
|
|
|
|
# ── classify_issue_type ───────────────────────────────────────────────────
|
|
|
|
class TestClassifyIssueType:
|
|
def test_classifies_bug_from_title(self):
|
|
issue = {"title": "Fix crash on startup", "body": "", "labels": []}
|
|
assert kr.classify_issue_type(issue) == "bug"
|
|
|
|
def test_classifies_feature_from_label(self):
|
|
issue = {"title": "Add dark mode", "body": "", "labels": [{"name": "enhancement"}]}
|
|
# label "enhancement" doesn't match any keyword directly, but "feature" and "add" are in title
|
|
assert kr.classify_issue_type(issue) == "feature"
|
|
|
|
def test_classifies_docs_from_label(self):
|
|
issue = {"title": "Update guide", "body": "", "labels": [{"name": "documentation"}]}
|
|
assert kr.classify_issue_type(issue) == "docs"
|
|
|
|
def test_label_match_stronger_than_title(self):
|
|
issue = {"title": "Something random", "body": "", "labels": [{"name": "bug"}]}
|
|
assert kr.classify_issue_type(issue) == "bug"
|
|
|
|
def test_kaizen_takes_precedence_with_both_labels(self):
|
|
issue = {"title": "Process improvement", "body": "", "labels": [{"name": "kaizen"}, {"name": "bug"}]}
|
|
# kaizen label gives +3, bug gives +3, tie goes to first seen? kaizen appears first in dict
|
|
assert kr.classify_issue_type(issue) == "kaizen"
|
|
|
|
def test_defaults_to_other(self):
|
|
issue = {"title": "Tidy up naming", "body": "No user-facing change", "labels": [{"name": "cleanup"}]}
|
|
assert kr.classify_issue_type(issue) == "other"
|
|
|
|
|
|
# ── is_max_attempts_candidate ─────────────────────────────────────────────
|
|
|
|
class TestIsMaxAttemptsCandidate:
|
|
def test_blocker_label_returns_true(self):
|
|
issue = {"labels": [{"name": "blocked"}], "comments": 0, "created_at": "2026-04-07T00:00:00Z"}
|
|
assert kr.is_max_attempts_candidate(issue) is True
|
|
|
|
def test_timeout_label_returns_true(self):
|
|
issue = {"labels": [{"name": "timeout"}], "comments": 0, "created_at": "2026-04-07T00:00:00Z"}
|
|
assert kr.is_max_attempts_candidate(issue) is True
|
|
|
|
def test_high_comment_count_returns_true(self):
|
|
issue = {"labels": [], "comments": 5, "created_at": "2026-04-07T00:00:00Z"}
|
|
assert kr.is_max_attempts_candidate(issue) is True
|
|
|
|
def test_fresh_issue_with_low_comments_returns_false(self):
|
|
now = datetime.now(timezone.utc)
|
|
issue = {"labels": [], "comments": 2, "created_at": now.isoformat()}
|
|
assert kr.is_max_attempts_candidate(issue) is False
|
|
|
|
def test_stale_age_returns_true(self):
|
|
old = datetime.now(timezone.utc) - timedelta(days=10)
|
|
issue = {"labels": [], "comments": 0, "created_at": old.isoformat()}
|
|
assert kr.is_max_attempts_candidate(issue) is True
|
|
|
|
|
|
# ── fmt_pct ───────────────────────────────────────────────────────────────
|
|
|
|
class TestFmtPct:
|
|
def test_basic_percentage(self):
|
|
assert kr.fmt_pct(3, 4) == "75%"
|
|
|
|
def test_zero_denominator(self):
|
|
assert kr.fmt_pct(0, 0) == "N/A"
|
|
|
|
def test_perfect_rate(self):
|
|
assert kr.fmt_pct(10, 10) == "100%"
|
|
|
|
|
|
# ── generate_suggestion ───────────────────────────────────────────────────
|
|
|
|
class TestGenerateSuggestion:
|
|
def test_agent_zero_success_rate(self):
|
|
metrics = {
|
|
"by_agent": {
|
|
"groq": {"successes": 0, "failures": 5, "closed": 0, "repos": ["timmy-home"]},
|
|
},
|
|
"by_repo": {},
|
|
"by_type": {},
|
|
"max_attempts_issues": [],
|
|
"closed_issues": [],
|
|
"merged_prs": [],
|
|
"closed_prs": [],
|
|
}
|
|
suggestion = kr.generate_suggestion(metrics, [])
|
|
assert "groq" in suggestion
|
|
assert "0%" in suggestion or "verify rate" in suggestion
|
|
|
|
def test_repo_with_most_failures(self):
|
|
metrics = {
|
|
"by_agent": {},
|
|
"by_repo": {
|
|
"the-nexus": {"successes": 2, "failures": 5, "closed": 2, "open": 3},
|
|
},
|
|
"by_type": {},
|
|
"max_attempts_issues": [],
|
|
"closed_issues": [],
|
|
"merged_prs": [],
|
|
"closed_prs": [],
|
|
}
|
|
suggestion = kr.generate_suggestion(metrics, [])
|
|
assert "the-nexus" in suggestion
|
|
assert "friction" in suggestion
|
|
|
|
def test_max_attempts_pattern(self):
|
|
metrics = {
|
|
"by_agent": {},
|
|
"by_repo": {},
|
|
"by_type": {},
|
|
"max_attempts_issues": [
|
|
{"type": "devops"}, {"type": "devops"}, {"type": "feature"}
|
|
],
|
|
"closed_issues": [],
|
|
"merged_prs": [],
|
|
"closed_prs": [],
|
|
}
|
|
suggestion = kr.generate_suggestion(metrics, [])
|
|
assert "devops" in suggestion
|
|
assert "max-attempts" in suggestion.lower() or "stale" in suggestion.lower()
|
|
|
|
def test_idle_agents(self):
|
|
metrics = {
|
|
"by_agent": {},
|
|
"by_repo": {},
|
|
"by_type": {},
|
|
"max_attempts_issues": [],
|
|
"closed_issues": [],
|
|
"merged_prs": [],
|
|
"closed_prs": [],
|
|
}
|
|
fleet = [{"name": "allegro", "active": True}, {"name": "ezra", "active": True}]
|
|
suggestion = kr.generate_suggestion(metrics, fleet)
|
|
assert "idle" in suggestion.lower() or "no assignments" in suggestion.lower()
|
|
|
|
def test_fallback_celebration(self):
|
|
metrics = {
|
|
"by_agent": {},
|
|
"by_repo": {},
|
|
"by_type": {},
|
|
"max_attempts_issues": [],
|
|
"closed_issues": [{}, {}, {}, {}, {}],
|
|
"merged_prs": [{}, {}, {}],
|
|
"closed_prs": [],
|
|
}
|
|
suggestion = kr.generate_suggestion(metrics, [])
|
|
assert "Strong cycle" in suggestion
|
|
|
|
def test_fallback_low_activity(self):
|
|
metrics = {
|
|
"by_agent": {},
|
|
"by_repo": {},
|
|
"by_type": {},
|
|
"max_attempts_issues": [],
|
|
"closed_issues": [],
|
|
"merged_prs": [],
|
|
"closed_prs": [],
|
|
}
|
|
suggestion = kr.generate_suggestion(metrics, [])
|
|
assert "Low activity" in suggestion or "idle" in suggestion.lower()
|
|
|
|
|
|
# ── build_report ──────────────────────────────────────────────────────────
|
|
|
|
class TestBuildReport:
|
|
def test_report_contains_numbers_section(self):
|
|
metrics = {
|
|
"closed_issues": [{}, {}],
|
|
"merged_prs": [{}],
|
|
"closed_prs": [],
|
|
"max_attempts_issues": [],
|
|
"by_agent": {"ezra": {"successes": 2, "failures": 0, "repos": ["timmy-config"]}},
|
|
"by_repo": {"timmy-config": {"successes": 2, "failures": 0, "open": 1}},
|
|
"by_type": {"feature": {"successes": 2, "failures": 0, "total": 2}},
|
|
}
|
|
report = kr.build_report(metrics, "Do better.", "2026-04-06T00:00:00+00:00")
|
|
assert "## Numbers" in report
|
|
assert "Issues closed:** 2" in report
|
|
assert "PRs merged:** 1" in report
|
|
assert "## By Agent" in report
|
|
assert "## By Repo" in report
|
|
assert "## By Issue Type" in report
|
|
assert "Do better." in report
|
|
|
|
def test_report_skips_empty_repos(self):
|
|
metrics = {
|
|
"closed_issues": [],
|
|
"merged_prs": [],
|
|
"closed_prs": [],
|
|
"max_attempts_issues": [],
|
|
"by_agent": {},
|
|
"by_repo": {"unused-repo": {"successes": 0, "failures": 0, "open": 0}},
|
|
"by_type": {},
|
|
}
|
|
report = kr.build_report(metrics, "Nudge.", "2026-04-06T00:00:00+00:00")
|
|
assert "unused-repo" not in report
|
|
|
|
def test_report_truncates_max_attempts(self):
|
|
metrics = {
|
|
"closed_issues": [],
|
|
"merged_prs": [],
|
|
"closed_prs": [],
|
|
"max_attempts_issues": [{"repo": "r", "number": i, "type": "bug", "assignee": "a", "title": f"T{i}"} for i in range(15)],
|
|
"by_agent": {},
|
|
"by_repo": {},
|
|
"by_type": {},
|
|
}
|
|
report = kr.build_report(metrics, "Fix it.", "2026-04-06T00:00:00+00:00")
|
|
assert "and 5 more" in report
|
|
|
|
|
|
# ── telegram_send ─────────────────────────────────────────────────────────
|
|
|
|
class TestTelegramSend:
|
|
def test_short_message_sent_in_one_piece(self):
|
|
with patch("urllib.request.urlopen") as mock_urlopen:
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b'{"ok": true}'
|
|
mock_urlopen.return_value.__enter__.return_value = mock_resp
|
|
|
|
results = kr.telegram_send("Hello", "fake-token", "123")
|
|
assert len(results) == 1
|
|
assert results[0]["ok"] is True
|
|
# Verify payload
|
|
call_args = mock_urlopen.call_args
|
|
req = call_args[0][0]
|
|
payload = json.loads(req.data.decode())
|
|
assert payload["text"] == "Hello"
|
|
assert payload["chat_id"] == "123"
|
|
|
|
def test_long_message_chunked(self):
|
|
big_text = "Line\n" * 2000 # ~10k chars
|
|
with patch("urllib.request.urlopen") as mock_urlopen:
|
|
mock_resp = MagicMock()
|
|
mock_resp.read.return_value = b'{"ok": true}'
|
|
mock_urlopen.return_value.__enter__.return_value = mock_resp
|
|
|
|
results = kr.telegram_send(big_text, "fake-token", "123")
|
|
assert len(results) >= 2
|
|
# First chunk should have a part prefix
|
|
req = mock_urlopen.call_args_list[0][0][0]
|
|
payload = json.loads(req.data.decode())
|
|
assert "(part 1" in payload["text"]
|
|
|
|
|
|
# ── load helpers ──────────────────────────────────────────────────────────
|
|
|
|
class TestLoadHelpers:
|
|
def test_load_json_missing_returns_none(self, tmp_path):
|
|
missing = tmp_path / "does_not_exist.json"
|
|
assert kr.load_json(missing) is None
|
|
|
|
def test_load_json_valid(self, tmp_path):
|
|
p = tmp_path / "data.json"
|
|
p.write_text('{"a": 1}')
|
|
assert kr.load_json(p) == {"a": 1}
|
|
|
|
def test_iso_day_ago_format(self):
|
|
s = kr.iso_day_ago(1)
|
|
# Should be a valid ISO timestamp string
|
|
dt = datetime.fromisoformat(s)
|
|
now = datetime.now(timezone.utc)
|
|
assert now - dt < timedelta(days=2)
|