5989600d80
Some checks failed
Forge CI / smoke-and-build (pull_request) Failing after 1m1s
Empirical audit: cron error rate peaks at 18:00 (9.4%) vs 4.0% at 09:00.
During configured high-error windows, automatically route cron jobs to
more capable models when the user is not present to correct errors.
- agent/smart_model_routing.py: resolve_cron_model() + _hour_in_window()
- cron/scheduler.py: wired into run_job() after base model resolution
- tests/test_cron_model_routing.py: 16 tests
Config:
cron_model_routing:
enabled: true
fallback_model: "anthropic/claude-sonnet-4"
fallback_provider: "openrouter"
windows:
- {start_hour: 17, end_hour: 22, reason: evening_error_peak}
- {start_hour: 2, end_hour: 5, reason: overnight_api_instability}
Features: midnight-wrap, per-window overrides, first-match-wins,
graceful degradation on malformed config.
Closes #317
129 lines
5.5 KiB
Python
129 lines
5.5 KiB
Python
"""Tests for time-aware cron model routing — Issue #317."""
|
|
|
|
import pytest
|
|
from datetime import datetime
|
|
|
|
from agent.smart_model_routing import resolve_cron_model, _hour_in_window
|
|
|
|
|
|
class TestHourInWindow:
|
|
"""Hour-in-window detection including midnight wrap."""
|
|
|
|
def test_normal_window(self):
|
|
assert _hour_in_window(18, 17, 22) is True
|
|
assert _hour_in_window(16, 17, 22) is False
|
|
assert _hour_in_window(22, 17, 22) is False
|
|
|
|
def test_midnight_wrap(self):
|
|
assert _hour_in_window(23, 22, 6) is True
|
|
assert _hour_in_window(3, 22, 6) is True
|
|
assert _hour_in_window(10, 22, 6) is False
|
|
|
|
def test_edge_cases(self):
|
|
assert _hour_in_window(0, 0, 24) is True
|
|
assert _hour_in_window(23, 0, 24) is True
|
|
assert _hour_in_window(0, 22, 6) is True
|
|
assert _hour_in_window(5, 22, 6) is True
|
|
assert _hour_in_window(6, 22, 6) is False
|
|
|
|
|
|
class TestResolveCronModel:
|
|
"""Time-aware model resolution for cron jobs."""
|
|
|
|
def _config(self, **overrides):
|
|
base = {
|
|
"enabled": True,
|
|
"fallback_model": "anthropic/claude-sonnet-4",
|
|
"fallback_provider": "openrouter",
|
|
"windows": [
|
|
{"start_hour": 17, "end_hour": 22, "reason": "evening_error_peak"},
|
|
],
|
|
}
|
|
base.update(overrides)
|
|
return base
|
|
|
|
def test_disabled_returns_base(self):
|
|
result = resolve_cron_model("mimo", {"enabled": False}, now=datetime(2026, 4, 12, 18, 0))
|
|
assert result["model"] == "mimo"
|
|
assert result["overridden"] is False
|
|
|
|
def test_no_config_returns_base(self):
|
|
result = resolve_cron_model("mimo", None)
|
|
assert result["model"] == "mimo"
|
|
assert result["overridden"] is False
|
|
|
|
def test_no_windows_returns_base(self):
|
|
result = resolve_cron_model("mimo", {"enabled": True, "windows": []}, now=datetime(2026, 4, 12, 18, 0))
|
|
assert result["overridden"] is False
|
|
|
|
def test_evening_window_overrides(self):
|
|
result = resolve_cron_model("mimo", self._config(), now=datetime(2026, 4, 12, 18, 0))
|
|
assert result["model"] == "anthropic/claude-sonnet-4"
|
|
assert result["provider"] == "openrouter"
|
|
assert result["overridden"] is True
|
|
assert "evening_error_peak" in result["reason"]
|
|
assert "hour=18" in result["reason"]
|
|
|
|
def test_outside_window_keeps_base(self):
|
|
result = resolve_cron_model("mimo", self._config(), now=datetime(2026, 4, 12, 9, 0))
|
|
assert result["model"] == "mimo"
|
|
assert result["overridden"] is False
|
|
|
|
def test_window_boundary_start_inclusive(self):
|
|
result = resolve_cron_model("mimo", self._config(), now=datetime(2026, 4, 12, 17, 0))
|
|
assert result["overridden"] is True
|
|
|
|
def test_window_boundary_end_exclusive(self):
|
|
result = resolve_cron_model("mimo", self._config(), now=datetime(2026, 4, 12, 22, 0))
|
|
assert result["overridden"] is False
|
|
|
|
def test_midnight_window(self):
|
|
config = self._config(windows=[{"start_hour": 22, "end_hour": 6, "reason": "overnight"}])
|
|
assert resolve_cron_model("mimo", config, now=datetime(2026, 4, 12, 23, 0))["overridden"] is True
|
|
assert resolve_cron_model("mimo", config, now=datetime(2026, 4, 13, 3, 0))["overridden"] is True
|
|
assert resolve_cron_model("mimo", config, now=datetime(2026, 4, 12, 10, 0))["overridden"] is False
|
|
|
|
def test_per_window_model_override(self):
|
|
config = self._config(windows=[{
|
|
"start_hour": 17, "end_hour": 22,
|
|
"model": "anthropic/claude-opus-4-6", "provider": "anthropic", "reason": "peak",
|
|
}])
|
|
result = resolve_cron_model("mimo", config, now=datetime(2026, 4, 12, 18, 0))
|
|
assert result["model"] == "anthropic/claude-opus-4-6"
|
|
assert result["provider"] == "anthropic"
|
|
|
|
def test_first_matching_window_wins(self):
|
|
config = self._config(windows=[
|
|
{"start_hour": 17, "end_hour": 20, "model": "strong-1", "provider": "p1", "reason": "w1"},
|
|
{"start_hour": 19, "end_hour": 22, "model": "strong-2", "provider": "p2", "reason": "w2"},
|
|
])
|
|
result = resolve_cron_model("mimo", config, now=datetime(2026, 4, 12, 19, 0))
|
|
assert result["model"] == "strong-1"
|
|
|
|
def test_no_fallback_model_keeps_base(self):
|
|
config = {"enabled": True, "windows": [{"start_hour": 17, "end_hour": 22, "reason": "test"}]}
|
|
result = resolve_cron_model("mimo", config, now=datetime(2026, 4, 12, 18, 0))
|
|
assert result["overridden"] is False
|
|
assert result["model"] == "mimo"
|
|
|
|
def test_malformed_windows_skipped(self):
|
|
config = self._config(windows=[
|
|
"not-a-dict",
|
|
{"start_hour": 17},
|
|
{"end_hour": 22},
|
|
{"start_hour": "bad", "end_hour": "bad"},
|
|
{"start_hour": 17, "end_hour": 22, "reason": "valid"},
|
|
])
|
|
result = resolve_cron_model("mimo", config, now=datetime(2026, 4, 12, 18, 0))
|
|
assert result["overridden"] is True
|
|
assert "valid" in result["reason"]
|
|
|
|
def test_multiple_windows_coverage(self):
|
|
config = self._config(windows=[
|
|
{"start_hour": 17, "end_hour": 22, "reason": "evening"},
|
|
{"start_hour": 2, "end_hour": 5, "reason": "overnight"},
|
|
])
|
|
assert resolve_cron_model("mimo", config, now=datetime(2026, 4, 12, 20, 0))["overridden"] is True
|
|
assert resolve_cron_model("mimo", config, now=datetime(2026, 4, 13, 3, 0))["overridden"] is True
|
|
assert resolve_cron_model("mimo", config, now=datetime(2026, 4, 12, 10, 0))["overridden"] is False
|