feat: keep live snapshots usable through refresh failures (#149)

tests/test_live_snapshot.py

@@ -11,10 +11,14 @@ def reset_live_snapshot_task():
     main._live_snapshot_task = None
     main._live_snapshot_value = None
     main._live_snapshot_created_at = None
+    main._live_snapshot_failure_count = 0
+    main._live_snapshot_retry_at = None
     yield
     main._live_snapshot_task = None
     main._live_snapshot_value = None
     main._live_snapshot_created_at = None
+    main._live_snapshot_failure_count = 0
+    main._live_snapshot_retry_at = None
 
 
 def payload(response):
@@ -238,6 +242,157 @@ async def test_stale_snapshot_returns_immediately_while_one_refresh_revalidates(
     assert builds == 2
 
 
+@pytest.mark.anyio
+async def test_failed_revalidation_enters_cooldown_and_keeps_last_snapshot(monkeypatch):
+    now = 100.0
+    builds = 0
+
+    async def snapshot():
+        nonlocal builds
+        builds += 1
+        if builds > 1:
+            raise ConnectionError("private upstream failure")
+        return {
+            "context": {"generation": 1},
+            "events": [{"id": 1}],
+            "notifications": [],
+            "sections": {
+                "context": "fresh",
+                "events": "fresh",
+                "notifications": "fresh",
+            },
+        }
+
+    monkeypatch.setattr(main.time, "monotonic", lambda: now)
+    monkeypatch.setattr(main, "_build_live_snapshot", snapshot)
+
+    await main.live_snapshot()
+    now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1
+    stale = payload(await main.live_snapshot())
+    await asyncio.sleep(0)
+    degraded = payload(await main.live_snapshot())
+
+    assert stale["freshness"]["revalidating"] is True
+    assert degraded["context"] == {"generation": 1}
+    assert degraded["freshness"]["stale"] is True
+    assert degraded["freshness"]["degraded"] is True
+    assert degraded["freshness"]["last_refresh_failed"] is True
+    assert degraded["freshness"]["revalidating"] is False
+    assert degraded["freshness"]["retry_in_seconds"] > 0
+    assert builds == 2
+    assert "private" not in json.dumps(degraded)
+
+
+@pytest.mark.anyio
+async def test_partial_refresh_updates_fresh_sections_and_retains_failed_sections(monkeypatch):
+    now = 100.0
+    builds = 0
+
+    async def snapshot():
+        nonlocal builds
+        builds += 1
+        if builds == 1:
+            return {
+                "context": {"generation": 1},
+                "events": [{"id": "last-known"}],
+                "notifications": [{"id": 7}],
+                "notification_pagination": {"page": 1, "total": 1, "has_more": False},
+                "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"},
+            }
+        return {
+            "context": {"generation": 2},
+            "events": None,
+            "notifications": [{"id": 8}],
+            "notification_pagination": {"page": 1, "total": 1, "has_more": False},
+            "sections": {
+                "context": "fresh",
+                "events": "temporarily unavailable",
+                "notifications": "fresh",
+            },
+        }
+
+    monkeypatch.setattr(main.time, "monotonic", lambda: now)
+    monkeypatch.setattr(main, "_build_live_snapshot", snapshot)
+
+    await main.live_snapshot()
+    now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1
+    await main.live_snapshot()
+    await asyncio.sleep(0)
+    result = payload(await main.live_snapshot())
+
+    assert result["context"] == {"generation": 2}
+    assert result["events"] == [{"id": "last-known"}]
+    assert result["notifications"] == [{"id": 8}]
+    assert result["sections"] == {
+        "context": "fresh",
+        "events": "stale",
+        "notifications": "fresh",
+    }
+    assert result["freshness"]["degraded"] is True
+    assert result["freshness"]["revalidating"] is False
+
+
+@pytest.mark.anyio
+async def test_retry_window_starts_one_refresh_and_success_clears_degraded_state(monkeypatch):
+    now = 100.0
+    builds = 0
+    retry_started = asyncio.Event()
+    release_retry = asyncio.Event()
+
+    async def snapshot():
+        nonlocal builds
+        builds += 1
+        if builds == 1:
+            return {
+                "context": {"generation": 1}, "events": [], "notifications": [],
+                "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"},
+            }
+        if builds == 2:
+            return {
+                "context": None, "events": [], "notifications": [],
+                "sections": {
+                    "context": "temporarily unavailable",
+                    "events": "fresh",
+                    "notifications": "fresh",
+                },
+            }
+        retry_started.set()
+        await release_retry.wait()
+        return {
+            "context": {"generation": 3}, "events": [], "notifications": [],
+            "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"},
+        }
+
+    monkeypatch.setattr(main.time, "monotonic", lambda: now)
+    monkeypatch.setattr(main, "_build_live_snapshot", snapshot)
+
+    await main.live_snapshot()
+    now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1
+    await main.live_snapshot()
+    await asyncio.sleep(0)
+    now += main.LIVE_SNAPSHOT_RETRY_BASE_SECONDS
+
+    first = asyncio.create_task(main.live_snapshot())
+    second = asyncio.create_task(main.live_snapshot())
+    await asyncio.sleep(0)
+    await asyncio.sleep(0)
+    assert retry_started.is_set()
+    assert builds == 3
+    assert payload(await first)["freshness"]["revalidating"] is True
+    assert payload(await second)["freshness"]["revalidating"] is True
+
+    release_retry.set()
+    refresh_task = main._live_snapshot_task
+    assert refresh_task is not None
+    await refresh_task
+    recovered = payload(await main.live_snapshot())
+
+    assert recovered["context"] == {"generation": 3}
+    assert recovered["freshness"]["degraded"] is False
+    assert recovered["freshness"]["last_refresh_failed"] is False
+    assert recovered["freshness"]["retry_in_seconds"] == 0
+
+
 @pytest.mark.anyio
 async def test_cancelling_one_waiter_does_not_cancel_the_shared_snapshot(monkeypatch):
     started = asyncio.Event()

tests/test_shared_context_snapshot.py

@@ -35,3 +35,12 @@ async def test_dashboard_announces_when_recent_snapshot_is_revalidating():
 
     assert "snapshot.freshness?.revalidating" in html
     assert "Refreshing · showing recent snapshot" in html
+
+
+@pytest.mark.anyio
+async def test_dashboard_announces_failed_refresh_and_retry_without_blanking_panels():
+    html = await dashboard()
+
+    assert "snapshot.freshness?.degraded" in html
+    assert "Refresh failed · showing last known data" in html
+    assert "snapshot.freshness.retry_in_seconds" in html