feat: keep live snapshots usable through refresh failures (#149)

frontend/index.html

@@ -715,7 +715,11 @@ textarea { resize: vertical; min-height: 120px; }
     } else {
       setEventStreamStatus('Update failed · showing last activity');
     }
-    if (snapshot.freshness?.revalidating) {
+    if (snapshot.freshness?.degraded && !snapshot.freshness.revalidating) {
+      const retrySeconds = Number(snapshot.freshness.retry_in_seconds) || 0;
+      setEventStreamStatus('Refresh failed · showing last known data' +
+        (retrySeconds > 0 ? ' · retrying in ' + retrySeconds + 's' : ''));
+    } else if (snapshot.freshness?.revalidating) {
       setEventStreamStatus('Refreshing · showing recent snapshot');
     }
   }

src/main.py

@@ -57,10 +57,14 @@ NOTIFICATION_PAGE_TIMEOUT_SECONDS = 5.0
 BULK_NOTIFICATION_CONCURRENCY = 5
 BULK_NOTIFICATION_DEADLINE_SECONDS = 6.0
 LIVE_SNAPSHOT_FRESHNESS_SECONDS = 8.0
+LIVE_SNAPSHOT_RETRY_BASE_SECONDS = 5.0
+LIVE_SNAPSHOT_RETRY_MAX_SECONDS = 60.0
 FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend"
 _live_snapshot_task: asyncio.Task | None = None
 _live_snapshot_value: dict | None = None
 _live_snapshot_created_at: float | None = None
+_live_snapshot_failure_count = 0
+_live_snapshot_retry_at: float | None = None
 _read_notification_ids: set[int] = set()
 
 
@@ -292,12 +296,52 @@ async def _build_live_snapshot_before_deadline() -> dict:
         return await _build_live_snapshot()
 
 
+def _record_live_snapshot_failure() -> None:
+    global _live_snapshot_failure_count, _live_snapshot_retry_at
+    _live_snapshot_failure_count += 1
+    delay = min(
+        LIVE_SNAPSHOT_RETRY_MAX_SECONDS,
+        LIVE_SNAPSHOT_RETRY_BASE_SECONDS * (2 ** (_live_snapshot_failure_count - 1)),
+    )
+    _live_snapshot_retry_at = time.monotonic() + delay
+
+
+def _merge_live_snapshot(previous: dict | None, refreshed: dict) -> dict:
+    if previous is None:
+        return refreshed
+    merged = dict(refreshed)
+    sections = dict(refreshed.get("sections") or {})
+    for section in ("context", "events", "notifications"):
+        if sections.get(section) == "fresh":
+            continue
+        if previous.get(section) is not None:
+            merged[section] = previous[section]
+            sections[section] = "stale"
+            if section == "notifications":
+                merged["notification_pagination"] = previous.get(
+                    "notification_pagination"
+                )
+    merged["sections"] = sections
+    return merged
+
+
 async def _refresh_live_snapshot() -> dict:
     global _live_snapshot_value, _live_snapshot_created_at
-    result = await _build_live_snapshot_before_deadline()
+    global _live_snapshot_failure_count, _live_snapshot_retry_at
+    try:
+        result = await _build_live_snapshot_before_deadline()
+    except Exception:
+        _record_live_snapshot_failure()
+        raise
+    result = _merge_live_snapshot(_live_snapshot_value, result)
     result = _without_read_notifications(result)
     _live_snapshot_value = result
     _live_snapshot_created_at = time.monotonic()
+    if any(state != "fresh" for state in result.get("sections", {}).values()):
+        _record_live_snapshot_failure()
+    else:
+        _live_snapshot_failure_count = 0
+        _live_snapshot_retry_at = None
     return result
 
 
@@ -322,11 +366,19 @@ def _live_snapshot_payload(value: dict, *, stale: bool, revalidating: bool) -> d
         if _live_snapshot_created_at is not None
         else 0.0
     )
+    retry_in_seconds = (
+        max(0.0, _live_snapshot_retry_at - time.monotonic())
+        if _live_snapshot_retry_at is not None
+        else 0.0
+    )
     payload["freshness"] = {
         "age_seconds": round(age, 3),
         "fresh_for_seconds": LIVE_SNAPSHOT_FRESHNESS_SECONDS,
         "stale": stale,
         "revalidating": revalidating,
+        "degraded": _live_snapshot_retry_at is not None,
+        "last_refresh_failed": _live_snapshot_retry_at is not None,
+        "retry_in_seconds": math.ceil(retry_in_seconds),
     }
     return payload
 
@@ -374,6 +426,22 @@ async def live_snapshot() -> JSONResponse:
     """Return a freshness-bounded snapshot and share identical upstream loads."""
     global _live_snapshot_task, _live_snapshot_value, _live_snapshot_created_at
     now = time.monotonic()
+    if (
+        _live_snapshot_value is not None
+        and _live_snapshot_retry_at is not None
+    ):
+        if now < _live_snapshot_retry_at:
+            return JSONResponse(
+                _live_snapshot_payload(
+                    _live_snapshot_value, stale=True, revalidating=False
+                )
+            )
+        _start_live_snapshot_refresh()
+        return JSONResponse(
+            _live_snapshot_payload(
+                _live_snapshot_value, stale=True, revalidating=True
+            )
+        )
     if (
         _live_snapshot_value is not None
         and _live_snapshot_created_at is not None

tests/test_live_snapshot.py

@@ -11,10 +11,14 @@ def reset_live_snapshot_task():
     main._live_snapshot_task = None
     main._live_snapshot_value = None
     main._live_snapshot_created_at = None
+    main._live_snapshot_failure_count = 0
+    main._live_snapshot_retry_at = None
     yield
     main._live_snapshot_task = None
     main._live_snapshot_value = None
     main._live_snapshot_created_at = None
+    main._live_snapshot_failure_count = 0
+    main._live_snapshot_retry_at = None
 
 
 def payload(response):
@@ -238,6 +242,157 @@ async def test_stale_snapshot_returns_immediately_while_one_refresh_revalidates(
     assert builds == 2
 
 
+@pytest.mark.anyio
+async def test_failed_revalidation_enters_cooldown_and_keeps_last_snapshot(monkeypatch):
+    now = 100.0
+    builds = 0
+
+    async def snapshot():
+        nonlocal builds
+        builds += 1
+        if builds > 1:
+            raise ConnectionError("private upstream failure")
+        return {
+            "context": {"generation": 1},
+            "events": [{"id": 1}],
+            "notifications": [],
+            "sections": {
+                "context": "fresh",
+                "events": "fresh",
+                "notifications": "fresh",
+            },
+        }
+
+    monkeypatch.setattr(main.time, "monotonic", lambda: now)
+    monkeypatch.setattr(main, "_build_live_snapshot", snapshot)
+
+    await main.live_snapshot()
+    now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1
+    stale = payload(await main.live_snapshot())
+    await asyncio.sleep(0)
+    degraded = payload(await main.live_snapshot())
+
+    assert stale["freshness"]["revalidating"] is True
+    assert degraded["context"] == {"generation": 1}
+    assert degraded["freshness"]["stale"] is True
+    assert degraded["freshness"]["degraded"] is True
+    assert degraded["freshness"]["last_refresh_failed"] is True
+    assert degraded["freshness"]["revalidating"] is False
+    assert degraded["freshness"]["retry_in_seconds"] > 0
+    assert builds == 2
+    assert "private" not in json.dumps(degraded)
+
+
+@pytest.mark.anyio
+async def test_partial_refresh_updates_fresh_sections_and_retains_failed_sections(monkeypatch):
+    now = 100.0
+    builds = 0
+
+    async def snapshot():
+        nonlocal builds
+        builds += 1
+        if builds == 1:
+            return {
+                "context": {"generation": 1},
+                "events": [{"id": "last-known"}],
+                "notifications": [{"id": 7}],
+                "notification_pagination": {"page": 1, "total": 1, "has_more": False},
+                "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"},
+            }
+        return {
+            "context": {"generation": 2},
+            "events": None,
+            "notifications": [{"id": 8}],
+            "notification_pagination": {"page": 1, "total": 1, "has_more": False},
+            "sections": {
+                "context": "fresh",
+                "events": "temporarily unavailable",
+                "notifications": "fresh",
+            },
+        }
+
+    monkeypatch.setattr(main.time, "monotonic", lambda: now)
+    monkeypatch.setattr(main, "_build_live_snapshot", snapshot)
+
+    await main.live_snapshot()
+    now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1
+    await main.live_snapshot()
+    await asyncio.sleep(0)
+    result = payload(await main.live_snapshot())
+
+    assert result["context"] == {"generation": 2}
+    assert result["events"] == [{"id": "last-known"}]
+    assert result["notifications"] == [{"id": 8}]
+    assert result["sections"] == {
+        "context": "fresh",
+        "events": "stale",
+        "notifications": "fresh",
+    }
+    assert result["freshness"]["degraded"] is True
+    assert result["freshness"]["revalidating"] is False
+
+
+@pytest.mark.anyio
+async def test_retry_window_starts_one_refresh_and_success_clears_degraded_state(monkeypatch):
+    now = 100.0
+    builds = 0
+    retry_started = asyncio.Event()
+    release_retry = asyncio.Event()
+
+    async def snapshot():
+        nonlocal builds
+        builds += 1
+        if builds == 1:
+            return {
+                "context": {"generation": 1}, "events": [], "notifications": [],
+                "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"},
+            }
+        if builds == 2:
+            return {
+                "context": None, "events": [], "notifications": [],
+                "sections": {
+                    "context": "temporarily unavailable",
+                    "events": "fresh",
+                    "notifications": "fresh",
+                },
+            }
+        retry_started.set()
+        await release_retry.wait()
+        return {
+            "context": {"generation": 3}, "events": [], "notifications": [],
+            "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"},
+        }
+
+    monkeypatch.setattr(main.time, "monotonic", lambda: now)
+    monkeypatch.setattr(main, "_build_live_snapshot", snapshot)
+
+    await main.live_snapshot()
+    now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1
+    await main.live_snapshot()
+    await asyncio.sleep(0)
+    now += main.LIVE_SNAPSHOT_RETRY_BASE_SECONDS
+
+    first = asyncio.create_task(main.live_snapshot())
+    second = asyncio.create_task(main.live_snapshot())
+    await asyncio.sleep(0)
+    await asyncio.sleep(0)
+    assert retry_started.is_set()
+    assert builds == 3
+    assert payload(await first)["freshness"]["revalidating"] is True
+    assert payload(await second)["freshness"]["revalidating"] is True
+
+    release_retry.set()
+    refresh_task = main._live_snapshot_task
+    assert refresh_task is not None
+    await refresh_task
+    recovered = payload(await main.live_snapshot())
+
+    assert recovered["context"] == {"generation": 3}
+    assert recovered["freshness"]["degraded"] is False
+    assert recovered["freshness"]["last_refresh_failed"] is False
+    assert recovered["freshness"]["retry_in_seconds"] == 0
+
+
 @pytest.mark.anyio
 async def test_cancelling_one_waiter_does_not_cancel_the_shared_snapshot(monkeypatch):
     started = asyncio.Event()

tests/test_shared_context_snapshot.py

@@ -35,3 +35,12 @@ async def test_dashboard_announces_when_recent_snapshot_is_revalidating():
 
     assert "snapshot.freshness?.revalidating" in html
     assert "Refreshing · showing recent snapshot" in html
+
+
+@pytest.mark.anyio
+async def test_dashboard_announces_failed_refresh_and_retry_without_blanking_panels():
+    html = await dashboard()
+
+    assert "snapshot.freshness?.degraded" in html
+    assert "Refresh failed · showing last known data" in html
+    assert "snapshot.freshness.retry_in_seconds" in html