From 8ee7dbf79ef39360b68a97db47275dbd17115b6e Mon Sep 17 00:00:00 2001 From: timmy Date: Thu, 6 Aug 2026 23:54:55 +0000 Subject: [PATCH] feat: keep live snapshots usable through refresh failures (#149) --- frontend/index.html | 6 +- src/main.py | 70 +++++++++++- tests/test_live_snapshot.py | 155 ++++++++++++++++++++++++++ tests/test_shared_context_snapshot.py | 9 ++ 4 files changed, 238 insertions(+), 2 deletions(-) diff --git a/frontend/index.html b/frontend/index.html index e8989bc..3f08bcc 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -715,7 +715,11 @@ textarea { resize: vertical; min-height: 120px; } } else { setEventStreamStatus('Update failed · showing last activity'); } - if (snapshot.freshness?.revalidating) { + if (snapshot.freshness?.degraded && !snapshot.freshness.revalidating) { + const retrySeconds = Number(snapshot.freshness.retry_in_seconds) || 0; + setEventStreamStatus('Refresh failed · showing last known data' + + (retrySeconds > 0 ? ' · retrying in ' + retrySeconds + 's' : '')); + } else if (snapshot.freshness?.revalidating) { setEventStreamStatus('Refreshing · showing recent snapshot'); } } diff --git a/src/main.py b/src/main.py index 37478d1..7ff0048 100644 --- a/src/main.py +++ b/src/main.py @@ -57,10 +57,14 @@ NOTIFICATION_PAGE_TIMEOUT_SECONDS = 5.0 BULK_NOTIFICATION_CONCURRENCY = 5 BULK_NOTIFICATION_DEADLINE_SECONDS = 6.0 LIVE_SNAPSHOT_FRESHNESS_SECONDS = 8.0 +LIVE_SNAPSHOT_RETRY_BASE_SECONDS = 5.0 +LIVE_SNAPSHOT_RETRY_MAX_SECONDS = 60.0 FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend" _live_snapshot_task: asyncio.Task | None = None _live_snapshot_value: dict | None = None _live_snapshot_created_at: float | None = None +_live_snapshot_failure_count = 0 +_live_snapshot_retry_at: float | None = None _read_notification_ids: set[int] = set() @@ -292,12 +296,52 @@ async def _build_live_snapshot_before_deadline() -> dict: return await _build_live_snapshot() +def _record_live_snapshot_failure() -> None: + global _live_snapshot_failure_count, _live_snapshot_retry_at + _live_snapshot_failure_count += 1 + delay = min( + LIVE_SNAPSHOT_RETRY_MAX_SECONDS, + LIVE_SNAPSHOT_RETRY_BASE_SECONDS * (2 ** (_live_snapshot_failure_count - 1)), + ) + _live_snapshot_retry_at = time.monotonic() + delay + + +def _merge_live_snapshot(previous: dict | None, refreshed: dict) -> dict: + if previous is None: + return refreshed + merged = dict(refreshed) + sections = dict(refreshed.get("sections") or {}) + for section in ("context", "events", "notifications"): + if sections.get(section) == "fresh": + continue + if previous.get(section) is not None: + merged[section] = previous[section] + sections[section] = "stale" + if section == "notifications": + merged["notification_pagination"] = previous.get( + "notification_pagination" + ) + merged["sections"] = sections + return merged + + async def _refresh_live_snapshot() -> dict: global _live_snapshot_value, _live_snapshot_created_at - result = await _build_live_snapshot_before_deadline() + global _live_snapshot_failure_count, _live_snapshot_retry_at + try: + result = await _build_live_snapshot_before_deadline() + except Exception: + _record_live_snapshot_failure() + raise + result = _merge_live_snapshot(_live_snapshot_value, result) result = _without_read_notifications(result) _live_snapshot_value = result _live_snapshot_created_at = time.monotonic() + if any(state != "fresh" for state in result.get("sections", {}).values()): + _record_live_snapshot_failure() + else: + _live_snapshot_failure_count = 0 + _live_snapshot_retry_at = None return result @@ -322,11 +366,19 @@ def _live_snapshot_payload(value: dict, *, stale: bool, revalidating: bool) -> d if _live_snapshot_created_at is not None else 0.0 ) + retry_in_seconds = ( + max(0.0, _live_snapshot_retry_at - time.monotonic()) + if _live_snapshot_retry_at is not None + else 0.0 + ) payload["freshness"] = { "age_seconds": round(age, 3), "fresh_for_seconds": LIVE_SNAPSHOT_FRESHNESS_SECONDS, "stale": stale, "revalidating": revalidating, + "degraded": _live_snapshot_retry_at is not None, + "last_refresh_failed": _live_snapshot_retry_at is not None, + "retry_in_seconds": math.ceil(retry_in_seconds), } return payload @@ -374,6 +426,22 @@ async def live_snapshot() -> JSONResponse: """Return a freshness-bounded snapshot and share identical upstream loads.""" global _live_snapshot_task, _live_snapshot_value, _live_snapshot_created_at now = time.monotonic() + if ( + _live_snapshot_value is not None + and _live_snapshot_retry_at is not None + ): + if now < _live_snapshot_retry_at: + return JSONResponse( + _live_snapshot_payload( + _live_snapshot_value, stale=True, revalidating=False + ) + ) + _start_live_snapshot_refresh() + return JSONResponse( + _live_snapshot_payload( + _live_snapshot_value, stale=True, revalidating=True + ) + ) if ( _live_snapshot_value is not None and _live_snapshot_created_at is not None diff --git a/tests/test_live_snapshot.py b/tests/test_live_snapshot.py index 36c581c..22d16e2 100644 --- a/tests/test_live_snapshot.py +++ b/tests/test_live_snapshot.py @@ -11,10 +11,14 @@ def reset_live_snapshot_task(): main._live_snapshot_task = None main._live_snapshot_value = None main._live_snapshot_created_at = None + main._live_snapshot_failure_count = 0 + main._live_snapshot_retry_at = None yield main._live_snapshot_task = None main._live_snapshot_value = None main._live_snapshot_created_at = None + main._live_snapshot_failure_count = 0 + main._live_snapshot_retry_at = None def payload(response): @@ -238,6 +242,157 @@ async def test_stale_snapshot_returns_immediately_while_one_refresh_revalidates( assert builds == 2 +@pytest.mark.anyio +async def test_failed_revalidation_enters_cooldown_and_keeps_last_snapshot(monkeypatch): + now = 100.0 + builds = 0 + + async def snapshot(): + nonlocal builds + builds += 1 + if builds > 1: + raise ConnectionError("private upstream failure") + return { + "context": {"generation": 1}, + "events": [{"id": 1}], + "notifications": [], + "sections": { + "context": "fresh", + "events": "fresh", + "notifications": "fresh", + }, + } + + monkeypatch.setattr(main.time, "monotonic", lambda: now) + monkeypatch.setattr(main, "_build_live_snapshot", snapshot) + + await main.live_snapshot() + now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1 + stale = payload(await main.live_snapshot()) + await asyncio.sleep(0) + degraded = payload(await main.live_snapshot()) + + assert stale["freshness"]["revalidating"] is True + assert degraded["context"] == {"generation": 1} + assert degraded["freshness"]["stale"] is True + assert degraded["freshness"]["degraded"] is True + assert degraded["freshness"]["last_refresh_failed"] is True + assert degraded["freshness"]["revalidating"] is False + assert degraded["freshness"]["retry_in_seconds"] > 0 + assert builds == 2 + assert "private" not in json.dumps(degraded) + + +@pytest.mark.anyio +async def test_partial_refresh_updates_fresh_sections_and_retains_failed_sections(monkeypatch): + now = 100.0 + builds = 0 + + async def snapshot(): + nonlocal builds + builds += 1 + if builds == 1: + return { + "context": {"generation": 1}, + "events": [{"id": "last-known"}], + "notifications": [{"id": 7}], + "notification_pagination": {"page": 1, "total": 1, "has_more": False}, + "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"}, + } + return { + "context": {"generation": 2}, + "events": None, + "notifications": [{"id": 8}], + "notification_pagination": {"page": 1, "total": 1, "has_more": False}, + "sections": { + "context": "fresh", + "events": "temporarily unavailable", + "notifications": "fresh", + }, + } + + monkeypatch.setattr(main.time, "monotonic", lambda: now) + monkeypatch.setattr(main, "_build_live_snapshot", snapshot) + + await main.live_snapshot() + now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1 + await main.live_snapshot() + await asyncio.sleep(0) + result = payload(await main.live_snapshot()) + + assert result["context"] == {"generation": 2} + assert result["events"] == [{"id": "last-known"}] + assert result["notifications"] == [{"id": 8}] + assert result["sections"] == { + "context": "fresh", + "events": "stale", + "notifications": "fresh", + } + assert result["freshness"]["degraded"] is True + assert result["freshness"]["revalidating"] is False + + +@pytest.mark.anyio +async def test_retry_window_starts_one_refresh_and_success_clears_degraded_state(monkeypatch): + now = 100.0 + builds = 0 + retry_started = asyncio.Event() + release_retry = asyncio.Event() + + async def snapshot(): + nonlocal builds + builds += 1 + if builds == 1: + return { + "context": {"generation": 1}, "events": [], "notifications": [], + "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"}, + } + if builds == 2: + return { + "context": None, "events": [], "notifications": [], + "sections": { + "context": "temporarily unavailable", + "events": "fresh", + "notifications": "fresh", + }, + } + retry_started.set() + await release_retry.wait() + return { + "context": {"generation": 3}, "events": [], "notifications": [], + "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"}, + } + + monkeypatch.setattr(main.time, "monotonic", lambda: now) + monkeypatch.setattr(main, "_build_live_snapshot", snapshot) + + await main.live_snapshot() + now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1 + await main.live_snapshot() + await asyncio.sleep(0) + now += main.LIVE_SNAPSHOT_RETRY_BASE_SECONDS + + first = asyncio.create_task(main.live_snapshot()) + second = asyncio.create_task(main.live_snapshot()) + await asyncio.sleep(0) + await asyncio.sleep(0) + assert retry_started.is_set() + assert builds == 3 + assert payload(await first)["freshness"]["revalidating"] is True + assert payload(await second)["freshness"]["revalidating"] is True + + release_retry.set() + refresh_task = main._live_snapshot_task + assert refresh_task is not None + await refresh_task + recovered = payload(await main.live_snapshot()) + + assert recovered["context"] == {"generation": 3} + assert recovered["freshness"]["degraded"] is False + assert recovered["freshness"]["last_refresh_failed"] is False + assert recovered["freshness"]["retry_in_seconds"] == 0 + + @pytest.mark.anyio async def test_cancelling_one_waiter_does_not_cancel_the_shared_snapshot(monkeypatch): started = asyncio.Event() diff --git a/tests/test_shared_context_snapshot.py b/tests/test_shared_context_snapshot.py index f78787a..897551b 100644 --- a/tests/test_shared_context_snapshot.py +++ b/tests/test_shared_context_snapshot.py @@ -35,3 +35,12 @@ async def test_dashboard_announces_when_recent_snapshot_is_revalidating(): assert "snapshot.freshness?.revalidating" in html assert "Refreshing · showing recent snapshot" in html + + +@pytest.mark.anyio +async def test_dashboard_announces_failed_refresh_and_retry_without_blanking_panels(): + html = await dashboard() + + assert "snapshot.freshness?.degraded" in html + assert "Refresh failed · showing last known data" in html + assert "snapshot.freshness.retry_in_seconds" in html