diff --git a/frontend/index.html b/frontend/index.html
index e8989bc..3f08bcc 100644
--- a/frontend/index.html
+++ b/frontend/index.html
@@ -715,7 +715,11 @@ textarea { resize: vertical; min-height: 120px; }
} else {
setEventStreamStatus('Update failed · showing last activity');
}
- if (snapshot.freshness?.revalidating) {
+ if (snapshot.freshness?.degraded && !snapshot.freshness.revalidating) {
+ const retrySeconds = Number(snapshot.freshness.retry_in_seconds) || 0;
+ setEventStreamStatus('Refresh failed · showing last known data' +
+ (retrySeconds > 0 ? ' · retrying in ' + retrySeconds + 's' : ''));
+ } else if (snapshot.freshness?.revalidating) {
setEventStreamStatus('Refreshing · showing recent snapshot');
}
}
diff --git a/src/main.py b/src/main.py
index 37478d1..7ff0048 100644
--- a/src/main.py
+++ b/src/main.py
@@ -57,10 +57,14 @@ NOTIFICATION_PAGE_TIMEOUT_SECONDS = 5.0
BULK_NOTIFICATION_CONCURRENCY = 5
BULK_NOTIFICATION_DEADLINE_SECONDS = 6.0
LIVE_SNAPSHOT_FRESHNESS_SECONDS = 8.0
+LIVE_SNAPSHOT_RETRY_BASE_SECONDS = 5.0
+LIVE_SNAPSHOT_RETRY_MAX_SECONDS = 60.0
FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend"
_live_snapshot_task: asyncio.Task | None = None
_live_snapshot_value: dict | None = None
_live_snapshot_created_at: float | None = None
+_live_snapshot_failure_count = 0
+_live_snapshot_retry_at: float | None = None
_read_notification_ids: set[int] = set()
@@ -292,12 +296,52 @@ async def _build_live_snapshot_before_deadline() -> dict:
return await _build_live_snapshot()
+def _record_live_snapshot_failure() -> None:
+ global _live_snapshot_failure_count, _live_snapshot_retry_at
+ _live_snapshot_failure_count += 1
+ delay = min(
+ LIVE_SNAPSHOT_RETRY_MAX_SECONDS,
+ LIVE_SNAPSHOT_RETRY_BASE_SECONDS * (2 ** (_live_snapshot_failure_count - 1)),
+ )
+ _live_snapshot_retry_at = time.monotonic() + delay
+
+
+def _merge_live_snapshot(previous: dict | None, refreshed: dict) -> dict:
+ if previous is None:
+ return refreshed
+ merged = dict(refreshed)
+ sections = dict(refreshed.get("sections") or {})
+ for section in ("context", "events", "notifications"):
+ if sections.get(section) == "fresh":
+ continue
+ if previous.get(section) is not None:
+ merged[section] = previous[section]
+ sections[section] = "stale"
+ if section == "notifications":
+ merged["notification_pagination"] = previous.get(
+ "notification_pagination"
+ )
+ merged["sections"] = sections
+ return merged
+
+
async def _refresh_live_snapshot() -> dict:
global _live_snapshot_value, _live_snapshot_created_at
- result = await _build_live_snapshot_before_deadline()
+ global _live_snapshot_failure_count, _live_snapshot_retry_at
+ try:
+ result = await _build_live_snapshot_before_deadline()
+ except Exception:
+ _record_live_snapshot_failure()
+ raise
+ result = _merge_live_snapshot(_live_snapshot_value, result)
result = _without_read_notifications(result)
_live_snapshot_value = result
_live_snapshot_created_at = time.monotonic()
+ if any(state != "fresh" for state in result.get("sections", {}).values()):
+ _record_live_snapshot_failure()
+ else:
+ _live_snapshot_failure_count = 0
+ _live_snapshot_retry_at = None
return result
@@ -322,11 +366,19 @@ def _live_snapshot_payload(value: dict, *, stale: bool, revalidating: bool) -> d
if _live_snapshot_created_at is not None
else 0.0
)
+ retry_in_seconds = (
+ max(0.0, _live_snapshot_retry_at - time.monotonic())
+ if _live_snapshot_retry_at is not None
+ else 0.0
+ )
payload["freshness"] = {
"age_seconds": round(age, 3),
"fresh_for_seconds": LIVE_SNAPSHOT_FRESHNESS_SECONDS,
"stale": stale,
"revalidating": revalidating,
+ "degraded": _live_snapshot_retry_at is not None,
+ "last_refresh_failed": _live_snapshot_retry_at is not None,
+ "retry_in_seconds": math.ceil(retry_in_seconds),
}
return payload
@@ -374,6 +426,22 @@ async def live_snapshot() -> JSONResponse:
"""Return a freshness-bounded snapshot and share identical upstream loads."""
global _live_snapshot_task, _live_snapshot_value, _live_snapshot_created_at
now = time.monotonic()
+ if (
+ _live_snapshot_value is not None
+ and _live_snapshot_retry_at is not None
+ ):
+ if now < _live_snapshot_retry_at:
+ return JSONResponse(
+ _live_snapshot_payload(
+ _live_snapshot_value, stale=True, revalidating=False
+ )
+ )
+ _start_live_snapshot_refresh()
+ return JSONResponse(
+ _live_snapshot_payload(
+ _live_snapshot_value, stale=True, revalidating=True
+ )
+ )
if (
_live_snapshot_value is not None
and _live_snapshot_created_at is not None
diff --git a/tests/test_live_snapshot.py b/tests/test_live_snapshot.py
index 36c581c..22d16e2 100644
--- a/tests/test_live_snapshot.py
+++ b/tests/test_live_snapshot.py
@@ -11,10 +11,14 @@ def reset_live_snapshot_task():
main._live_snapshot_task = None
main._live_snapshot_value = None
main._live_snapshot_created_at = None
+ main._live_snapshot_failure_count = 0
+ main._live_snapshot_retry_at = None
yield
main._live_snapshot_task = None
main._live_snapshot_value = None
main._live_snapshot_created_at = None
+ main._live_snapshot_failure_count = 0
+ main._live_snapshot_retry_at = None
def payload(response):
@@ -238,6 +242,157 @@ async def test_stale_snapshot_returns_immediately_while_one_refresh_revalidates(
assert builds == 2
+@pytest.mark.anyio
+async def test_failed_revalidation_enters_cooldown_and_keeps_last_snapshot(monkeypatch):
+ now = 100.0
+ builds = 0
+
+ async def snapshot():
+ nonlocal builds
+ builds += 1
+ if builds > 1:
+ raise ConnectionError("private upstream failure")
+ return {
+ "context": {"generation": 1},
+ "events": [{"id": 1}],
+ "notifications": [],
+ "sections": {
+ "context": "fresh",
+ "events": "fresh",
+ "notifications": "fresh",
+ },
+ }
+
+ monkeypatch.setattr(main.time, "monotonic", lambda: now)
+ monkeypatch.setattr(main, "_build_live_snapshot", snapshot)
+
+ await main.live_snapshot()
+ now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1
+ stale = payload(await main.live_snapshot())
+ await asyncio.sleep(0)
+ degraded = payload(await main.live_snapshot())
+
+ assert stale["freshness"]["revalidating"] is True
+ assert degraded["context"] == {"generation": 1}
+ assert degraded["freshness"]["stale"] is True
+ assert degraded["freshness"]["degraded"] is True
+ assert degraded["freshness"]["last_refresh_failed"] is True
+ assert degraded["freshness"]["revalidating"] is False
+ assert degraded["freshness"]["retry_in_seconds"] > 0
+ assert builds == 2
+ assert "private" not in json.dumps(degraded)
+
+
+@pytest.mark.anyio
+async def test_partial_refresh_updates_fresh_sections_and_retains_failed_sections(monkeypatch):
+ now = 100.0
+ builds = 0
+
+ async def snapshot():
+ nonlocal builds
+ builds += 1
+ if builds == 1:
+ return {
+ "context": {"generation": 1},
+ "events": [{"id": "last-known"}],
+ "notifications": [{"id": 7}],
+ "notification_pagination": {"page": 1, "total": 1, "has_more": False},
+ "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"},
+ }
+ return {
+ "context": {"generation": 2},
+ "events": None,
+ "notifications": [{"id": 8}],
+ "notification_pagination": {"page": 1, "total": 1, "has_more": False},
+ "sections": {
+ "context": "fresh",
+ "events": "temporarily unavailable",
+ "notifications": "fresh",
+ },
+ }
+
+ monkeypatch.setattr(main.time, "monotonic", lambda: now)
+ monkeypatch.setattr(main, "_build_live_snapshot", snapshot)
+
+ await main.live_snapshot()
+ now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1
+ await main.live_snapshot()
+ await asyncio.sleep(0)
+ result = payload(await main.live_snapshot())
+
+ assert result["context"] == {"generation": 2}
+ assert result["events"] == [{"id": "last-known"}]
+ assert result["notifications"] == [{"id": 8}]
+ assert result["sections"] == {
+ "context": "fresh",
+ "events": "stale",
+ "notifications": "fresh",
+ }
+ assert result["freshness"]["degraded"] is True
+ assert result["freshness"]["revalidating"] is False
+
+
+@pytest.mark.anyio
+async def test_retry_window_starts_one_refresh_and_success_clears_degraded_state(monkeypatch):
+ now = 100.0
+ builds = 0
+ retry_started = asyncio.Event()
+ release_retry = asyncio.Event()
+
+ async def snapshot():
+ nonlocal builds
+ builds += 1
+ if builds == 1:
+ return {
+ "context": {"generation": 1}, "events": [], "notifications": [],
+ "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"},
+ }
+ if builds == 2:
+ return {
+ "context": None, "events": [], "notifications": [],
+ "sections": {
+ "context": "temporarily unavailable",
+ "events": "fresh",
+ "notifications": "fresh",
+ },
+ }
+ retry_started.set()
+ await release_retry.wait()
+ return {
+ "context": {"generation": 3}, "events": [], "notifications": [],
+ "sections": {"context": "fresh", "events": "fresh", "notifications": "fresh"},
+ }
+
+ monkeypatch.setattr(main.time, "monotonic", lambda: now)
+ monkeypatch.setattr(main, "_build_live_snapshot", snapshot)
+
+ await main.live_snapshot()
+ now += main.LIVE_SNAPSHOT_FRESHNESS_SECONDS + 1
+ await main.live_snapshot()
+ await asyncio.sleep(0)
+ now += main.LIVE_SNAPSHOT_RETRY_BASE_SECONDS
+
+ first = asyncio.create_task(main.live_snapshot())
+ second = asyncio.create_task(main.live_snapshot())
+ await asyncio.sleep(0)
+ await asyncio.sleep(0)
+ assert retry_started.is_set()
+ assert builds == 3
+ assert payload(await first)["freshness"]["revalidating"] is True
+ assert payload(await second)["freshness"]["revalidating"] is True
+
+ release_retry.set()
+ refresh_task = main._live_snapshot_task
+ assert refresh_task is not None
+ await refresh_task
+ recovered = payload(await main.live_snapshot())
+
+ assert recovered["context"] == {"generation": 3}
+ assert recovered["freshness"]["degraded"] is False
+ assert recovered["freshness"]["last_refresh_failed"] is False
+ assert recovered["freshness"]["retry_in_seconds"] == 0
+
+
@pytest.mark.anyio
async def test_cancelling_one_waiter_does_not_cancel_the_shared_snapshot(monkeypatch):
started = asyncio.Event()
diff --git a/tests/test_shared_context_snapshot.py b/tests/test_shared_context_snapshot.py
index f78787a..897551b 100644
--- a/tests/test_shared_context_snapshot.py
+++ b/tests/test_shared_context_snapshot.py
@@ -35,3 +35,12 @@ async def test_dashboard_announces_when_recent_snapshot_is_revalidating():
assert "snapshot.freshness?.revalidating" in html
assert "Refreshing · showing recent snapshot" in html
+
+
+@pytest.mark.anyio
+async def test_dashboard_announces_failed_refresh_and_retry_without_blanking_panels():
+ html = await dashboard()
+
+ assert "snapshot.freshness?.degraded" in html
+ assert "Refresh failed · showing last known data" in html
+ assert "snapshot.freshness.retry_in_seconds" in html