Preserve healthy live sections during partial upstream timeouts #1430

Merged
timmy merged 1 commits from timmy/1429-section-isolated-live-snapshot-deadlines into main 2026-08-26 11:55:23 +00:00
2 changed files with 115 additions and 16 deletions

View File

@ -286,6 +286,7 @@ app = FastAPI(title="Stackchain Dashboard", lifespan=lifespan)
app.add_middleware(RequestBodyLimitMiddleware, limit_for=request_body_limit)
app.add_middleware(NegotiatedGZipMiddleware, minimum_size=1_024)
CONTEXT_TIMEOUT_SECONDS = 5.0
LIVE_SNAPSHOT_DEADLINE_GRACE_SECONDS = 0.1
EVENT_STREAM_TIMEOUT_SECONDS = 5.0
READINESS_TIMEOUT_SECONDS = 5.0
READINESS_INTERVAL_SECONDS = max(
@ -4832,28 +4833,43 @@ async def _load_context_for_user(user_data: dict) -> dict:
async def _build_live_snapshot(sections: set[str] | None = None) -> dict:
requested = set(sections or LIVE_SNAPSHOT_SECTIONS)
results: dict[str, object] = {}
user_data: dict | None = None
user_task: asyncio.Task | None = None
if requested & {"context", "events"}:
user_task = asyncio.create_task(current_user())
async def load_user_section(section: str) -> object:
assert user_task is not None
user_data = await asyncio.shield(user_task)
if not isinstance(user_data, dict) or not user_data.get("login"):
raise ContextPayloadError("Gitea current-user response was invalid")
if section == "context":
return await _load_context_for_user(user_data)
return await activity_events(user_data)
async def load_before_deadline(load: Awaitable[Any]) -> object:
try:
user_data = await current_user()
if not isinstance(user_data, dict) or not user_data.get("login"):
raise ContextPayloadError("Gitea current-user response was invalid")
async with asyncio.timeout(CONTEXT_TIMEOUT_SECONDS):
return await load
except Exception as exc:
for section in requested & {"context", "events"}:
results[section] = exc
return exc
loads: dict[str, Awaitable[Any]] = {}
if "context" in requested and "context" not in results:
assert user_data is not None
loads["context"] = _load_context_for_user(user_data)
if "events" in requested and "events" not in results:
assert user_data is not None
loads["events"] = activity_events(user_data)
if "context" in requested:
loads["context"] = load_user_section("context")
if "events" in requested:
loads["events"] = load_user_section("events")
if "notifications" in requested:
loads["notifications"] = notifications()
if loads:
loaded = await asyncio.gather(*loads.values(), return_exceptions=True)
results.update(zip(loads, loaded))
try:
if loads:
loaded = await asyncio.gather(
*(load_before_deadline(load) for load in loads.values())
)
results.update(zip(loads, loaded))
finally:
if user_task is not None and not user_task.done():
user_task.cancel()
await asyncio.gather(user_task, return_exceptions=True)
context_result = results.get("context")
events_result = results.get("events")
@ -4892,7 +4908,9 @@ async def _build_live_snapshot(sections: set[str] | None = None) -> dict:
async def _build_live_snapshot_before_deadline(sections: set[str]) -> dict:
async with asyncio.timeout(CONTEXT_TIMEOUT_SECONDS):
async with asyncio.timeout(
CONTEXT_TIMEOUT_SECONDS + LIVE_SNAPSHOT_DEADLINE_GRACE_SECONDS
):
return await _build_live_snapshot(sections)

View File

@ -103,6 +103,87 @@ async def test_live_snapshot_fetches_user_once_and_updates_work_and_activity(mon
assert result["revisions"]["context"].endswith(".1")
@pytest.mark.anyio
async def test_notification_timeout_preserves_healthy_context_and_events(monkeypatch):
async def user():
return {"id": 1, "login": "timmy"}
async def empty():
return []
async def events(_authenticated_user):
return [{"type": "push"}]
async def stalled_notifications():
await asyncio.Event().wait()
monkeypatch.setattr(main, "CONTEXT_TIMEOUT_SECONDS", 0.01)
monkeypatch.setattr(main, "current_user", user)
monkeypatch.setattr(main, "repos", empty)
monkeypatch.setattr(main, "issues", empty)
monkeypatch.setattr(main, "pull_requests", empty)
monkeypatch.setattr(main, "activity_events", events)
monkeypatch.setattr(main, "notifications", stalled_notifications)
response = await main.live_snapshot()
result = payload(response)
assert response.status_code == 200
assert result["context"]["user"]["login"] == "timmy"
assert result["events"] == [{"type": "push"}]
assert result["notifications"] is None
assert result["sections"] == {
"context": "fresh",
"events": "fresh",
"notifications": "temporarily unavailable",
}
assert result["freshness"]["sections"]["context"]["degraded"] is False
assert result["freshness"]["sections"]["events"]["degraded"] is False
assert result["freshness"]["sections"]["notifications"]["degraded"] is True
assert main._live_section_failure_count == {
"context": 0,
"events": 0,
"notifications": 1,
}
@pytest.mark.anyio
async def test_identity_timeout_preserves_independent_notifications(monkeypatch):
identity_cancelled = asyncio.Event()
async def stalled_user():
try:
await asyncio.Event().wait()
finally:
identity_cancelled.set()
async def updates():
return [{"id": 42, "title": "Mentioned you"}]
monkeypatch.setattr(main, "CONTEXT_TIMEOUT_SECONDS", 0.01)
monkeypatch.setattr(main, "current_user", stalled_user)
monkeypatch.setattr(main, "notifications", updates)
response = await main.live_snapshot()
result = payload(response)
assert response.status_code == 200
assert result["context"] is None
assert result["events"] is None
assert result["notifications"] == [{"id": 42, "title": "Mentioned you"}]
assert result["sections"] == {
"context": "temporarily unavailable",
"events": "temporarily unavailable",
"notifications": "fresh",
}
assert main._live_section_failure_count == {
"context": 1,
"events": 1,
"notifications": 0,
}
await asyncio.wait_for(identity_cancelled.wait(), timeout=0.1)
@pytest.mark.anyio
async def test_live_snapshot_omits_section_bodies_at_known_revisions(monkeypatch):
async def user():