feat: isolate live snapshot section deadlines (Closes #1429)
All checks were successful
CI / lint (pull_request) Successful in 4m2s
CI / build-release (pull_request) Successful in 7s
CI / browser-journey (pull_request) Successful in 7m12s
CI / release-candidate (pull_request) Has been skipped

This commit is contained in:
timmy 2026-08-26 11:27:35 +00:00
parent 065cc46553
commit dc9e64e98e
2 changed files with 115 additions and 16 deletions

View File

@ -286,6 +286,7 @@ app = FastAPI(title="Stackchain Dashboard", lifespan=lifespan)
app.add_middleware(RequestBodyLimitMiddleware, limit_for=request_body_limit) app.add_middleware(RequestBodyLimitMiddleware, limit_for=request_body_limit)
app.add_middleware(NegotiatedGZipMiddleware, minimum_size=1_024) app.add_middleware(NegotiatedGZipMiddleware, minimum_size=1_024)
CONTEXT_TIMEOUT_SECONDS = 5.0 CONTEXT_TIMEOUT_SECONDS = 5.0
LIVE_SNAPSHOT_DEADLINE_GRACE_SECONDS = 0.1
EVENT_STREAM_TIMEOUT_SECONDS = 5.0 EVENT_STREAM_TIMEOUT_SECONDS = 5.0
READINESS_TIMEOUT_SECONDS = 5.0 READINESS_TIMEOUT_SECONDS = 5.0
READINESS_INTERVAL_SECONDS = max( READINESS_INTERVAL_SECONDS = max(
@ -4832,28 +4833,43 @@ async def _load_context_for_user(user_data: dict) -> dict:
async def _build_live_snapshot(sections: set[str] | None = None) -> dict: async def _build_live_snapshot(sections: set[str] | None = None) -> dict:
requested = set(sections or LIVE_SNAPSHOT_SECTIONS) requested = set(sections or LIVE_SNAPSHOT_SECTIONS)
results: dict[str, object] = {} results: dict[str, object] = {}
user_data: dict | None = None user_task: asyncio.Task | None = None
if requested & {"context", "events"}: if requested & {"context", "events"}:
user_task = asyncio.create_task(current_user())
async def load_user_section(section: str) -> object:
assert user_task is not None
user_data = await asyncio.shield(user_task)
if not isinstance(user_data, dict) or not user_data.get("login"):
raise ContextPayloadError("Gitea current-user response was invalid")
if section == "context":
return await _load_context_for_user(user_data)
return await activity_events(user_data)
async def load_before_deadline(load: Awaitable[Any]) -> object:
try: try:
user_data = await current_user() async with asyncio.timeout(CONTEXT_TIMEOUT_SECONDS):
if not isinstance(user_data, dict) or not user_data.get("login"): return await load
raise ContextPayloadError("Gitea current-user response was invalid")
except Exception as exc: except Exception as exc:
for section in requested & {"context", "events"}: return exc
results[section] = exc
loads: dict[str, Awaitable[Any]] = {} loads: dict[str, Awaitable[Any]] = {}
if "context" in requested and "context" not in results: if "context" in requested:
assert user_data is not None loads["context"] = load_user_section("context")
loads["context"] = _load_context_for_user(user_data) if "events" in requested:
if "events" in requested and "events" not in results: loads["events"] = load_user_section("events")
assert user_data is not None
loads["events"] = activity_events(user_data)
if "notifications" in requested: if "notifications" in requested:
loads["notifications"] = notifications() loads["notifications"] = notifications()
if loads: try:
loaded = await asyncio.gather(*loads.values(), return_exceptions=True) if loads:
results.update(zip(loads, loaded)) loaded = await asyncio.gather(
*(load_before_deadline(load) for load in loads.values())
)
results.update(zip(loads, loaded))
finally:
if user_task is not None and not user_task.done():
user_task.cancel()
await asyncio.gather(user_task, return_exceptions=True)
context_result = results.get("context") context_result = results.get("context")
events_result = results.get("events") events_result = results.get("events")
@ -4892,7 +4908,9 @@ async def _build_live_snapshot(sections: set[str] | None = None) -> dict:
async def _build_live_snapshot_before_deadline(sections: set[str]) -> dict: async def _build_live_snapshot_before_deadline(sections: set[str]) -> dict:
async with asyncio.timeout(CONTEXT_TIMEOUT_SECONDS): async with asyncio.timeout(
CONTEXT_TIMEOUT_SECONDS + LIVE_SNAPSHOT_DEADLINE_GRACE_SECONDS
):
return await _build_live_snapshot(sections) return await _build_live_snapshot(sections)

View File

@ -103,6 +103,87 @@ async def test_live_snapshot_fetches_user_once_and_updates_work_and_activity(mon
assert result["revisions"]["context"].endswith(".1") assert result["revisions"]["context"].endswith(".1")
@pytest.mark.anyio
async def test_notification_timeout_preserves_healthy_context_and_events(monkeypatch):
async def user():
return {"id": 1, "login": "timmy"}
async def empty():
return []
async def events(_authenticated_user):
return [{"type": "push"}]
async def stalled_notifications():
await asyncio.Event().wait()
monkeypatch.setattr(main, "CONTEXT_TIMEOUT_SECONDS", 0.01)
monkeypatch.setattr(main, "current_user", user)
monkeypatch.setattr(main, "repos", empty)
monkeypatch.setattr(main, "issues", empty)
monkeypatch.setattr(main, "pull_requests", empty)
monkeypatch.setattr(main, "activity_events", events)
monkeypatch.setattr(main, "notifications", stalled_notifications)
response = await main.live_snapshot()
result = payload(response)
assert response.status_code == 200
assert result["context"]["user"]["login"] == "timmy"
assert result["events"] == [{"type": "push"}]
assert result["notifications"] is None
assert result["sections"] == {
"context": "fresh",
"events": "fresh",
"notifications": "temporarily unavailable",
}
assert result["freshness"]["sections"]["context"]["degraded"] is False
assert result["freshness"]["sections"]["events"]["degraded"] is False
assert result["freshness"]["sections"]["notifications"]["degraded"] is True
assert main._live_section_failure_count == {
"context": 0,
"events": 0,
"notifications": 1,
}
@pytest.mark.anyio
async def test_identity_timeout_preserves_independent_notifications(monkeypatch):
identity_cancelled = asyncio.Event()
async def stalled_user():
try:
await asyncio.Event().wait()
finally:
identity_cancelled.set()
async def updates():
return [{"id": 42, "title": "Mentioned you"}]
monkeypatch.setattr(main, "CONTEXT_TIMEOUT_SECONDS", 0.01)
monkeypatch.setattr(main, "current_user", stalled_user)
monkeypatch.setattr(main, "notifications", updates)
response = await main.live_snapshot()
result = payload(response)
assert response.status_code == 200
assert result["context"] is None
assert result["events"] is None
assert result["notifications"] == [{"id": 42, "title": "Mentioned you"}]
assert result["sections"] == {
"context": "temporarily unavailable",
"events": "temporarily unavailable",
"notifications": "fresh",
}
assert main._live_section_failure_count == {
"context": 1,
"events": 1,
"notifications": 0,
}
await asyncio.wait_for(identity_cancelled.wait(), timeout=0.1)
@pytest.mark.anyio @pytest.mark.anyio
async def test_live_snapshot_omits_section_bodies_at_known_revisions(monkeypatch): async def test_live_snapshot_omits_section_bodies_at_known_revisions(monkeypatch):
async def user(): async def user():