Preserve healthy live sections during partial upstream timeouts #1430
50
src/main.py
50
src/main.py
|
|
@ -286,6 +286,7 @@ app = FastAPI(title="Stackchain Dashboard", lifespan=lifespan)
|
|||
app.add_middleware(RequestBodyLimitMiddleware, limit_for=request_body_limit)
|
||||
app.add_middleware(NegotiatedGZipMiddleware, minimum_size=1_024)
|
||||
CONTEXT_TIMEOUT_SECONDS = 5.0
|
||||
LIVE_SNAPSHOT_DEADLINE_GRACE_SECONDS = 0.1
|
||||
EVENT_STREAM_TIMEOUT_SECONDS = 5.0
|
||||
READINESS_TIMEOUT_SECONDS = 5.0
|
||||
READINESS_INTERVAL_SECONDS = max(
|
||||
|
|
@ -4832,28 +4833,43 @@ async def _load_context_for_user(user_data: dict) -> dict:
|
|||
async def _build_live_snapshot(sections: set[str] | None = None) -> dict:
|
||||
requested = set(sections or LIVE_SNAPSHOT_SECTIONS)
|
||||
results: dict[str, object] = {}
|
||||
user_data: dict | None = None
|
||||
user_task: asyncio.Task | None = None
|
||||
if requested & {"context", "events"}:
|
||||
user_task = asyncio.create_task(current_user())
|
||||
|
||||
async def load_user_section(section: str) -> object:
|
||||
assert user_task is not None
|
||||
user_data = await asyncio.shield(user_task)
|
||||
if not isinstance(user_data, dict) or not user_data.get("login"):
|
||||
raise ContextPayloadError("Gitea current-user response was invalid")
|
||||
if section == "context":
|
||||
return await _load_context_for_user(user_data)
|
||||
return await activity_events(user_data)
|
||||
|
||||
async def load_before_deadline(load: Awaitable[Any]) -> object:
|
||||
try:
|
||||
user_data = await current_user()
|
||||
if not isinstance(user_data, dict) or not user_data.get("login"):
|
||||
raise ContextPayloadError("Gitea current-user response was invalid")
|
||||
async with asyncio.timeout(CONTEXT_TIMEOUT_SECONDS):
|
||||
return await load
|
||||
except Exception as exc:
|
||||
for section in requested & {"context", "events"}:
|
||||
results[section] = exc
|
||||
return exc
|
||||
|
||||
loads: dict[str, Awaitable[Any]] = {}
|
||||
if "context" in requested and "context" not in results:
|
||||
assert user_data is not None
|
||||
loads["context"] = _load_context_for_user(user_data)
|
||||
if "events" in requested and "events" not in results:
|
||||
assert user_data is not None
|
||||
loads["events"] = activity_events(user_data)
|
||||
if "context" in requested:
|
||||
loads["context"] = load_user_section("context")
|
||||
if "events" in requested:
|
||||
loads["events"] = load_user_section("events")
|
||||
if "notifications" in requested:
|
||||
loads["notifications"] = notifications()
|
||||
if loads:
|
||||
loaded = await asyncio.gather(*loads.values(), return_exceptions=True)
|
||||
results.update(zip(loads, loaded))
|
||||
try:
|
||||
if loads:
|
||||
loaded = await asyncio.gather(
|
||||
*(load_before_deadline(load) for load in loads.values())
|
||||
)
|
||||
results.update(zip(loads, loaded))
|
||||
finally:
|
||||
if user_task is not None and not user_task.done():
|
||||
user_task.cancel()
|
||||
await asyncio.gather(user_task, return_exceptions=True)
|
||||
|
||||
context_result = results.get("context")
|
||||
events_result = results.get("events")
|
||||
|
|
@ -4892,7 +4908,9 @@ async def _build_live_snapshot(sections: set[str] | None = None) -> dict:
|
|||
|
||||
|
||||
async def _build_live_snapshot_before_deadline(sections: set[str]) -> dict:
|
||||
async with asyncio.timeout(CONTEXT_TIMEOUT_SECONDS):
|
||||
async with asyncio.timeout(
|
||||
CONTEXT_TIMEOUT_SECONDS + LIVE_SNAPSHOT_DEADLINE_GRACE_SECONDS
|
||||
):
|
||||
return await _build_live_snapshot(sections)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -103,6 +103,87 @@ async def test_live_snapshot_fetches_user_once_and_updates_work_and_activity(mon
|
|||
assert result["revisions"]["context"].endswith(".1")
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_notification_timeout_preserves_healthy_context_and_events(monkeypatch):
|
||||
async def user():
|
||||
return {"id": 1, "login": "timmy"}
|
||||
|
||||
async def empty():
|
||||
return []
|
||||
|
||||
async def events(_authenticated_user):
|
||||
return [{"type": "push"}]
|
||||
|
||||
async def stalled_notifications():
|
||||
await asyncio.Event().wait()
|
||||
|
||||
monkeypatch.setattr(main, "CONTEXT_TIMEOUT_SECONDS", 0.01)
|
||||
monkeypatch.setattr(main, "current_user", user)
|
||||
monkeypatch.setattr(main, "repos", empty)
|
||||
monkeypatch.setattr(main, "issues", empty)
|
||||
monkeypatch.setattr(main, "pull_requests", empty)
|
||||
monkeypatch.setattr(main, "activity_events", events)
|
||||
monkeypatch.setattr(main, "notifications", stalled_notifications)
|
||||
|
||||
response = await main.live_snapshot()
|
||||
result = payload(response)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert result["context"]["user"]["login"] == "timmy"
|
||||
assert result["events"] == [{"type": "push"}]
|
||||
assert result["notifications"] is None
|
||||
assert result["sections"] == {
|
||||
"context": "fresh",
|
||||
"events": "fresh",
|
||||
"notifications": "temporarily unavailable",
|
||||
}
|
||||
assert result["freshness"]["sections"]["context"]["degraded"] is False
|
||||
assert result["freshness"]["sections"]["events"]["degraded"] is False
|
||||
assert result["freshness"]["sections"]["notifications"]["degraded"] is True
|
||||
assert main._live_section_failure_count == {
|
||||
"context": 0,
|
||||
"events": 0,
|
||||
"notifications": 1,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_identity_timeout_preserves_independent_notifications(monkeypatch):
|
||||
identity_cancelled = asyncio.Event()
|
||||
|
||||
async def stalled_user():
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
identity_cancelled.set()
|
||||
|
||||
async def updates():
|
||||
return [{"id": 42, "title": "Mentioned you"}]
|
||||
|
||||
monkeypatch.setattr(main, "CONTEXT_TIMEOUT_SECONDS", 0.01)
|
||||
monkeypatch.setattr(main, "current_user", stalled_user)
|
||||
monkeypatch.setattr(main, "notifications", updates)
|
||||
|
||||
response = await main.live_snapshot()
|
||||
result = payload(response)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert result["context"] is None
|
||||
assert result["events"] is None
|
||||
assert result["notifications"] == [{"id": 42, "title": "Mentioned you"}]
|
||||
assert result["sections"] == {
|
||||
"context": "temporarily unavailable",
|
||||
"events": "temporarily unavailable",
|
||||
"notifications": "fresh",
|
||||
}
|
||||
assert main._live_section_failure_count == {
|
||||
"context": 1,
|
||||
"events": 1,
|
||||
"notifications": 0,
|
||||
}
|
||||
await asyncio.wait_for(identity_cancelled.wait(), timeout=0.1)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_live_snapshot_omits_section_bodies_at_known_revisions(monkeypatch):
|
||||
async def user():
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user