Isolate unexpected Web Push device failures #656
|
|
@ -219,8 +219,17 @@ async def dispatch_unread_updates(
|
||||||
count += 1
|
count += 1
|
||||||
return count
|
return count
|
||||||
|
|
||||||
|
async def dispatch_device_safely(delivery) -> int:
|
||||||
|
try:
|
||||||
|
return await dispatch_device(delivery)
|
||||||
|
except Exception:
|
||||||
|
# Device-specific validation, persistence, or provider failures
|
||||||
|
# must not cancel healthy siblings. Leave any uncheckpointed
|
||||||
|
# revisions unseen so a later poll can retry them.
|
||||||
|
return 0
|
||||||
|
|
||||||
counts = await asyncio.gather(
|
counts = await asyncio.gather(
|
||||||
*(dispatch_device(delivery) for delivery in deliveries)
|
*(dispatch_device_safely(delivery) for delivery in deliveries)
|
||||||
)
|
)
|
||||||
return sum(counts)
|
return sum(counts)
|
||||||
finally:
|
finally:
|
||||||
|
|
|
||||||
|
|
@ -2,6 +2,7 @@ import json
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import sqlite3
|
import sqlite3
|
||||||
|
import time
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
@ -567,6 +568,75 @@ async def test_transient_endpoint_failure_does_not_block_healthy_devices(tmp_pat
|
||||||
assert store.claim_unseen({8})[0].session_id == "session-a"
|
assert store.claim_unseen({8})[0].session_id == "session-a"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_unexpected_device_failure_waits_for_siblings_and_retries_only_that_device(tmp_path):
|
||||||
|
path = tmp_path / "push.sqlite3"
|
||||||
|
store = PushSubscriptionStore(path)
|
||||||
|
competitor = PushSubscriptionStore(path)
|
||||||
|
for session_id in ("failing-device", "healthy-device"):
|
||||||
|
store.upsert(session_id, {
|
||||||
|
"endpoint": f"https://push.example/{session_id}",
|
||||||
|
"keys": {"p256dh": "public-key", "auth": "auth-secret"},
|
||||||
|
})
|
||||||
|
healthy_started = asyncio.Event()
|
||||||
|
release_healthy = asyncio.Event()
|
||||||
|
sent = []
|
||||||
|
|
||||||
|
async def unread():
|
||||||
|
return {"items": [{"id": 8, "updated_at": "r1"}], "complete": True}
|
||||||
|
|
||||||
|
async def validate(endpoint):
|
||||||
|
if endpoint.endswith("failing-device"):
|
||||||
|
raise TimeoutError("endpoint validation unavailable")
|
||||||
|
return endpoint
|
||||||
|
|
||||||
|
async def send(subscription, _payload):
|
||||||
|
sent.append(subscription["endpoint"])
|
||||||
|
healthy_started.set()
|
||||||
|
await release_healthy.wait()
|
||||||
|
|
||||||
|
config = PushConfiguration("public", "private", "mailto:ops@example.com")
|
||||||
|
dispatch = asyncio.create_task(dispatch_unread_updates(
|
||||||
|
store,
|
||||||
|
config,
|
||||||
|
unread,
|
||||||
|
send,
|
||||||
|
endpoint_validator=validate,
|
||||||
|
))
|
||||||
|
await asyncio.wait_for(healthy_started.wait(), timeout=0.2)
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
|
||||||
|
assert dispatch.done() is False
|
||||||
|
assert competitor.acquire_dispatch_lease(
|
||||||
|
"competing-worker", now=time.time(), lease_seconds=60
|
||||||
|
) is False
|
||||||
|
|
||||||
|
release_healthy.set()
|
||||||
|
assert await dispatch == 1
|
||||||
|
remaining = store.claim_unseen({8: "r1"})
|
||||||
|
assert [(item.session_id, item.thread_ids) for item in remaining] == [
|
||||||
|
("failing-device", (8,))
|
||||||
|
]
|
||||||
|
|
||||||
|
retried = []
|
||||||
|
|
||||||
|
async def validate_recovered(endpoint):
|
||||||
|
return endpoint
|
||||||
|
|
||||||
|
async def send_retry(subscription, _payload):
|
||||||
|
retried.append(subscription["endpoint"])
|
||||||
|
|
||||||
|
assert await dispatch_unread_updates(
|
||||||
|
store,
|
||||||
|
config,
|
||||||
|
unread,
|
||||||
|
send_retry,
|
||||||
|
endpoint_validator=validate_recovered,
|
||||||
|
) == 1
|
||||||
|
assert retried == ["https://push.example/failing-device"]
|
||||||
|
assert sent == ["https://push.example/healthy-device"]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_transient_failure_stops_that_device_until_the_next_poll(tmp_path):
|
async def test_transient_failure_stops_that_device_until_the_next_poll(tmp_path):
|
||||||
store = PushSubscriptionStore(tmp_path / "push.sqlite3")
|
store = PushSubscriptionStore(tmp_path / "push.sqlite3")
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user