feat: verify per-device Push delivery health (Closes #1120)
All checks were successful
CI / lint (pull_request) Successful in 2m53s
CI / build-release (pull_request) Successful in 6s
CI / browser-journey (pull_request) Successful in 3m1s
CI / release-candidate (pull_request) Has been skipped

This commit is contained in:
timmy 2026-08-19 09:00:06 +00:00
parent 1047f3ecdc
commit 13e73bd342
8 changed files with 395 additions and 7 deletions

View File

@ -7924,6 +7924,7 @@
const controller = createPushNotifications({
control:qs('#push-updates'),
status:qs('#push-update-status'),
testControl:qs('#push-test'),
deadlineControl:qs('#push-deadlines'),
deadlineStatus:qs('#push-deadline-status'),
deadlineHour:qs('#push-deadline-hour'),

View File

@ -190,6 +190,7 @@
<label for="delivery-receipts"><input id="delivery-receipts" type="checkbox" /> Notify me when queued work finishes</label>
<label class="push-update-control" for="push-updates"><input id="push-updates" type="checkbox" /> Notify me about new updates</label>
<span class="small" id="push-update-status" role="status" aria-live="polite"></span>
<button class="secondary" id="push-test" type="button" hidden>Send test notification</button>
<label class="push-update-control" for="push-deadlines"><input id="push-deadlines" type="checkbox" /> Notify me about deadlines</label>
<label for="push-deadline-hour">Reminder hour <select id="push-deadline-hour" aria-label="Deadline reminder local hour"></select></label>
<label for="push-deadline-days">Warn me <select id="push-deadline-days" aria-label="Deadline reminder horizon"><option value="0">Due today</option><option value="2" selected>Next 2 days</option><option value="7">Next 7 days</option></select></label>

View File

@ -2,12 +2,41 @@
if (typeof module === 'object' && module.exports) module.exports = factory;
else root.createPushNotifications = factory;
})(typeof self !== 'undefined' ? self : this, function createPushNotifications({
control, status, deadlineControl, deadlineStatus, deadlineHour, deadlineDays,
control, status, testControl, deadlineControl, deadlineStatus, deadlineHour, deadlineDays,
deadlineSnooze, deadlineSnoozeStatus, deadlineSnoozeReview, onReviewDeadlines,
notification, serviceWorker, fetchJson,
}) {
let configuration = null;
function renderDeliveryHealth() {
const health = Object.values(configuration?.delivery_health || {});
const degraded = health.find(item => item?.state === 'degraded');
if (testControl) testControl.hidden = !configuration?.subscribed;
if (configuration?.subscribed && degraded) {
const count = Number(degraded.consecutive_failures || 1);
status.textContent = `Update notifications need attention after ${count} failed ${count === 1 ? 'delivery' : 'deliveries'}. Send a test notification.`;
return;
}
status.textContent = configuration?.subscribed
? 'New update notifications enabled for this device.'
: 'New update notifications are off for this device.';
}
async function testDelivery() {
if (!testControl) return;
testControl.disabled = true;
status.textContent = 'Sending a test notification…';
try {
await fetchJson('api/v1/push-subscription/test', {method:'POST'});
configuration.delivery_health = {unread:{state:'healthy',consecutive_failures:0}};
status.textContent = 'Test delivered. Update notifications are working on this device.';
} catch (error) {
status.textContent = 'Test delivery failed. Check your connection, then try again.';
} finally {
testControl.disabled = false;
}
}
function formattedHour(value) {
return `${String(Number(value)).padStart(2, '0')}:00`;
}
@ -67,6 +96,7 @@
await fetchJson('api/v1/push-subscription', {method:'DELETE'});
await subscription?.unsubscribe?.();
control.checked = false;
if (testControl) testControl.hidden = true;
if (deadlineControl) deadlineControl.checked = false;
status.textContent = 'New update notifications are off for this device.';
if (deadlineStatus) deadlineStatus.textContent = 'Deadline reminders are off for this device.';
@ -95,6 +125,7 @@
control.checked = true;
status.textContent = 'New update notifications enabled for this device.';
configuration.subscribed = true;
if (testControl) testControl.hidden = false;
return subscription;
}
@ -163,6 +194,7 @@
async function init() {
if (!control || !notification || !serviceWorker) return;
control.addEventListener('change', change);
testControl?.addEventListener('click', testDelivery);
deadlineControl?.addEventListener('change', changeDeadline);
deadlineSnoozeReview?.addEventListener('click', reviewSnoozedDeadlines);
configuration = await fetchJson('api/v1/push-subscription');
@ -176,9 +208,7 @@
if (deadlineControl) deadlineControl.checked = Boolean(configuration.deadline_enabled);
if (deadlineHour) deadlineHour.value = String(configuration.reminder_hour ?? 9);
if (deadlineDays) deadlineDays.value = String(configuration.reminder_days ?? 2);
status.textContent = configuration.subscribed
? 'New update notifications enabled for this device.'
: 'New update notifications are off for this device.';
renderDeliveryHealth();
if (deadlineStatus) deadlineStatus.textContent = configuration.deadline_enabled
? enabledDeadlineText(configuration.reminder_hour, configuration.reminder_days)
: 'Deadline reminders are off for this device.';

View File

@ -3,6 +3,7 @@ import base64
import binascii
import hashlib
import hmac
import json
import math
import os
import re
@ -55,6 +56,7 @@ from src.push_notifications import (
PushConfiguration,
dispatch_deadline_reminders,
dispatch_unread_updates,
send_web_push,
)
from src.push_endpoint_policy import UnsafePushEndpoint, validate_public_push_endpoint
from src.push_subscription_store import build_push_subscription_store
@ -2159,6 +2161,9 @@ async def push_status(request: Request):
preferences = await asyncio.to_thread(
_push_subscription_store.deadline_preferences, device_id, now=time.time()
)
delivery_health = await asyncio.to_thread(
_push_subscription_store.delivery_health, device_id
)
return {
"available": configuration.enabled,
"subscribed": subscribed,
@ -2168,6 +2173,7 @@ async def push_status(request: Request):
"reminder_hour": preferences["reminder_hour"],
"reminder_days": preferences["reminder_days"],
"snoozed_until": preferences["snoozed_until"],
"delivery_health": delivery_health,
}
@ -2221,6 +2227,52 @@ async def unsubscribe_push(request: Request):
return {"subscribed": False}
@app.post("/api/v1/push-subscription/test")
async def test_push_notification(request: Request):
configuration = _push_configuration()
if not configuration.enabled:
raise HTTPException(status_code=503, detail="Push notifications are not configured")
device_id = await dashboard_auth.session_management_id(
request.state.dashboard_session
)
subscription = await asyncio.to_thread(
_push_subscription_store.subscription_for_session, device_id
)
if subscription is None:
raise HTTPException(status_code=409, detail="Enable device notifications first")
payload = json.dumps({
"title": "Stackchain notifications are working",
"body": "This device can receive private work alerts.",
"route": "#/device-setup",
"tag": "stackchain-push-test",
}, separators=(",", ":"))
try:
await send_web_push(subscription, payload, configuration)
except Exception as error:
status = getattr(getattr(error, "response", None), "status_code", None)
if isinstance(error, UnsafePushEndpoint) or status in {404, 410}:
await asyncio.to_thread(_push_subscription_store.delete_session, device_id)
raise HTTPException(
status_code=409, detail="Re-enable notifications for this device"
) from error
reason = "timeout" if isinstance(error, (asyncio.TimeoutError, TimeoutError)) else "provider"
await asyncio.to_thread(
_push_subscription_store.mark_delivery_failed,
device_id,
"unread",
reason,
)
raise HTTPException(
status_code=503,
detail="Test notification delivery failed",
headers={"Retry-After": "1"},
) from error
await asyncio.to_thread(
_push_subscription_store.mark_delivery_succeeded, device_id, "unread"
)
return {"delivery_state": "healthy", "delivered": True}
@app.put("/api/v1/push-subscription/deadlines")
async def update_deadline_reminders(payload: DeadlineReminderPayload, request: Request):
device_id = await dashboard_auth.session_management_id(

View File

@ -57,6 +57,12 @@ class _PinnedHTTPSAdapter(requests.adapters.HTTPAdapter):
return host, tls
def _delivery_failure_reason(error: Exception) -> str:
if isinstance(error, (asyncio.TimeoutError, TimeoutError)):
return "timeout"
return "provider"
async def send_web_push(
subscription: dict,
payload: str,
@ -214,10 +220,20 @@ async def dispatch_unread_updates(
await asyncio.to_thread(
store.delete_session, delivery.session_id
)
else:
await asyncio.to_thread(
store.mark_delivery_failed,
delivery.session_id,
"unread",
_delivery_failure_reason(error),
)
can_send_digest = False
# Leave this device's transient failures unseen for a later
# poll instead of paying the endpoint deadline repeatedly.
break
await asyncio.to_thread(
store.mark_delivery_succeeded, delivery.session_id, "unread"
)
await asyncio.to_thread(
store.mark_delivered,
delivery.session_id,
@ -262,12 +278,21 @@ async def dispatch_unread_updates(
store.delete_session, delivery.session_id
)
else:
await asyncio.to_thread(
store.mark_delivery_failed,
delivery.session_id,
"unread",
_delivery_failure_reason(error),
)
await asyncio.to_thread(
store.mark_digest_pending,
delivery.session_id,
overflow_revisions,
)
return count
await asyncio.to_thread(
store.mark_delivery_succeeded, delivery.session_id, "unread"
)
await asyncio.to_thread(
store.mark_delivered,
delivery.session_id,
@ -279,10 +304,19 @@ async def dispatch_unread_updates(
async def dispatch_device_safely(delivery) -> int:
try:
return await dispatch_device(delivery)
except Exception:
except Exception as error:
# Device-specific validation, persistence, or provider failures
# must not cancel healthy siblings. Leave any uncheckpointed
# revisions unseen so a later poll can retry them.
try:
await asyncio.to_thread(
store.mark_delivery_failed,
delivery.session_id,
"unread",
_delivery_failure_reason(error),
)
except Exception:
pass
return 0
counts = await asyncio.gather(
@ -461,9 +495,20 @@ async def _dispatch_deadline_reminders_unlocked(
)
await asyncio.wait_for(operation, timeout=send_timeout_seconds)
except Exception as error:
if isinstance(error, UnsafePushEndpoint):
status = getattr(getattr(error, "response", None), "status_code", None)
if isinstance(error, UnsafePushEndpoint) or status in {404, 410}:
await asyncio.to_thread(store.delete_session, device.session_id)
else:
await asyncio.to_thread(
store.mark_delivery_failed,
device.session_id,
"deadline",
_delivery_failure_reason(error),
)
return 0
await asyncio.to_thread(
store.mark_delivery_succeeded, device.session_id, "deadline"
)
await asyncio.to_thread(
store.mark_deadline_reminder_delivered, device.session_id, local_day
)

View File

@ -49,6 +49,9 @@ class DisabledPushSubscriptionStore:
def is_subscribed(self, session_id: str) -> bool:
return False
def subscription_for_session(self, session_id: str) -> dict | None:
return None
def deadline_preferences(
self, session_id: str, *, now: float | None = None
) -> dict:
@ -102,6 +105,15 @@ class DisabledPushSubscriptionStore:
def mark_delivered(self, *args, **kwargs) -> None:
return None
def delivery_health(self, *args, **kwargs) -> dict:
return {}
def mark_delivery_failed(self, *args, **kwargs) -> None:
return None
def mark_delivery_succeeded(self, *args, **kwargs) -> None:
return None
def build_push_subscription_store(
path: str | Path, *, push_enabled: bool
@ -183,6 +195,17 @@ class PushSubscriptionStore:
FOREIGN KEY (session_id) REFERENCES push_subscriptions(session_id)
ON DELETE CASCADE
);
CREATE TABLE IF NOT EXISTS push_delivery_health (
session_id TEXT NOT NULL,
channel TEXT NOT NULL,
consecutive_failures INTEGER NOT NULL DEFAULT 0,
last_attempted_at REAL NOT NULL,
last_succeeded_at REAL,
reason TEXT,
PRIMARY KEY (session_id, channel),
FOREIGN KEY (session_id) REFERENCES push_subscriptions(session_id)
ON DELETE CASCADE
);
"""
)
delivery_columns = {
@ -316,6 +339,67 @@ class PushSubscriptionStore:
"SELECT 1 FROM push_subscriptions WHERE session_id = ?", (session_id,)
).fetchone() is not None
def subscription_for_session(self, session_id: str) -> dict | None:
with self._connect() as connection:
row = connection.execute(
"SELECT subscription_json FROM push_subscriptions WHERE session_id = ?",
(session_id,),
).fetchone()
return self._open_subscription(session_id, row[0]) if row else None
def delivery_health(self, session_id: str) -> dict:
with self._connect() as connection:
rows = connection.execute(
"""SELECT channel, consecutive_failures, last_attempted_at,
last_succeeded_at, reason
FROM push_delivery_health WHERE session_id = ? ORDER BY channel""",
(session_id,),
).fetchall()
return {
row[0]: {
"state": "degraded" if row[1] else "healthy",
"consecutive_failures": row[1],
"last_attempted_at": row[2],
"last_succeeded_at": row[3],
"reason": row[4],
}
for row in rows
}
def mark_delivery_failed(
self, session_id: str, channel: str, reason: str, *, now: float | None = None
) -> None:
attempted_at = time.time() if now is None else now
with self._connect() as connection:
connection.execute(
"""INSERT INTO push_delivery_health(
session_id, channel, consecutive_failures, last_attempted_at, reason
) VALUES (?, ?, 1, ?, ?)
ON CONFLICT(session_id, channel) DO UPDATE SET
consecutive_failures = consecutive_failures + 1,
last_attempted_at = excluded.last_attempted_at,
reason = excluded.reason""",
(session_id, channel, attempted_at, reason),
)
def mark_delivery_succeeded(
self, session_id: str, channel: str, *, now: float | None = None
) -> None:
attempted_at = time.time() if now is None else now
with self._connect() as connection:
connection.execute(
"""INSERT INTO push_delivery_health(
session_id, channel, consecutive_failures, last_attempted_at,
last_succeeded_at, reason
) VALUES (?, ?, 0, ?, ?, NULL)
ON CONFLICT(session_id, channel) DO UPDATE SET
consecutive_failures = 0,
last_attempted_at = excluded.last_attempted_at,
last_succeeded_at = excluded.last_succeeded_at,
reason = NULL""",
(session_id, channel, attempted_at, attempted_at),
)
def set_deadline_preferences(
self, session_id: str, *, enabled: bool, timezone: str, reminder_hour: int,
reminder_days: int = 2,

View File

@ -22,6 +22,7 @@ const deadlineHour = {value:'9', disabled:false, addEventListener:(_name, callba
const deadlineDays = {value:'2', disabled:false, addEventListener:(_name, callback) => state.deadlineDaysChange = callback};
const deadlineStatus = {set textContent(value) { state.deadlineText = value; }, get textContent() { return state.deadlineText; }};
const status = {set textContent(value) { state.text = value; }, get textContent() { return state.text; }};
const testControl = {hidden:true, disabled:false, addEventListener:(_name, callback) => state.testDelivery = callback};
const deadlineSnooze = {hidden:true};
const deadlineSnoozeStatus = {set textContent(value) { state.snoozeText = value; }, get textContent() { return state.snoozeText; }};
const deadlineSnoozeReview = {disabled:false, addEventListener:(_name, callback) => state.reviewSnooze = callback};
@ -31,7 +32,7 @@ const registration = {pushManager:{
subscribe: async options => { state.subscriptions.push(options); state.current=existing; return existing; },
}};
const feature = createPushNotifications({
control, status, deadlineControl, deadlineStatus, deadlineHour, deadlineDays,
control, status, testControl, deadlineControl, deadlineStatus, deadlineHour, deadlineDays,
deadlineSnooze, deadlineSnoozeStatus, deadlineSnoozeReview,
onReviewDeadlines:() => { state.reviewed = true; },
notification: {permission:'default', requestPermission:async () => { state.prompts += 1; return state.permission || 'granted'; }},
@ -69,6 +70,27 @@ process.stdout.write(JSON.stringify(state));
assert result["text"] == "New update notifications enabled for this device."
def test_degraded_device_can_run_a_test_notification_and_show_recovery():
result = run_scenario("""
state.server = {available:true,subscribed:true,public_key:'AQID',delivery_health:{unread:{state:'degraded',consecutive_failures:3,reason:'timeout'}}};
state.current = existing;
await feature.init();
const degraded = {text:state.text,hidden:testControl.hidden};
state.server = {delivery_state:'healthy',delivered:true};
await state.testDelivery();
process.stdout.write(JSON.stringify({degraded,text:state.text,hidden:testControl.hidden,disabled:testControl.disabled,requests:state.requests}));
""")
assert result["degraded"] == {
"text": "Update notifications need attention after 3 failed deliveries. Send a test notification.",
"hidden": False,
}
assert result["requests"][-1][0:2] == ["api/v1/push-subscription/test", "POST"]
assert result["text"] == "Test delivered. Update notifications are working on this device."
assert result["hidden"] is False
assert result["disabled"] is False
def test_deadline_opt_in_reuses_subscription_and_sends_local_timezone_without_second_prompt():
result = run_scenario("""
state.current = existing;
@ -220,6 +242,7 @@ def test_mobile_dashboard_mounts_opt_in_and_precaches_its_controller():
assert 'id="push-updates"' in html
assert 'id="push-update-status"' in html
assert 'id="push-test"' in html
assert 'id="push-deadlines"' in html
assert 'id="push-deadline-hour"' in html
assert 'id="push-deadline-days"' in html
@ -227,6 +250,7 @@ def test_mobile_dashboard_mounts_opt_in_and_precaches_its_controller():
assert 'id="push-deadline-status"' in html
assert '<script src="static/push-notifications.js"></script>' in html
assert "createPushNotifications({" in dashboard
assert "testControl:qs('#push-test')" in dashboard
assert "BASE + 'static/push-notifications.js'" in worker
assert ".push-update-control" in css and "min-height:44px" in css
assert "pywebpush==" in requirements

View File

@ -200,6 +200,46 @@ def test_subscription_store_delivers_each_unread_thread_once_per_device(tmp_path
assert second == []
def test_delivery_health_is_bounded_per_channel_and_recovers_after_success(tmp_path):
store = PushSubscriptionStore(tmp_path / "push.sqlite3")
store.upsert("session-a", {
"endpoint": "https://push.example/device-a",
"keys": {"p256dh": "public-key", "auth": "auth-secret"},
})
store.mark_delivery_failed("session-a", "unread", "timeout", now=100)
store.mark_delivery_failed("session-a", "unread", "provider", now=110)
store.mark_delivery_failed("session-a", "deadline", "provider", now=120)
assert store.delivery_health("session-a") == {
"unread": {
"state": "degraded",
"consecutive_failures": 2,
"last_attempted_at": 110,
"last_succeeded_at": None,
"reason": "provider",
},
"deadline": {
"state": "degraded",
"consecutive_failures": 1,
"last_attempted_at": 120,
"last_succeeded_at": None,
"reason": "provider",
},
}
store.mark_delivery_succeeded("session-a", "unread", now=130)
assert store.delivery_health("session-a")["unread"] == {
"state": "healthy",
"consecutive_failures": 0,
"last_attempted_at": 130,
"last_succeeded_at": 130,
"reason": None,
}
assert store.delivery_health("session-a")["deadline"]["state"] == "degraded"
def test_subscription_store_reopens_a_delivered_thread_when_its_revision_changes(tmp_path):
store = PushSubscriptionStore(tmp_path / "push.sqlite3")
store.upsert("session-a", {
@ -918,6 +958,42 @@ async def test_transient_endpoint_failure_does_not_block_healthy_devices(tmp_pat
assert store.claim_unseen({8})[0].session_id == "session-a"
@pytest.mark.anyio
async def test_unread_dispatch_records_degraded_health_then_recovers(tmp_path):
store = PushSubscriptionStore(tmp_path / "push.sqlite3")
store.upsert("session-a", {
"endpoint": "https://push.example/device-a",
"keys": {"p256dh": "public-key", "auth": "auth-secret"},
})
revision = 0
async def unread():
nonlocal revision
revision += 1
return {"items": [{"id": 42, "updated_at": f"r{revision}"}], "complete": True}
async def timeout_send(_subscription, _payload):
raise asyncio.TimeoutError
config = PushConfiguration("public", "private", "mailto:ops@example.com")
for _ in range(3):
assert await dispatch_unread_updates(store, config, unread, timeout_send) == 0
health = store.delivery_health("session-a")["unread"]
assert health["state"] == "degraded"
assert health["consecutive_failures"] == 3
assert health["reason"] == "timeout"
async def succeed(_subscription, _payload):
return None
assert await dispatch_unread_updates(store, config, unread, succeed) == 1
recovered = store.delivery_health("session-a")["unread"]
assert recovered["state"] == "healthy"
assert recovered["consecutive_failures"] == 0
assert recovered["last_succeeded_at"] is not None
@pytest.mark.anyio
async def test_unexpected_device_failure_waits_for_siblings_and_retries_only_that_device(tmp_path):
path = tmp_path / "push.sqlite3"
@ -967,6 +1043,7 @@ async def test_unexpected_device_failure_waits_for_siblings_and_retries_only_tha
assert [(item.session_id, item.thread_ids) for item in remaining] == [
("failing-device", (8,))
]
assert store.delivery_health("failing-device")["unread"]["reason"] == "timeout"
retried = []
@ -1184,6 +1261,7 @@ async def test_authenticated_device_can_subscribe_report_status_and_unsubscribe(
"reminder_hour": 9,
"reminder_days": 2,
"snoozed_until": None,
"delivery_health": {},
}
assert await main.subscribe_push(payload, request) == {"subscribed": True}
assert (await main.push_status(request))["subscribed"] is True
@ -1191,6 +1269,79 @@ async def test_authenticated_device_can_subscribe_report_status_and_unsubscribe(
assert (await main.push_status(request))["subscribed"] is False
@pytest.mark.anyio
async def test_authenticated_device_can_send_a_real_test_notification(tmp_path, monkeypatch):
store = PushSubscriptionStore(tmp_path / "push.sqlite3")
subscription = {
"endpoint": "https://push.example/device-a",
"keys": {"p256dh": "public-key", "auth": "auth-secret"},
}
store.upsert("session-a", subscription)
monkeypatch.setattr(main, "_push_subscription_store", store)
configuration = PushConfiguration("public", "private", "mailto:ops@example.com")
monkeypatch.setattr(main, "_push_configuration", lambda: configuration)
async def management_id(_session):
return "session-a"
sent = []
async def send(delivery, payload, config):
sent.append((delivery, json.loads(payload), config))
monkeypatch.setattr(main.dashboard_auth, "session_management_id", management_id)
monkeypatch.setattr(main, "send_web_push", send, raising=False)
request = SimpleNamespace(state=SimpleNamespace(dashboard_session=object()))
assert await main.test_push_notification(request) == {
"delivery_state": "healthy",
"delivered": True,
}
assert sent == [(
subscription,
{
"title": "Stackchain notifications are working",
"body": "This device can receive private work alerts.",
"route": "#/device-setup",
"tag": "stackchain-push-test",
},
configuration,
)]
assert store.delivery_health("session-a")["unread"]["state"] == "healthy"
@pytest.mark.anyio
async def test_test_notification_reports_provider_failure_as_degraded(tmp_path, monkeypatch):
store = PushSubscriptionStore(tmp_path / "push.sqlite3")
store.upsert("session-a", {
"endpoint": "https://push.example/device-a",
"keys": {"p256dh": "public-key", "auth": "auth-secret"},
})
monkeypatch.setattr(main, "_push_subscription_store", store)
monkeypatch.setattr(
main, "_push_configuration",
lambda: PushConfiguration("public", "private", "mailto:ops@example.com"),
)
async def management_id(_session):
return "session-a"
async def fail(*_args):
raise asyncio.TimeoutError
monkeypatch.setattr(main.dashboard_auth, "session_management_id", management_id)
monkeypatch.setattr(main, "send_web_push", fail)
request = SimpleNamespace(state=SimpleNamespace(dashboard_session=object()))
with pytest.raises(main.HTTPException) as raised:
await main.test_push_notification(request)
assert raised.value.status_code == 503
assert raised.value.detail == "Test notification delivery failed"
assert store.is_subscribed("session-a") is True
assert store.delivery_health("session-a")["unread"]["reason"] == "timeout"
@pytest.mark.anyio
async def test_authenticated_device_can_enable_deadline_reminders(tmp_path, monkeypatch):
store = PushSubscriptionStore(tmp_path / "push.sqlite3")