feat: verify per-device Push delivery health (Closes #1120)

tests/test_push_frontend.py

@@ -22,6 +22,7 @@ const deadlineHour = {value:'9', disabled:false, addEventListener:(_name, callba
 const deadlineDays = {value:'2', disabled:false, addEventListener:(_name, callback) => state.deadlineDaysChange = callback};
 const deadlineStatus = {set textContent(value) { state.deadlineText = value; }, get textContent() { return state.deadlineText; }};
 const status = {set textContent(value) { state.text = value; }, get textContent() { return state.text; }};
+const testControl = {hidden:true, disabled:false, addEventListener:(_name, callback) => state.testDelivery = callback};
 const deadlineSnooze = {hidden:true};
 const deadlineSnoozeStatus = {set textContent(value) { state.snoozeText = value; }, get textContent() { return state.snoozeText; }};
 const deadlineSnoozeReview = {disabled:false, addEventListener:(_name, callback) => state.reviewSnooze = callback};
@@ -31,7 +32,7 @@ const registration = {pushManager:{
   subscribe: async options => { state.subscriptions.push(options); state.current=existing; return existing; },
 }};
 const feature = createPushNotifications({
-  control, status, deadlineControl, deadlineStatus, deadlineHour, deadlineDays,
+  control, status, testControl, deadlineControl, deadlineStatus, deadlineHour, deadlineDays,
   deadlineSnooze, deadlineSnoozeStatus, deadlineSnoozeReview,
   onReviewDeadlines:() => { state.reviewed = true; },
   notification: {permission:'default', requestPermission:async () => { state.prompts += 1; return state.permission || 'granted'; }},
@@ -69,6 +70,27 @@ process.stdout.write(JSON.stringify(state));
     assert result["text"] == "New update notifications enabled for this device."
 
 
+def test_degraded_device_can_run_a_test_notification_and_show_recovery():
+    result = run_scenario("""
+state.server = {available:true,subscribed:true,public_key:'AQID',delivery_health:{unread:{state:'degraded',consecutive_failures:3,reason:'timeout'}}};
+state.current = existing;
+await feature.init();
+const degraded = {text:state.text,hidden:testControl.hidden};
+state.server = {delivery_state:'healthy',delivered:true};
+await state.testDelivery();
+process.stdout.write(JSON.stringify({degraded,text:state.text,hidden:testControl.hidden,disabled:testControl.disabled,requests:state.requests}));
+""")
+
+    assert result["degraded"] == {
+        "text": "Update notifications need attention after 3 failed deliveries. Send a test notification.",
+        "hidden": False,
+    }
+    assert result["requests"][-1][0:2] == ["api/v1/push-subscription/test", "POST"]
+    assert result["text"] == "Test delivered. Update notifications are working on this device."
+    assert result["hidden"] is False
+    assert result["disabled"] is False
+
+
 def test_deadline_opt_in_reuses_subscription_and_sends_local_timezone_without_second_prompt():
     result = run_scenario("""
 state.current = existing;
@@ -220,6 +242,7 @@ def test_mobile_dashboard_mounts_opt_in_and_precaches_its_controller():
 
     assert 'id="push-updates"' in html
     assert 'id="push-update-status"' in html
+    assert 'id="push-test"' in html
     assert 'id="push-deadlines"' in html
     assert 'id="push-deadline-hour"' in html
     assert 'id="push-deadline-days"' in html
@@ -227,6 +250,7 @@ def test_mobile_dashboard_mounts_opt_in_and_precaches_its_controller():
     assert 'id="push-deadline-status"' in html
     assert '<script src="static/push-notifications.js"></script>' in html
     assert "createPushNotifications({" in dashboard
+    assert "testControl:qs('#push-test')" in dashboard
     assert "BASE + 'static/push-notifications.js'" in worker
     assert ".push-update-control" in css and "min-height:44px" in css
     assert "pywebpush==" in requirements

tests/test_push_notifications.py

@@ -200,6 +200,46 @@ def test_subscription_store_delivers_each_unread_thread_once_per_device(tmp_path
     assert second == []
 
 
+def test_delivery_health_is_bounded_per_channel_and_recovers_after_success(tmp_path):
+    store = PushSubscriptionStore(tmp_path / "push.sqlite3")
+    store.upsert("session-a", {
+        "endpoint": "https://push.example/device-a",
+        "keys": {"p256dh": "public-key", "auth": "auth-secret"},
+    })
+
+    store.mark_delivery_failed("session-a", "unread", "timeout", now=100)
+    store.mark_delivery_failed("session-a", "unread", "provider", now=110)
+    store.mark_delivery_failed("session-a", "deadline", "provider", now=120)
+
+    assert store.delivery_health("session-a") == {
+        "unread": {
+            "state": "degraded",
+            "consecutive_failures": 2,
+            "last_attempted_at": 110,
+            "last_succeeded_at": None,
+            "reason": "provider",
+        },
+        "deadline": {
+            "state": "degraded",
+            "consecutive_failures": 1,
+            "last_attempted_at": 120,
+            "last_succeeded_at": None,
+            "reason": "provider",
+        },
+    }
+
+    store.mark_delivery_succeeded("session-a", "unread", now=130)
+
+    assert store.delivery_health("session-a")["unread"] == {
+        "state": "healthy",
+        "consecutive_failures": 0,
+        "last_attempted_at": 130,
+        "last_succeeded_at": 130,
+        "reason": None,
+    }
+    assert store.delivery_health("session-a")["deadline"]["state"] == "degraded"
+
+
 def test_subscription_store_reopens_a_delivered_thread_when_its_revision_changes(tmp_path):
     store = PushSubscriptionStore(tmp_path / "push.sqlite3")
     store.upsert("session-a", {
@@ -918,6 +958,42 @@ async def test_transient_endpoint_failure_does_not_block_healthy_devices(tmp_pat
     assert store.claim_unseen({8})[0].session_id == "session-a"
 
 
+@pytest.mark.anyio
+async def test_unread_dispatch_records_degraded_health_then_recovers(tmp_path):
+    store = PushSubscriptionStore(tmp_path / "push.sqlite3")
+    store.upsert("session-a", {
+        "endpoint": "https://push.example/device-a",
+        "keys": {"p256dh": "public-key", "auth": "auth-secret"},
+    })
+    revision = 0
+
+    async def unread():
+        nonlocal revision
+        revision += 1
+        return {"items": [{"id": 42, "updated_at": f"r{revision}"}], "complete": True}
+
+    async def timeout_send(_subscription, _payload):
+        raise asyncio.TimeoutError
+
+    config = PushConfiguration("public", "private", "mailto:ops@example.com")
+    for _ in range(3):
+        assert await dispatch_unread_updates(store, config, unread, timeout_send) == 0
+
+    health = store.delivery_health("session-a")["unread"]
+    assert health["state"] == "degraded"
+    assert health["consecutive_failures"] == 3
+    assert health["reason"] == "timeout"
+
+    async def succeed(_subscription, _payload):
+        return None
+
+    assert await dispatch_unread_updates(store, config, unread, succeed) == 1
+    recovered = store.delivery_health("session-a")["unread"]
+    assert recovered["state"] == "healthy"
+    assert recovered["consecutive_failures"] == 0
+    assert recovered["last_succeeded_at"] is not None
+
+
 @pytest.mark.anyio
 async def test_unexpected_device_failure_waits_for_siblings_and_retries_only_that_device(tmp_path):
     path = tmp_path / "push.sqlite3"
@@ -967,6 +1043,7 @@ async def test_unexpected_device_failure_waits_for_siblings_and_retries_only_tha
     assert [(item.session_id, item.thread_ids) for item in remaining] == [
         ("failing-device", (8,))
     ]
+    assert store.delivery_health("failing-device")["unread"]["reason"] == "timeout"
 
     retried = []
 
@@ -1184,6 +1261,7 @@ async def test_authenticated_device_can_subscribe_report_status_and_unsubscribe(
         "reminder_hour": 9,
         "reminder_days": 2,
         "snoozed_until": None,
+        "delivery_health": {},
     }
     assert await main.subscribe_push(payload, request) == {"subscribed": True}
     assert (await main.push_status(request))["subscribed"] is True
@@ -1191,6 +1269,79 @@ async def test_authenticated_device_can_subscribe_report_status_and_unsubscribe(
     assert (await main.push_status(request))["subscribed"] is False
 
 
+@pytest.mark.anyio
+async def test_authenticated_device_can_send_a_real_test_notification(tmp_path, monkeypatch):
+    store = PushSubscriptionStore(tmp_path / "push.sqlite3")
+    subscription = {
+        "endpoint": "https://push.example/device-a",
+        "keys": {"p256dh": "public-key", "auth": "auth-secret"},
+    }
+    store.upsert("session-a", subscription)
+    monkeypatch.setattr(main, "_push_subscription_store", store)
+    configuration = PushConfiguration("public", "private", "mailto:ops@example.com")
+    monkeypatch.setattr(main, "_push_configuration", lambda: configuration)
+
+    async def management_id(_session):
+        return "session-a"
+
+    sent = []
+
+    async def send(delivery, payload, config):
+        sent.append((delivery, json.loads(payload), config))
+
+    monkeypatch.setattr(main.dashboard_auth, "session_management_id", management_id)
+    monkeypatch.setattr(main, "send_web_push", send, raising=False)
+    request = SimpleNamespace(state=SimpleNamespace(dashboard_session=object()))
+
+    assert await main.test_push_notification(request) == {
+        "delivery_state": "healthy",
+        "delivered": True,
+    }
+    assert sent == [(
+        subscription,
+        {
+            "title": "Stackchain notifications are working",
+            "body": "This device can receive private work alerts.",
+            "route": "#/device-setup",
+            "tag": "stackchain-push-test",
+        },
+        configuration,
+    )]
+    assert store.delivery_health("session-a")["unread"]["state"] == "healthy"
+
+
+@pytest.mark.anyio
+async def test_test_notification_reports_provider_failure_as_degraded(tmp_path, monkeypatch):
+    store = PushSubscriptionStore(tmp_path / "push.sqlite3")
+    store.upsert("session-a", {
+        "endpoint": "https://push.example/device-a",
+        "keys": {"p256dh": "public-key", "auth": "auth-secret"},
+    })
+    monkeypatch.setattr(main, "_push_subscription_store", store)
+    monkeypatch.setattr(
+        main, "_push_configuration",
+        lambda: PushConfiguration("public", "private", "mailto:ops@example.com"),
+    )
+
+    async def management_id(_session):
+        return "session-a"
+
+    async def fail(*_args):
+        raise asyncio.TimeoutError
+
+    monkeypatch.setattr(main.dashboard_auth, "session_management_id", management_id)
+    monkeypatch.setattr(main, "send_web_push", fail)
+    request = SimpleNamespace(state=SimpleNamespace(dashboard_session=object()))
+
+    with pytest.raises(main.HTTPException) as raised:
+        await main.test_push_notification(request)
+
+    assert raised.value.status_code == 503
+    assert raised.value.detail == "Test notification delivery failed"
+    assert store.is_subscribed("session-a") is True
+    assert store.delivery_health("session-a")["unread"]["reason"] == "timeout"
+
+
 @pytest.mark.anyio
 async def test_authenticated_device_can_enable_deadline_reminders(tmp_path, monkeypatch):
     store = PushSubscriptionStore(tmp_path / "push.sqlite3")