Merge pull request 'Retry Today and Later sync automatically after transient failures' (#366) from timmy/365-planning-sync-auto-retry into main

frontend/dashboard.js

@@ -148,12 +148,13 @@
       refreshMyWorkView();
       warmTodayOffline();
     },
-    onStatus: state => {
+    onStatus: (state, detail = {}) => {
       const status = qs('#today-sync-status');
       status.textContent = state === 'saved' ? 'Today saved to account.' :
+        (state === 'retrying' ? `Today saved on this device · retrying in ${Math.ceil(detail.delayMs / 1000)}s.` :
           (state === 'pending' ? 'Today saved on this device · sync pending.' :
           (state === 'full' ? 'Another device filled Today · showing its saved plan.' :
-            'Today sync unavailable · changes stay on this device.'));
+            'Today sync unavailable · changes stay on this device.')));
     },
   });
   todaySync.startLifecycle({ window, document });
@@ -180,10 +181,11 @@
       if (!planningOwnerLogin || !laterWork.adopt(records)) return;
       refreshMyWorkView();
     },
-    onStatus: state => {
+    onStatus: (state, detail = {}) => {
       qs('#later-sync-status').textContent = state === 'saved' ? 'Later saved to account.' :
+        (state === 'retrying' ? `Later saved on this device · retrying in ${Math.ceil(detail.delayMs / 1000)}s.` :
           (state === 'pending' ? 'Later saved on this device · sync pending.' :
-          'Later sync unavailable · changes stay on this device.');
+            'Later sync unavailable · changes stay on this device.'));
     },
   });
   laterSync.startLifecycle({ window, document });
@@ -198,6 +200,8 @@
       const error = new Error(payload.error || payload.detail?.message || payload.detail || 'Review request failed.');
       error.status = response.status;
       error.code = payload.detail?.code;
+      const retryAfter = response.headers.get('Retry-After');
+      error.retryAfter = retryAfter === null ? undefined : Number(retryAfter);
       throw error;
     }
     return payload;

frontend/later-sync.js

@@ -1,10 +1,34 @@
-function createLaterSync({ storage, getLogin, fetchJson, onRemoteRecords, onStatus, createOperationId, createChannel }) {
+function createLaterSync({ storage, getLogin, fetchJson, onRemoteRecords, onStatus, createOperationId, createChannel,
+  setTimer = globalThis.setTimeout, clearTimer = globalThis.clearTimeout, retryBaseMs = 1000, retryMaxMs = 30000 }) {
   const prefix = 'stackchain.later-sync.v1.';
   const migrationPrefix = 'stackchain.later-sync-migrated.v1.';
   const snapshotPrefix = 'stackchain.later-sync-snapshot.v1.';
   let flushing = null;
   let channel = null;
   let channelKey = '';
+  let retryTimer = null;
+  let retryAttempt = 0;
+
+  function cancelRetry() {
+    if (retryTimer !== null) clearTimer?.(retryTimer);
+    retryTimer = null;
+  }
+
+  function scheduleRetry(error, ownerKey) {
+    if (retryTimer !== null || !pending().length || !ownerKey) return;
+    const advised = Number(error?.retryAfter);
+    const delayMs = Number.isFinite(advised) && advised >= 0
+      ? advised * 1000
+      : Math.min(retryMaxMs, retryBaseMs * (2 ** retryAttempt));
+    retryAttempt += 1;
+    onStatus?.('retrying', { delayMs });
+    retryTimer = setTimer?.(async () => {
+      retryTimer = null;
+      if (key() !== ownerKey) return false;
+      return flush();
+    }, delayMs);
+    retryTimer?.unref?.();
+  }
 
   function key() {
     const login = String(getLogin?.() || '').trim().toLowerCase();
@@ -118,12 +142,14 @@ function createLaterSync({ storage, getLogin, fetchJson, onRemoteRecords, onStat
   }
 
   async function run() {
-    if (!key()) return false;
+    const ownerKey = key();
+    if (!ownerKey) return false;
     ensureChannel();
     try {
       let plan = await fetchJson('api/v1/later');
       let operations = pending();
       while (operations.length) {
+        if (key() !== ownerKey) return false;
         const operation = operations[0];
         plan = await fetchJson('api/v1/later', {
           method: 'PATCH',
@@ -139,9 +165,12 @@ function createLaterSync({ storage, getLogin, fetchJson, onRemoteRecords, onStat
       }
       adopt(plan);
       onStatus?.(pending().length ? 'pending' : 'saved');
+      retryAttempt = 0;
+      cancelRetry();
       return true;
-    } catch (_error) {
+    } catch (error) {
-      onStatus?.(pending().length ? 'pending' : 'error');
+      if (pending().length) scheduleRetry(error, ownerKey);
+      else onStatus?.('error');
       return false;
     }
   }

frontend/service-worker.js

@@ -1,6 +1,6 @@
 const BASE = new URL('./', self.location.href).pathname;
 importScripts(BASE + 'static/background-issue-sync.js');
-const CACHE = 'stackchain-dashboard-shell-v48';
+const CACHE = 'stackchain-dashboard-shell-v49';
 const OUTAGE_STATUSES = new Set([500, 502, 503, 504]);
 const NAVIGATION_TIMEOUT_MS = self.__STACKCHAIN_NAVIGATION_TIMEOUT_MS || 4000;
 const SHELL = [

frontend/today-sync.js

@@ -1,10 +1,34 @@
-function createTodaySync({ storage, getLogin, fetchJson, onRemoteIds, onStatus, createOperationId, createChannel }) {
+function createTodaySync({ storage, getLogin, fetchJson, onRemoteIds, onStatus, createOperationId, createChannel,
+  setTimer = globalThis.setTimeout, clearTimer = globalThis.clearTimeout, retryBaseMs = 1000, retryMaxMs = 30000 }) {
   const prefix = 'stackchain.today-sync.v1.';
   const migrationPrefix = 'stackchain.today-sync-migrated.v1.';
   const snapshotPrefix = 'stackchain.today-sync-snapshot.v1.';
   let flushing = null;
   let channel = null;
   let channelKey = '';
+  let retryTimer = null;
+  let retryAttempt = 0;
+
+  function cancelRetry() {
+    if (retryTimer !== null) clearTimer?.(retryTimer);
+    retryTimer = null;
+  }
+
+  function scheduleRetry(error, ownerKey) {
+    if (retryTimer !== null || !pending().length || !ownerKey) return;
+    const advised = Number(error?.retryAfter);
+    const delayMs = Number.isFinite(advised) && advised >= 0
+      ? advised * 1000
+      : Math.min(retryMaxMs, retryBaseMs * (2 ** retryAttempt));
+    retryAttempt += 1;
+    onStatus?.('retrying', { delayMs });
+    retryTimer = setTimer?.(async () => {
+      retryTimer = null;
+      if (key() !== ownerKey) return false;
+      return flush();
+    }, delayMs);
+    retryTimer?.unref?.();
+  }
 
   function key() {
     const login = String(getLogin?.() || '').trim().toLowerCase();
@@ -114,12 +138,14 @@ function createTodaySync({ storage, getLogin, fetchJson, onRemoteIds, onStatus,
   }
 
   async function run() {
-    if (!key()) return false;
+    const ownerKey = key();
+    if (!ownerKey) return false;
     ensureChannel();
     try {
       let plan = await fetchJson('api/v1/today');
       const operations = pending();
       for (const operation of operations) {
+        if (key() !== ownerKey) return false;
         try {
           plan = await fetchJson('api/v1/today', {
             method: 'PATCH',
@@ -132,6 +158,8 @@ function createTodaySync({ storage, getLogin, fetchJson, onRemoteIds, onStatus,
           save(rejected.filter(candidate => candidate.operation_id !== operation.operation_id));
           adopt(plan);
           onStatus?.('full');
+          retryAttempt = 0;
+          cancelRetry();
           return false;
         }
         const remaining = pending();
@@ -139,9 +167,12 @@ function createTodaySync({ storage, getLogin, fetchJson, onRemoteIds, onStatus,
       }
       adopt(plan);
       onStatus?.(pending().length ? 'pending' : 'saved');
+      retryAttempt = 0;
+      cancelRetry();
       return true;
-    } catch (_error) {
+    } catch (error) {
-      onStatus?.(pending().length ? 'pending' : 'error');
+      if (pending().length) scheduleRetry(error, ownerKey);
+      else onStatus?.('error');
       return false;
     }
   }

src/main.py

@@ -897,7 +897,9 @@ async def get_today_plan():
         return await asyncio.to_thread(_today_store().get, login)
     except (OSError, sqlite3.Error):
         raise HTTPException(
-            status_code=503, detail="Today synchronization is unavailable"
+            status_code=503,
+            detail="Today synchronization is unavailable",
+            headers={"Retry-After": "1"},
         )
 
 
@@ -917,7 +919,9 @@ async def update_today_plan(payload: TodayOperation):
         raise HTTPException(status_code=409, detail="Today is limited to 5 items")
     except (OSError, sqlite3.Error):
         raise HTTPException(
-            status_code=503, detail="Today synchronization is unavailable"
+            status_code=503,
+            detail="Today synchronization is unavailable",
+            headers={"Retry-After": "1"},
         )
 
 
@@ -928,7 +932,9 @@ async def get_later_plan():
         return await asyncio.to_thread(_later_store().get, login)
     except (OSError, sqlite3.Error):
         raise HTTPException(
-            status_code=503, detail="Later synchronization is unavailable"
+            status_code=503,
+            detail="Later synchronization is unavailable",
+            headers={"Retry-After": "1"},
         )
 
 
@@ -948,7 +954,9 @@ async def update_later_plan(payload: LaterOperation):
         raise HTTPException(status_code=422, detail=str(error))
     except (OSError, sqlite3.Error):
         raise HTTPException(
-            status_code=503, detail="Later synchronization is unavailable"
+            status_code=503,
+            detail="Later synchronization is unavailable",
+            headers={"Retry-After": "1"},
         )
 
 

tests/test_later_store.py

@@ -1,3 +1,5 @@
+import sqlite3
+
 import httpx
 import pytest
 
@@ -5,6 +7,31 @@ from src import main
 from src.later_store import LaterStore
 
 
+@pytest.mark.anyio
+async def test_transient_later_store_failure_tells_clients_when_to_retry(monkeypatch):
+    async def user():
+        return {"login": "timmy"}
+
+    class BusyStore:
+        def apply(self, *_args, **_kwargs):
+            raise sqlite3.OperationalError("database is busy")
+
+    monkeypatch.setattr(main, "current_user", user)
+    monkeypatch.setattr(main, "_later_store", lambda: BusyStore())
+    payload = main.LaterOperation(
+        operation_id="retry-me",
+        action="defer",
+        item_id="issue:r:1:",
+        wake_at="2026-08-10T09:00:00.000Z",
+    )
+
+    with pytest.raises(main.HTTPException) as raised:
+        await main.update_later_plan(payload)
+
+    assert raised.value.status_code == 503
+    assert raised.value.headers == {"Retry-After": "1"}
+
+
 def test_deferrals_are_durable_revisioned_idempotent_and_account_scoped(tmp_path):
     path = tmp_path / "later.sqlite3"
     store = LaterStore(path)

tests/test_later_sync.py

@@ -83,7 +83,34 @@ sync.enqueue('restore','issue:r:2:');
                 "wake_at": None,
             }
         ],
-        "status": "pending",
+        "status": "retrying",
+    }
+
+
+def test_retry_is_single_flight_and_does_not_cross_account_boundary():
+    script = f"""
+const createLaterSync = require({json.dumps(str(LATER_SYNC))});
+const values=new Map(); const timers=[]; const requests=[]; let login='timmy';
+const sync=createLaterSync({{
+ storage:{{getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v),removeItem:k=>values.delete(k)}},
+ getLogin:()=>login,createOperationId:()=> 'stable-later-op',
+ setTimer:(callback,delay)=>{{timers.push({{callback,delay}});return timers.length}},clearTimer:()=>{{}},
+ fetchJson:async (_url,options={{}})=>{{requests.push(options.method||'GET');throw new Error('offline')}},
+ onRemoteRecords:()=>{{}},onStatus:()=>{{}},
+}});
+sync.enqueue('defer','issue:r:2:','2026-08-10T09:00:00.000Z');
+(async()=>{{
+ const first=sync.flush(); const same=sync.flush(); await Promise.all([first,same]);
+ const scheduledBeforeSwitch=timers.length;
+ login='alexander'; await timers[0].callback();
+ process.stdout.write(JSON.stringify({{scheduledBeforeSwitch,timers:timers.length,requests,pendingForAlexander:sync.pending()}}));
+}})();
+"""
+    assert run_node(script) == {
+        "scheduledBeforeSwitch": 1,
+        "timers": 1,
+        "requests": ["GET"],
+        "pendingForAlexander": [],
     }
 
 
@@ -206,5 +233,5 @@ async def test_dashboard_syncs_every_later_change_and_exposes_account_status():
 def test_later_sync_ships_atomically_in_the_offline_shell():
     source = (Path(__file__).parents[1] / "frontend" / "service-worker.js").read_text()
 
-    assert "stackchain-dashboard-shell-v48" in source
+    assert "stackchain-dashboard-shell-v49" in source
     assert "BASE + 'static/later-sync.js'" in source

tests/test_markdown_renderer.py

@@ -137,4 +137,4 @@ def test_markdown_work_bodies_are_mobile_safe_block_containers():
     assert ".markdown-content { min-width:0; max-width:100%; overflow-wrap:anywhere;" in css
     assert ".markdown-content pre { max-width:100%; overflow-x:auto;" in css
     assert ".markdown-content a { min-height:44px;" in css
-    assert "stackchain-dashboard-shell-v48" in worker
+    assert "stackchain-dashboard-shell-v49" in worker

tests/test_mobile_composer_integration.py

@@ -35,4 +35,4 @@ def test_offline_shell_contains_every_local_dashboard_runtime_asset():
     shell_assets = set(re.findall(r"BASE \+ '([^']+)'", worker.split("async function sessionCsrf", 1)[0]))
 
     assert local_assets <= shell_assets, f"Offline shell is missing: {sorted(local_assets - shell_assets)}"
-    assert "stackchain-dashboard-shell-v48" in worker
+    assert "stackchain-dashboard-shell-v49" in worker

tests/test_service_worker.py

@@ -108,7 +108,7 @@ async function dispatchNotificationClick(route) {{
 def test_navigation_deadline_ships_in_a_new_shell_cache():
     source = WORKER.read_text()
 
-    assert "stackchain-dashboard-shell-v48" in source
+    assert "stackchain-dashboard-shell-v49" in source
     assert "BASE + 'static/dashboard.css'" in source
     assert "BASE + 'static/dashboard.js'" in source
     assert "BASE + 'static/install-app.js'" in source
@@ -117,21 +117,21 @@ def test_navigation_deadline_ships_in_a_new_shell_cache():
 def test_today_convergence_ships_in_a_new_shell_cache():
     source = WORKER.read_text()
 
-    assert "stackchain-dashboard-shell-v48" in source
+    assert "stackchain-dashboard-shell-v49" in source
     assert "BASE + 'static/today-sync.js'" in source
 
 
 def test_mobile_search_viewport_ships_in_a_new_offline_shell():
     source = WORKER.read_text()
 
-    assert "stackchain-dashboard-shell-v48" in source
+    assert "stackchain-dashboard-shell-v49" in source
     assert "BASE + 'static/mobile-search-viewport.js'" in source
 
 
 def test_update_ownership_flow_ships_atomically_in_a_new_offline_shell():
     source = WORKER.read_text()
 
-    assert "stackchain-dashboard-shell-v48" in source
+    assert "stackchain-dashboard-shell-v49" in source
     assert "BASE + 'static/update-ownership.js'" in source
 
 

tests/test_today_store.py

@@ -1,3 +1,5 @@
+import sqlite3
+
 import pytest
 import httpx
 
@@ -5,6 +7,28 @@ from src import main
 from src.today_store import TodayPlanFull, TodayStore
 
 
+@pytest.mark.anyio
+async def test_transient_today_store_failure_tells_clients_when_to_retry(monkeypatch):
+    async def user():
+        return {"login": "timmy"}
+
+    class BusyStore:
+        def apply(self, *_args, **_kwargs):
+            raise sqlite3.OperationalError("database is locked")
+
+    monkeypatch.setattr(main, "current_user", user)
+    monkeypatch.setattr(main, "_today_store", lambda: BusyStore())
+    payload = main.TodayOperation(
+        operation_id="retry-me", action="add", item_id="issue:r:1:"
+    )
+
+    with pytest.raises(main.HTTPException) as raised:
+        await main.update_today_plan(payload)
+
+    assert raised.value.status_code == 503
+    assert raised.value.headers == {"Retry-After": "1"}
+
+
 def test_operations_are_durable_ordered_idempotent_and_account_scoped(tmp_path):
     path = tmp_path / "today.sqlite3"
     store = TodayStore(path, limit=3)

tests/test_today_sync.py

@@ -83,10 +83,49 @@ sync.enqueue('move', 'issue:r:2:', 'up');
             "item_id": "issue:r:2:",
             "direction": "up",
         }],
-        "status": "pending",
+        "status": "retrying",
     }
 
 
+def test_retryable_failure_replays_automatically_once_with_same_operation_id():
+    script = f"""
+const createTodaySync = require({json.dumps(str(TODAY_SYNC))});
+const values = new Map(); const timers = []; const statuses = []; const patches = [];
+let attempts = 0;
+const sync = createTodaySync({{
+  storage: {{getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v),removeItem:k=>values.delete(k)}},
+  getLogin:()=> 'timmy', createOperationId:()=> 'stable-op',
+  setTimer:(callback,delay)=>{{timers.push({{callback,delay}});return timers.length}}, clearTimer:()=>{{}},
+  fetchJson:async (_url,options={{}})=>{{
+    attempts += 1;
+    if (attempts === 1) {{ const error = new Error('busy'); error.status=503; error.retryAfter=2; throw error; }}
+    if (options.method) patches.push(JSON.parse(options.body));
+    return options.method ? {{revision:1,ids:['issue:r:2:']}} : {{revision:0,ids:[]}};
+  }},
+  onRemoteIds:()=>{{}}, onStatus:(state,detail)=>statuses.push([state,detail?.delayMs||null]),
+}});
+sync.enqueue('add','issue:r:2:');
+(async()=>{{
+  await sync.flush();
+  const scheduled = timers.map(timer=>timer.delay);
+  await timers[0].callback();
+  process.stdout.write(JSON.stringify({{scheduled,attempts,patches,pending:sync.pending(),statuses}}));
+}})();
+"""
+    result = json.loads(
+        subprocess.run(["node", "-e", script], check=True, capture_output=True, text=True).stdout
+    )
+
+    assert result["scheduled"] == [2000]
+    assert result["attempts"] == 3
+    assert result["patches"] == [{
+        "operation_id": "stable-op", "action": "add", "item_id": "issue:r:2:", "direction": None,
+    }]
+    assert result["pending"] == []
+    assert ["retrying", 2000] in result["statuses"]
+    assert result["statuses"][-1] == ["saved", None]
+
+
 def test_duplicate_pending_retirements_collapse_to_one_effective_remove():
     script = f"""
 const createTodaySync = require({json.dumps(str(TODAY_SYNC))});

tests/test_today_work.py

@@ -16,6 +16,15 @@ def run_node(script):
     ).stdout
 
 
+@pytest.mark.anyio
+async def test_dashboard_surfaces_automatic_planning_retry_guidance():
+    html = await dashboard()
+
+    assert "error.retryAfter = retryAfter === null ? undefined : Number(retryAfter)" in html
+    assert "Today saved on this device · retrying" in html
+    assert "Later saved on this device · retrying" in html
+
+
 def test_today_queue_is_account_scoped_ordered_unique_and_bounded():
     script = f"""
 const createTodayWork = require({json.dumps(str(TODAY_WORK))});