feat: retry planning sync automatically (#365)
All checks were successful
CI / lint (pull_request) Successful in 43s
CI / build-release (pull_request) Successful in 5s
CI / release-candidate (pull_request) Has been skipped

This commit is contained in:
timmy 2026-08-09 03:00:12 +00:00
parent 6dfc02158d
commit 901691a93b
13 changed files with 226 additions and 28 deletions

View File

@ -148,12 +148,13 @@
refreshMyWorkView();
warmTodayOffline();
},
onStatus: state => {
onStatus: (state, detail = {}) => {
const status = qs('#today-sync-status');
status.textContent = state === 'saved' ? 'Today saved to account.' :
(state === 'pending' ? 'Today saved on this device · sync pending.' :
(state === 'retrying' ? `Today saved on this device · retrying in ${Math.ceil(detail.delayMs / 1000)}s.` :
(state === 'pending' ? 'Today saved on this device · sync pending.' :
(state === 'full' ? 'Another device filled Today · showing its saved plan.' :
'Today sync unavailable · changes stay on this device.'));
'Today sync unavailable · changes stay on this device.')));
},
});
todaySync.startLifecycle({ window, document });
@ -180,10 +181,11 @@
if (!planningOwnerLogin || !laterWork.adopt(records)) return;
refreshMyWorkView();
},
onStatus: state => {
onStatus: (state, detail = {}) => {
qs('#later-sync-status').textContent = state === 'saved' ? 'Later saved to account.' :
(state === 'pending' ? 'Later saved on this device · sync pending.' :
'Later sync unavailable · changes stay on this device.');
(state === 'retrying' ? `Later saved on this device · retrying in ${Math.ceil(detail.delayMs / 1000)}s.` :
(state === 'pending' ? 'Later saved on this device · sync pending.' :
'Later sync unavailable · changes stay on this device.'));
},
});
laterSync.startLifecycle({ window, document });
@ -198,6 +200,8 @@
const error = new Error(payload.error || payload.detail?.message || payload.detail || 'Review request failed.');
error.status = response.status;
error.code = payload.detail?.code;
const retryAfter = response.headers.get('Retry-After');
error.retryAfter = retryAfter === null ? undefined : Number(retryAfter);
throw error;
}
return payload;

View File

@ -1,10 +1,34 @@
function createLaterSync({ storage, getLogin, fetchJson, onRemoteRecords, onStatus, createOperationId, createChannel }) {
function createLaterSync({ storage, getLogin, fetchJson, onRemoteRecords, onStatus, createOperationId, createChannel,
setTimer = globalThis.setTimeout, clearTimer = globalThis.clearTimeout, retryBaseMs = 1000, retryMaxMs = 30000 }) {
const prefix = 'stackchain.later-sync.v1.';
const migrationPrefix = 'stackchain.later-sync-migrated.v1.';
const snapshotPrefix = 'stackchain.later-sync-snapshot.v1.';
let flushing = null;
let channel = null;
let channelKey = '';
let retryTimer = null;
let retryAttempt = 0;
function cancelRetry() {
if (retryTimer !== null) clearTimer?.(retryTimer);
retryTimer = null;
}
function scheduleRetry(error, ownerKey) {
if (retryTimer !== null || !pending().length || !ownerKey) return;
const advised = Number(error?.retryAfter);
const delayMs = Number.isFinite(advised) && advised >= 0
? advised * 1000
: Math.min(retryMaxMs, retryBaseMs * (2 ** retryAttempt));
retryAttempt += 1;
onStatus?.('retrying', { delayMs });
retryTimer = setTimer?.(async () => {
retryTimer = null;
if (key() !== ownerKey) return false;
return flush();
}, delayMs);
retryTimer?.unref?.();
}
function key() {
const login = String(getLogin?.() || '').trim().toLowerCase();
@ -118,12 +142,14 @@ function createLaterSync({ storage, getLogin, fetchJson, onRemoteRecords, onStat
}
async function run() {
if (!key()) return false;
const ownerKey = key();
if (!ownerKey) return false;
ensureChannel();
try {
let plan = await fetchJson('api/v1/later');
let operations = pending();
while (operations.length) {
if (key() !== ownerKey) return false;
const operation = operations[0];
plan = await fetchJson('api/v1/later', {
method: 'PATCH',
@ -139,9 +165,12 @@ function createLaterSync({ storage, getLogin, fetchJson, onRemoteRecords, onStat
}
adopt(plan);
onStatus?.(pending().length ? 'pending' : 'saved');
retryAttempt = 0;
cancelRetry();
return true;
} catch (_error) {
onStatus?.(pending().length ? 'pending' : 'error');
} catch (error) {
if (pending().length) scheduleRetry(error, ownerKey);
else onStatus?.('error');
return false;
}
}

View File

@ -1,6 +1,6 @@
const BASE = new URL('./', self.location.href).pathname;
importScripts(BASE + 'static/background-issue-sync.js');
const CACHE = 'stackchain-dashboard-shell-v48';
const CACHE = 'stackchain-dashboard-shell-v49';
const OUTAGE_STATUSES = new Set([500, 502, 503, 504]);
const NAVIGATION_TIMEOUT_MS = self.__STACKCHAIN_NAVIGATION_TIMEOUT_MS || 4000;
const SHELL = [

View File

@ -1,10 +1,34 @@
function createTodaySync({ storage, getLogin, fetchJson, onRemoteIds, onStatus, createOperationId, createChannel }) {
function createTodaySync({ storage, getLogin, fetchJson, onRemoteIds, onStatus, createOperationId, createChannel,
setTimer = globalThis.setTimeout, clearTimer = globalThis.clearTimeout, retryBaseMs = 1000, retryMaxMs = 30000 }) {
const prefix = 'stackchain.today-sync.v1.';
const migrationPrefix = 'stackchain.today-sync-migrated.v1.';
const snapshotPrefix = 'stackchain.today-sync-snapshot.v1.';
let flushing = null;
let channel = null;
let channelKey = '';
let retryTimer = null;
let retryAttempt = 0;
function cancelRetry() {
if (retryTimer !== null) clearTimer?.(retryTimer);
retryTimer = null;
}
function scheduleRetry(error, ownerKey) {
if (retryTimer !== null || !pending().length || !ownerKey) return;
const advised = Number(error?.retryAfter);
const delayMs = Number.isFinite(advised) && advised >= 0
? advised * 1000
: Math.min(retryMaxMs, retryBaseMs * (2 ** retryAttempt));
retryAttempt += 1;
onStatus?.('retrying', { delayMs });
retryTimer = setTimer?.(async () => {
retryTimer = null;
if (key() !== ownerKey) return false;
return flush();
}, delayMs);
retryTimer?.unref?.();
}
function key() {
const login = String(getLogin?.() || '').trim().toLowerCase();
@ -114,12 +138,14 @@ function createTodaySync({ storage, getLogin, fetchJson, onRemoteIds, onStatus,
}
async function run() {
if (!key()) return false;
const ownerKey = key();
if (!ownerKey) return false;
ensureChannel();
try {
let plan = await fetchJson('api/v1/today');
const operations = pending();
for (const operation of operations) {
if (key() !== ownerKey) return false;
try {
plan = await fetchJson('api/v1/today', {
method: 'PATCH',
@ -132,6 +158,8 @@ function createTodaySync({ storage, getLogin, fetchJson, onRemoteIds, onStatus,
save(rejected.filter(candidate => candidate.operation_id !== operation.operation_id));
adopt(plan);
onStatus?.('full');
retryAttempt = 0;
cancelRetry();
return false;
}
const remaining = pending();
@ -139,9 +167,12 @@ function createTodaySync({ storage, getLogin, fetchJson, onRemoteIds, onStatus,
}
adopt(plan);
onStatus?.(pending().length ? 'pending' : 'saved');
retryAttempt = 0;
cancelRetry();
return true;
} catch (_error) {
onStatus?.(pending().length ? 'pending' : 'error');
} catch (error) {
if (pending().length) scheduleRetry(error, ownerKey);
else onStatus?.('error');
return false;
}
}

View File

@ -897,7 +897,9 @@ async def get_today_plan():
return await asyncio.to_thread(_today_store().get, login)
except (OSError, sqlite3.Error):
raise HTTPException(
status_code=503, detail="Today synchronization is unavailable"
status_code=503,
detail="Today synchronization is unavailable",
headers={"Retry-After": "1"},
)
@ -917,7 +919,9 @@ async def update_today_plan(payload: TodayOperation):
raise HTTPException(status_code=409, detail="Today is limited to 5 items")
except (OSError, sqlite3.Error):
raise HTTPException(
status_code=503, detail="Today synchronization is unavailable"
status_code=503,
detail="Today synchronization is unavailable",
headers={"Retry-After": "1"},
)
@ -928,7 +932,9 @@ async def get_later_plan():
return await asyncio.to_thread(_later_store().get, login)
except (OSError, sqlite3.Error):
raise HTTPException(
status_code=503, detail="Later synchronization is unavailable"
status_code=503,
detail="Later synchronization is unavailable",
headers={"Retry-After": "1"},
)
@ -948,7 +954,9 @@ async def update_later_plan(payload: LaterOperation):
raise HTTPException(status_code=422, detail=str(error))
except (OSError, sqlite3.Error):
raise HTTPException(
status_code=503, detail="Later synchronization is unavailable"
status_code=503,
detail="Later synchronization is unavailable",
headers={"Retry-After": "1"},
)

View File

@ -1,3 +1,5 @@
import sqlite3
import httpx
import pytest
@ -5,6 +7,31 @@ from src import main
from src.later_store import LaterStore
@pytest.mark.anyio
async def test_transient_later_store_failure_tells_clients_when_to_retry(monkeypatch):
async def user():
return {"login": "timmy"}
class BusyStore:
def apply(self, *_args, **_kwargs):
raise sqlite3.OperationalError("database is busy")
monkeypatch.setattr(main, "current_user", user)
monkeypatch.setattr(main, "_later_store", lambda: BusyStore())
payload = main.LaterOperation(
operation_id="retry-me",
action="defer",
item_id="issue:r:1:",
wake_at="2026-08-10T09:00:00.000Z",
)
with pytest.raises(main.HTTPException) as raised:
await main.update_later_plan(payload)
assert raised.value.status_code == 503
assert raised.value.headers == {"Retry-After": "1"}
def test_deferrals_are_durable_revisioned_idempotent_and_account_scoped(tmp_path):
path = tmp_path / "later.sqlite3"
store = LaterStore(path)

View File

@ -83,7 +83,34 @@ sync.enqueue('restore','issue:r:2:');
"wake_at": None,
}
],
"status": "pending",
"status": "retrying",
}
def test_retry_is_single_flight_and_does_not_cross_account_boundary():
script = f"""
const createLaterSync = require({json.dumps(str(LATER_SYNC))});
const values=new Map(); const timers=[]; const requests=[]; let login='timmy';
const sync=createLaterSync({{
storage:{{getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v),removeItem:k=>values.delete(k)}},
getLogin:()=>login,createOperationId:()=> 'stable-later-op',
setTimer:(callback,delay)=>{{timers.push({{callback,delay}});return timers.length}},clearTimer:()=>{{}},
fetchJson:async (_url,options={{}})=>{{requests.push(options.method||'GET');throw new Error('offline')}},
onRemoteRecords:()=>{{}},onStatus:()=>{{}},
}});
sync.enqueue('defer','issue:r:2:','2026-08-10T09:00:00.000Z');
(async()=>{{
const first=sync.flush(); const same=sync.flush(); await Promise.all([first,same]);
const scheduledBeforeSwitch=timers.length;
login='alexander'; await timers[0].callback();
process.stdout.write(JSON.stringify({{scheduledBeforeSwitch,timers:timers.length,requests,pendingForAlexander:sync.pending()}}));
}})();
"""
assert run_node(script) == {
"scheduledBeforeSwitch": 1,
"timers": 1,
"requests": ["GET"],
"pendingForAlexander": [],
}
@ -206,5 +233,5 @@ async def test_dashboard_syncs_every_later_change_and_exposes_account_status():
def test_later_sync_ships_atomically_in_the_offline_shell():
source = (Path(__file__).parents[1] / "frontend" / "service-worker.js").read_text()
assert "stackchain-dashboard-shell-v48" in source
assert "stackchain-dashboard-shell-v49" in source
assert "BASE + 'static/later-sync.js'" in source

View File

@ -137,4 +137,4 @@ def test_markdown_work_bodies_are_mobile_safe_block_containers():
assert ".markdown-content { min-width:0; max-width:100%; overflow-wrap:anywhere;" in css
assert ".markdown-content pre { max-width:100%; overflow-x:auto;" in css
assert ".markdown-content a { min-height:44px;" in css
assert "stackchain-dashboard-shell-v48" in worker
assert "stackchain-dashboard-shell-v49" in worker

View File

@ -35,4 +35,4 @@ def test_offline_shell_contains_every_local_dashboard_runtime_asset():
shell_assets = set(re.findall(r"BASE \+ '([^']+)'", worker.split("async function sessionCsrf", 1)[0]))
assert local_assets <= shell_assets, f"Offline shell is missing: {sorted(local_assets - shell_assets)}"
assert "stackchain-dashboard-shell-v48" in worker
assert "stackchain-dashboard-shell-v49" in worker

View File

@ -108,7 +108,7 @@ async function dispatchNotificationClick(route) {{
def test_navigation_deadline_ships_in_a_new_shell_cache():
source = WORKER.read_text()
assert "stackchain-dashboard-shell-v48" in source
assert "stackchain-dashboard-shell-v49" in source
assert "BASE + 'static/dashboard.css'" in source
assert "BASE + 'static/dashboard.js'" in source
assert "BASE + 'static/install-app.js'" in source
@ -117,21 +117,21 @@ def test_navigation_deadline_ships_in_a_new_shell_cache():
def test_today_convergence_ships_in_a_new_shell_cache():
source = WORKER.read_text()
assert "stackchain-dashboard-shell-v48" in source
assert "stackchain-dashboard-shell-v49" in source
assert "BASE + 'static/today-sync.js'" in source
def test_mobile_search_viewport_ships_in_a_new_offline_shell():
source = WORKER.read_text()
assert "stackchain-dashboard-shell-v48" in source
assert "stackchain-dashboard-shell-v49" in source
assert "BASE + 'static/mobile-search-viewport.js'" in source
def test_update_ownership_flow_ships_atomically_in_a_new_offline_shell():
source = WORKER.read_text()
assert "stackchain-dashboard-shell-v48" in source
assert "stackchain-dashboard-shell-v49" in source
assert "BASE + 'static/update-ownership.js'" in source

View File

@ -1,3 +1,5 @@
import sqlite3
import pytest
import httpx
@ -5,6 +7,28 @@ from src import main
from src.today_store import TodayPlanFull, TodayStore
@pytest.mark.anyio
async def test_transient_today_store_failure_tells_clients_when_to_retry(monkeypatch):
async def user():
return {"login": "timmy"}
class BusyStore:
def apply(self, *_args, **_kwargs):
raise sqlite3.OperationalError("database is locked")
monkeypatch.setattr(main, "current_user", user)
monkeypatch.setattr(main, "_today_store", lambda: BusyStore())
payload = main.TodayOperation(
operation_id="retry-me", action="add", item_id="issue:r:1:"
)
with pytest.raises(main.HTTPException) as raised:
await main.update_today_plan(payload)
assert raised.value.status_code == 503
assert raised.value.headers == {"Retry-After": "1"}
def test_operations_are_durable_ordered_idempotent_and_account_scoped(tmp_path):
path = tmp_path / "today.sqlite3"
store = TodayStore(path, limit=3)

View File

@ -83,10 +83,49 @@ sync.enqueue('move', 'issue:r:2:', 'up');
"item_id": "issue:r:2:",
"direction": "up",
}],
"status": "pending",
"status": "retrying",
}
def test_retryable_failure_replays_automatically_once_with_same_operation_id():
script = f"""
const createTodaySync = require({json.dumps(str(TODAY_SYNC))});
const values = new Map(); const timers = []; const statuses = []; const patches = [];
let attempts = 0;
const sync = createTodaySync({{
storage: {{getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v),removeItem:k=>values.delete(k)}},
getLogin:()=> 'timmy', createOperationId:()=> 'stable-op',
setTimer:(callback,delay)=>{{timers.push({{callback,delay}});return timers.length}}, clearTimer:()=>{{}},
fetchJson:async (_url,options={{}})=>{{
attempts += 1;
if (attempts === 1) {{ const error = new Error('busy'); error.status=503; error.retryAfter=2; throw error; }}
if (options.method) patches.push(JSON.parse(options.body));
return options.method ? {{revision:1,ids:['issue:r:2:']}} : {{revision:0,ids:[]}};
}},
onRemoteIds:()=>{{}}, onStatus:(state,detail)=>statuses.push([state,detail?.delayMs||null]),
}});
sync.enqueue('add','issue:r:2:');
(async()=>{{
await sync.flush();
const scheduled = timers.map(timer=>timer.delay);
await timers[0].callback();
process.stdout.write(JSON.stringify({{scheduled,attempts,patches,pending:sync.pending(),statuses}}));
}})();
"""
result = json.loads(
subprocess.run(["node", "-e", script], check=True, capture_output=True, text=True).stdout
)
assert result["scheduled"] == [2000]
assert result["attempts"] == 3
assert result["patches"] == [{
"operation_id": "stable-op", "action": "add", "item_id": "issue:r:2:", "direction": None,
}]
assert result["pending"] == []
assert ["retrying", 2000] in result["statuses"]
assert result["statuses"][-1] == ["saved", None]
def test_duplicate_pending_retirements_collapse_to_one_effective_remove():
script = f"""
const createTodaySync = require({json.dumps(str(TODAY_SYNC))});

View File

@ -16,6 +16,15 @@ def run_node(script):
).stdout
@pytest.mark.anyio
async def test_dashboard_surfaces_automatic_planning_retry_guidance():
html = await dashboard()
assert "error.retryAfter = retryAfter === null ? undefined : Number(retryAfter)" in html
assert "Today saved on this device · retrying" in html
assert "Later saved on this device · retrying" in html
def test_today_queue_is_account_scoped_ordered_unique_and_bounded():
script = f"""
const createTodayWork = require({json.dumps(str(TODAY_WORK))});