fix: recover stalled background delivery (#305)

tests/test_background_issue_sync.py

@@ -359,6 +359,83 @@ const fetchJson = async url => {{
     assert output["error"] == "Gitea unavailable"
 
 
+def test_stalled_delivery_times_out_releases_claim_and_retries_with_same_idempotency_key():
+    script = f"""
+const createBackgroundIssueSync = require({json.dumps(str(SYNC))});
+const item = {{id:'capture-timeout',operationId:'stable-operation',ownerLogin:'timmy',repository:'o/r',title:'Retry',body:'Later',labelIds:[]}};
+const state = {{queued:true,released:[],completed:[],keys:[],attempts:0}};
+const store = {{
+  claimNext: async () => state.queued ? (state.queued=false, {{...item}}) : null,
+  complete: async id => state.completed.push(id),
+  release: async id => {{state.released.push(id); state.queued=true;}},
+  fail: async () => {{}}, countBlocked: async () => 0,
+}};
+const fetchJson = async (url, options={{}}) => {{
+  if (url === 'api/v1/background-identity') return {{login:'timmy'}};
+  state.keys.push(options.headers['Idempotency-Key']);
+  state.attempts += 1;
+  if (state.attempts === 1) return new Promise(() => {{}});
+  return {{repository:'o/r',number:9}};
+}};
+(async () => {{
+  const sync=createBackgroundIssueSync({{store,fetchJson,requestTimeoutMs:10}});
+  let firstError='';
+  try {{ await sync.flush(); }} catch (error) {{ firstError=error.message; }}
+  const second=await sync.flush();
+  process.stdout.write(JSON.stringify({{state,firstError,second}}));
+}})();
+"""
+    output = run_node(script)
+
+    assert output["firstError"] == "Background request timed out."
+    assert output["state"]["released"] == ["capture-timeout"]
+    assert output["state"]["completed"] == ["capture-timeout"]
+    assert output["state"]["keys"] == ["stable-operation", "stable-operation"]
+    assert output["second"]["confirmed"][0]["number"] == 9
+
+
+def test_purge_cancels_stalled_delivery_without_reclaiming_it():
+    script = f"""
+const createBackgroundIssueSync = require({json.dumps(str(SYNC))});
+const items=[
+  {{id:'capture-purge-1',operationId:'capture-purge-1',ownerLogin:'timmy',repository:'o/r',title:'Private',body:'Draft',labelIds:[]}},
+  {{id:'capture-purge-2',operationId:'capture-purge-2',ownerLogin:'timmy',repository:'o/r',title:'Private 2',body:'Draft',labelIds:[]}},
+];
+const state={{released:[],closed:0,mutations:0}};
+let mutationStarted;
+const started=new Promise(resolve => mutationStarted=resolve);
+const store={{
+  claimBatch:async()=>items.map(item=>({{...item}})),
+  complete:async()=>{{}},
+  release:async id=>state.released.push(id),
+  fail:async()=>{{}}, countBlocked:async()=>0, close:async()=>{{state.closed+=1;}},
+}};
+const fetchJson=async url=>{{
+  if(url==='api/v1/background-identity')return{{login:'timmy'}};
+  state.mutations+=1;mutationStarted();return new Promise(()=>{{}});
+}};
+(async()=>{{
+  const sync=createBackgroundIssueSync({{store,fetchJson,maxConcurrency:1,requestTimeoutMs:1000}});
+  sync.flush().catch(()=>{{}});
+  await started;
+  const outcome=await Promise.race([
+    sync.purge().then(()=> 'purged'),
+    new Promise(resolve=>setTimeout(()=>resolve('blocked'),80)),
+  ]);
+  process.stdout.write(JSON.stringify({{state,outcome}}));
+  process.exit(0);
+}})();
+"""
+    output = run_node(script)
+
+    assert output["outcome"] == "purged"
+    assert output["state"] == {
+        "released": ["capture-purge-1", "capture-purge-2"],
+        "closed": 1,
+        "mutations": 1,
+    }
+
+
 def test_bounded_flush_delivers_healthy_records_after_transient_failure_without_same_run_retry():
     script = f"""
 const createBackgroundIssueSync = require({json.dumps(str(SYNC))});

tests/test_service_worker.py

@@ -20,7 +20,7 @@ const cache = {{
   put: async (request, response) => {{ state.puts.push(String(request.url || request)); }},
 }};
 const context = {{
-  URL, Request, Response,
+  URL, Request, Response, Headers, AbortController,
   console,
   self: {{
     location: {{ href: 'https://forge.example/dashboard/service-worker.js', origin: 'https://forge.example' }},
@@ -52,7 +52,7 @@ const context = {{
   }},
 }};
 vm.createContext(context);
-vm.runInContext(fs.readFileSync({json.dumps(str(WORKER))}, 'utf8'), context);
+vm.runInContext(fs.readFileSync({json.dumps(str(WORKER))}, 'utf8') + '\\nself.__testFetchJson = fetchJson; self.__testWithSessionCsrf = withSessionCsrf;', context);
 async function dispatch(name, request) {{
   let pending;
   let response;
@@ -95,7 +95,7 @@ async function dispatchNotificationClick(route) {{
 def test_strict_browser_assets_ship_in_a_new_shell_cache():
     source = WORKER.read_text()
 
-    assert "stackchain-dashboard-shell-v29" in source
+    assert "stackchain-dashboard-shell-v30" in source
     assert "BASE + 'static/dashboard.css'" in source
     assert "BASE + 'static/dashboard.js'" in source
     assert "BASE + 'static/install-app.js'" in source
@@ -124,6 +124,31 @@ def test_authenticated_page_message_resumes_queued_background_delivery():
     assert result["backgroundFlushes"] == 1
 
 
+def test_background_mutation_abort_also_cancels_stalled_csrf_lookup():
+    result = run_worker_scenario(
+        """
+  let mutations=0;
+  context.fetch=(request, options={})=>{
+    if(String(request).includes('/api/v1/session')){
+      return new Promise((resolve,reject)=>options.signal?.addEventListener('abort',()=>reject(new Error('aborted')), {once:true}));
+    }
+    mutations+=1;
+    return Promise.resolve(new Response('{}',{status:200,headers:{'Content-Type':'application/json'}}));
+  };
+  const controller=new AbortController();
+  const request=context.self.__testWithSessionCsrf(() =>
+    context.self.__testFetchJson('/dashboard/api/v1/repos/o/r/issues',{method:'POST',signal:controller.signal})
+  ).then(()=> 'completed', error=>error.message);
+  controller.abort();
+  const outcome=await Promise.race([request,new Promise(resolve=>setTimeout(()=>resolve('blocked'),40))]);
+  process.stdout.write(JSON.stringify({outcome,mutations}));
+  process.exit(0);
+"""
+    )
+
+    assert result == {"outcome": "aborted", "mutations": 0}
+
+
 def test_device_purge_message_stops_worker_outbox_and_acknowledges_completion():
     result = run_worker_scenario(
         """
@@ -180,8 +205,9 @@ def test_opted_in_background_sync_notifies_privately_and_receipt_tap_focuses_rou
 def test_background_mutations_obtain_session_bound_csrf_proof():
     source = WORKER.read_text()
 
-    assert "async function sessionCsrf()" in source
+    assert "async function sessionCsrf(signal)" in source
     assert "BASE + 'api/v1/session'" in source
+    assert "signal," in source
     assert "headers.set('X-CSRF-Token', csrf)" in source
 
 
@@ -189,8 +215,11 @@ def test_one_session_bound_csrf_proof_is_reused_for_a_background_drain():
     source = WORKER.read_text()
 
     assert "async function withSessionCsrf(work)" in source
-    assert "batchedCsrf = sessionCsrf()" in source
-    assert "await (batchedCsrf || sessionCsrf())" in source
+    assert "batchedCsrfController = new AbortController()" in source
+    assert "batchedCsrf = sessionCsrf(batchedCsrfController.signal)" in source
+    assert "batchedCsrfController.abort()" in source
+    assert "await (batchedCsrf || sessionCsrf(options.signal))" in source
+    assert "if (options.signal?.aborted)" in source
     assert "batch: withSessionCsrf" in source