Merge pull request 'Recover orphaned authored operations safely' (#320) from timmy/319-recover-orphaned-authored-operations into main
All checks were successful
CI / lint (push) Successful in 32s
Release / release-candidate (push) Successful in 5s
CI / build-frontend (push) Successful in 5s

This commit is contained in:
rockachopa 2026-08-08 16:41:43 +00:00
commit d6c7a495ed
15 changed files with 321 additions and 18 deletions

View File

@ -77,6 +77,7 @@ function createAuthoredOutbox({ storage, fetchJson, coordinator, backgroundSync,
status: 'queued',
};
delete updated.error;
delete updated.deliveryState;
return updated;
}));
return updated;
@ -134,6 +135,7 @@ function createAuthoredOutbox({ storage, fetchJson, coordinator, backgroundSync,
...candidate,
status: 'attention',
error: String(error.message || 'Message needs attention').slice(0, 240),
...(error.code === 'delivery_uncertain' ? { deliveryState: 'uncertain' } : {}),
} : candidate));
}
return { error, transient: !permanent };
@ -170,7 +172,17 @@ function createAuthoredOutbox({ storage, fetchJson, coordinator, backgroundSync,
if (!currentLogin || item.ownerLogin !== currentLogin) {
return { confirmed: [], remaining: read(), blocked: 1 };
}
const queued = item.status === 'attention' ? update(id, item) : item;
let queued = item;
if (item.status === 'attention') {
queued = {
...item,
operationId: String(makeId()).slice(0, 128),
status: 'queued',
};
delete queued.error;
delete queued.deliveryState;
write(read().map(candidate => candidate.id === id ? queued : candidate));
}
const outcome = await sendItem(queued, currentLogin);
return { confirmed: outcome.result ? [outcome.result] : [], remaining: read(), blocked: 0 };
}
@ -189,6 +201,7 @@ function createAuthoredOutbox({ storage, fetchJson, coordinator, backgroundSync,
...item,
status: 'attention',
error: String(background.error || 'Message needs attention').slice(0, 240),
...(background.deliveryState ? { deliveryState: background.deliveryState } : {}),
}];
return [item];
});

View File

@ -182,7 +182,10 @@ function createIssueSyncStore({ transaction, indexedDB = globalThis.indexedDB, n
claimBatch,
complete: id => update(id, item => ({ ...item, status: 'sent', claimUntil: 0 })),
release: id => update(id, item => ({ ...item, status: 'queued', claimUntil: 0 })),
fail: (id, error) => update(id, item => ({ ...item, status: 'attention', claimUntil: 0, error })),
fail: (id, error, deliveryState) => update(id, item => ({
...item, status: 'attention', claimUntil: 0, error,
...(deliveryState ? { deliveryState } : {}),
})),
snapshot: () => transact(async records =>
(await records.getAll()).filter(item => item.recordType !== 'receipt-preference')),
countBlocked: ownerLogin => transact(async records =>
@ -301,7 +304,11 @@ function createBackgroundIssueSync({
throw error;
}
if (status >= 400 && status < 500) {
await store.fail(item.id, String(error?.message || 'Issue needs attention').slice(0, 240));
await store.fail(
item.id,
String(error?.message || 'Issue needs attention').slice(0, 240),
error?.code === 'delivery_uncertain' ? 'uncertain' : undefined,
);
return { attention: true, error, receipt: receiptFor(item, 'attention') };
}
await store.release(item.id);

View File

@ -144,8 +144,9 @@
const response = await fetch(url, options);
const payload = await response.json().catch(() => ({}));
if (!response.ok) {
const error = new Error(payload.error || payload.detail || 'Review request failed.');
const error = new Error(payload.error || payload.detail?.message || payload.detail || 'Review request failed.');
error.status = response.status;
error.code = payload.detail?.code;
throw error;
}
return payload;
@ -709,16 +710,17 @@
list.innerHTML = lastDrafts.length ? lastDrafts.map((item, index) => {
const isOutbox = item.kind === 'issue-outbox' || item.kind === 'authored-outbox';
const isUnfiled = item.kind === 'unfiled-issue';
const sendLabel = item.delivery_state === 'uncertain' ? 'Verified not posted — retry' : 'Send now';
const outboxActions = item.quarantined ?
'<button class="draft-copy" data-draft-index="' + index + '" type="button">Copy content</button>' +
'<button class="draft-discard" data-draft-index="' + index + '" type="button">Discard</button>' :
item.kind === 'issue-outbox' ?
'<button class="draft-edit" data-draft-index="' + index + '" type="button">Edit</button>' +
'<button class="draft-send" data-draft-index="' + index + '" type="button">Send now</button>' +
'<button class="draft-send" data-draft-index="' + index + '" type="button">' + sendLabel + '</button>' +
'<button class="draft-discard" data-draft-index="' + index + '" type="button">Discard</button>' :
item.kind === 'authored-outbox' ?
'<button class="draft-resume" data-draft-index="' + index + '" type="button">Open message</button>' +
'<button class="draft-send" data-draft-index="' + index + '" type="button">Send now</button>' +
'<button class="draft-send" data-draft-index="' + index + '" type="button">' + sendLabel + '</button>' +
'<button class="draft-discard" data-draft-index="' + index + '" type="button">Discard</button>' :
'<button class="draft-resume" data-draft-index="' + index + '" type="button">' +
(isUnfiled ? 'Choose repository' : 'Resume draft') + '</button>' +

View File

@ -113,7 +113,9 @@ function createDraftInbox({ storage, getCurrentLogin = () => '', now = () => Dat
outbox_id: item.id,
kind: 'issue-outbox',
status: item.status === 'attention' ? 'attention' : 'queued',
label: item.status === 'attention' ? 'Needs attention' : 'Queued issue',
label: item.deliveryState === 'uncertain' ? 'Verify delivery' :
(item.status === 'attention' ? 'Needs attention' : 'Queued issue'),
delivery_state: item.deliveryState,
repository: item.repository,
title: textPreview(item.title) || 'Untitled queued issue',
preview: textPreview([item.title, item.error || item.body].filter(Boolean).join(' — ')),
@ -144,7 +146,9 @@ function createDraftInbox({ storage, getCurrentLogin = () => '', now = () => Dat
outbox_id: item.id,
kind: 'authored-outbox',
status: item.status === 'attention' ? 'attention' : 'queued',
label: item.status === 'attention' ? 'Needs attention' : 'Queued message',
label: item.deliveryState === 'uncertain' ? 'Verify delivery' :
(item.status === 'attention' ? 'Needs attention' : 'Queued message'),
delivery_state: item.deliveryState,
repository: isUpdate ? '' : item.repository,
title: target,
preview: textPreview([item.error, item.body].filter(Boolean).join(' — ')),

View File

@ -89,6 +89,7 @@ function createIssueOutbox({ storage, fetchJson, coordinator, backgroundSync, ge
if (nextMilestoneId === undefined) delete updated.milestoneId;
if (nextDueDate === undefined) delete updated.dueDate;
delete updated.error;
delete updated.deliveryState;
return updated;
}), mirror);
return updated;
@ -152,6 +153,7 @@ function createIssueOutbox({ storage, fetchJson, coordinator, backgroundSync, ge
if (status >= 400 && status < 500) {
write(read().map(candidate => candidate.id === item.id ? {
...candidate, status: 'attention', error: String(error.message || 'Issue needs attention').slice(0, 240),
...(error.code === 'delivery_uncertain' ? { deliveryState: 'uncertain' } : {}),
} : candidate));
}
return { error, transient: !(status >= 400 && status < 500) };
@ -188,8 +190,18 @@ function createIssueOutbox({ storage, fetchJson, coordinator, backgroundSync, ge
if (!currentLogin || item.ownerLogin !== currentLogin) {
return { confirmed: [], remaining: read(), blocked: 1 };
}
update(id, item);
const result = await sendItem({ ...item, status: 'queued' }, currentLogin);
let queued = item;
if (item.status === 'attention') {
queued = {
...item,
operationId: String(operationId()).slice(0, 128),
status: 'queued',
};
delete queued.error;
delete queued.deliveryState;
write(read().map(candidate => candidate.id === id ? queued : candidate));
}
const result = await sendItem(queued, currentLogin);
return { confirmed: result.issue ? [result.issue] : [], remaining: read(), blocked: 0 };
}
@ -205,6 +217,7 @@ function createIssueOutbox({ storage, fetchJson, coordinator, backgroundSync, ge
if (background?.status === 'sent') return [];
if (background?.status === 'attention') return [{
...item, status: 'attention', error: String(background.error || 'Issue needs attention').slice(0, 240),
...(background.deliveryState ? { deliveryState: background.deliveryState } : {}),
}];
return [item];
});

View File

@ -1,6 +1,6 @@
const BASE = new URL('./', self.location.href).pathname;
importScripts(BASE + 'static/background-issue-sync.js');
const CACHE = 'stackchain-dashboard-shell-v33';
const CACHE = 'stackchain-dashboard-shell-v34';
const OUTAGE_STATUSES = new Set([500, 502, 503, 504]);
const SHELL = [
BASE,
@ -86,8 +86,9 @@ async function fetchJson(url, options = {}) {
const response = await fetch(new URL(url, self.location.origin), requestOptions);
const payload = await response.json().catch(() => ({}));
if (!response.ok) {
const error = new Error(payload.error || payload.detail || 'Background issue delivery failed.');
const error = new Error(payload.error || payload.detail?.message || payload.detail || 'Background issue delivery failed.');
error.status = response.status;
error.code = payload.detail?.code;
throw error;
}
return payload;

View File

@ -78,7 +78,7 @@ class IdempotencyLedger:
(now - self.ttl_seconds,),
)
row = connection.execute(
"SELECT fingerprint, status, response_json FROM idempotency_operations "
"SELECT fingerprint, status, response_json, created_at FROM idempotency_operations "
"WHERE key = ?",
(key,),
).fetchone()
@ -87,9 +87,13 @@ class IdempotencyLedger:
return Reservation("conflict")
if row[1] == "completed":
return Reservation("completed", json.loads(row[2]))
if row[3] <= now - self.ttl_seconds:
return Reservation("uncertain")
return Reservation("pending")
count = connection.execute(
"SELECT COUNT(*) FROM idempotency_operations"
"SELECT COUNT(*) FROM idempotency_operations "
"WHERE status = 'completed' OR created_at > ?",
(now - self.ttl_seconds,),
).fetchone()[0]
if count >= self.max_entries:
completed = connection.execute(

View File

@ -314,9 +314,22 @@ async def _run_idempotent_authored_action(
detail="Authored action queue is busy; please retry",
headers={"Retry-After": "1"},
)
existing = _authored_action_operations.get(idempotency_key)
if reservation.state == "pending":
if reservation.state == "uncertain" and (
existing is None or existing[0] != fingerprint
):
operation.close()
raise HTTPException(
status_code=422,
detail={
"code": "delivery_uncertain",
"message": (
"Delivery could not be confirmed. Verify it was not posted before retrying."
),
},
)
if reservation.state in {"pending", "uncertain"}:
operation.close()
if existing is None or existing[0] != fingerprint:
raise HTTPException(

View File

@ -116,6 +116,54 @@ async def test_notification_reply_retry_recovers_after_caller_timeout(monkeypatc
assert calls == 1
@pytest.mark.anyio
async def test_live_authored_task_remains_joinable_after_reservation_age_threshold(
monkeypatch, tmp_path
):
calls = 0
now = [1_000.0]
release = asyncio.Event()
async def reply(_thread_id, body):
nonlocal calls
calls += 1
await release.wait()
return {"id": 319, "body": body}
monkeypatch.setattr(
main,
"_idempotency_ledger",
IdempotencyLedger(
tmp_path / "live.sqlite3",
ttl_seconds=1,
max_entries=256,
clock=lambda: now[0],
),
)
monkeypatch.setattr(main, "NOTIFICATION_MUTATION_TIMEOUT_SECONDS", 0.01)
monkeypatch.setattr(main.gitea_proxy, "reply_to_notification", reply)
transport = httpx.ASGITransport(app=main.app)
headers = {"Idempotency-Key": "live-after-threshold-319"}
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
timed_out = await client.post(
"/api/v1/notifications/42/reply", json={"body": "Join me"}, headers=headers,
)
now[0] += 2
monkeypatch.setattr(main, "NOTIFICATION_MUTATION_TIMEOUT_SECONDS", 1.0)
joined = asyncio.create_task(client.post(
"/api/v1/notifications/42/reply", json={"body": "Join me"}, headers=headers,
))
await asyncio.sleep(0.02)
assert not joined.done(), "a live local operation must be joined instead of rejected as stale"
release.set()
recovered = await joined
assert timed_out.status_code == 503
assert recovered.status_code == 201
assert calls == 1
@pytest.mark.anyio
async def test_review_replays_same_key_and_rejects_changed_head(monkeypatch):
calls = []
@ -233,6 +281,51 @@ async def test_orphaned_pending_comment_fails_closed_without_upstream_retry(
assert calls == 0
@pytest.mark.anyio
async def test_stale_orphaned_comment_returns_non_retryable_uncertain_delivery(
monkeypatch, tmp_path
):
calls = 0
now = [1_000.0]
async def assigned(_repository, _number):
return True
async def comment(_repository, _number, _body):
nonlocal calls
calls += 1
return {"id": 319}
ledger = IdempotencyLedger(
tmp_path / "stale-orphan.sqlite3",
ttl_seconds=10,
max_entries=256,
clock=lambda: now[0],
)
fingerprint = ("issue-comment", "stackchain/api", 7, "Verify first")
assert ledger.reserve("stale-orphan-319", fingerprint).state == "reserved"
now[0] += 11
monkeypatch.setattr(main, "_idempotency_ledger", ledger)
monkeypatch.setattr(main.gitea_proxy, "is_assigned_issue", assigned)
monkeypatch.setattr(main.gitea_proxy, "comment_on_issue", comment)
transport = httpx.ASGITransport(app=main.app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
response = await client.post(
"/api/v1/repos/stackchain/api/issues/7/comments",
json={"body": "Verify first"},
headers={"Idempotency-Key": "stale-orphan-319"},
)
assert response.status_code == 422
assert response.json()["detail"] == {
"code": "delivery_uncertain",
"message": "Delivery could not be confirmed. Verify it was not posted before retrying.",
}
assert "Retry-After" not in response.headers
assert calls == 0
@pytest.mark.anyio
async def test_slow_ledger_reservation_does_not_block_health_requests(monkeypatch):
async def assigned(_repository, _number):

View File

@ -113,6 +113,37 @@ outbox.flush('timmy').then(async first => {{
assert output["remaining"][1]["status"] == "queued"
def test_authored_outbox_requires_deliberate_rekey_after_uncertain_delivery():
script = f"""
const createAuthoredOutbox = require({json.dumps(str(OUTBOX))});
const values = new Map(); let sequence=0; const calls=[];
const storage = {{getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v)}};
const outbox = createAuthoredOutbox({{
storage, getOwnerLogin:()=>'timmy', createOperationId:()=> 'verified-' + (++sequence),
fetchJson:async (_url, options) => {{
const key=options.headers['Idempotency-Key']; calls.push(key);
if (key === 'uncertain-key') {{ const e=new Error('Verify it was not posted before retrying.'); e.status=422; e.code='delivery_uncertain'; throw e; }}
return {{id:calls.length}};
}},
}});
const uncertain=outbox.enqueue({{kind:'issue-comment',repository:'o/r',number:1,body:'Check',operationId:'uncertain-key'}});
outbox.enqueue({{kind:'pull-comment',repository:'o/r',number:2,body:'Continue',operationId:'other-key'}});
(async()=>{{
const first=await outbox.flush('timmy');
const attention=outbox.list()[0];
const retried=await outbox.retry(uncertain.id,'timmy');
process.stdout.write(JSON.stringify({{first,attention,retried,calls,remaining:outbox.list()}}));
}})();
"""
output = run_node(script)
assert output["calls"] == ["uncertain-key", "other-key", "verified-1"]
assert len(output["first"]["confirmed"]) == 1
assert output["attention"]["status"] == "attention"
assert output["attention"]["deliveryState"] == "uncertain"
assert output["remaining"] == []
def test_authored_outbox_retry_is_single_flight_and_edit_rotates_identity():
script = f"""
const createAuthoredOutbox = require({json.dumps(str(OUTBOX))});
@ -182,6 +213,21 @@ setTimeout(() => {{
]
def test_authored_outbox_reconciles_uncertain_worker_delivery_state():
script = f"""
const createAuthoredOutbox = require({json.dumps(str(OUTBOX))});
const values=new Map(); const storage={{getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v)}};
const outbox=createAuthoredOutbox({{storage,getOwnerLogin:()=>'timmy'}});
const item=outbox.enqueue({{kind:'issue-comment',repository:'o/r',number:7,body:'Maybe',operationId:'maybe'}});
const reconciled=outbox.reconcileBackground([{{...item,status:'attention',deliveryState:'uncertain',error:'Verify first'}}]);
process.stdout.write(JSON.stringify(reconciled[0]));
"""
output = run_node(script)
assert output["status"] == "attention"
assert output["deliveryState"] == "uncertain"
def test_authored_outbox_waits_for_durable_background_admission():
script = f"""
const createAuthoredOutbox = require({json.dumps(str(OUTBOX))});

View File

@ -585,6 +585,31 @@ const fetchJson = async url => {{
assert output["result"]["attention"] == 1
def test_uncertain_background_delivery_preserves_verification_state():
script = f"""
const createBackgroundIssueSync = require({json.dumps(str(SYNC))});
let claimed=false; const state={{failed:[]}};
const store={{
claimNext:async()=>claimed?null:(claimed=true,{{id:'maybe',operationId:'maybe',ownerLogin:'timmy',repository:'o/r',title:'Maybe',labelIds:[]}}),
fail:async(id,message,deliveryState)=>state.failed.push({{id,message,deliveryState}}),
release:async()=>{{}}, countBlocked:async()=>0,
}};
const fetchJson=async url=>{{
if(url==='api/v1/background-identity') return {{login:'timmy'}};
const error=new Error('Verify first'); error.status=422; error.code='delivery_uncertain'; throw error;
}};
(async()=>{{
await createBackgroundIssueSync({{store,fetchJson}}).flush();
process.stdout.write(JSON.stringify(state));
}})();
"""
output = run_node(script)
assert output["failed"] == [
{"id": "maybe", "message": "Verify first", "deliveryState": "uncertain"}
]
def test_issue_store_atomically_grants_one_delivery_claim():
script = f"""
const createBackgroundIssueSync = require({json.dumps(str(SYNC))});

View File

@ -121,6 +121,22 @@ process.stdout.write(JSON.stringify(drafts));
assert output[1]["label"] == "Queued issue"
def test_draft_inbox_exposes_uncertain_delivery_for_explicit_user_verification():
script = f"""
const createDraftInbox = require({json.dumps(str(DRAFTS))});
const values = new Map([['stackchain.authored-outbox.v1', JSON.stringify({{version:2,items:[
{{id:'uncertain',kind:'issue-comment',repository:'o/r',number:7,body:'Possibly posted',ownerLogin:'timmy',status:'attention',deliveryState:'uncertain',error:'Verify it was not posted before retrying.',queuedAt:200}}
]}})]]);
const storage = {{get length(){{return values.size}},key:i=>Array.from(values.keys())[i]||null,getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v)}};
const drafts = createDraftInbox({{storage,getCurrentLogin:()=>'timmy'}}).list();
process.stdout.write(JSON.stringify(drafts[0]));
"""
output = run_node(script)
assert output["delivery_state"] == "uncertain"
assert output["label"] == "Verify delivery"
def test_draft_inbox_marks_mismatched_and_legacy_outbox_content_copy_only():
script = f"""
const createDraftInbox = require({json.dumps(str(DRAFTS))});
@ -160,6 +176,9 @@ async def test_mobile_dashboard_exposes_touch_safe_draft_recovery_lane():
assert '.draft-actions button { min-height:44px;' in html
assert 'createDraftInbox({ storage: localStorage' in html
assert "captureDraft.repository && !repositories.includes(captureDraft.repository)" in html
assert "Verified not posted — retry" in html
assert "payload.detail?.message" in html
assert "error.code = payload.detail?.code" in html
@pytest.mark.anyio

View File

@ -71,3 +71,21 @@ def test_independent_connections_atomically_reserve_one_operation(tmp_path):
states = list(executor.map(reserve, ledgers))
assert sorted(states) == ["pending", "reserved"]
def test_stale_pending_reservation_becomes_uncertain_without_consuming_capacity(tmp_path):
now = [1_000.0]
ledger = IdempotencyLedger(
tmp_path / "orphaned.sqlite3",
ttl_seconds=10,
max_entries=1,
clock=lambda: now[0],
)
fingerprint = ("issue-comment", "stackchain/api", 7, "Verify first")
assert ledger.reserve("orphan", fingerprint).state == "reserved"
now[0] += 11
assert ledger.reserve("orphan", fingerprint).state == "uncertain"
assert ledger.reserve("unrelated", ("issue-comment", "stackchain/api", 8, "Continue")).state == "reserved"
assert ledger.reserve("orphan", ("issue-comment", "stackchain/api", 7, "Changed")).state == "conflict"

View File

@ -139,6 +139,34 @@ outbox.flush('timmy').then(async () => {{
assert output["remaining"] == []
def test_issue_outbox_rekeys_only_after_user_retries_uncertain_delivery():
script = f"""
const createIssueOutbox = require({json.dumps(str(OUTBOX))});
const values = new Map(); let sequence=0; const calls=[];
const storage = {{getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v)}};
const outbox = createIssueOutbox({{
storage, getOwnerLogin:()=>'timmy', createOperationId:()=> 'operation-' + (++sequence),
fetchJson:async (_url, options) => {{
const key=options.headers['Idempotency-Key']; calls.push(key);
if (key === 'operation-1') {{ const e=new Error('Verify it was not posted before retrying.'); e.status=422; e.code='delivery_uncertain'; throw e; }}
return {{number:319}};
}},
}});
const item=outbox.enqueue({{repository:'o/r',title:'Possibly sent',body:'Check first'}});
(async()=>{{
await outbox.flush('timmy'); const attention=outbox.list()[0];
await outbox.retry(item.id,'timmy');
process.stdout.write(JSON.stringify({{attention,calls,remaining:outbox.list()}}));
}})();
"""
output = run_node(script)
assert output["attention"]["status"] == "attention"
assert output["attention"]["deliveryState"] == "uncertain"
assert output["calls"] == ["operation-1", "operation-2"]
assert output["remaining"] == []
def test_issue_outbox_rotates_operation_id_only_when_delivery_payload_changes():
script = f"""
const createIssueOutbox = require({json.dumps(str(OUTBOX))});

View File

@ -97,7 +97,7 @@ async function dispatchNotificationClick(route) {{
def test_share_target_sign_in_fix_ships_in_a_new_shell_cache():
source = WORKER.read_text()
assert "stackchain-dashboard-shell-v33" in source
assert "stackchain-dashboard-shell-v34" in source
assert "BASE + 'static/dashboard.css'" in source
assert "BASE + 'static/dashboard.js'" in source
assert "BASE + 'static/install-app.js'" in source
@ -106,7 +106,7 @@ def test_share_target_sign_in_fix_ships_in_a_new_shell_cache():
def test_mobile_search_viewport_ships_in_a_new_offline_shell():
source = WORKER.read_text()
assert "stackchain-dashboard-shell-v33" in source
assert "stackchain-dashboard-shell-v34" in source
assert "BASE + 'static/mobile-search-viewport.js'" in source
@ -122,6 +122,23 @@ def test_background_sync_event_flushes_closed_app_issue_outbox_only_for_its_tag(
assert result["backgroundFlushes"] == 1
def test_background_delivery_preserves_structured_uncertain_error():
result = run_worker_scenario(
"""
context.fetch=async()=>new Response(JSON.stringify({detail:{code:'delivery_uncertain',message:'Verify it was not posted before retrying.'}}),{status:422,headers:{'Content-Type':'application/json'}});
const outcome=await context.self.__testFetchJson('/dashboard/api/v1/repos/o/r/issues',{method:'POST'})
.then(()=>({ok:true}),error=>({status:error.status,code:error.code,message:error.message}));
process.stdout.write(JSON.stringify(outcome));
"""
)
assert result == {
"status": 422,
"code": "delivery_uncertain",
"message": "Verify it was not posted before retrying.",
}
def test_authenticated_page_message_resumes_queued_background_delivery():
result = run_worker_scenario(
"""