Recover orphaned authored operations safely #320
|
|
@ -77,6 +77,7 @@ function createAuthoredOutbox({ storage, fetchJson, coordinator, backgroundSync,
|
||||||
status: 'queued',
|
status: 'queued',
|
||||||
};
|
};
|
||||||
delete updated.error;
|
delete updated.error;
|
||||||
|
delete updated.deliveryState;
|
||||||
return updated;
|
return updated;
|
||||||
}));
|
}));
|
||||||
return updated;
|
return updated;
|
||||||
|
|
@ -134,6 +135,7 @@ function createAuthoredOutbox({ storage, fetchJson, coordinator, backgroundSync,
|
||||||
...candidate,
|
...candidate,
|
||||||
status: 'attention',
|
status: 'attention',
|
||||||
error: String(error.message || 'Message needs attention').slice(0, 240),
|
error: String(error.message || 'Message needs attention').slice(0, 240),
|
||||||
|
...(error.code === 'delivery_uncertain' ? { deliveryState: 'uncertain' } : {}),
|
||||||
} : candidate));
|
} : candidate));
|
||||||
}
|
}
|
||||||
return { error, transient: !permanent };
|
return { error, transient: !permanent };
|
||||||
|
|
@ -170,7 +172,17 @@ function createAuthoredOutbox({ storage, fetchJson, coordinator, backgroundSync,
|
||||||
if (!currentLogin || item.ownerLogin !== currentLogin) {
|
if (!currentLogin || item.ownerLogin !== currentLogin) {
|
||||||
return { confirmed: [], remaining: read(), blocked: 1 };
|
return { confirmed: [], remaining: read(), blocked: 1 };
|
||||||
}
|
}
|
||||||
const queued = item.status === 'attention' ? update(id, item) : item;
|
let queued = item;
|
||||||
|
if (item.status === 'attention') {
|
||||||
|
queued = {
|
||||||
|
...item,
|
||||||
|
operationId: String(makeId()).slice(0, 128),
|
||||||
|
status: 'queued',
|
||||||
|
};
|
||||||
|
delete queued.error;
|
||||||
|
delete queued.deliveryState;
|
||||||
|
write(read().map(candidate => candidate.id === id ? queued : candidate));
|
||||||
|
}
|
||||||
const outcome = await sendItem(queued, currentLogin);
|
const outcome = await sendItem(queued, currentLogin);
|
||||||
return { confirmed: outcome.result ? [outcome.result] : [], remaining: read(), blocked: 0 };
|
return { confirmed: outcome.result ? [outcome.result] : [], remaining: read(), blocked: 0 };
|
||||||
}
|
}
|
||||||
|
|
@ -189,6 +201,7 @@ function createAuthoredOutbox({ storage, fetchJson, coordinator, backgroundSync,
|
||||||
...item,
|
...item,
|
||||||
status: 'attention',
|
status: 'attention',
|
||||||
error: String(background.error || 'Message needs attention').slice(0, 240),
|
error: String(background.error || 'Message needs attention').slice(0, 240),
|
||||||
|
...(background.deliveryState ? { deliveryState: background.deliveryState } : {}),
|
||||||
}];
|
}];
|
||||||
return [item];
|
return [item];
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -182,7 +182,10 @@ function createIssueSyncStore({ transaction, indexedDB = globalThis.indexedDB, n
|
||||||
claimBatch,
|
claimBatch,
|
||||||
complete: id => update(id, item => ({ ...item, status: 'sent', claimUntil: 0 })),
|
complete: id => update(id, item => ({ ...item, status: 'sent', claimUntil: 0 })),
|
||||||
release: id => update(id, item => ({ ...item, status: 'queued', claimUntil: 0 })),
|
release: id => update(id, item => ({ ...item, status: 'queued', claimUntil: 0 })),
|
||||||
fail: (id, error) => update(id, item => ({ ...item, status: 'attention', claimUntil: 0, error })),
|
fail: (id, error, deliveryState) => update(id, item => ({
|
||||||
|
...item, status: 'attention', claimUntil: 0, error,
|
||||||
|
...(deliveryState ? { deliveryState } : {}),
|
||||||
|
})),
|
||||||
snapshot: () => transact(async records =>
|
snapshot: () => transact(async records =>
|
||||||
(await records.getAll()).filter(item => item.recordType !== 'receipt-preference')),
|
(await records.getAll()).filter(item => item.recordType !== 'receipt-preference')),
|
||||||
countBlocked: ownerLogin => transact(async records =>
|
countBlocked: ownerLogin => transact(async records =>
|
||||||
|
|
@ -301,7 +304,11 @@ function createBackgroundIssueSync({
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
if (status >= 400 && status < 500) {
|
if (status >= 400 && status < 500) {
|
||||||
await store.fail(item.id, String(error?.message || 'Issue needs attention').slice(0, 240));
|
await store.fail(
|
||||||
|
item.id,
|
||||||
|
String(error?.message || 'Issue needs attention').slice(0, 240),
|
||||||
|
error?.code === 'delivery_uncertain' ? 'uncertain' : undefined,
|
||||||
|
);
|
||||||
return { attention: true, error, receipt: receiptFor(item, 'attention') };
|
return { attention: true, error, receipt: receiptFor(item, 'attention') };
|
||||||
}
|
}
|
||||||
await store.release(item.id);
|
await store.release(item.id);
|
||||||
|
|
|
||||||
|
|
@ -144,8 +144,9 @@
|
||||||
const response = await fetch(url, options);
|
const response = await fetch(url, options);
|
||||||
const payload = await response.json().catch(() => ({}));
|
const payload = await response.json().catch(() => ({}));
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const error = new Error(payload.error || payload.detail || 'Review request failed.');
|
const error = new Error(payload.error || payload.detail?.message || payload.detail || 'Review request failed.');
|
||||||
error.status = response.status;
|
error.status = response.status;
|
||||||
|
error.code = payload.detail?.code;
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
return payload;
|
return payload;
|
||||||
|
|
@ -709,16 +710,17 @@
|
||||||
list.innerHTML = lastDrafts.length ? lastDrafts.map((item, index) => {
|
list.innerHTML = lastDrafts.length ? lastDrafts.map((item, index) => {
|
||||||
const isOutbox = item.kind === 'issue-outbox' || item.kind === 'authored-outbox';
|
const isOutbox = item.kind === 'issue-outbox' || item.kind === 'authored-outbox';
|
||||||
const isUnfiled = item.kind === 'unfiled-issue';
|
const isUnfiled = item.kind === 'unfiled-issue';
|
||||||
|
const sendLabel = item.delivery_state === 'uncertain' ? 'Verified not posted — retry' : 'Send now';
|
||||||
const outboxActions = item.quarantined ?
|
const outboxActions = item.quarantined ?
|
||||||
'<button class="draft-copy" data-draft-index="' + index + '" type="button">Copy content</button>' +
|
'<button class="draft-copy" data-draft-index="' + index + '" type="button">Copy content</button>' +
|
||||||
'<button class="draft-discard" data-draft-index="' + index + '" type="button">Discard</button>' :
|
'<button class="draft-discard" data-draft-index="' + index + '" type="button">Discard</button>' :
|
||||||
item.kind === 'issue-outbox' ?
|
item.kind === 'issue-outbox' ?
|
||||||
'<button class="draft-edit" data-draft-index="' + index + '" type="button">Edit</button>' +
|
'<button class="draft-edit" data-draft-index="' + index + '" type="button">Edit</button>' +
|
||||||
'<button class="draft-send" data-draft-index="' + index + '" type="button">Send now</button>' +
|
'<button class="draft-send" data-draft-index="' + index + '" type="button">' + sendLabel + '</button>' +
|
||||||
'<button class="draft-discard" data-draft-index="' + index + '" type="button">Discard</button>' :
|
'<button class="draft-discard" data-draft-index="' + index + '" type="button">Discard</button>' :
|
||||||
item.kind === 'authored-outbox' ?
|
item.kind === 'authored-outbox' ?
|
||||||
'<button class="draft-resume" data-draft-index="' + index + '" type="button">Open message</button>' +
|
'<button class="draft-resume" data-draft-index="' + index + '" type="button">Open message</button>' +
|
||||||
'<button class="draft-send" data-draft-index="' + index + '" type="button">Send now</button>' +
|
'<button class="draft-send" data-draft-index="' + index + '" type="button">' + sendLabel + '</button>' +
|
||||||
'<button class="draft-discard" data-draft-index="' + index + '" type="button">Discard</button>' :
|
'<button class="draft-discard" data-draft-index="' + index + '" type="button">Discard</button>' :
|
||||||
'<button class="draft-resume" data-draft-index="' + index + '" type="button">' +
|
'<button class="draft-resume" data-draft-index="' + index + '" type="button">' +
|
||||||
(isUnfiled ? 'Choose repository' : 'Resume draft') + '</button>' +
|
(isUnfiled ? 'Choose repository' : 'Resume draft') + '</button>' +
|
||||||
|
|
|
||||||
|
|
@ -113,7 +113,9 @@ function createDraftInbox({ storage, getCurrentLogin = () => '', now = () => Dat
|
||||||
outbox_id: item.id,
|
outbox_id: item.id,
|
||||||
kind: 'issue-outbox',
|
kind: 'issue-outbox',
|
||||||
status: item.status === 'attention' ? 'attention' : 'queued',
|
status: item.status === 'attention' ? 'attention' : 'queued',
|
||||||
label: item.status === 'attention' ? 'Needs attention' : 'Queued issue',
|
label: item.deliveryState === 'uncertain' ? 'Verify delivery' :
|
||||||
|
(item.status === 'attention' ? 'Needs attention' : 'Queued issue'),
|
||||||
|
delivery_state: item.deliveryState,
|
||||||
repository: item.repository,
|
repository: item.repository,
|
||||||
title: textPreview(item.title) || 'Untitled queued issue',
|
title: textPreview(item.title) || 'Untitled queued issue',
|
||||||
preview: textPreview([item.title, item.error || item.body].filter(Boolean).join(' — ')),
|
preview: textPreview([item.title, item.error || item.body].filter(Boolean).join(' — ')),
|
||||||
|
|
@ -144,7 +146,9 @@ function createDraftInbox({ storage, getCurrentLogin = () => '', now = () => Dat
|
||||||
outbox_id: item.id,
|
outbox_id: item.id,
|
||||||
kind: 'authored-outbox',
|
kind: 'authored-outbox',
|
||||||
status: item.status === 'attention' ? 'attention' : 'queued',
|
status: item.status === 'attention' ? 'attention' : 'queued',
|
||||||
label: item.status === 'attention' ? 'Needs attention' : 'Queued message',
|
label: item.deliveryState === 'uncertain' ? 'Verify delivery' :
|
||||||
|
(item.status === 'attention' ? 'Needs attention' : 'Queued message'),
|
||||||
|
delivery_state: item.deliveryState,
|
||||||
repository: isUpdate ? '' : item.repository,
|
repository: isUpdate ? '' : item.repository,
|
||||||
title: target,
|
title: target,
|
||||||
preview: textPreview([item.error, item.body].filter(Boolean).join(' — ')),
|
preview: textPreview([item.error, item.body].filter(Boolean).join(' — ')),
|
||||||
|
|
|
||||||
|
|
@ -89,6 +89,7 @@ function createIssueOutbox({ storage, fetchJson, coordinator, backgroundSync, ge
|
||||||
if (nextMilestoneId === undefined) delete updated.milestoneId;
|
if (nextMilestoneId === undefined) delete updated.milestoneId;
|
||||||
if (nextDueDate === undefined) delete updated.dueDate;
|
if (nextDueDate === undefined) delete updated.dueDate;
|
||||||
delete updated.error;
|
delete updated.error;
|
||||||
|
delete updated.deliveryState;
|
||||||
return updated;
|
return updated;
|
||||||
}), mirror);
|
}), mirror);
|
||||||
return updated;
|
return updated;
|
||||||
|
|
@ -152,6 +153,7 @@ function createIssueOutbox({ storage, fetchJson, coordinator, backgroundSync, ge
|
||||||
if (status >= 400 && status < 500) {
|
if (status >= 400 && status < 500) {
|
||||||
write(read().map(candidate => candidate.id === item.id ? {
|
write(read().map(candidate => candidate.id === item.id ? {
|
||||||
...candidate, status: 'attention', error: String(error.message || 'Issue needs attention').slice(0, 240),
|
...candidate, status: 'attention', error: String(error.message || 'Issue needs attention').slice(0, 240),
|
||||||
|
...(error.code === 'delivery_uncertain' ? { deliveryState: 'uncertain' } : {}),
|
||||||
} : candidate));
|
} : candidate));
|
||||||
}
|
}
|
||||||
return { error, transient: !(status >= 400 && status < 500) };
|
return { error, transient: !(status >= 400 && status < 500) };
|
||||||
|
|
@ -188,8 +190,18 @@ function createIssueOutbox({ storage, fetchJson, coordinator, backgroundSync, ge
|
||||||
if (!currentLogin || item.ownerLogin !== currentLogin) {
|
if (!currentLogin || item.ownerLogin !== currentLogin) {
|
||||||
return { confirmed: [], remaining: read(), blocked: 1 };
|
return { confirmed: [], remaining: read(), blocked: 1 };
|
||||||
}
|
}
|
||||||
update(id, item);
|
let queued = item;
|
||||||
const result = await sendItem({ ...item, status: 'queued' }, currentLogin);
|
if (item.status === 'attention') {
|
||||||
|
queued = {
|
||||||
|
...item,
|
||||||
|
operationId: String(operationId()).slice(0, 128),
|
||||||
|
status: 'queued',
|
||||||
|
};
|
||||||
|
delete queued.error;
|
||||||
|
delete queued.deliveryState;
|
||||||
|
write(read().map(candidate => candidate.id === id ? queued : candidate));
|
||||||
|
}
|
||||||
|
const result = await sendItem(queued, currentLogin);
|
||||||
return { confirmed: result.issue ? [result.issue] : [], remaining: read(), blocked: 0 };
|
return { confirmed: result.issue ? [result.issue] : [], remaining: read(), blocked: 0 };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -205,6 +217,7 @@ function createIssueOutbox({ storage, fetchJson, coordinator, backgroundSync, ge
|
||||||
if (background?.status === 'sent') return [];
|
if (background?.status === 'sent') return [];
|
||||||
if (background?.status === 'attention') return [{
|
if (background?.status === 'attention') return [{
|
||||||
...item, status: 'attention', error: String(background.error || 'Issue needs attention').slice(0, 240),
|
...item, status: 'attention', error: String(background.error || 'Issue needs attention').slice(0, 240),
|
||||||
|
...(background.deliveryState ? { deliveryState: background.deliveryState } : {}),
|
||||||
}];
|
}];
|
||||||
return [item];
|
return [item];
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
const BASE = new URL('./', self.location.href).pathname;
|
const BASE = new URL('./', self.location.href).pathname;
|
||||||
importScripts(BASE + 'static/background-issue-sync.js');
|
importScripts(BASE + 'static/background-issue-sync.js');
|
||||||
const CACHE = 'stackchain-dashboard-shell-v33';
|
const CACHE = 'stackchain-dashboard-shell-v34';
|
||||||
const OUTAGE_STATUSES = new Set([500, 502, 503, 504]);
|
const OUTAGE_STATUSES = new Set([500, 502, 503, 504]);
|
||||||
const SHELL = [
|
const SHELL = [
|
||||||
BASE,
|
BASE,
|
||||||
|
|
@ -86,8 +86,9 @@ async function fetchJson(url, options = {}) {
|
||||||
const response = await fetch(new URL(url, self.location.origin), requestOptions);
|
const response = await fetch(new URL(url, self.location.origin), requestOptions);
|
||||||
const payload = await response.json().catch(() => ({}));
|
const payload = await response.json().catch(() => ({}));
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const error = new Error(payload.error || payload.detail || 'Background issue delivery failed.');
|
const error = new Error(payload.error || payload.detail?.message || payload.detail || 'Background issue delivery failed.');
|
||||||
error.status = response.status;
|
error.status = response.status;
|
||||||
|
error.code = payload.detail?.code;
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
return payload;
|
return payload;
|
||||||
|
|
|
||||||
|
|
@ -78,7 +78,7 @@ class IdempotencyLedger:
|
||||||
(now - self.ttl_seconds,),
|
(now - self.ttl_seconds,),
|
||||||
)
|
)
|
||||||
row = connection.execute(
|
row = connection.execute(
|
||||||
"SELECT fingerprint, status, response_json FROM idempotency_operations "
|
"SELECT fingerprint, status, response_json, created_at FROM idempotency_operations "
|
||||||
"WHERE key = ?",
|
"WHERE key = ?",
|
||||||
(key,),
|
(key,),
|
||||||
).fetchone()
|
).fetchone()
|
||||||
|
|
@ -87,9 +87,13 @@ class IdempotencyLedger:
|
||||||
return Reservation("conflict")
|
return Reservation("conflict")
|
||||||
if row[1] == "completed":
|
if row[1] == "completed":
|
||||||
return Reservation("completed", json.loads(row[2]))
|
return Reservation("completed", json.loads(row[2]))
|
||||||
|
if row[3] <= now - self.ttl_seconds:
|
||||||
|
return Reservation("uncertain")
|
||||||
return Reservation("pending")
|
return Reservation("pending")
|
||||||
count = connection.execute(
|
count = connection.execute(
|
||||||
"SELECT COUNT(*) FROM idempotency_operations"
|
"SELECT COUNT(*) FROM idempotency_operations "
|
||||||
|
"WHERE status = 'completed' OR created_at > ?",
|
||||||
|
(now - self.ttl_seconds,),
|
||||||
).fetchone()[0]
|
).fetchone()[0]
|
||||||
if count >= self.max_entries:
|
if count >= self.max_entries:
|
||||||
completed = connection.execute(
|
completed = connection.execute(
|
||||||
|
|
|
||||||
17
src/main.py
17
src/main.py
|
|
@ -314,9 +314,22 @@ async def _run_idempotent_authored_action(
|
||||||
detail="Authored action queue is busy; please retry",
|
detail="Authored action queue is busy; please retry",
|
||||||
headers={"Retry-After": "1"},
|
headers={"Retry-After": "1"},
|
||||||
)
|
)
|
||||||
|
|
||||||
existing = _authored_action_operations.get(idempotency_key)
|
existing = _authored_action_operations.get(idempotency_key)
|
||||||
if reservation.state == "pending":
|
if reservation.state == "uncertain" and (
|
||||||
|
existing is None or existing[0] != fingerprint
|
||||||
|
):
|
||||||
|
operation.close()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=422,
|
||||||
|
detail={
|
||||||
|
"code": "delivery_uncertain",
|
||||||
|
"message": (
|
||||||
|
"Delivery could not be confirmed. Verify it was not posted before retrying."
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
if reservation.state in {"pending", "uncertain"}:
|
||||||
operation.close()
|
operation.close()
|
||||||
if existing is None or existing[0] != fingerprint:
|
if existing is None or existing[0] != fingerprint:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
|
|
|
||||||
|
|
@ -116,6 +116,54 @@ async def test_notification_reply_retry_recovers_after_caller_timeout(monkeypatc
|
||||||
assert calls == 1
|
assert calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_live_authored_task_remains_joinable_after_reservation_age_threshold(
|
||||||
|
monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
calls = 0
|
||||||
|
now = [1_000.0]
|
||||||
|
release = asyncio.Event()
|
||||||
|
|
||||||
|
async def reply(_thread_id, body):
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
await release.wait()
|
||||||
|
return {"id": 319, "body": body}
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
main,
|
||||||
|
"_idempotency_ledger",
|
||||||
|
IdempotencyLedger(
|
||||||
|
tmp_path / "live.sqlite3",
|
||||||
|
ttl_seconds=1,
|
||||||
|
max_entries=256,
|
||||||
|
clock=lambda: now[0],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(main, "NOTIFICATION_MUTATION_TIMEOUT_SECONDS", 0.01)
|
||||||
|
monkeypatch.setattr(main.gitea_proxy, "reply_to_notification", reply)
|
||||||
|
transport = httpx.ASGITransport(app=main.app)
|
||||||
|
headers = {"Idempotency-Key": "live-after-threshold-319"}
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||||
|
timed_out = await client.post(
|
||||||
|
"/api/v1/notifications/42/reply", json={"body": "Join me"}, headers=headers,
|
||||||
|
)
|
||||||
|
now[0] += 2
|
||||||
|
monkeypatch.setattr(main, "NOTIFICATION_MUTATION_TIMEOUT_SECONDS", 1.0)
|
||||||
|
joined = asyncio.create_task(client.post(
|
||||||
|
"/api/v1/notifications/42/reply", json={"body": "Join me"}, headers=headers,
|
||||||
|
))
|
||||||
|
await asyncio.sleep(0.02)
|
||||||
|
assert not joined.done(), "a live local operation must be joined instead of rejected as stale"
|
||||||
|
release.set()
|
||||||
|
recovered = await joined
|
||||||
|
|
||||||
|
assert timed_out.status_code == 503
|
||||||
|
assert recovered.status_code == 201
|
||||||
|
assert calls == 1
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_review_replays_same_key_and_rejects_changed_head(monkeypatch):
|
async def test_review_replays_same_key_and_rejects_changed_head(monkeypatch):
|
||||||
calls = []
|
calls = []
|
||||||
|
|
@ -233,6 +281,51 @@ async def test_orphaned_pending_comment_fails_closed_without_upstream_retry(
|
||||||
assert calls == 0
|
assert calls == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_stale_orphaned_comment_returns_non_retryable_uncertain_delivery(
|
||||||
|
monkeypatch, tmp_path
|
||||||
|
):
|
||||||
|
calls = 0
|
||||||
|
now = [1_000.0]
|
||||||
|
|
||||||
|
async def assigned(_repository, _number):
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def comment(_repository, _number, _body):
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
return {"id": 319}
|
||||||
|
|
||||||
|
ledger = IdempotencyLedger(
|
||||||
|
tmp_path / "stale-orphan.sqlite3",
|
||||||
|
ttl_seconds=10,
|
||||||
|
max_entries=256,
|
||||||
|
clock=lambda: now[0],
|
||||||
|
)
|
||||||
|
fingerprint = ("issue-comment", "stackchain/api", 7, "Verify first")
|
||||||
|
assert ledger.reserve("stale-orphan-319", fingerprint).state == "reserved"
|
||||||
|
now[0] += 11
|
||||||
|
monkeypatch.setattr(main, "_idempotency_ledger", ledger)
|
||||||
|
monkeypatch.setattr(main.gitea_proxy, "is_assigned_issue", assigned)
|
||||||
|
monkeypatch.setattr(main.gitea_proxy, "comment_on_issue", comment)
|
||||||
|
transport = httpx.ASGITransport(app=main.app)
|
||||||
|
|
||||||
|
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||||
|
response = await client.post(
|
||||||
|
"/api/v1/repos/stackchain/api/issues/7/comments",
|
||||||
|
json={"body": "Verify first"},
|
||||||
|
headers={"Idempotency-Key": "stale-orphan-319"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 422
|
||||||
|
assert response.json()["detail"] == {
|
||||||
|
"code": "delivery_uncertain",
|
||||||
|
"message": "Delivery could not be confirmed. Verify it was not posted before retrying.",
|
||||||
|
}
|
||||||
|
assert "Retry-After" not in response.headers
|
||||||
|
assert calls == 0
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_slow_ledger_reservation_does_not_block_health_requests(monkeypatch):
|
async def test_slow_ledger_reservation_does_not_block_health_requests(monkeypatch):
|
||||||
async def assigned(_repository, _number):
|
async def assigned(_repository, _number):
|
||||||
|
|
|
||||||
|
|
@ -113,6 +113,37 @@ outbox.flush('timmy').then(async first => {{
|
||||||
assert output["remaining"][1]["status"] == "queued"
|
assert output["remaining"][1]["status"] == "queued"
|
||||||
|
|
||||||
|
|
||||||
|
def test_authored_outbox_requires_deliberate_rekey_after_uncertain_delivery():
|
||||||
|
script = f"""
|
||||||
|
const createAuthoredOutbox = require({json.dumps(str(OUTBOX))});
|
||||||
|
const values = new Map(); let sequence=0; const calls=[];
|
||||||
|
const storage = {{getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v)}};
|
||||||
|
const outbox = createAuthoredOutbox({{
|
||||||
|
storage, getOwnerLogin:()=>'timmy', createOperationId:()=> 'verified-' + (++sequence),
|
||||||
|
fetchJson:async (_url, options) => {{
|
||||||
|
const key=options.headers['Idempotency-Key']; calls.push(key);
|
||||||
|
if (key === 'uncertain-key') {{ const e=new Error('Verify it was not posted before retrying.'); e.status=422; e.code='delivery_uncertain'; throw e; }}
|
||||||
|
return {{id:calls.length}};
|
||||||
|
}},
|
||||||
|
}});
|
||||||
|
const uncertain=outbox.enqueue({{kind:'issue-comment',repository:'o/r',number:1,body:'Check',operationId:'uncertain-key'}});
|
||||||
|
outbox.enqueue({{kind:'pull-comment',repository:'o/r',number:2,body:'Continue',operationId:'other-key'}});
|
||||||
|
(async()=>{{
|
||||||
|
const first=await outbox.flush('timmy');
|
||||||
|
const attention=outbox.list()[0];
|
||||||
|
const retried=await outbox.retry(uncertain.id,'timmy');
|
||||||
|
process.stdout.write(JSON.stringify({{first,attention,retried,calls,remaining:outbox.list()}}));
|
||||||
|
}})();
|
||||||
|
"""
|
||||||
|
output = run_node(script)
|
||||||
|
|
||||||
|
assert output["calls"] == ["uncertain-key", "other-key", "verified-1"]
|
||||||
|
assert len(output["first"]["confirmed"]) == 1
|
||||||
|
assert output["attention"]["status"] == "attention"
|
||||||
|
assert output["attention"]["deliveryState"] == "uncertain"
|
||||||
|
assert output["remaining"] == []
|
||||||
|
|
||||||
|
|
||||||
def test_authored_outbox_retry_is_single_flight_and_edit_rotates_identity():
|
def test_authored_outbox_retry_is_single_flight_and_edit_rotates_identity():
|
||||||
script = f"""
|
script = f"""
|
||||||
const createAuthoredOutbox = require({json.dumps(str(OUTBOX))});
|
const createAuthoredOutbox = require({json.dumps(str(OUTBOX))});
|
||||||
|
|
@ -182,6 +213,21 @@ setTimeout(() => {{
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_authored_outbox_reconciles_uncertain_worker_delivery_state():
|
||||||
|
script = f"""
|
||||||
|
const createAuthoredOutbox = require({json.dumps(str(OUTBOX))});
|
||||||
|
const values=new Map(); const storage={{getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v)}};
|
||||||
|
const outbox=createAuthoredOutbox({{storage,getOwnerLogin:()=>'timmy'}});
|
||||||
|
const item=outbox.enqueue({{kind:'issue-comment',repository:'o/r',number:7,body:'Maybe',operationId:'maybe'}});
|
||||||
|
const reconciled=outbox.reconcileBackground([{{...item,status:'attention',deliveryState:'uncertain',error:'Verify first'}}]);
|
||||||
|
process.stdout.write(JSON.stringify(reconciled[0]));
|
||||||
|
"""
|
||||||
|
output = run_node(script)
|
||||||
|
|
||||||
|
assert output["status"] == "attention"
|
||||||
|
assert output["deliveryState"] == "uncertain"
|
||||||
|
|
||||||
|
|
||||||
def test_authored_outbox_waits_for_durable_background_admission():
|
def test_authored_outbox_waits_for_durable_background_admission():
|
||||||
script = f"""
|
script = f"""
|
||||||
const createAuthoredOutbox = require({json.dumps(str(OUTBOX))});
|
const createAuthoredOutbox = require({json.dumps(str(OUTBOX))});
|
||||||
|
|
|
||||||
|
|
@ -585,6 +585,31 @@ const fetchJson = async url => {{
|
||||||
assert output["result"]["attention"] == 1
|
assert output["result"]["attention"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_uncertain_background_delivery_preserves_verification_state():
|
||||||
|
script = f"""
|
||||||
|
const createBackgroundIssueSync = require({json.dumps(str(SYNC))});
|
||||||
|
let claimed=false; const state={{failed:[]}};
|
||||||
|
const store={{
|
||||||
|
claimNext:async()=>claimed?null:(claimed=true,{{id:'maybe',operationId:'maybe',ownerLogin:'timmy',repository:'o/r',title:'Maybe',labelIds:[]}}),
|
||||||
|
fail:async(id,message,deliveryState)=>state.failed.push({{id,message,deliveryState}}),
|
||||||
|
release:async()=>{{}}, countBlocked:async()=>0,
|
||||||
|
}};
|
||||||
|
const fetchJson=async url=>{{
|
||||||
|
if(url==='api/v1/background-identity') return {{login:'timmy'}};
|
||||||
|
const error=new Error('Verify first'); error.status=422; error.code='delivery_uncertain'; throw error;
|
||||||
|
}};
|
||||||
|
(async()=>{{
|
||||||
|
await createBackgroundIssueSync({{store,fetchJson}}).flush();
|
||||||
|
process.stdout.write(JSON.stringify(state));
|
||||||
|
}})();
|
||||||
|
"""
|
||||||
|
output = run_node(script)
|
||||||
|
|
||||||
|
assert output["failed"] == [
|
||||||
|
{"id": "maybe", "message": "Verify first", "deliveryState": "uncertain"}
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_issue_store_atomically_grants_one_delivery_claim():
|
def test_issue_store_atomically_grants_one_delivery_claim():
|
||||||
script = f"""
|
script = f"""
|
||||||
const createBackgroundIssueSync = require({json.dumps(str(SYNC))});
|
const createBackgroundIssueSync = require({json.dumps(str(SYNC))});
|
||||||
|
|
|
||||||
|
|
@ -121,6 +121,22 @@ process.stdout.write(JSON.stringify(drafts));
|
||||||
assert output[1]["label"] == "Queued issue"
|
assert output[1]["label"] == "Queued issue"
|
||||||
|
|
||||||
|
|
||||||
|
def test_draft_inbox_exposes_uncertain_delivery_for_explicit_user_verification():
|
||||||
|
script = f"""
|
||||||
|
const createDraftInbox = require({json.dumps(str(DRAFTS))});
|
||||||
|
const values = new Map([['stackchain.authored-outbox.v1', JSON.stringify({{version:2,items:[
|
||||||
|
{{id:'uncertain',kind:'issue-comment',repository:'o/r',number:7,body:'Possibly posted',ownerLogin:'timmy',status:'attention',deliveryState:'uncertain',error:'Verify it was not posted before retrying.',queuedAt:200}}
|
||||||
|
]}})]]);
|
||||||
|
const storage = {{get length(){{return values.size}},key:i=>Array.from(values.keys())[i]||null,getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v)}};
|
||||||
|
const drafts = createDraftInbox({{storage,getCurrentLogin:()=>'timmy'}}).list();
|
||||||
|
process.stdout.write(JSON.stringify(drafts[0]));
|
||||||
|
"""
|
||||||
|
output = run_node(script)
|
||||||
|
|
||||||
|
assert output["delivery_state"] == "uncertain"
|
||||||
|
assert output["label"] == "Verify delivery"
|
||||||
|
|
||||||
|
|
||||||
def test_draft_inbox_marks_mismatched_and_legacy_outbox_content_copy_only():
|
def test_draft_inbox_marks_mismatched_and_legacy_outbox_content_copy_only():
|
||||||
script = f"""
|
script = f"""
|
||||||
const createDraftInbox = require({json.dumps(str(DRAFTS))});
|
const createDraftInbox = require({json.dumps(str(DRAFTS))});
|
||||||
|
|
@ -160,6 +176,9 @@ async def test_mobile_dashboard_exposes_touch_safe_draft_recovery_lane():
|
||||||
assert '.draft-actions button { min-height:44px;' in html
|
assert '.draft-actions button { min-height:44px;' in html
|
||||||
assert 'createDraftInbox({ storage: localStorage' in html
|
assert 'createDraftInbox({ storage: localStorage' in html
|
||||||
assert "captureDraft.repository && !repositories.includes(captureDraft.repository)" in html
|
assert "captureDraft.repository && !repositories.includes(captureDraft.repository)" in html
|
||||||
|
assert "Verified not posted — retry" in html
|
||||||
|
assert "payload.detail?.message" in html
|
||||||
|
assert "error.code = payload.detail?.code" in html
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
|
|
|
||||||
|
|
@ -71,3 +71,21 @@ def test_independent_connections_atomically_reserve_one_operation(tmp_path):
|
||||||
states = list(executor.map(reserve, ledgers))
|
states = list(executor.map(reserve, ledgers))
|
||||||
|
|
||||||
assert sorted(states) == ["pending", "reserved"]
|
assert sorted(states) == ["pending", "reserved"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_stale_pending_reservation_becomes_uncertain_without_consuming_capacity(tmp_path):
|
||||||
|
now = [1_000.0]
|
||||||
|
ledger = IdempotencyLedger(
|
||||||
|
tmp_path / "orphaned.sqlite3",
|
||||||
|
ttl_seconds=10,
|
||||||
|
max_entries=1,
|
||||||
|
clock=lambda: now[0],
|
||||||
|
)
|
||||||
|
fingerprint = ("issue-comment", "stackchain/api", 7, "Verify first")
|
||||||
|
|
||||||
|
assert ledger.reserve("orphan", fingerprint).state == "reserved"
|
||||||
|
now[0] += 11
|
||||||
|
|
||||||
|
assert ledger.reserve("orphan", fingerprint).state == "uncertain"
|
||||||
|
assert ledger.reserve("unrelated", ("issue-comment", "stackchain/api", 8, "Continue")).state == "reserved"
|
||||||
|
assert ledger.reserve("orphan", ("issue-comment", "stackchain/api", 7, "Changed")).state == "conflict"
|
||||||
|
|
|
||||||
|
|
@ -139,6 +139,34 @@ outbox.flush('timmy').then(async () => {{
|
||||||
assert output["remaining"] == []
|
assert output["remaining"] == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_issue_outbox_rekeys_only_after_user_retries_uncertain_delivery():
|
||||||
|
script = f"""
|
||||||
|
const createIssueOutbox = require({json.dumps(str(OUTBOX))});
|
||||||
|
const values = new Map(); let sequence=0; const calls=[];
|
||||||
|
const storage = {{getItem:k=>values.get(k)||null,setItem:(k,v)=>values.set(k,v)}};
|
||||||
|
const outbox = createIssueOutbox({{
|
||||||
|
storage, getOwnerLogin:()=>'timmy', createOperationId:()=> 'operation-' + (++sequence),
|
||||||
|
fetchJson:async (_url, options) => {{
|
||||||
|
const key=options.headers['Idempotency-Key']; calls.push(key);
|
||||||
|
if (key === 'operation-1') {{ const e=new Error('Verify it was not posted before retrying.'); e.status=422; e.code='delivery_uncertain'; throw e; }}
|
||||||
|
return {{number:319}};
|
||||||
|
}},
|
||||||
|
}});
|
||||||
|
const item=outbox.enqueue({{repository:'o/r',title:'Possibly sent',body:'Check first'}});
|
||||||
|
(async()=>{{
|
||||||
|
await outbox.flush('timmy'); const attention=outbox.list()[0];
|
||||||
|
await outbox.retry(item.id,'timmy');
|
||||||
|
process.stdout.write(JSON.stringify({{attention,calls,remaining:outbox.list()}}));
|
||||||
|
}})();
|
||||||
|
"""
|
||||||
|
output = run_node(script)
|
||||||
|
|
||||||
|
assert output["attention"]["status"] == "attention"
|
||||||
|
assert output["attention"]["deliveryState"] == "uncertain"
|
||||||
|
assert output["calls"] == ["operation-1", "operation-2"]
|
||||||
|
assert output["remaining"] == []
|
||||||
|
|
||||||
|
|
||||||
def test_issue_outbox_rotates_operation_id_only_when_delivery_payload_changes():
|
def test_issue_outbox_rotates_operation_id_only_when_delivery_payload_changes():
|
||||||
script = f"""
|
script = f"""
|
||||||
const createIssueOutbox = require({json.dumps(str(OUTBOX))});
|
const createIssueOutbox = require({json.dumps(str(OUTBOX))});
|
||||||
|
|
|
||||||
|
|
@ -97,7 +97,7 @@ async function dispatchNotificationClick(route) {{
|
||||||
def test_share_target_sign_in_fix_ships_in_a_new_shell_cache():
|
def test_share_target_sign_in_fix_ships_in_a_new_shell_cache():
|
||||||
source = WORKER.read_text()
|
source = WORKER.read_text()
|
||||||
|
|
||||||
assert "stackchain-dashboard-shell-v33" in source
|
assert "stackchain-dashboard-shell-v34" in source
|
||||||
assert "BASE + 'static/dashboard.css'" in source
|
assert "BASE + 'static/dashboard.css'" in source
|
||||||
assert "BASE + 'static/dashboard.js'" in source
|
assert "BASE + 'static/dashboard.js'" in source
|
||||||
assert "BASE + 'static/install-app.js'" in source
|
assert "BASE + 'static/install-app.js'" in source
|
||||||
|
|
@ -106,7 +106,7 @@ def test_share_target_sign_in_fix_ships_in_a_new_shell_cache():
|
||||||
def test_mobile_search_viewport_ships_in_a_new_offline_shell():
|
def test_mobile_search_viewport_ships_in_a_new_offline_shell():
|
||||||
source = WORKER.read_text()
|
source = WORKER.read_text()
|
||||||
|
|
||||||
assert "stackchain-dashboard-shell-v33" in source
|
assert "stackchain-dashboard-shell-v34" in source
|
||||||
assert "BASE + 'static/mobile-search-viewport.js'" in source
|
assert "BASE + 'static/mobile-search-viewport.js'" in source
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -122,6 +122,23 @@ def test_background_sync_event_flushes_closed_app_issue_outbox_only_for_its_tag(
|
||||||
assert result["backgroundFlushes"] == 1
|
assert result["backgroundFlushes"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_background_delivery_preserves_structured_uncertain_error():
|
||||||
|
result = run_worker_scenario(
|
||||||
|
"""
|
||||||
|
context.fetch=async()=>new Response(JSON.stringify({detail:{code:'delivery_uncertain',message:'Verify it was not posted before retrying.'}}),{status:422,headers:{'Content-Type':'application/json'}});
|
||||||
|
const outcome=await context.self.__testFetchJson('/dashboard/api/v1/repos/o/r/issues',{method:'POST'})
|
||||||
|
.then(()=>({ok:true}),error=>({status:error.status,code:error.code,message:error.message}));
|
||||||
|
process.stdout.write(JSON.stringify(outcome));
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == {
|
||||||
|
"status": 422,
|
||||||
|
"code": "delivery_uncertain",
|
||||||
|
"message": "Verify it was not posted before retrying.",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def test_authenticated_page_message_resumes_queued_background_delivery():
|
def test_authenticated_page_message_resumes_queued_background_delivery():
|
||||||
result = run_worker_scenario(
|
result = run_worker_scenario(
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user