Make screenshot uploads retry-safe #472
|
|
@ -293,7 +293,11 @@
|
||||||
'api/v1/repos/' + repository + '/issues/' + encodeURIComponent(payload.number) + '/attachments',
|
'api/v1/repos/' + repository + '/issues/' + encodeURIComponent(payload.number) + '/attachments',
|
||||||
{
|
{
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: { Accept: 'application/json', 'Content-Type': 'application/json' },
|
headers: {
|
||||||
|
Accept: 'application/json',
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
'Idempotency-Key': payload.operation_id,
|
||||||
|
},
|
||||||
body: JSON.stringify({
|
body: JSON.stringify({
|
||||||
filename: payload.filename,
|
filename: payload.filename,
|
||||||
content_type: payload.content_type,
|
content_type: payload.content_type,
|
||||||
|
|
|
||||||
|
|
@ -11,9 +11,16 @@
|
||||||
function create(options) {
|
function create(options) {
|
||||||
const readDataUrl = options.readDataUrl;
|
const readDataUrl = options.readDataUrl;
|
||||||
const upload = options.upload;
|
const upload = options.upload;
|
||||||
|
const createOperationId = options.createOperationId || (() => {
|
||||||
|
if (typeof globalThis !== 'undefined' && globalThis.crypto?.randomUUID) {
|
||||||
|
return globalThis.crypto.randomUUID();
|
||||||
|
}
|
||||||
|
return Date.now().toString(36) + '-' + Math.random().toString(36).slice(2);
|
||||||
|
});
|
||||||
let selected = null;
|
let selected = null;
|
||||||
let confirmed = null;
|
let confirmed = null;
|
||||||
let serialized = null;
|
let serialized = null;
|
||||||
|
let operationId = null;
|
||||||
|
|
||||||
function select(file) {
|
function select(file) {
|
||||||
if (!file || !IMAGE_TYPES.has(file.type)) {
|
if (!file || !IMAGE_TYPES.has(file.type)) {
|
||||||
|
|
@ -25,6 +32,7 @@
|
||||||
selected = file;
|
selected = file;
|
||||||
confirmed = null;
|
confirmed = null;
|
||||||
serialized = null;
|
serialized = null;
|
||||||
|
operationId = createOperationId();
|
||||||
return state();
|
return state();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -32,6 +40,7 @@
|
||||||
selected = null;
|
selected = null;
|
||||||
confirmed = null;
|
confirmed = null;
|
||||||
serialized = null;
|
serialized = null;
|
||||||
|
operationId = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function restore(value) {
|
function restore(value) {
|
||||||
|
|
@ -80,6 +89,7 @@
|
||||||
filename: attachment.filename,
|
filename: attachment.filename,
|
||||||
content_type: attachment.contentType,
|
content_type: attachment.contentType,
|
||||||
data: attachment.data,
|
data: attachment.data,
|
||||||
|
operation_id: operationId,
|
||||||
});
|
});
|
||||||
if (!confirmed || typeof confirmed.markdown !== 'string' || !confirmed.markdown) {
|
if (!confirmed || typeof confirmed.markdown !== 'string' || !confirmed.markdown) {
|
||||||
confirmed = null;
|
confirmed = null;
|
||||||
|
|
|
||||||
33
src/main.py
33
src/main.py
|
|
@ -1,6 +1,7 @@
|
||||||
import asyncio
|
import asyncio
|
||||||
import base64
|
import base64
|
||||||
import binascii
|
import binascii
|
||||||
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import math
|
import math
|
||||||
import os
|
import os
|
||||||
|
|
@ -2776,20 +2777,40 @@ async def attach_to_assigned_issue(
|
||||||
owner: str,
|
owner: str,
|
||||||
repo: str,
|
repo: str,
|
||||||
number: int = PathParam(gt=0),
|
number: int = PathParam(gt=0),
|
||||||
|
idempotency_key: str | None = Header(default=None, max_length=128),
|
||||||
):
|
):
|
||||||
repository = f"{owner}/{repo}"
|
repository = f"{owner}/{repo}"
|
||||||
try:
|
try:
|
||||||
content = attachment.content()
|
content = attachment.content()
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||||
|
async def upload_attachment():
|
||||||
|
result = await gitea_proxy.upload_assigned_issue_attachment(
|
||||||
|
repository,
|
||||||
|
number,
|
||||||
|
attachment.filename,
|
||||||
|
attachment.content_type,
|
||||||
|
content,
|
||||||
|
)
|
||||||
|
safe_name = (
|
||||||
|
result["name"].replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]")
|
||||||
|
.replace("\r", " ").replace("\n", " ")
|
||||||
|
)
|
||||||
|
safe_url = result["url"].replace("<", "%3C").replace(">", "%3E")
|
||||||
|
result["markdown"] = f""
|
||||||
|
return result
|
||||||
|
|
||||||
try:
|
try:
|
||||||
result = await asyncio.wait_for(
|
result = await _run_idempotent_authored_action(
|
||||||
gitea_proxy.upload_assigned_issue_attachment(
|
upload_attachment(),
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
fingerprint=(
|
||||||
|
"issue-attachment",
|
||||||
repository,
|
repository,
|
||||||
number,
|
number,
|
||||||
attachment.filename,
|
attachment.filename,
|
||||||
attachment.content_type,
|
attachment.content_type,
|
||||||
content,
|
hashlib.sha256(content).hexdigest(),
|
||||||
),
|
),
|
||||||
timeout=ISSUE_ACTION_TIMEOUT_SECONDS,
|
timeout=ISSUE_ACTION_TIMEOUT_SECONDS,
|
||||||
)
|
)
|
||||||
|
|
@ -2803,12 +2824,6 @@ async def attach_to_assigned_issue(
|
||||||
status_code=503,
|
status_code=503,
|
||||||
headers={"Retry-After": "1"},
|
headers={"Retry-After": "1"},
|
||||||
)
|
)
|
||||||
safe_name = (
|
|
||||||
result["name"].replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]")
|
|
||||||
.replace("\r", " ").replace("\n", " ")
|
|
||||||
)
|
|
||||||
safe_url = result["url"].replace("<", "%3C").replace(">", "%3E")
|
|
||||||
result["markdown"] = f""
|
|
||||||
return JSONResponse(result, status_code=201)
|
return JSONResponse(result, status_code=201)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,7 @@ const attachment = require({json.dumps(str(ATTACHMENT))});
|
||||||
const calls = [];
|
const calls = [];
|
||||||
const file = {{name:'checkout.png', type:'image/png', size:8}};
|
const file = {{name:'checkout.png', type:'image/png', size:8}};
|
||||||
const controller = attachment.create({{
|
const controller = attachment.create({{
|
||||||
|
createOperationId: () => 'attachment-comment-471',
|
||||||
readDataUrl: async selected => {{ calls.push('read:' + selected.name); return 'data:image/png;base64,iVBORw0KGgo='; }},
|
readDataUrl: async selected => {{ calls.push('read:' + selected.name); return 'data:image/png;base64,iVBORw0KGgo='; }},
|
||||||
upload: async payload => {{ calls.push(payload); return {{markdown:''}}; }},
|
upload: async payload => {{ calls.push(payload); return {{markdown:''}}; }},
|
||||||
}});
|
}});
|
||||||
|
|
@ -47,6 +48,7 @@ controller.select(file);
|
||||||
"filename": "checkout.png",
|
"filename": "checkout.png",
|
||||||
"content_type": "image/png",
|
"content_type": "image/png",
|
||||||
"data": "iVBORw0KGgo=",
|
"data": "iVBORw0KGgo=",
|
||||||
|
"operation_id": "attachment-comment-471",
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
assert output["state"] == {
|
assert output["state"] == {
|
||||||
|
|
@ -54,6 +56,32 @@ controller.select(file);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_mobile_attachment_retry_reuses_operation_key_until_file_changes():
|
||||||
|
script = f"""
|
||||||
|
const attachment = require({json.dumps(str(ATTACHMENT))});
|
||||||
|
const keys=[];
|
||||||
|
let sequence=0;
|
||||||
|
let fail=true;
|
||||||
|
const controller=attachment.create({{
|
||||||
|
createOperationId:()=> 'attachment-' + (++sequence),
|
||||||
|
readDataUrl:async file=>'data:'+file.type+';base64,iVBORw0KGgo=',
|
||||||
|
upload:async payload=>{{keys.push(payload.operation_id);if(fail){{fail=false;throw new Error('offline');}}return {{markdown:''}};}},
|
||||||
|
}});
|
||||||
|
controller.select({{name:'first.png',type:'image/png',size:8}});
|
||||||
|
(async()=>{{
|
||||||
|
try{{await controller.prepareComment({{repository:'stackchain/api',number:17}},'Evidence');}}catch(_error){{}}
|
||||||
|
await controller.prepareComment({{repository:'stackchain/api',number:17}},'Evidence');
|
||||||
|
controller.select({{name:'replacement.png',type:'image/png',size:8}});
|
||||||
|
await controller.prepareComment({{repository:'stackchain/api',number:17}},'Replacement');
|
||||||
|
process.stdout.write(JSON.stringify(keys));
|
||||||
|
}})().catch(error=>{{console.error(error);process.exit(1);}});
|
||||||
|
"""
|
||||||
|
|
||||||
|
assert json.loads(run_node(script)) == [
|
||||||
|
"attachment-1", "attachment-1", "attachment-2"
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def test_selected_screenshot_serializes_for_durable_issue_capture_without_uploading():
|
def test_selected_screenshot_serializes_for_durable_issue_capture_without_uploading():
|
||||||
script = f"""
|
script = f"""
|
||||||
const attachment = require({json.dumps(str(ATTACHMENT))});
|
const attachment = require({json.dumps(str(ATTACHMENT))});
|
||||||
|
|
@ -165,6 +193,7 @@ def test_issue_comment_actions_upload_before_posting_and_clear_after_acceptance(
|
||||||
assert "issueAttachmentController.prepareComment(item, body)" in source
|
assert "issueAttachmentController.prepareComment(item, body)" in source
|
||||||
assert "issueAttachmentController.prepareComment(selectedIssue, body)" in source
|
assert "issueAttachmentController.prepareComment(selectedIssue, body)" in source
|
||||||
assert source.count("issueAttachmentController.clear();") >= 2
|
assert source.count("issueAttachmentController.clear();") >= 2
|
||||||
|
assert "'Idempotency-Key': payload.operation_id" in source
|
||||||
|
|
||||||
|
|
||||||
def test_closing_issue_sheet_cannot_carry_a_screenshot_to_another_issue():
|
def test_closing_issue_sheet_cannot_carry_a_screenshot_to_another_issue():
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
import asyncio
|
||||||
import base64
|
import base64
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
@ -9,6 +10,15 @@ from src import gitea_proxy, main
|
||||||
PNG_BYTES = b"\x89PNG\r\n\x1a\n" + b"mobile screenshot"
|
PNG_BYTES = b"\x89PNG\r\n\x1a\n" + b"mobile screenshot"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def clear_attachment_idempotency():
|
||||||
|
main._authored_action_operations.clear()
|
||||||
|
main._idempotency_ledger.clear()
|
||||||
|
yield
|
||||||
|
main._authored_action_operations.clear()
|
||||||
|
main._idempotency_ledger.clear()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_attachment_endpoint_uploads_valid_screenshot_to_assigned_issue(monkeypatch):
|
async def test_attachment_endpoint_uploads_valid_screenshot_to_assigned_issue(monkeypatch):
|
||||||
calls = []
|
calls = []
|
||||||
|
|
@ -48,6 +58,118 @@ async def test_attachment_endpoint_uploads_valid_screenshot_to_assigned_issue(mo
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_attachment_endpoint_replays_confirmed_upload_for_same_key(monkeypatch):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
async def upload(repository, number, filename, content_type, content):
|
||||||
|
calls.append((repository, number, filename, content_type, content))
|
||||||
|
return {
|
||||||
|
"name": filename,
|
||||||
|
"url": "https://forge.example/attachments/checkout.png",
|
||||||
|
"size": len(content),
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(main.gitea_proxy, "upload_assigned_issue_attachment", upload)
|
||||||
|
payload = {
|
||||||
|
"filename": "checkout.png",
|
||||||
|
"content_type": "image/png",
|
||||||
|
"data": base64.b64encode(PNG_BYTES).decode("ascii"),
|
||||||
|
}
|
||||||
|
transport = httpx.ASGITransport(app=main.app)
|
||||||
|
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||||
|
first = await client.post(
|
||||||
|
"/api/v1/repos/stackchain/api/issues/17/attachments",
|
||||||
|
json=payload,
|
||||||
|
headers={"Idempotency-Key": "attachment-retry-471"},
|
||||||
|
)
|
||||||
|
replay = await client.post(
|
||||||
|
"/api/v1/repos/stackchain/api/issues/17/attachments",
|
||||||
|
json=payload,
|
||||||
|
headers={"Idempotency-Key": "attachment-retry-471"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert first.status_code == replay.status_code == 201
|
||||||
|
assert replay.json() == first.json()
|
||||||
|
assert len(calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_attachment_endpoint_coalesces_concurrent_upload_retries(monkeypatch):
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def upload(_repository, _number, filename, _content_type, content):
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
await asyncio.sleep(0.05)
|
||||||
|
return {
|
||||||
|
"name": filename,
|
||||||
|
"url": "https://forge.example/attachments/checkout.png",
|
||||||
|
"size": len(content),
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(main.gitea_proxy, "upload_assigned_issue_attachment", upload)
|
||||||
|
payload = {
|
||||||
|
"filename": "checkout.png",
|
||||||
|
"content_type": "image/png",
|
||||||
|
"data": base64.b64encode(PNG_BYTES).decode("ascii"),
|
||||||
|
}
|
||||||
|
transport = httpx.ASGITransport(app=main.app)
|
||||||
|
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||||
|
first, retry = await asyncio.gather(*[
|
||||||
|
client.post(
|
||||||
|
"/api/v1/repos/stackchain/api/issues/17/attachments",
|
||||||
|
json=payload,
|
||||||
|
headers={"Idempotency-Key": "attachment-concurrent-471"},
|
||||||
|
)
|
||||||
|
for _ in range(2)
|
||||||
|
])
|
||||||
|
|
||||||
|
assert first.status_code == retry.status_code == 201
|
||||||
|
assert first.json() == retry.json()
|
||||||
|
assert calls == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.anyio
|
||||||
|
async def test_attachment_endpoint_rejects_changed_upload_for_used_key(monkeypatch):
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
async def upload(_repository, _number, filename, _content_type, content):
|
||||||
|
nonlocal calls
|
||||||
|
calls += 1
|
||||||
|
return {
|
||||||
|
"name": filename,
|
||||||
|
"url": "https://forge.example/attachments/checkout.png",
|
||||||
|
"size": len(content),
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(main.gitea_proxy, "upload_assigned_issue_attachment", upload)
|
||||||
|
transport = httpx.ASGITransport(app=main.app)
|
||||||
|
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||||
|
accepted = await client.post(
|
||||||
|
"/api/v1/repos/stackchain/api/issues/17/attachments",
|
||||||
|
json={
|
||||||
|
"filename": "checkout.png",
|
||||||
|
"content_type": "image/png",
|
||||||
|
"data": base64.b64encode(PNG_BYTES).decode("ascii"),
|
||||||
|
},
|
||||||
|
headers={"Idempotency-Key": "attachment-conflict-471"},
|
||||||
|
)
|
||||||
|
conflict = await client.post(
|
||||||
|
"/api/v1/repos/stackchain/api/issues/17/attachments",
|
||||||
|
json={
|
||||||
|
"filename": "checkout.png",
|
||||||
|
"content_type": "image/png",
|
||||||
|
"data": base64.b64encode(PNG_BYTES + b" changed").decode("ascii"),
|
||||||
|
},
|
||||||
|
headers={"Idempotency-Key": "attachment-conflict-471"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert accepted.status_code == 201
|
||||||
|
assert conflict.status_code == 409
|
||||||
|
assert calls == 1
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.anyio
|
@pytest.mark.anyio
|
||||||
async def test_attachment_endpoint_admits_a_normal_phone_screenshot(monkeypatch):
|
async def test_attachment_endpoint_admits_a_normal_phone_screenshot(monkeypatch):
|
||||||
screenshot = b"\x89PNG\r\n\x1a\n" + (b"x" * (100 * 1024))
|
screenshot = b"\x89PNG\r\n\x1a\n" + (b"x" * (100 * 1024))
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user