fix: fence background delivery claims (Closes #479)
This commit is contained in:
timmy committed
1 parent
8206856d94
commit
31b0e79aa1
3 files changed
+244
-43
No files matched your search
@@ -129,6 +129,40 @@ createBackgroundIssueSync({{store,fetchJson}}).flush().then(()=>process.stdout.w
|
||||
assert all(len(key) <= 128 for key in keys)
|
||||
|
||||
|
||||
def test_capture_renews_claim_before_every_network_stage_and_fences_completion():
|
||||
script = f"""
|
||||
const createBackgroundIssueSync=require({json.dumps(str(SYNC))});
|
||||
const item={{id:'capture',claimToken:'claim-7',operationId:'capture',ownerLogin:'timmy',repository:'o/r',title:'Bug',body:'',labelIds:[],attachment:{{filename:'a.png',contentType:'image/png',data:'abc'}}}};
|
||||
const events=[];let claimed=false;
|
||||
const store={{
|
||||
supportsClaimTokens:true,
|
||||
claimNext:async()=>claimed?null:(claimed=true,item),
|
||||
renew:async(id,token)=>{{events.push(['renew',id,token]);return item;}},
|
||||
checkpoint:async(id,token,transform)=>{{events.push(['checkpoint',id,token]);transform(item);return true;}},
|
||||
complete:async(id,token)=>events.push(['complete',id,token]),
|
||||
release:async(id,token)=>events.push(['release',id,token]),
|
||||
fail:async()=>{{}},countBlocked:async()=>0,
|
||||
}};
|
||||
const fetchJson=async(url,options={{}})=>{{
|
||||
if(url==='api/v1/background-identity')return{{login:'timmy'}};
|
||||
events.push(['network',options.headers['Idempotency-Key']]);
|
||||
if(url.endsWith('/issues'))return{{number:3}};
|
||||
if(url.endsWith('/attachments'))return{{markdown:''}};
|
||||
return{{id:4}};
|
||||
}};
|
||||
(async()=>{{await createBackgroundIssueSync({{store,fetchJson}}).flush();process.stdout.write(JSON.stringify(events));}})();
|
||||
"""
|
||||
events = run_node(script)
|
||||
|
||||
networks = [index for index, event in enumerate(events) if event[0] == "network"]
|
||||
assert [events[index - 1][0] for index in networks] == ["renew", "renew", "renew"]
|
||||
assert [events[index][1] for index in networks] == [
|
||||
"capture", "capture:attachment", "capture:attachment-comment"
|
||||
]
|
||||
assert events[-1] == ["complete", "capture", "claim-7"]
|
||||
assert all(event[2] == "claim-7" for event in events if event[0] in {"renew", "checkpoint"})
|
||||
|
||||
|
||||
def test_screenshot_comment_retry_reuses_upload_checkpoint_and_posts_combined_body_once():
|
||||
script = f"""
|
||||
const createBackgroundIssueSync=require({json.dumps(str(SYNC))});
|
||||
@@ -758,6 +792,52 @@ const fetchJson = async (url, options={{}}) => {{
|
||||
assert output["error"] == "Temporary outage"
|
||||
|
||||
|
||||
def test_bounded_flush_claims_just_in_time_and_keeps_unadmitted_records_queued():
|
||||
script = f"""
|
||||
const createBackgroundIssueSync=require({json.dumps(str(SYNC))});
|
||||
const records=new Map();let tail=Promise.resolve();
|
||||
const transaction=work=>{{const run=tail.then(()=>work({{
|
||||
get:async id=>records.get(id),
|
||||
getAll:async()=>[...records.values()].map(value=>({{...value}})),
|
||||
put:async value=>records.set(value.id,{{...value}}),delete:async id=>records.delete(id),
|
||||
}}));tail=run.catch(()=>{{}});return run;}};
|
||||
let allow=false;const waiting=[];const attempted=[];
|
||||
const fetchJson=async(url,options={{}})=>{{
|
||||
if(url==='api/v1/background-identity')return{{login:'timmy'}};
|
||||
attempted.push(options.headers['Idempotency-Key']);
|
||||
if(!allow)await new Promise(resolve=>waiting.push(resolve));
|
||||
return{{number:attempted.length}};
|
||||
}};
|
||||
(async()=>{{
|
||||
const store=createBackgroundIssueSync.createIssueSyncStore({{transaction,createToken:(()=>{{let n=0;return()=>`token-${{++n}}`;}})()}});
|
||||
await store.reconcile(Array.from({{length:4}},(_,i)=>({{id:`issue-${{i}}`,operationId:`issue-${{i}}`,ownerLogin:'timmy',status:'queued',repository:'o/r',title:'Issue',labelIds:[]}})),'issue');
|
||||
await store.reconcile(Array.from({{length:2}},(_,i)=>({{id:`message-${{i}}`,operationId:`message-${{i}}`,ownerLogin:'timmy',status:'queued',kind:'issue-comment',repository:'o/r',number:i,body:'Reply'}})),'authored');
|
||||
const sync=createBackgroundIssueSync({{store,fetchJson,maxConcurrency:2,batchSize:5}});
|
||||
const flushing=sync.flush();
|
||||
while(waiting.length<2)await new Promise(resolve=>setTimeout(resolve,0));
|
||||
const during=await store.snapshot();
|
||||
allow=true;waiting.splice(0).forEach(resolve=>resolve());
|
||||
await flushing;
|
||||
const after=await store.snapshot();
|
||||
process.stdout.write(JSON.stringify({{attempted,during,after}}));
|
||||
}})();
|
||||
"""
|
||||
output = run_node(script)
|
||||
|
||||
assert [item["id"] for item in output["during"] if item["status"] == "sending"] == [
|
||||
"issue-0", "message-0"
|
||||
]
|
||||
assert [item["id"] for item in output["during"] if item["status"] == "queued"] == [
|
||||
"issue-1", "issue-2", "issue-3", "message-1"
|
||||
]
|
||||
assert output["attempted"] == [
|
||||
"issue-0", "message-0", "issue-1", "message-1", "issue-2"
|
||||
]
|
||||
unadmitted = next(item for item in output["after"] if item["id"] == "issue-3")
|
||||
assert unadmitted["status"] == "queued"
|
||||
assert "claimToken" not in unadmitted
|
||||
|
||||
|
||||
def test_bounded_flush_stops_admission_and_releases_unstarted_claims_on_auth_loss():
|
||||
script = f"""
|
||||
const createBackgroundIssueSync = require({json.dumps(str(SYNC))});
|
||||
@@ -812,18 +892,15 @@ const transaction=work=>{{const run=tail.then(()=>work({{
|
||||
{{id:'message-1',ownerLogin:'timmy',status:'queued'}},
|
||||
{{id:'message-2',ownerLogin:'timmy',status:'queued'}},
|
||||
], 'authored');
|
||||
const claimed=await store.claimBatch('timmy', 4);
|
||||
process.stdout.write(JSON.stringify({{claimed,snapshot:await store.snapshot()}}));
|
||||
const planned=await store.planBatch('timmy', 4);
|
||||
process.stdout.write(JSON.stringify({{planned,snapshot:await store.snapshot()}}));
|
||||
}})();
|
||||
"""
|
||||
output = run_node(script)
|
||||
|
||||
assert [item["id"] for item in output["claimed"]] == [
|
||||
"issue-1", "message-1", "issue-2", "message-2"
|
||||
]
|
||||
assert all(item["status"] == "sending" for item in output["claimed"])
|
||||
assert all(item["claimUntil"] == 5100 for item in output["claimed"])
|
||||
assert next(item for item in output["snapshot"] if item["id"] == "issue-3")["status"] == "queued"
|
||||
assert output["planned"] == ["issue-1", "message-1", "issue-2", "message-2"]
|
||||
assert all(item["status"] == "queued" for item in output["snapshot"])
|
||||
assert all("claimUntil" not in item for item in output["snapshot"])
|
||||
|
||||
|
||||
def test_permanent_delivery_failure_marks_issue_for_foreground_attention():
|
||||
@@ -908,6 +985,47 @@ const transaction = work => {{
|
||||
assert output["records"][0]["claimUntil"] == 6000
|
||||
|
||||
|
||||
def test_issue_store_renews_and_fences_claim_mutations_with_unique_tokens():
|
||||
script = f"""
|
||||
const createBackgroundIssueSync = require({json.dumps(str(SYNC))});
|
||||
const records=new Map();let tail=Promise.resolve();let timestamp=100;
|
||||
const transaction=work=>{{const run=tail.then(()=>work({{
|
||||
get:async id=>records.get(id),
|
||||
getAll:async()=>[...records.values()].map(value=>({{...value}})),
|
||||
put:async value=>records.set(value.id,{{...value}}),delete:async id=>records.delete(id),
|
||||
}}));tail=run.catch(()=>{{}});return run;}};
|
||||
(async()=>{{
|
||||
const tokens=['claim-a','claim-b'];
|
||||
const store=createBackgroundIssueSync.createIssueSyncStore({{
|
||||
transaction,now:()=>timestamp,claimMs:30,createToken:()=>tokens.shift(),
|
||||
}});
|
||||
await store.reconcile([{{id:'lease',ownerLogin:'timmy',status:'queued'}}]);
|
||||
const first=await store.claimNext('timmy');
|
||||
timestamp=120;
|
||||
const renewed=await store.renew('lease',first.claimToken);
|
||||
timestamp=151;
|
||||
const second=await store.claimNext('timmy');
|
||||
const staleComplete=await store.complete('lease',first.claimToken);
|
||||
const staleRelease=await store.release('lease',first.claimToken);
|
||||
const staleFail=await store.fail('lease',first.claimToken,'old failure');
|
||||
const currentComplete=await store.complete('lease',second.claimToken,{{number:9}});
|
||||
process.stdout.write(JSON.stringify({{first,renewed,second,staleComplete,staleRelease,staleFail,currentComplete,record:records.get('lease')}}));
|
||||
}})();
|
||||
"""
|
||||
output = run_node(script)
|
||||
|
||||
assert output["first"]["claimToken"] == "claim-a"
|
||||
assert output["first"]["claimUntil"] == 130
|
||||
assert output["renewed"]["claimUntil"] == 150
|
||||
assert output["second"]["claimToken"] == "claim-b"
|
||||
assert output["second"]["claimUntil"] == 181
|
||||
assert output["staleComplete"] is False
|
||||
assert output["staleRelease"] is False
|
||||
assert output["staleFail"] is False
|
||||
assert output["currentComplete"] is True
|
||||
assert output["record"]["status"] == "sent"
|
||||
|
||||
|
||||
def test_foreground_and_worker_race_still_posts_one_issue():
|
||||
script = f"""
|
||||
const createBackgroundIssueSync = require({json.dumps(str(SYNC))});
|
||||
@@ -978,8 +1096,8 @@ const transaction=work=>{{const run=tail.then(()=>work({{
|
||||
(async()=>{{
|
||||
const store=createBackgroundIssueSync.createIssueSyncStore({{transaction,now:()=>100}});
|
||||
await store.reconcile([{{id:'done',ownerLogin:'timmy',status:'queued'}}]);
|
||||
await store.claimNext('timmy');
|
||||
await store.complete('done');
|
||||
const claimed=await store.claimNext('timmy');
|
||||
await store.complete('done', claimed.claimToken);
|
||||
const replay=await store.claimNext('timmy');
|
||||
const snapshot=await store.snapshot();
|
||||
process.stdout.write(JSON.stringify({{replay,snapshot}}));
|
||||
|
||||
Reference in new issue
Block a user