fix: keep authored actions responsive under ledger contention (#248)

tests/test_authored_idempotency.py

@@ -1,4 +1,7 @@
 import asyncio
+import sqlite3
+import threading
+import time
 
 import httpx
 import pytest
@@ -228,3 +231,170 @@ async def test_orphaned_pending_comment_fails_closed_without_upstream_retry(
         "This action may still be completing; verify its result before retrying"
     )
     assert calls == 0
+
+
+@pytest.mark.anyio
+async def test_slow_ledger_reservation_does_not_block_health_requests(monkeypatch):
+    async def assigned(_repository, _number):
+        return True
+
+    async def comment(_repository, _number, body):
+        return {"id": 248, "body": body}
+
+    started = threading.Event()
+    original_reserve = main._idempotency_ledger.reserve
+
+    def slow_reserve(key, fingerprint):
+        started.set()
+        time.sleep(0.2)
+        return original_reserve(key, fingerprint)
+
+    monkeypatch.setattr(main._idempotency_ledger, "reserve", slow_reserve)
+    monkeypatch.setattr(main.gitea_proxy, "is_assigned_issue", assigned)
+    monkeypatch.setattr(main.gitea_proxy, "comment_on_issue", comment)
+    transport = httpx.ASGITransport(app=main.app)
+
+    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+        started_at = asyncio.get_running_loop().time()
+        authored = asyncio.create_task(client.post(
+            "/api/v1/repos/stackchain/api/issues/248/comments",
+            json={"body": "Keep the dashboard responsive"},
+            headers={"Idempotency-Key": "contention-248"},
+        ))
+        assert await asyncio.to_thread(started.wait, 1)
+        health = await client.get("/healthz")
+        health_elapsed = asyncio.get_running_loop().time() - started_at
+        authored_response = await authored
+
+    assert health.status_code == 200
+    assert health_elapsed < 0.15
+    assert authored_response.status_code == 201
+
+
+@pytest.mark.anyio
+async def test_locked_ledger_returns_bounded_retryable_response(monkeypatch, tmp_path):
+    upstream_calls = 0
+
+    async def assigned(_repository, _number):
+        return True
+
+    async def comment(_repository, _number, body):
+        nonlocal upstream_calls
+        upstream_calls += 1
+        return {"id": 248, "body": body}
+
+    database = tmp_path / "contended.sqlite3"
+    ledger = IdempotencyLedger(
+        database,
+        ttl_seconds=600,
+        max_entries=256,
+        lock_timeout_seconds=0.05,
+    )
+    monkeypatch.setattr(main, "_idempotency_ledger", ledger)
+    monkeypatch.setattr(main.gitea_proxy, "is_assigned_issue", assigned)
+    monkeypatch.setattr(main.gitea_proxy, "comment_on_issue", comment)
+    blocker = sqlite3.connect(database)
+    blocker.execute("BEGIN IMMEDIATE")
+    transport = httpx.ASGITransport(app=main.app)
+
+    try:
+        async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+            started_at = asyncio.get_running_loop().time()
+            response = await client.post(
+                "/api/v1/repos/stackchain/api/issues/248/comments",
+                json={"body": "Retry without freezing"},
+                headers={"Idempotency-Key": "locked-ledger-248"},
+            )
+            elapsed = asyncio.get_running_loop().time() - started_at
+    finally:
+        blocker.rollback()
+        blocker.close()
+
+    assert response.status_code == 503
+    assert response.headers["Retry-After"] == "1"
+    assert response.json()["detail"] == "Authored action queue is busy; please retry"
+    assert elapsed < 0.5
+    assert upstream_calls == 0
+
+
+@pytest.mark.anyio
+async def test_slow_ledger_completion_does_not_block_health_requests(monkeypatch):
+    async def assigned(_repository, _number):
+        return True
+
+    async def comment(_repository, _number, body):
+        return {"id": 249, "body": body}
+
+    started = threading.Event()
+    original_complete = main._idempotency_ledger.complete
+
+    def slow_complete(key, response):
+        started.set()
+        time.sleep(0.2)
+        return original_complete(key, response)
+
+    monkeypatch.setattr(main._idempotency_ledger, "complete", slow_complete)
+    monkeypatch.setattr(main.gitea_proxy, "is_assigned_issue", assigned)
+    monkeypatch.setattr(main.gitea_proxy, "comment_on_issue", comment)
+    transport = httpx.ASGITransport(app=main.app)
+
+    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+        started_at = asyncio.get_running_loop().time()
+        authored = asyncio.create_task(client.post(
+            "/api/v1/repos/stackchain/api/issues/248/comments",
+            json={"body": "Persist without freezing"},
+            headers={"Idempotency-Key": "slow-completion-248"},
+        ))
+        assert await asyncio.to_thread(started.wait, 1)
+        health = await client.get("/healthz")
+        health_elapsed = asyncio.get_running_loop().time() - started_at
+        authored_response = await authored
+
+    assert health.status_code == 200
+    assert health_elapsed < 0.15
+    assert authored_response.status_code == 201
+
+
+@pytest.mark.anyio
+async def test_locked_completion_fails_closed_with_retry_guidance(monkeypatch, tmp_path):
+    database = tmp_path / "completion-contention.sqlite3"
+    ledger = IdempotencyLedger(
+        database, ttl_seconds=600, max_entries=256, lock_timeout_seconds=0.05
+    )
+    blocker = None
+
+    async def assigned(_repository, _number):
+        return True
+
+    async def comment(_repository, _number, body):
+        nonlocal blocker
+        blocker = sqlite3.connect(database)
+        blocker.execute("BEGIN IMMEDIATE")
+        return {"id": 250, "body": body}
+
+    monkeypatch.setattr(main, "_idempotency_ledger", ledger)
+    monkeypatch.setattr(main.gitea_proxy, "is_assigned_issue", assigned)
+    monkeypatch.setattr(main.gitea_proxy, "comment_on_issue", comment)
+    transport = httpx.ASGITransport(app=main.app)
+
+    try:
+        async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+            response = await client.post(
+                "/api/v1/repos/stackchain/api/issues/248/comments",
+                json={"body": "The upstream mutation may already exist"},
+                headers={"Idempotency-Key": "locked-completion-248"},
+            )
+    finally:
+        if blocker is not None:
+            blocker.rollback()
+            blocker.close()
+
+    assert response.status_code == 503
+    assert response.headers["Retry-After"] == "5"
+    assert response.json()["detail"] == (
+        "This action may have completed; verify its result before retrying"
+    )
+    assert ledger.reserve(
+        "locked-completion-248",
+        ("issue-comment", "stackchain/api", 248, "The upstream mutation may already exist"),
+    ).state == "pending"