feat: resume release runs from checkpoints (#331)

README.md

@@ -296,6 +296,9 @@ without advancing the release. The equivalent environment variables are
 `RELEASE_AGENT_TIMEOUT`, `RELEASE_TEST_TIMEOUT`, and
 `RELEASE_TERMINATION_GRACE`; every value must be positive.
 
-Durable state defaults to `.release-engine/state.json`. Full behavior and
-safety gates are documented in
+Durable state defaults to `.release-engine/state.json`. Successful checkpoints
+record the repository, issue branch, and exact commit SHA. A restart resumes at
+tests, push, or PR creation only after validating that identity and a clean
+worktree; a pushed checkpoint also verifies the remote branch SHA. Full
+behavior and safety gates are documented in
 [`docs/release-engine-spec.md`](docs/release-engine-spec.md).

docs/release-engine-spec.md

@@ -23,6 +23,18 @@ Convert one eligible Gitea issue into a tested, traceable pull request without d
 
 Terminal failure states include `claim_failed`, `agent_failed`, `agent_timed_out`, `tests_failed`, `tests_timed_out`, and `push_failed`. A rerun resumes from persisted state and never opens a duplicate PR.
 
+Successful checkpoints are schema-versioned and bind the repository, issue,
+deterministic branch, and exact commit SHA. On restart, the engine:
+
+- resumes `agent_complete` at tests;
+- resumes `tests_passed` at push; and
+- resumes `pushed` at PR creation after verifying the remote branch SHA.
+
+Before skipping any stage, it checks out the recorded branch, verifies its
+local `HEAD`, and requires a clean worktree. Missing identity, an unsupported
+schema, or any repository/issue/branch/SHA mismatch fails closed instead of
+repeating or advancing delivery.
+
 ## Queue selection
 
 1. Only open issues; pull requests are excluded.
@@ -63,6 +75,8 @@ Terminal failure states include `claim_failed`, `agent_failed`, `agent_timed_out
 
 - `--dry-run` performs discovery and planning only: no claim, git mutation, agent command, push, or PR.
 - State is written atomically after each successful transition.
+- Resumable state carries repository and commit identity; successful stages are
+  skipped only after local (and, after push, remote) Git verification.
 - One invocation handles at most one issue.
 - Missing token, dirty worktree, failed claim verification, failed tests, or missing evidence blocks PR creation.
 
@@ -79,3 +93,6 @@ Terminal failure states include `claim_failed`, `agent_failed`, `agent_timed_out
 9. A timed-out command terminates its descendants and returns within its deadline plus grace period.
 10. Agent and test timeouts persist distinct terminal states and block every later delivery stage.
 11. Non-positive deadline configuration aborts before issue claim.
+12. Restarts from `agent_complete`, `tests_passed`, and `pushed` skip only the
+    completed stages and open exactly one linked PR.
+13. Dirty or identity-mismatched checkpoints stop before push or PR creation.

src/release_engine.py

@@ -57,6 +57,9 @@ class RunState:
     evidence: str = ""
     pr_number: int | None = None
     pr_url: str = ""
+    schema_version: int = 1
+    repo: str = ""
+    head_sha: str = ""
 
 
 class Client(Protocol):
@@ -249,6 +252,7 @@ class ReleaseEngine:
 
         issue = select_issue(self.client.list_open_issues(self.config.repo), self.config.agent, issue_number)
         branch = branch_name(self.config.agent, issue)
+        head_sha = ""
 
         existing_pr = self.client.find_open_pr(self.config.repo, branch)
         if existing_pr:
@@ -272,61 +276,121 @@ class ReleaseEngine:
 
         agent_command = parse_command(self.config.agent_command, "agent command")
         test_command = parse_command(self.config.test_command, "test command")
-
-        self.client.claim_issue(self.config.repo, issue.number, self.config.agent)
-        verified = self.client.get_issue(self.config.repo, issue.number)
-        if verified.assignee != self.config.agent:
-            raise RuntimeError(
-                f"claim verification failed: expected {self.config.agent}, got {verified.assignee}"
+        checkpoint = self.store.load()
+        resumable_statuses = {"agent_complete", "tests_passed", "pushed"}
+        if checkpoint is not None and checkpoint.status in resumable_statuses:
+            if (
+                checkpoint.schema_version != 1
+                or checkpoint.repo != self.config.repo
+                or checkpoint.issue != issue.number
+                or checkpoint.branch != branch
+                or not checkpoint.head_sha
+            ):
+                raise RuntimeError("release checkpoint does not match this run")
+            branch_result = self.runner.run(["git", "checkout", branch], self.config.repo_path)
+            if branch_result.returncode != 0:
+                raise RuntimeError(f"checkpoint branch checkout failed: {branch_result.stderr.strip()}")
+            head_result = self.runner.run(["git", "rev-parse", "HEAD"], self.config.repo_path)
+            if head_result.returncode != 0 or head_result.stdout.strip() != checkpoint.head_sha:
+                raise RuntimeError("release checkpoint commit does not match the local branch")
+            status_result = self.runner.run(
+                ["git", "status", "--porcelain"], self.config.repo_path
             )
-        self.store.save(RunState(issue.number, branch, "claimed"))
+            if status_result.returncode != 0 or status_result.stdout.strip():
+                raise RuntimeError("release checkpoint worktree is not clean")
+            head_sha = checkpoint.head_sha
+            evidence = checkpoint.evidence
+        else:
+            checkpoint = None
+            evidence = ""
+            self.client.claim_issue(self.config.repo, issue.number, self.config.agent)
+            verified = self.client.get_issue(self.config.repo, issue.number)
+            if verified.assignee != self.config.agent:
+                raise RuntimeError(
+                    f"claim verification failed: expected {self.config.agent}, got {verified.assignee}"
+                )
+            self.store.save(RunState(issue.number, branch, "claimed"))
 
-        branch_result = self.runner.run(["git", "checkout", "-B", branch], self.config.repo_path)
-        if branch_result.returncode != 0:
-            raise RuntimeError(f"branch creation failed: {branch_result.stderr.strip()}")
+            branch_result = self.runner.run(["git", "checkout", "-B", branch], self.config.repo_path)
+            if branch_result.returncode != 0:
+                raise RuntimeError(f"branch creation failed: {branch_result.stderr.strip()}")
 
-        agent_result = self.runner.run(
-            agent_command,
-            self.config.repo_path,
-            {
-                "RELEASE_ISSUE_NUMBER": str(issue.number),
-                "RELEASE_ISSUE_TITLE": issue.title,
-                "RELEASE_ISSUE_BODY": issue.body,
-                "RELEASE_REPO": self.config.repo,
-                "RELEASE_BRANCH": branch,
-            },
-            timeout_seconds=self.config.agent_timeout_seconds,
-            termination_grace_seconds=self.config.termination_grace_seconds,
-        )
-        if agent_result.timed_out:
-            evidence = (agent_result.stdout + "\n" + agent_result.stderr).strip()[-2000:]
-            self.store.save(RunState(issue.number, branch, "agent_timed_out", evidence))
-            raise RuntimeError(f"coding agent timed out: {evidence}")
-        if agent_result.returncode != 0:
-            self.store.save(RunState(issue.number, branch, "agent_failed", agent_result.stderr[-2000:]))
-            raise RuntimeError(f"coding agent failed: {agent_result.stderr.strip()}")
-        self.store.save(RunState(issue.number, branch, "agent_complete"))
+            agent_result = self.runner.run(
+                agent_command,
+                self.config.repo_path,
+                {
+                    "RELEASE_ISSUE_NUMBER": str(issue.number),
+                    "RELEASE_ISSUE_TITLE": issue.title,
+                    "RELEASE_ISSUE_BODY": issue.body,
+                    "RELEASE_REPO": self.config.repo,
+                    "RELEASE_BRANCH": branch,
+                },
+                timeout_seconds=self.config.agent_timeout_seconds,
+                termination_grace_seconds=self.config.termination_grace_seconds,
+            )
+            if agent_result.timed_out:
+                evidence = (agent_result.stdout + "\n" + agent_result.stderr).strip()[-2000:]
+                self.store.save(RunState(issue.number, branch, "agent_timed_out", evidence))
+                raise RuntimeError(f"coding agent timed out: {evidence}")
+            if agent_result.returncode != 0:
+                self.store.save(RunState(issue.number, branch, "agent_failed", agent_result.stderr[-2000:]))
+                raise RuntimeError(f"coding agent failed: {agent_result.stderr.strip()}")
+            head_result = self.runner.run(["git", "rev-parse", "HEAD"], self.config.repo_path)
+            head_sha = head_result.stdout.strip()
+            if head_result.returncode != 0 or not head_sha:
+                raise RuntimeError("could not identify the release checkpoint commit")
+            self.store.save(RunState(
+                issue.number, branch, "agent_complete",
+                repo=self.config.repo, head_sha=head_sha,
+            ))
 
-        test_result = self.runner.run(
-            test_command,
-            self.config.repo_path,
-            timeout_seconds=self.config.test_timeout_seconds,
-            termination_grace_seconds=self.config.termination_grace_seconds,
-        )
-        evidence = (test_result.stdout + "\n" + test_result.stderr).strip()[-4000:]
-        if test_result.timed_out:
-            self.store.save(RunState(issue.number, branch, "tests_timed_out", evidence))
-            raise RuntimeError(f"tests timed out: {evidence}")
-        if test_result.returncode != 0:
-            self.store.save(RunState(issue.number, branch, "tests_failed", evidence))
-            raise RuntimeError(f"tests failed: {evidence}")
-        self.store.save(RunState(issue.number, branch, "tests_passed", evidence))
+        if checkpoint is None or checkpoint.status == "agent_complete":
+            test_result = self.runner.run(
+                test_command,
+                self.config.repo_path,
+                timeout_seconds=self.config.test_timeout_seconds,
+                termination_grace_seconds=self.config.termination_grace_seconds,
+            )
+            evidence = (test_result.stdout + "\n" + test_result.stderr).strip()[-4000:]
+            if test_result.timed_out:
+                self.store.save(RunState(
+                    issue.number, branch, "tests_timed_out", evidence,
+                    repo=self.config.repo, head_sha=head_sha,
+                ))
+                raise RuntimeError(f"tests timed out: {evidence}")
+            if test_result.returncode != 0:
+                self.store.save(RunState(
+                    issue.number, branch, "tests_failed", evidence,
+                    repo=self.config.repo, head_sha=head_sha,
+                ))
+                raise RuntimeError(f"tests failed: {evidence}")
+            self.store.save(RunState(
+                issue.number, branch, "tests_passed", evidence,
+                repo=self.config.repo, head_sha=head_sha,
+            ))
 
-        push_result = self.runner.run(["git", "push", "-u", "origin", branch], self.config.repo_path)
-        if push_result.returncode != 0:
-            self.store.save(RunState(issue.number, branch, "push_failed", push_result.stderr[-2000:]))
-            raise RuntimeError(f"push failed: {push_result.stderr.strip()}")
-        self.store.save(RunState(issue.number, branch, "pushed", evidence))
+        if checkpoint is not None and checkpoint.status == "pushed":
+            remote_result = self.runner.run(
+                ["git", "ls-remote", "--heads", "origin", branch],
+                self.config.repo_path,
+            )
+            expected_remote = f"{checkpoint.head_sha}\trefs/heads/{branch}"
+            if remote_result.returncode != 0 or remote_result.stdout.strip() != expected_remote:
+                raise RuntimeError("release checkpoint commit does not match the remote branch")
+        else:
+            push_result = self.runner.run(
+                ["git", "push", "-u", "origin", branch], self.config.repo_path
+            )
+            if push_result.returncode != 0:
+                self.store.save(RunState(
+                    issue.number, branch, "push_failed", push_result.stderr[-2000:],
+                    repo=self.config.repo, head_sha=head_sha,
+                ))
+                raise RuntimeError(f"push failed: {push_result.stderr.strip()}")
+            self.store.save(RunState(
+                issue.number, branch, "pushed", evidence,
+                repo=self.config.repo, head_sha=head_sha,
+            ))
 
         pr_body = (
             f"Closes #{issue.number}\n\n"
@@ -351,6 +415,8 @@ class ReleaseEngine:
             evidence,
             pr_number=pr["number"],
             pr_url=pr.get("html_url", ""),
+            repo=self.config.repo,
+            head_sha=head_sha,
         )
         self.store.save(final)
         return {

tests/test_release_engine.py

@@ -312,6 +312,7 @@ def test_failed_tests_block_push_and_pr(tmp_path):
     runner = FakeRunner([
         CommandResult(0, "", ""),
         CommandResult(0, "changed", ""),
+        CommandResult(0, "abc123\n", ""),
         CommandResult(1, "1 failed", ""),
     ])
     engine = ReleaseEngine(config(tmp_path), client, runner)
@@ -329,6 +330,7 @@ def test_test_timeout_records_terminal_state_and_blocks_push(tmp_path):
     runner = FakeRunner([
         CommandResult(0, "", ""),
         CommandResult(0, "implemented", ""),
+        CommandResult(0, "abc123\n", ""),
         CommandResult(-15, "3 passed before stall", "", timed_out=True),
     ])
     cfg = config(tmp_path)
@@ -340,7 +342,7 @@ def test_test_timeout_records_terminal_state_and_blocks_push(tmp_path):
     assert state is not None
     assert state.status == "tests_timed_out"
     assert "3 passed before stall" in state.evidence
-    assert runner.options[2] == {
+    assert runner.options[3] == {
         "timeout_seconds": cfg.test_timeout_seconds,
         "termination_grace_seconds": cfg.termination_grace_seconds,
     }
@@ -354,6 +356,7 @@ def test_passing_run_pushes_and_opens_linked_pr(tmp_path):
     runner = FakeRunner([
         CommandResult(0, "", ""),
         CommandResult(0, "implemented", ""),
+        CommandResult(0, "abc123\n", ""),
         CommandResult(0, "8 passed", ""),
         CommandResult(0, "", ""),
     ])
@@ -403,3 +406,222 @@ def test_state_store_round_trip_is_atomic(tmp_path):
     store.save(expected)
     assert store.load() == expected
     assert not (tmp_path / "state.json.tmp").exists()
+
+
+def test_restart_from_agent_complete_skips_claim_and_coding(tmp_path):
+    issue = Issue(331, "Resume releases", "body", "open", [], "timmy")
+    client = FakeClient([issue])
+    cfg = config(tmp_path)
+    branch = "timmy/331-resume-releases"
+    cfg.state_path.write_text(json.dumps({
+        "schema_version": 1,
+        "repo": cfg.repo,
+        "issue": issue.number,
+        "branch": branch,
+        "status": "agent_complete",
+        "evidence": "",
+        "pr_number": None,
+        "pr_url": "",
+        "head_sha": "abc123",
+    }))
+    runner = FakeRunner([
+        CommandResult(0, "", ""),
+        CommandResult(0, "abc123\n", ""),
+        CommandResult(0, "", ""),
+        CommandResult(0, "8 passed", ""),
+        CommandResult(0, "", ""),
+    ])
+
+    result = ReleaseEngine(cfg, client, runner).execute(issue_number=331)
+
+    assert result["tests"] == "passed"
+    assert client.claimed == []
+    assert ["agent", "--issue-from-env"] not in runner.commands
+    assert runner.commands[:3] == [
+        ["git", "checkout", branch],
+        ["git", "rev-parse", "HEAD"],
+        ["git", "status", "--porcelain"],
+    ]
+
+
+def test_restart_from_tests_passed_skips_claim_coding_and_tests(tmp_path):
+    issue = Issue(331, "Resume releases", "body", "open", [], "timmy")
+    client = FakeClient([issue])
+    cfg = config(tmp_path)
+    branch = "timmy/331-resume-releases"
+    StateStore(cfg.state_path).save(RunState(
+        issue=issue.number,
+        branch=branch,
+        status="tests_passed",
+        evidence="604 passed",
+        repo=cfg.repo,
+        head_sha="abc123",
+    ))
+    runner = FakeRunner([
+        CommandResult(0, "", ""),
+        CommandResult(0, "abc123\n", ""),
+        CommandResult(0, "", ""),
+        CommandResult(0, "", ""),
+    ])
+
+    result = ReleaseEngine(cfg, client, runner).execute(issue_number=331)
+
+    assert result["tests"] == "passed"
+    assert client.claimed == []
+    assert ["agent", "--issue-from-env"] not in runner.commands
+    assert ["pytest", "-q"] not in runner.commands
+    assert runner.commands == [
+        ["git", "checkout", branch],
+        ["git", "rev-parse", "HEAD"],
+        ["git", "status", "--porcelain"],
+        ["git", "push", "-u", "origin", branch],
+    ]
+
+
+def test_restart_from_pushed_verifies_remote_and_only_opens_pr(tmp_path):
+    issue = Issue(331, "Resume releases", "body", "open", [], "timmy")
+    client = FakeClient([issue])
+    cfg = config(tmp_path)
+    branch = "timmy/331-resume-releases"
+    StateStore(cfg.state_path).save(RunState(
+        issue=issue.number,
+        branch=branch,
+        status="pushed",
+        evidence="604 passed",
+        repo=cfg.repo,
+        head_sha="abc123",
+    ))
+    runner = FakeRunner([
+        CommandResult(0, "", ""),
+        CommandResult(0, "abc123\n", ""),
+        CommandResult(0, "", ""),
+        CommandResult(0, f"abc123\trefs/heads/{branch}\n", ""),
+    ])
+
+    result = ReleaseEngine(cfg, client, runner).execute(issue_number=331)
+
+    assert result["pr_number"] == 44
+    assert client.claimed == []
+    assert len(client.created_prs) == 1
+    assert runner.commands == [
+        ["git", "checkout", branch],
+        ["git", "rev-parse", "HEAD"],
+        ["git", "status", "--porcelain"],
+        ["git", "ls-remote", "--heads", "origin", branch],
+    ]
+
+
+def test_fresh_run_persists_repository_and_commit_identity(tmp_path):
+    issue = Issue(331, "Resume releases", "body", "open", [], None)
+    client = FakeClient([issue])
+    cfg = config(tmp_path)
+    runner = FakeRunner([
+        CommandResult(0, "", ""),
+        CommandResult(0, "implemented", ""),
+        CommandResult(0, "abc123\n", ""),
+        CommandResult(0, "604 passed", ""),
+        CommandResult(0, "", ""),
+    ])
+
+    ReleaseEngine(cfg, client, runner).execute(issue_number=331)
+
+    state = StateStore(cfg.state_path).load()
+    assert state is not None
+    assert state.status == "pr_opened"
+    assert state.schema_version == 1
+    assert state.repo == cfg.repo
+    assert state.head_sha == "abc123"
+    assert ["git", "rev-parse", "HEAD"] in runner.commands
+
+
+def test_resume_fails_closed_when_checkpoint_worktree_is_dirty(tmp_path):
+    issue = Issue(331, "Resume releases", "body", "open", [], "timmy")
+    client = FakeClient([issue])
+    cfg = config(tmp_path)
+    branch = "timmy/331-resume-releases"
+    StateStore(cfg.state_path).save(RunState(
+        issue=issue.number,
+        branch=branch,
+        status="tests_passed",
+        evidence="604 passed",
+        repo=cfg.repo,
+        head_sha="abc123",
+    ))
+    runner = FakeRunner([
+        CommandResult(0, "", ""),
+        CommandResult(0, "abc123\n", ""),
+        CommandResult(0, " M src/release_engine.py\n", ""),
+        CommandResult(0, "", ""),
+    ])
+
+    with pytest.raises(RuntimeError, match="worktree is not clean"):
+        ReleaseEngine(cfg, client, runner).execute(issue_number=331)
+
+    assert not any(command[:2] == ["git", "push"] for command in runner.commands)
+    assert client.created_prs == []
+
+
+def test_resume_fails_closed_when_checkpoint_targets_another_repository(tmp_path):
+    issue = Issue(331, "Resume releases", "body", "open", [], "timmy")
+    client = FakeClient([issue])
+    cfg = config(tmp_path)
+    StateStore(cfg.state_path).save(RunState(
+        issue=issue.number,
+        branch="timmy/331-resume-releases",
+        status="tests_passed",
+        repo="other/repository",
+        head_sha="abc123",
+    ))
+
+    with pytest.raises(RuntimeError, match="checkpoint does not match"):
+        ReleaseEngine(cfg, client, FakeRunner([])).execute(issue_number=331)
+
+    assert client.created_prs == []
+
+
+def test_resume_fails_closed_when_local_commit_changed(tmp_path):
+    issue = Issue(331, "Resume releases", "body", "open", [], "timmy")
+    client = FakeClient([issue])
+    cfg = config(tmp_path)
+    StateStore(cfg.state_path).save(RunState(
+        issue=issue.number,
+        branch="timmy/331-resume-releases",
+        status="tests_passed",
+        repo=cfg.repo,
+        head_sha="abc123",
+    ))
+    runner = FakeRunner([
+        CommandResult(0, "", ""),
+        CommandResult(0, "different-sha\n", ""),
+    ])
+
+    with pytest.raises(RuntimeError, match="commit does not match the local branch"):
+        ReleaseEngine(cfg, client, runner).execute(issue_number=331)
+
+    assert client.created_prs == []
+
+
+def test_resume_fails_closed_when_remote_commit_changed(tmp_path):
+    issue = Issue(331, "Resume releases", "body", "open", [], "timmy")
+    client = FakeClient([issue])
+    cfg = config(tmp_path)
+    branch = "timmy/331-resume-releases"
+    StateStore(cfg.state_path).save(RunState(
+        issue=issue.number,
+        branch=branch,
+        status="pushed",
+        evidence="604 passed",
+        repo=cfg.repo,
+        head_sha="abc123",
+    ))
+    runner = FakeRunner([
+        CommandResult(0, "", ""),
+        CommandResult(0, "abc123\n", ""),
+        CommandResult(0, "", ""),
+        CommandResult(0, f"different-sha\trefs/heads/{branch}\n", ""),
+    ])
+
+    with pytest.raises(RuntimeError, match="commit does not match the remote branch"):
+        ReleaseEngine(cfg, client, runner).execute(issue_number=331)
+
+    assert client.created_prs == []