feat: resume release runs from checkpoints (#331)

src/release_engine.py

@@ -57,6 +57,9 @@ class RunState:
     evidence: str = ""
     pr_number: int | None = None
     pr_url: str = ""
+    schema_version: int = 1
+    repo: str = ""
+    head_sha: str = ""
 
 
 class Client(Protocol):
@@ -249,6 +252,7 @@ class ReleaseEngine:
 
         issue = select_issue(self.client.list_open_issues(self.config.repo), self.config.agent, issue_number)
         branch = branch_name(self.config.agent, issue)
+        head_sha = ""
 
         existing_pr = self.client.find_open_pr(self.config.repo, branch)
         if existing_pr:
@@ -272,61 +276,121 @@ class ReleaseEngine:
 
         agent_command = parse_command(self.config.agent_command, "agent command")
         test_command = parse_command(self.config.test_command, "test command")
-
-        self.client.claim_issue(self.config.repo, issue.number, self.config.agent)
-        verified = self.client.get_issue(self.config.repo, issue.number)
-        if verified.assignee != self.config.agent:
-            raise RuntimeError(
-                f"claim verification failed: expected {self.config.agent}, got {verified.assignee}"
+        checkpoint = self.store.load()
+        resumable_statuses = {"agent_complete", "tests_passed", "pushed"}
+        if checkpoint is not None and checkpoint.status in resumable_statuses:
+            if (
+                checkpoint.schema_version != 1
+                or checkpoint.repo != self.config.repo
+                or checkpoint.issue != issue.number
+                or checkpoint.branch != branch
+                or not checkpoint.head_sha
+            ):
+                raise RuntimeError("release checkpoint does not match this run")
+            branch_result = self.runner.run(["git", "checkout", branch], self.config.repo_path)
+            if branch_result.returncode != 0:
+                raise RuntimeError(f"checkpoint branch checkout failed: {branch_result.stderr.strip()}")
+            head_result = self.runner.run(["git", "rev-parse", "HEAD"], self.config.repo_path)
+            if head_result.returncode != 0 or head_result.stdout.strip() != checkpoint.head_sha:
+                raise RuntimeError("release checkpoint commit does not match the local branch")
+            status_result = self.runner.run(
+                ["git", "status", "--porcelain"], self.config.repo_path
             )
-        self.store.save(RunState(issue.number, branch, "claimed"))
+            if status_result.returncode != 0 or status_result.stdout.strip():
+                raise RuntimeError("release checkpoint worktree is not clean")
+            head_sha = checkpoint.head_sha
+            evidence = checkpoint.evidence
+        else:
+            checkpoint = None
+            evidence = ""
+            self.client.claim_issue(self.config.repo, issue.number, self.config.agent)
+            verified = self.client.get_issue(self.config.repo, issue.number)
+            if verified.assignee != self.config.agent:
+                raise RuntimeError(
+                    f"claim verification failed: expected {self.config.agent}, got {verified.assignee}"
+                )
+            self.store.save(RunState(issue.number, branch, "claimed"))
 
-        branch_result = self.runner.run(["git", "checkout", "-B", branch], self.config.repo_path)
-        if branch_result.returncode != 0:
-            raise RuntimeError(f"branch creation failed: {branch_result.stderr.strip()}")
+            branch_result = self.runner.run(["git", "checkout", "-B", branch], self.config.repo_path)
+            if branch_result.returncode != 0:
+                raise RuntimeError(f"branch creation failed: {branch_result.stderr.strip()}")
 
-        agent_result = self.runner.run(
-            agent_command,
-            self.config.repo_path,
-            {
-                "RELEASE_ISSUE_NUMBER": str(issue.number),
-                "RELEASE_ISSUE_TITLE": issue.title,
-                "RELEASE_ISSUE_BODY": issue.body,
-                "RELEASE_REPO": self.config.repo,
-                "RELEASE_BRANCH": branch,
-            },
-            timeout_seconds=self.config.agent_timeout_seconds,
-            termination_grace_seconds=self.config.termination_grace_seconds,
-        )
-        if agent_result.timed_out:
-            evidence = (agent_result.stdout + "\n" + agent_result.stderr).strip()[-2000:]
-            self.store.save(RunState(issue.number, branch, "agent_timed_out", evidence))
-            raise RuntimeError(f"coding agent timed out: {evidence}")
-        if agent_result.returncode != 0:
-            self.store.save(RunState(issue.number, branch, "agent_failed", agent_result.stderr[-2000:]))
-            raise RuntimeError(f"coding agent failed: {agent_result.stderr.strip()}")
-        self.store.save(RunState(issue.number, branch, "agent_complete"))
+            agent_result = self.runner.run(
+                agent_command,
+                self.config.repo_path,
+                {
+                    "RELEASE_ISSUE_NUMBER": str(issue.number),
+                    "RELEASE_ISSUE_TITLE": issue.title,
+                    "RELEASE_ISSUE_BODY": issue.body,
+                    "RELEASE_REPO": self.config.repo,
+                    "RELEASE_BRANCH": branch,
+                },
+                timeout_seconds=self.config.agent_timeout_seconds,
+                termination_grace_seconds=self.config.termination_grace_seconds,
+            )
+            if agent_result.timed_out:
+                evidence = (agent_result.stdout + "\n" + agent_result.stderr).strip()[-2000:]
+                self.store.save(RunState(issue.number, branch, "agent_timed_out", evidence))
+                raise RuntimeError(f"coding agent timed out: {evidence}")
+            if agent_result.returncode != 0:
+                self.store.save(RunState(issue.number, branch, "agent_failed", agent_result.stderr[-2000:]))
+                raise RuntimeError(f"coding agent failed: {agent_result.stderr.strip()}")
+            head_result = self.runner.run(["git", "rev-parse", "HEAD"], self.config.repo_path)
+            head_sha = head_result.stdout.strip()
+            if head_result.returncode != 0 or not head_sha:
+                raise RuntimeError("could not identify the release checkpoint commit")
+            self.store.save(RunState(
+                issue.number, branch, "agent_complete",
+                repo=self.config.repo, head_sha=head_sha,
+            ))
 
-        test_result = self.runner.run(
-            test_command,
-            self.config.repo_path,
-            timeout_seconds=self.config.test_timeout_seconds,
-            termination_grace_seconds=self.config.termination_grace_seconds,
-        )
-        evidence = (test_result.stdout + "\n" + test_result.stderr).strip()[-4000:]
-        if test_result.timed_out:
-            self.store.save(RunState(issue.number, branch, "tests_timed_out", evidence))
-            raise RuntimeError(f"tests timed out: {evidence}")
-        if test_result.returncode != 0:
-            self.store.save(RunState(issue.number, branch, "tests_failed", evidence))
-            raise RuntimeError(f"tests failed: {evidence}")
-        self.store.save(RunState(issue.number, branch, "tests_passed", evidence))
+        if checkpoint is None or checkpoint.status == "agent_complete":
+            test_result = self.runner.run(
+                test_command,
+                self.config.repo_path,
+                timeout_seconds=self.config.test_timeout_seconds,
+                termination_grace_seconds=self.config.termination_grace_seconds,
+            )
+            evidence = (test_result.stdout + "\n" + test_result.stderr).strip()[-4000:]
+            if test_result.timed_out:
+                self.store.save(RunState(
+                    issue.number, branch, "tests_timed_out", evidence,
+                    repo=self.config.repo, head_sha=head_sha,
+                ))
+                raise RuntimeError(f"tests timed out: {evidence}")
+            if test_result.returncode != 0:
+                self.store.save(RunState(
+                    issue.number, branch, "tests_failed", evidence,
+                    repo=self.config.repo, head_sha=head_sha,
+                ))
+                raise RuntimeError(f"tests failed: {evidence}")
+            self.store.save(RunState(
+                issue.number, branch, "tests_passed", evidence,
+                repo=self.config.repo, head_sha=head_sha,
+            ))
 
-        push_result = self.runner.run(["git", "push", "-u", "origin", branch], self.config.repo_path)
-        if push_result.returncode != 0:
-            self.store.save(RunState(issue.number, branch, "push_failed", push_result.stderr[-2000:]))
-            raise RuntimeError(f"push failed: {push_result.stderr.strip()}")
-        self.store.save(RunState(issue.number, branch, "pushed", evidence))
+        if checkpoint is not None and checkpoint.status == "pushed":
+            remote_result = self.runner.run(
+                ["git", "ls-remote", "--heads", "origin", branch],
+                self.config.repo_path,
+            )
+            expected_remote = f"{checkpoint.head_sha}\trefs/heads/{branch}"
+            if remote_result.returncode != 0 or remote_result.stdout.strip() != expected_remote:
+                raise RuntimeError("release checkpoint commit does not match the remote branch")
+        else:
+            push_result = self.runner.run(
+                ["git", "push", "-u", "origin", branch], self.config.repo_path
+            )
+            if push_result.returncode != 0:
+                self.store.save(RunState(
+                    issue.number, branch, "push_failed", push_result.stderr[-2000:],
+                    repo=self.config.repo, head_sha=head_sha,
+                ))
+                raise RuntimeError(f"push failed: {push_result.stderr.strip()}")
+            self.store.save(RunState(
+                issue.number, branch, "pushed", evidence,
+                repo=self.config.repo, head_sha=head_sha,
+            ))
 
         pr_body = (
             f"Closes #{issue.number}\n\n"
@@ -351,6 +415,8 @@ class ReleaseEngine:
             evidence,
             pr_number=pr["number"],
             pr_url=pr.get("html_url", ""),
+            repo=self.config.repo,
+            head_sha=head_sha,
         )
         self.store.save(final)
         return {