From 503d6f4988efa515348f4e67ebefeaddbc5cd5ac Mon Sep 17 00:00:00 2001 From: timmy Date: Sat, 8 Aug 2026 19:18:27 +0000 Subject: [PATCH] feat: resume release runs from checkpoints (#331) --- README.md | 7 +- docs/release-engine-spec.md | 17 +++ src/release_engine.py | 166 ++++++++++++++++++-------- tests/test_release_engine.py | 224 ++++++++++++++++++++++++++++++++++- 4 files changed, 361 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 69b9482..0b1d3ba 100644 --- a/README.md +++ b/README.md @@ -296,6 +296,9 @@ without advancing the release. The equivalent environment variables are `RELEASE_AGENT_TIMEOUT`, `RELEASE_TEST_TIMEOUT`, and `RELEASE_TERMINATION_GRACE`; every value must be positive. -Durable state defaults to `.release-engine/state.json`. Full behavior and -safety gates are documented in +Durable state defaults to `.release-engine/state.json`. Successful checkpoints +record the repository, issue branch, and exact commit SHA. A restart resumes at +tests, push, or PR creation only after validating that identity and a clean +worktree; a pushed checkpoint also verifies the remote branch SHA. Full +behavior and safety gates are documented in [`docs/release-engine-spec.md`](docs/release-engine-spec.md). diff --git a/docs/release-engine-spec.md b/docs/release-engine-spec.md index 2d4c1cd..d9b0c29 100644 --- a/docs/release-engine-spec.md +++ b/docs/release-engine-spec.md @@ -23,6 +23,18 @@ Convert one eligible Gitea issue into a tested, traceable pull request without d Terminal failure states include `claim_failed`, `agent_failed`, `agent_timed_out`, `tests_failed`, `tests_timed_out`, and `push_failed`. A rerun resumes from persisted state and never opens a duplicate PR. +Successful checkpoints are schema-versioned and bind the repository, issue, +deterministic branch, and exact commit SHA. On restart, the engine: + +- resumes `agent_complete` at tests; +- resumes `tests_passed` at push; and +- resumes `pushed` at PR creation after verifying the remote branch SHA. + +Before skipping any stage, it checks out the recorded branch, verifies its +local `HEAD`, and requires a clean worktree. Missing identity, an unsupported +schema, or any repository/issue/branch/SHA mismatch fails closed instead of +repeating or advancing delivery. + ## Queue selection 1. Only open issues; pull requests are excluded. @@ -63,6 +75,8 @@ Terminal failure states include `claim_failed`, `agent_failed`, `agent_timed_out - `--dry-run` performs discovery and planning only: no claim, git mutation, agent command, push, or PR. - State is written atomically after each successful transition. +- Resumable state carries repository and commit identity; successful stages are + skipped only after local (and, after push, remote) Git verification. - One invocation handles at most one issue. - Missing token, dirty worktree, failed claim verification, failed tests, or missing evidence blocks PR creation. @@ -79,3 +93,6 @@ Terminal failure states include `claim_failed`, `agent_failed`, `agent_timed_out 9. A timed-out command terminates its descendants and returns within its deadline plus grace period. 10. Agent and test timeouts persist distinct terminal states and block every later delivery stage. 11. Non-positive deadline configuration aborts before issue claim. +12. Restarts from `agent_complete`, `tests_passed`, and `pushed` skip only the + completed stages and open exactly one linked PR. +13. Dirty or identity-mismatched checkpoints stop before push or PR creation. diff --git a/src/release_engine.py b/src/release_engine.py index e0f44e0..0d4d2b5 100644 --- a/src/release_engine.py +++ b/src/release_engine.py @@ -57,6 +57,9 @@ class RunState: evidence: str = "" pr_number: int | None = None pr_url: str = "" + schema_version: int = 1 + repo: str = "" + head_sha: str = "" class Client(Protocol): @@ -249,6 +252,7 @@ class ReleaseEngine: issue = select_issue(self.client.list_open_issues(self.config.repo), self.config.agent, issue_number) branch = branch_name(self.config.agent, issue) + head_sha = "" existing_pr = self.client.find_open_pr(self.config.repo, branch) if existing_pr: @@ -272,61 +276,121 @@ class ReleaseEngine: agent_command = parse_command(self.config.agent_command, "agent command") test_command = parse_command(self.config.test_command, "test command") - - self.client.claim_issue(self.config.repo, issue.number, self.config.agent) - verified = self.client.get_issue(self.config.repo, issue.number) - if verified.assignee != self.config.agent: - raise RuntimeError( - f"claim verification failed: expected {self.config.agent}, got {verified.assignee}" + checkpoint = self.store.load() + resumable_statuses = {"agent_complete", "tests_passed", "pushed"} + if checkpoint is not None and checkpoint.status in resumable_statuses: + if ( + checkpoint.schema_version != 1 + or checkpoint.repo != self.config.repo + or checkpoint.issue != issue.number + or checkpoint.branch != branch + or not checkpoint.head_sha + ): + raise RuntimeError("release checkpoint does not match this run") + branch_result = self.runner.run(["git", "checkout", branch], self.config.repo_path) + if branch_result.returncode != 0: + raise RuntimeError(f"checkpoint branch checkout failed: {branch_result.stderr.strip()}") + head_result = self.runner.run(["git", "rev-parse", "HEAD"], self.config.repo_path) + if head_result.returncode != 0 or head_result.stdout.strip() != checkpoint.head_sha: + raise RuntimeError("release checkpoint commit does not match the local branch") + status_result = self.runner.run( + ["git", "status", "--porcelain"], self.config.repo_path ) - self.store.save(RunState(issue.number, branch, "claimed")) + if status_result.returncode != 0 or status_result.stdout.strip(): + raise RuntimeError("release checkpoint worktree is not clean") + head_sha = checkpoint.head_sha + evidence = checkpoint.evidence + else: + checkpoint = None + evidence = "" + self.client.claim_issue(self.config.repo, issue.number, self.config.agent) + verified = self.client.get_issue(self.config.repo, issue.number) + if verified.assignee != self.config.agent: + raise RuntimeError( + f"claim verification failed: expected {self.config.agent}, got {verified.assignee}" + ) + self.store.save(RunState(issue.number, branch, "claimed")) - branch_result = self.runner.run(["git", "checkout", "-B", branch], self.config.repo_path) - if branch_result.returncode != 0: - raise RuntimeError(f"branch creation failed: {branch_result.stderr.strip()}") + branch_result = self.runner.run(["git", "checkout", "-B", branch], self.config.repo_path) + if branch_result.returncode != 0: + raise RuntimeError(f"branch creation failed: {branch_result.stderr.strip()}") - agent_result = self.runner.run( - agent_command, - self.config.repo_path, - { - "RELEASE_ISSUE_NUMBER": str(issue.number), - "RELEASE_ISSUE_TITLE": issue.title, - "RELEASE_ISSUE_BODY": issue.body, - "RELEASE_REPO": self.config.repo, - "RELEASE_BRANCH": branch, - }, - timeout_seconds=self.config.agent_timeout_seconds, - termination_grace_seconds=self.config.termination_grace_seconds, - ) - if agent_result.timed_out: - evidence = (agent_result.stdout + "\n" + agent_result.stderr).strip()[-2000:] - self.store.save(RunState(issue.number, branch, "agent_timed_out", evidence)) - raise RuntimeError(f"coding agent timed out: {evidence}") - if agent_result.returncode != 0: - self.store.save(RunState(issue.number, branch, "agent_failed", agent_result.stderr[-2000:])) - raise RuntimeError(f"coding agent failed: {agent_result.stderr.strip()}") - self.store.save(RunState(issue.number, branch, "agent_complete")) + agent_result = self.runner.run( + agent_command, + self.config.repo_path, + { + "RELEASE_ISSUE_NUMBER": str(issue.number), + "RELEASE_ISSUE_TITLE": issue.title, + "RELEASE_ISSUE_BODY": issue.body, + "RELEASE_REPO": self.config.repo, + "RELEASE_BRANCH": branch, + }, + timeout_seconds=self.config.agent_timeout_seconds, + termination_grace_seconds=self.config.termination_grace_seconds, + ) + if agent_result.timed_out: + evidence = (agent_result.stdout + "\n" + agent_result.stderr).strip()[-2000:] + self.store.save(RunState(issue.number, branch, "agent_timed_out", evidence)) + raise RuntimeError(f"coding agent timed out: {evidence}") + if agent_result.returncode != 0: + self.store.save(RunState(issue.number, branch, "agent_failed", agent_result.stderr[-2000:])) + raise RuntimeError(f"coding agent failed: {agent_result.stderr.strip()}") + head_result = self.runner.run(["git", "rev-parse", "HEAD"], self.config.repo_path) + head_sha = head_result.stdout.strip() + if head_result.returncode != 0 or not head_sha: + raise RuntimeError("could not identify the release checkpoint commit") + self.store.save(RunState( + issue.number, branch, "agent_complete", + repo=self.config.repo, head_sha=head_sha, + )) - test_result = self.runner.run( - test_command, - self.config.repo_path, - timeout_seconds=self.config.test_timeout_seconds, - termination_grace_seconds=self.config.termination_grace_seconds, - ) - evidence = (test_result.stdout + "\n" + test_result.stderr).strip()[-4000:] - if test_result.timed_out: - self.store.save(RunState(issue.number, branch, "tests_timed_out", evidence)) - raise RuntimeError(f"tests timed out: {evidence}") - if test_result.returncode != 0: - self.store.save(RunState(issue.number, branch, "tests_failed", evidence)) - raise RuntimeError(f"tests failed: {evidence}") - self.store.save(RunState(issue.number, branch, "tests_passed", evidence)) + if checkpoint is None or checkpoint.status == "agent_complete": + test_result = self.runner.run( + test_command, + self.config.repo_path, + timeout_seconds=self.config.test_timeout_seconds, + termination_grace_seconds=self.config.termination_grace_seconds, + ) + evidence = (test_result.stdout + "\n" + test_result.stderr).strip()[-4000:] + if test_result.timed_out: + self.store.save(RunState( + issue.number, branch, "tests_timed_out", evidence, + repo=self.config.repo, head_sha=head_sha, + )) + raise RuntimeError(f"tests timed out: {evidence}") + if test_result.returncode != 0: + self.store.save(RunState( + issue.number, branch, "tests_failed", evidence, + repo=self.config.repo, head_sha=head_sha, + )) + raise RuntimeError(f"tests failed: {evidence}") + self.store.save(RunState( + issue.number, branch, "tests_passed", evidence, + repo=self.config.repo, head_sha=head_sha, + )) - push_result = self.runner.run(["git", "push", "-u", "origin", branch], self.config.repo_path) - if push_result.returncode != 0: - self.store.save(RunState(issue.number, branch, "push_failed", push_result.stderr[-2000:])) - raise RuntimeError(f"push failed: {push_result.stderr.strip()}") - self.store.save(RunState(issue.number, branch, "pushed", evidence)) + if checkpoint is not None and checkpoint.status == "pushed": + remote_result = self.runner.run( + ["git", "ls-remote", "--heads", "origin", branch], + self.config.repo_path, + ) + expected_remote = f"{checkpoint.head_sha}\trefs/heads/{branch}" + if remote_result.returncode != 0 or remote_result.stdout.strip() != expected_remote: + raise RuntimeError("release checkpoint commit does not match the remote branch") + else: + push_result = self.runner.run( + ["git", "push", "-u", "origin", branch], self.config.repo_path + ) + if push_result.returncode != 0: + self.store.save(RunState( + issue.number, branch, "push_failed", push_result.stderr[-2000:], + repo=self.config.repo, head_sha=head_sha, + )) + raise RuntimeError(f"push failed: {push_result.stderr.strip()}") + self.store.save(RunState( + issue.number, branch, "pushed", evidence, + repo=self.config.repo, head_sha=head_sha, + )) pr_body = ( f"Closes #{issue.number}\n\n" @@ -351,6 +415,8 @@ class ReleaseEngine: evidence, pr_number=pr["number"], pr_url=pr.get("html_url", ""), + repo=self.config.repo, + head_sha=head_sha, ) self.store.save(final) return { diff --git a/tests/test_release_engine.py b/tests/test_release_engine.py index 9ad635a..e4620f9 100644 --- a/tests/test_release_engine.py +++ b/tests/test_release_engine.py @@ -312,6 +312,7 @@ def test_failed_tests_block_push_and_pr(tmp_path): runner = FakeRunner([ CommandResult(0, "", ""), CommandResult(0, "changed", ""), + CommandResult(0, "abc123\n", ""), CommandResult(1, "1 failed", ""), ]) engine = ReleaseEngine(config(tmp_path), client, runner) @@ -329,6 +330,7 @@ def test_test_timeout_records_terminal_state_and_blocks_push(tmp_path): runner = FakeRunner([ CommandResult(0, "", ""), CommandResult(0, "implemented", ""), + CommandResult(0, "abc123\n", ""), CommandResult(-15, "3 passed before stall", "", timed_out=True), ]) cfg = config(tmp_path) @@ -340,7 +342,7 @@ def test_test_timeout_records_terminal_state_and_blocks_push(tmp_path): assert state is not None assert state.status == "tests_timed_out" assert "3 passed before stall" in state.evidence - assert runner.options[2] == { + assert runner.options[3] == { "timeout_seconds": cfg.test_timeout_seconds, "termination_grace_seconds": cfg.termination_grace_seconds, } @@ -354,6 +356,7 @@ def test_passing_run_pushes_and_opens_linked_pr(tmp_path): runner = FakeRunner([ CommandResult(0, "", ""), CommandResult(0, "implemented", ""), + CommandResult(0, "abc123\n", ""), CommandResult(0, "8 passed", ""), CommandResult(0, "", ""), ]) @@ -403,3 +406,222 @@ def test_state_store_round_trip_is_atomic(tmp_path): store.save(expected) assert store.load() == expected assert not (tmp_path / "state.json.tmp").exists() + + +def test_restart_from_agent_complete_skips_claim_and_coding(tmp_path): + issue = Issue(331, "Resume releases", "body", "open", [], "timmy") + client = FakeClient([issue]) + cfg = config(tmp_path) + branch = "timmy/331-resume-releases" + cfg.state_path.write_text(json.dumps({ + "schema_version": 1, + "repo": cfg.repo, + "issue": issue.number, + "branch": branch, + "status": "agent_complete", + "evidence": "", + "pr_number": None, + "pr_url": "", + "head_sha": "abc123", + })) + runner = FakeRunner([ + CommandResult(0, "", ""), + CommandResult(0, "abc123\n", ""), + CommandResult(0, "", ""), + CommandResult(0, "8 passed", ""), + CommandResult(0, "", ""), + ]) + + result = ReleaseEngine(cfg, client, runner).execute(issue_number=331) + + assert result["tests"] == "passed" + assert client.claimed == [] + assert ["agent", "--issue-from-env"] not in runner.commands + assert runner.commands[:3] == [ + ["git", "checkout", branch], + ["git", "rev-parse", "HEAD"], + ["git", "status", "--porcelain"], + ] + + +def test_restart_from_tests_passed_skips_claim_coding_and_tests(tmp_path): + issue = Issue(331, "Resume releases", "body", "open", [], "timmy") + client = FakeClient([issue]) + cfg = config(tmp_path) + branch = "timmy/331-resume-releases" + StateStore(cfg.state_path).save(RunState( + issue=issue.number, + branch=branch, + status="tests_passed", + evidence="604 passed", + repo=cfg.repo, + head_sha="abc123", + )) + runner = FakeRunner([ + CommandResult(0, "", ""), + CommandResult(0, "abc123\n", ""), + CommandResult(0, "", ""), + CommandResult(0, "", ""), + ]) + + result = ReleaseEngine(cfg, client, runner).execute(issue_number=331) + + assert result["tests"] == "passed" + assert client.claimed == [] + assert ["agent", "--issue-from-env"] not in runner.commands + assert ["pytest", "-q"] not in runner.commands + assert runner.commands == [ + ["git", "checkout", branch], + ["git", "rev-parse", "HEAD"], + ["git", "status", "--porcelain"], + ["git", "push", "-u", "origin", branch], + ] + + +def test_restart_from_pushed_verifies_remote_and_only_opens_pr(tmp_path): + issue = Issue(331, "Resume releases", "body", "open", [], "timmy") + client = FakeClient([issue]) + cfg = config(tmp_path) + branch = "timmy/331-resume-releases" + StateStore(cfg.state_path).save(RunState( + issue=issue.number, + branch=branch, + status="pushed", + evidence="604 passed", + repo=cfg.repo, + head_sha="abc123", + )) + runner = FakeRunner([ + CommandResult(0, "", ""), + CommandResult(0, "abc123\n", ""), + CommandResult(0, "", ""), + CommandResult(0, f"abc123\trefs/heads/{branch}\n", ""), + ]) + + result = ReleaseEngine(cfg, client, runner).execute(issue_number=331) + + assert result["pr_number"] == 44 + assert client.claimed == [] + assert len(client.created_prs) == 1 + assert runner.commands == [ + ["git", "checkout", branch], + ["git", "rev-parse", "HEAD"], + ["git", "status", "--porcelain"], + ["git", "ls-remote", "--heads", "origin", branch], + ] + + +def test_fresh_run_persists_repository_and_commit_identity(tmp_path): + issue = Issue(331, "Resume releases", "body", "open", [], None) + client = FakeClient([issue]) + cfg = config(tmp_path) + runner = FakeRunner([ + CommandResult(0, "", ""), + CommandResult(0, "implemented", ""), + CommandResult(0, "abc123\n", ""), + CommandResult(0, "604 passed", ""), + CommandResult(0, "", ""), + ]) + + ReleaseEngine(cfg, client, runner).execute(issue_number=331) + + state = StateStore(cfg.state_path).load() + assert state is not None + assert state.status == "pr_opened" + assert state.schema_version == 1 + assert state.repo == cfg.repo + assert state.head_sha == "abc123" + assert ["git", "rev-parse", "HEAD"] in runner.commands + + +def test_resume_fails_closed_when_checkpoint_worktree_is_dirty(tmp_path): + issue = Issue(331, "Resume releases", "body", "open", [], "timmy") + client = FakeClient([issue]) + cfg = config(tmp_path) + branch = "timmy/331-resume-releases" + StateStore(cfg.state_path).save(RunState( + issue=issue.number, + branch=branch, + status="tests_passed", + evidence="604 passed", + repo=cfg.repo, + head_sha="abc123", + )) + runner = FakeRunner([ + CommandResult(0, "", ""), + CommandResult(0, "abc123\n", ""), + CommandResult(0, " M src/release_engine.py\n", ""), + CommandResult(0, "", ""), + ]) + + with pytest.raises(RuntimeError, match="worktree is not clean"): + ReleaseEngine(cfg, client, runner).execute(issue_number=331) + + assert not any(command[:2] == ["git", "push"] for command in runner.commands) + assert client.created_prs == [] + + +def test_resume_fails_closed_when_checkpoint_targets_another_repository(tmp_path): + issue = Issue(331, "Resume releases", "body", "open", [], "timmy") + client = FakeClient([issue]) + cfg = config(tmp_path) + StateStore(cfg.state_path).save(RunState( + issue=issue.number, + branch="timmy/331-resume-releases", + status="tests_passed", + repo="other/repository", + head_sha="abc123", + )) + + with pytest.raises(RuntimeError, match="checkpoint does not match"): + ReleaseEngine(cfg, client, FakeRunner([])).execute(issue_number=331) + + assert client.created_prs == [] + + +def test_resume_fails_closed_when_local_commit_changed(tmp_path): + issue = Issue(331, "Resume releases", "body", "open", [], "timmy") + client = FakeClient([issue]) + cfg = config(tmp_path) + StateStore(cfg.state_path).save(RunState( + issue=issue.number, + branch="timmy/331-resume-releases", + status="tests_passed", + repo=cfg.repo, + head_sha="abc123", + )) + runner = FakeRunner([ + CommandResult(0, "", ""), + CommandResult(0, "different-sha\n", ""), + ]) + + with pytest.raises(RuntimeError, match="commit does not match the local branch"): + ReleaseEngine(cfg, client, runner).execute(issue_number=331) + + assert client.created_prs == [] + + +def test_resume_fails_closed_when_remote_commit_changed(tmp_path): + issue = Issue(331, "Resume releases", "body", "open", [], "timmy") + client = FakeClient([issue]) + cfg = config(tmp_path) + branch = "timmy/331-resume-releases" + StateStore(cfg.state_path).save(RunState( + issue=issue.number, + branch=branch, + status="pushed", + evidence="604 passed", + repo=cfg.repo, + head_sha="abc123", + )) + runner = FakeRunner([ + CommandResult(0, "", ""), + CommandResult(0, "abc123\n", ""), + CommandResult(0, "", ""), + CommandResult(0, f"different-sha\trefs/heads/{branch}\n", ""), + ]) + + with pytest.raises(RuntimeError, match="commit does not match the remote branch"): + ReleaseEngine(cfg, client, runner).execute(issue_number=331) + + assert client.created_prs == [] -- 2.43.0