diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 4ce905a..4632a16 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: RELEASE_URL="${{ github.server_url }}/api/v1/repos/${{ github.repository }}/releases" (cd dist && sha256sum -c ./*.sha256) - python3 scripts/verify_release.py --input-dir dist --commit "$TARGET" + python3 scripts/verify_release.py --input-dir dist --commit "$TARGET" --repository . printf '{"tag_name":"%s","target_commitish":"%s","name":"Release Candidate %s","body":"CI-tested release candidate for commit %s. Verify downloads with the attached SHA-256 checksum.","draft":true,"prerelease":true}\n' \ "$TAG" "$TARGET" "$TAG" "$TARGET" > /tmp/release.json diff --git a/scripts/build_release.py b/scripts/build_release.py index a2ab0e1..d217878 100644 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Build a reproducible, verifiable Stackchain Dashboard release bundle.""" +"""Build a reproducible release bundle from an exact Git commit.""" from __future__ import annotations @@ -8,37 +8,88 @@ import gzip import hashlib import io import json +import re +import subprocess import tarfile -from pathlib import Path +from dataclasses import dataclass +from pathlib import Path, PurePosixPath RUNTIME_DIRECTORIES = ("docs", "frontend", "src") RUNTIME_FILES = ("README.md", "requirements.txt") +EXCLUDED_PARTS = {"__pycache__", ".pytest_cache"} +EXCLUDED_SUFFIXES = (".pyc", ".pyo") +COMMIT_PATTERN = re.compile(r"[0-9a-f]{40}") + + +@dataclass(frozen=True) +class RuntimeFile: + name: str + data: bytes + executable: bool def sha256(data: bytes) -> str: return hashlib.sha256(data).hexdigest() -def runtime_files(root: Path) -> list[Path]: - files = [root / name for name in RUNTIME_FILES if (root / name).is_file()] - for name in RUNTIME_DIRECTORIES: - directory = root / name - if directory.is_dir(): - files.extend(path for path in directory.rglob("*") if path.is_file()) - return sorted(files, key=lambda path: path.relative_to(root).as_posix()) +def _git(root: Path, *args: str, text: bool = False) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", "-C", str(root), *args], + capture_output=True, + check=False, + text=text, + ) -def manifest_for(root: Path, files: list[Path], commit: str) -> dict: +def runtime_files_at_commit(root: Path, commit: str) -> list[RuntimeFile]: + """Read the runtime allowlist directly from the declared Git tree.""" + root = root.resolve() + if not COMMIT_PATTERN.fullmatch(commit): + raise ValueError("declared commit must be a full 40-character Git object ID") + resolved = _git(root, "rev-parse", "--verify", f"{commit}^{{commit}}", text=True) + if resolved.returncode != 0 or resolved.stdout.strip() != commit: + raise ValueError("declared commit is unavailable") + + listing = _git(root, "ls-tree", "-rz", "--full-tree", commit) + if listing.returncode != 0: + raise ValueError("could not inspect the declared commit") + + files: list[RuntimeFile] = [] + for record in listing.stdout.split(b"\0"): + if not record: + continue + metadata, raw_name = record.split(b"\t", 1) + mode, object_type, object_id = metadata.decode("ascii").split() + name = raw_name.decode("utf-8") + path = PurePosixPath(name) + selected = name in RUNTIME_FILES or (path.parts and path.parts[0] in RUNTIME_DIRECTORIES) + if not selected or object_type != "blob": + continue + if mode not in {"100644", "100755"}: + raise ValueError(f"release member is not a regular file: {name}") + if EXCLUDED_PARTS.intersection(path.parts) or name.endswith(EXCLUDED_SUFFIXES): + raise ValueError(f"declared commit contains generated release member: {name}") + blob = _git(root, "cat-file", "blob", object_id) + if blob.returncode != 0: + raise ValueError(f"could not read release member: {name}") + files.append(RuntimeFile(name, blob.stdout, mode == "100755")) + + names = {item.name for item in files} + missing = [name for name in RUNTIME_FILES if name not in names] + if missing or not any(name.startswith("src/") for name in names) or not any( + name.startswith("frontend/") for name in names + ): + raise ValueError("declared commit is missing required runtime files") + return sorted(files, key=lambda item: item.name) + + +def manifest_for(files: list[RuntimeFile], commit: str) -> dict: return { "schema_version": 1, "commit": commit, "files": { - path.relative_to(root).as_posix(): { - "sha256": sha256(path.read_bytes()), - "size": path.stat().st_size, - } - for path in files + item.name: {"sha256": sha256(item.data), "size": len(item.data)} for item in files }, } @@ -58,19 +109,10 @@ def tar_info(name: str, size: int, epoch: int, executable: bool = False) -> tarf def build(root: Path, output_dir: Path, commit: str, epoch: int) -> tuple[Path, Path, Path]: - root = root.resolve() - files = runtime_files(root) - missing = [name for name in RUNTIME_FILES if not (root / name).is_file()] - if missing or not (root / "src").is_dir() or not (root / "frontend").is_dir(): - raise ValueError("release root is missing required runtime files") - - embedded_manifest = manifest_for(root, files, commit) + files = runtime_files_at_commit(root, commit) + embedded_manifest = manifest_for(files, commit) archive_members: dict[str, tuple[bytes, bool]] = { - path.relative_to(root).as_posix(): ( - path.read_bytes(), - bool(path.stat().st_mode & 0o111), - ) - for path in files + item.name: (item.data, item.executable) for item in files } archive_members["release-manifest.json"] = (json_bytes(embedded_manifest), False) @@ -112,7 +154,11 @@ def parse_args() -> argparse.Namespace: def main() -> None: args = parse_args() - for path in build(args.root, args.output_dir, args.commit, args.source_date_epoch): + try: + paths = build(args.root, args.output_dir, args.commit, args.source_date_epoch) + except (OSError, ValueError) as error: + raise SystemExit(f"release build failed: {error}") from error + for path in paths: print(path) diff --git a/scripts/verify_release.py b/scripts/verify_release.py index de471f6..daef506 100644 --- a/scripts/verify_release.py +++ b/scripts/verify_release.py @@ -10,6 +10,8 @@ import sys import tarfile from pathlib import Path, PurePosixPath +from build_release import manifest_for, runtime_files_at_commit + def one(directory: Path, pattern: str) -> Path: matches = sorted(directory.glob(pattern)) @@ -22,7 +24,7 @@ def digest(data: bytes) -> str: return hashlib.sha256(data).hexdigest() -def verify(input_dir: Path, commit: str) -> None: +def verify(input_dir: Path, commit: str, repository: Path) -> None: archive_path = one(input_dir, "*.tar.gz") manifest_path = one(input_dir, "*.manifest.json") checksum_path = one(input_dir, "*.sha256") @@ -45,6 +47,11 @@ def verify(input_dir: Path, commit: str) -> None: expected_files = manifest.get("files") if not isinstance(expected_files, dict): raise ValueError("manifest files must be an object") + source_files = runtime_files_at_commit(repository, commit) + source_manifest = manifest_for(source_files, commit)["files"] + if expected_files != source_manifest: + raise ValueError("bundle manifest does not match the declared Git commit") + source_by_name = {item.name: item for item in source_files} with tarfile.open(archive_path, "r:gz") as archive: members = archive.getmembers() for member in members: @@ -67,19 +74,22 @@ def verify(input_dir: Path, commit: str) -> None: data = bundled.read() if metadata != {"sha256": digest(data), "size": len(data)}: raise ValueError(f"bundle member failed verification: {name}") + if data != source_by_name[name].data: + raise ValueError(f"bundle member differs from the declared Git commit: {name}") def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input-dir", type=Path, required=True) parser.add_argument("--commit", required=True) + parser.add_argument("--repository", type=Path, default=Path.cwd()) return parser.parse_args() def main() -> None: args = parse_args() try: - verify(args.input_dir, args.commit) + verify(args.input_dir, args.commit, args.repository) except (OSError, ValueError, json.JSONDecodeError, tarfile.TarError) as error: print(f"release verification failed: {error}", file=sys.stderr) raise SystemExit(1) from error diff --git a/tests/test_ci_workflow.py b/tests/test_ci_workflow.py index 78f040d..432bebe 100644 --- a/tests/test_ci_workflow.py +++ b/tests/test_ci_workflow.py @@ -41,7 +41,10 @@ def test_release_promotion_waits_for_tests_and_bundle(): assert "needs: [lint, build-release, browser-journey]" in release assert "github.event_name == 'push'" in release assert "actions/download-artifact@v3" in release - assert 'python3 scripts/verify_release.py --input-dir dist --commit "$TARGET"' in release + assert ( + 'python3 scripts/verify_release.py --input-dir dist --commit "$TARGET" --repository .' + in release + ) assert release.index("sha256sum -c") < release.index("curl --fail-with-body") diff --git a/tests/test_release_bundle.py b/tests/test_release_bundle.py index 004cde9..77172a4 100644 --- a/tests/test_release_bundle.py +++ b/tests/test_release_bundle.py @@ -20,7 +20,24 @@ def _fixture(root: Path) -> None: (root / "README.md").write_text("# Stackchain\n") -def _build(source: Path, output: Path) -> subprocess.CompletedProcess[str]: +def _commit_fixture(source: Path) -> str: + subprocess.run(["git", "init", "-q", str(source)], check=True) + subprocess.run(["git", "-C", str(source), "config", "user.name", "Release Test"], check=True) + subprocess.run( + ["git", "-C", str(source), "config", "user.email", "release-test@example.invalid"], + check=True, + ) + subprocess.run(["git", "-C", str(source), "add", "."], check=True) + subprocess.run(["git", "-C", str(source), "commit", "-qm", "fixture"], check=True) + return subprocess.run( + ["git", "-C", str(source), "rev-parse", "HEAD"], + text=True, + capture_output=True, + check=True, + ).stdout.strip() + + +def _build(source: Path, output: Path, commit: str) -> subprocess.CompletedProcess[str]: return subprocess.run( [ sys.executable, @@ -30,7 +47,7 @@ def _build(source: Path, output: Path) -> subprocess.CompletedProcess[str]: "--output-dir", str(output), "--commit", - "a" * 40, + commit, "--source-date-epoch", "1720000000", ], @@ -40,14 +57,20 @@ def _build(source: Path, output: Path) -> subprocess.CompletedProcess[str]: ) -def test_release_bundle_is_reproducible(tmp_path): +def test_release_bundle_is_reproducible_from_declared_commit(tmp_path): source = tmp_path / "source" _fixture(source) + commit = _commit_fixture(source) - first = _build(source, tmp_path / "first") - second = _build(source, tmp_path / "second") - + first = _build(source, tmp_path / "first", commit) assert first.returncode == 0, first.stderr + + (source / "src" / "__pycache__").mkdir() + (source / "src" / "__pycache__" / "main.cpython-311.pyc").write_bytes(b"bytecode") + (source / "src" / "untracked-secret.txt").write_text("must not ship") + (source / "src" / "main.py").write_text("print('dirty worktree')\n") + second = _build(source, tmp_path / "second", commit) + assert second.returncode == 0, second.stderr first_archive = next((tmp_path / "first").glob("*.tar.gz")) second_archive = next((tmp_path / "second").glob("*.tar.gz")) @@ -55,7 +78,7 @@ def test_release_bundle_is_reproducible(tmp_path): manifest = json.loads(next((tmp_path / "first").glob("*.manifest.json")).read_text()) checksum = next((tmp_path / "first").glob("*.sha256")).read_text().split()[0] - assert manifest["commit"] == "a" * 40 + assert manifest["commit"] == commit assert manifest["artifact"]["sha256"] == checksum assert checksum == hashlib.sha256(first_archive.read_bytes()).hexdigest() assert sorted(manifest["files"]) == [ @@ -74,19 +97,40 @@ def test_release_bundle_is_reproducible(tmp_path): "requirements.txt", "src/main.py", ] - assert embedded["commit"] == "a" * 40 + assert embedded["commit"] == commit assert embedded["files"] == manifest["files"] +def test_release_bundle_rejects_unknown_commit(tmp_path): + source = tmp_path / "source" + _fixture(source) + _commit_fixture(source) + + built = _build(source, tmp_path / "dist", "a" * 40) + + assert built.returncode != 0 + assert "declared commit is unavailable" in built.stderr + + def test_release_bundle_verifier_enforces_integrity(tmp_path): source = tmp_path / "source" output = tmp_path / "dist" _fixture(source) - built = _build(source, output) + commit = _commit_fixture(source) + built = _build(source, output, commit) assert built.returncode == 0, built.stderr valid = subprocess.run( - [sys.executable, str(VERIFIER), "--input-dir", str(output), "--commit", "a" * 40], + [ + sys.executable, + str(VERIFIER), + "--input-dir", + str(output), + "--commit", + commit, + "--repository", + str(source), + ], text=True, capture_output=True, check=False, @@ -96,9 +140,18 @@ def test_release_bundle_verifier_enforces_integrity(tmp_path): archive = next(output.glob("*.tar.gz")) archive.write_bytes(archive.read_bytes() + b"tampered") tampered = subprocess.run( - [sys.executable, str(VERIFIER), "--input-dir", str(output), "--commit", "a" * 40], + [ + sys.executable, + str(VERIFIER), + "--input-dir", + str(output), + "--commit", + commit, + "--repository", + str(source), + ], text=True, capture_output=True, check=False, ) - assert tampered.returncode != 0 \ No newline at end of file + assert tampered.returncode != 0