From f8f01c4b2e96cadfcad972bbdf8466160f2e421a Mon Sep 17 00:00:00 2001 From: Timmy Date: Fri, 21 Aug 2026 14:32:31 +0000 Subject: [PATCH] feat: add hardened atomic staging deployment --- .gitea/workflows/quality.yml | 8 +- README.md | 13 + deploy/Caddyfile.staging.example | 36 +++ deploy/timmy-staging.env.example | 11 + deploy/timmy-staging.service | 61 +++++ docs/STAGING-RUNBOOK.md | 118 +++++++++ package.json | 5 +- scripts/build_release.py | 2 +- scripts/deploy_staging.py | 433 +++++++++++++++++++++++++++++++ src/vision-config.js | 2 +- tests/ci-workflow.test.js | 13 + tests/staging-config.test.js | 105 ++++++++ tests/staging-deploy.test.py | 344 ++++++++++++++++++++++++ tests/staging.acceptance.mjs | 98 +++++++ tests/vision-config.test.js | 5 + 15 files changed, 1248 insertions(+), 6 deletions(-) create mode 100644 deploy/Caddyfile.staging.example create mode 100644 deploy/timmy-staging.env.example create mode 100644 deploy/timmy-staging.service create mode 100644 docs/STAGING-RUNBOOK.md create mode 100755 scripts/deploy_staging.py create mode 100644 tests/staging-config.test.js create mode 100644 tests/staging-deploy.test.py create mode 100644 tests/staging.acceptance.mjs diff --git a/.gitea/workflows/quality.yml b/.gitea/workflows/quality.yml index d23b66a..c77cc35 100644 --- a/.gitea/workflows/quality.yml +++ b/.gitea/workflows/quality.yml @@ -30,7 +30,9 @@ jobs: - name: Install browser run: npx playwright install --with-deps chromium - name: Unit and security tests - run: npm test + run: | + npm test + python3 tests/staging-deploy.test.py -v - name: Mobile browser acceptance run: | npm start > /tmp/timmy-server.log 2>&1 & @@ -52,6 +54,8 @@ jobs: - name: Dependency audit run: npm audit --audit-level=high - name: Syntax checks - run: npm run check:syntax + run: | + npm run check:syntax + node --check tests/staging.acceptance.mjs - name: Diff hygiene run: npm run check:diff diff --git a/README.md b/README.md index c717dd5..eaa8686 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,19 @@ python3 scripts/build_release.py The builder clones the committed `main` tree into an isolated directory, runs unit/security and both mobile acceptance suites, audits dependencies, checks syntax and secrets, excludes model weights and sensitive/generated media, records a vertical feature demonstration from the working app, fully decodes and probes that MP4, and writes checksummed source/video artifacts plus `manifest.json` under `/root/timmy-releases/`. It refuses dirty or non-`main` source trees. +## Promote or roll back private staging + +The standard-library deployment tool verifies the declared archive SHA-256 before tar parsing, rejects unsafe or secret-bearing members, creates one immutable `releases/<40-character-commit>` directory, and atomically repoints `current`. Restart, bounded commit-specific health, or synthetic mobile smoke failures restore and verify the prior release automatically. + +```bash +python3 scripts/deploy_staging.py --dry-run promote --tag TAG --archive FILE --sha256 HASH --commit FULL_COMMIT +sudo python3 scripts/deploy_staging.py promote --tag TAG --archive FILE --sha256 HASH --commit FULL_COMMIT +python3 scripts/deploy_staging.py status +sudo python3 scripts/deploy_staging.py rollback --commit PRIOR_FULL_COMMIT +``` + +Commands are fixed argv arrays executed with `shell=False`; JSON argv overrides and `--root` exist for rootless fixtures. See [the staging runbook](docs/STAGING-RUNBOOK.md) for host prerequisites, mode-600 environment injection, Caddy validation, smoke credentials, logs, backups, removal, and the explicit no-live-change boundary. + ## Run with the self-hosted open-weight path ```bash diff --git a/deploy/Caddyfile.staging.example b/deploy/Caddyfile.staging.example new file mode 100644 index 0000000..991ae7f --- /dev/null +++ b/deploy/Caddyfile.staging.example @@ -0,0 +1,36 @@ +# Example complete site used for validation. Merge only the reviewed handles into +# the existing forge site. Route order is security-sensitive. +forge.example.invalid { + encode zstd gzip + + @git path /git /git/* + handle @git { + reverse_proxy 127.0.0.1:3000 + } + + handle_path /timmy-staging/* { + basic_auth { + staging {$TIMMY_STAGING_PASSWORD_HASH} + } + request_body { + max_size 8MB + } + header { + X-Content-Type-Options "nosniff" + X-Frame-Options "DENY" + Referrer-Policy "no-referrer" + Permissions-Policy "camera=(self), microphone=(), geolocation=()" + Content-Security-Policy "default-src 'self'; img-src 'self' data: blob:; style-src 'self'; script-src 'self'; connect-src 'self'; object-src 'none'; base-uri 'self'; frame-ancestors 'none'; form-action 'self'" + -Server + } + # handle_path strips the public prefix exactly once. Reconstruct it for + # Timmy because its validated base-path router must see that same prefix. + rewrite * /timmy-staging{uri} + reverse_proxy 127.0.0.1:4174 + } + + # Existing Gitea catchall belongs after the private staging route. + handle { + reverse_proxy 127.0.0.1:3000 + } +} diff --git a/deploy/timmy-staging.env.example b/deploy/timmy-staging.env.example new file mode 100644 index 0000000..be1c343 --- /dev/null +++ b/deploy/timmy-staging.env.example @@ -0,0 +1,11 @@ +# Copy with: sudo install -m 600 deploy/timmy-staging.env.example /etc/timmy-staging.env +# Replace release identity for every promotion. Keep this file root-owned and out of archives. +HOST=127.0.0.1 +PORT=4174 +TIMMY_BASE_PATH=/timmy-staging +TIMMY_STAGING_LABEL=true +TIMMY_RELEASE_TAG=CHANGE_ME +TIMMY_RELEASE_COMMIT=0000000000000000000000000000000000000000 +TIMMY_AGENT_ENABLED=false +TIMMY_VISION_ENABLED=false +NODE_ENV=production diff --git a/deploy/timmy-staging.service b/deploy/timmy-staging.service new file mode 100644 index 0000000..35df2ee --- /dev/null +++ b/deploy/timmy-staging.service @@ -0,0 +1,61 @@ +[Unit] +Description=Timmy private staging service +Documentation=file:/opt/timmy-staging/current/docs/STAGING-RUNBOOK.md +After=network.target + +[Service] +Type=simple +User=timmy-staging +Group=timmy-staging +UMask=0077 +WorkingDirectory=/opt/timmy-staging/current +Environment=NODE_ENV=production +Environment=HOST=127.0.0.1 +Environment=PORT=4174 +Environment=TIMMY_BASE_PATH=/timmy-staging +Environment=TIMMY_AGENT_ENABLED=false +Environment=TIMMY_VISION_ENABLED=false +EnvironmentFile=/etc/timmy-staging.env +ExecStart=/usr/bin/env TIMMY_AGENT_ENABLED=false TIMMY_VISION_ENABLED=false /usr/local/lib/timmy-staging/node server.mjs +Restart=on-failure +RestartSec=5s +TimeoutStartSec=30s +TimeoutStopSec=15s +KillSignal=SIGTERM + +# Filesystem and privilege boundary. Only state is writable. +NoNewPrivileges=true +PrivateTmp=true +PrivateDevices=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/timmy-staging +StateDirectory=timmy-staging +StateDirectoryMode=0700 +RestrictSUIDSGID=true +LockPersonality=true +RestrictNamespaces=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +ProtectClock=true +ProtectHostname=true +CapabilityBoundingSet= +AmbientCapabilities= + +# Process, syscall, network, and resource boundary. +RestrictRealtime=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallFilter=~@privileged @resources +ProtectProc=invisible +ProcSubset=pid +TasksMax=64 +MemoryMax=512M +CPUQuota=100% +LimitNOFILE=1024 + +[Install] +WantedBy=multi-user.target diff --git a/docs/STAGING-RUNBOOK.md b/docs/STAGING-RUNBOOK.md new file mode 100644 index 0000000..5c815d6 --- /dev/null +++ b/docs/STAGING-RUNBOOK.md @@ -0,0 +1,118 @@ +# Timmy private staging runbook + +This runbook is a reviewed host template, not approval to change a live host. Apply it only in a separately approved provisioning change. Staging is Phase 1: Hermes Agent and vision are off. + +## Prerequisites + +- Linux host with Node.js 22, systemd, and Caddy. +- Existing HTTPS origin and Gitea route inventory. +- A reviewed 40-character commit, source archive, and SHA-256 receipt from the release manifest. +- Root only for one-time account/unit/config installation; promotions use the narrowest available sudo policy. +- A fresh Caddy password hash delivered outside Git. Never put the password or hash in shell history, release notes, screenshots, or this repository. + +Confirm that `127.0.0.1:4174` is unused. Do not copy `.env`, credentials, model weights, raw media, or a mutable Git checkout into a release. + +## DNS and URL + +No DNS change is required for the approved subpage deployment. The private URL is `https://forge.alexanderwhitestone.com/timmy-staging/`. Caddy remains the only public listener; Node binds only to `127.0.0.1:4174`. Verify `/git/` before and after any separately approved Caddy reload. + +## Install + +These commands are reference commands for an approved maintenance window; they have not been run by this change: + +```bash +NODE_SOURCE="$(readlink -f "$(command -v node)")" +"$NODE_SOURCE" --version # must report v22.x before installation +sudo install -D -o root -g root -m 755 "$NODE_SOURCE" /usr/local/lib/timmy-staging/node +/usr/local/lib/timmy-staging/node --version +sudo useradd --system --home-dir /nonexistent --no-create-home --shell /usr/sbin/nologin timmy-staging +sudo install -d -o root -g root -m 755 /opt/timmy-staging/releases +sudo install -d -o timmy-staging -g timmy-staging -m 700 /var/lib/timmy-staging +sudo install -o root -g root -m 644 deploy/timmy-staging.service /etc/systemd/system/timmy-staging.service +sudo install -o root -g root -m 600 deploy/timmy-staging.env.example /etc/timmy-staging.env +sudo chmod 600 /etc/timmy-staging.env +sudo systemctl daemon-reload +``` + +Edit only `TIMMY_RELEASE_TAG` and `TIMMY_RELEASE_COMMIT` for the selected artifact. The unit deliberately applies `TIMMY_AGENT_ENABLED=false` and `TIMMY_VISION_ENABLED=false` on the `ExecStart` command after loading the environment file, so values in that file cannot enable either subsystem. Leave the loopback host, port, and base path unchanged. The environment file must stay root-owned mode 600 and absent from release archives. The copied Node 22 executable is root-owned at `/usr/local/lib/timmy-staging/node`, outside every home directory, so `ProtectHome=true` remains enforceable; replace it only through a separately reviewed Node upgrade. + +Generate Caddy basic-auth material interactively (for example, `caddy hash-password`) and inject it as `TIMMY_STAGING_PASSWORD_HASH`; do not commit the output. Merge the staging handle before the existing catchall, validate a temporary complete config, then use `caddy reload` rather than restarting unrelated services. + +## Promote an immutable release + +First inspect without mutation: + +```bash +python3 scripts/deploy_staging.py --dry-run promote \ + --tag daily-YYYY-MM-DD.N --archive /secure/inbox/timmy.tar.gz \ + --sha256 64_LOWERCASE_HEX_CHARACTERS --commit 40_LOWERCASE_HEX_CHARACTERS +``` + +Then run the same command without `--dry-run`. Promotion opens only a non-symlink regular source, copies it while hashing into a mode-400 file in a private temporary directory, and inspects and extracts only that verified copy before cleaning it. It rejects unsafe members and forbidden artifacts, extracts once to `/opt/timmy-staging/releases/`, and never overwrites that directory. It atomically swaps `current`, restarts only `timmy-staging.service`, checks bounded health, and invokes `npm run test:staging-smoke` as an argv array with `shell=False`. Restart, health, or smoke failure automatically restores and verifies the prior symlink. If there is no prior release, the tool removes `current` and stops the service instead of restarting it against a missing path; the failed release remains unreferenced as inert immutable evidence for operator review. + +The operator must update `/etc/timmy-staging.env` release identity to the same reviewed tag and commit before promotion. A narrow wrapper/sudo policy may set the smoke environment without granting arbitrary command execution: + +```bash +export TIMMY_STAGING_URL=https://forge.alexanderwhitestone.com/timmy-staging/ +export TIMMY_STAGING_USER=staging +# Read TIMMY_STAGING_PASSWORD from an approved secret channel; never paste it here. +sudo -E python3 scripts/deploy_staging.py promote --tag "$TAG" --archive "$ARCHIVE" --sha256 "$SHA256" --commit "$COMMIT" +``` + +## Smoke test + +The promotion runs the synthetic 390×844 Playwright suite. It checks edge auth, build identity, one dominant photo action, manual save, urgent-language interception, Journal visibility, export, no horizontal overflow, browser errors, and suspicious secret-bearing responses. It must never use a real stool photo or medical record. + +For an explicit rerun: + +```bash +TIMMY_STAGING_URL=https://forge.alexanderwhitestone.com/timmy-staging/ \ +TIMMY_EXPECT_RELEASE_TAG="$TAG" TIMMY_EXPECT_RELEASE_COMMIT="$COMMIT" \ +TIMMY_STAGING_USER=staging TIMMY_STAGING_PASSWORD="$(secret-reader)" \ +npm run test:staging-smoke +``` + +Also verify `curl --fail http://127.0.0.1:4174/timmy-staging/api/healthz`, authenticated public health, unauthenticated HTTP 401, `/git/`, and that no wildcard/public listener owns port 4174. + +## Status and logs + +```bash +python3 scripts/deploy_staging.py status +sudo systemctl status timmy-staging.service +sudo journalctl -u timmy-staging.service --since today --no-pager +# Only after an approved environment-file change: +sudo systemctl restart timmy-staging.service +``` + +Health and logs may show bounded release identity, but must not show environment values, cookies, passwords, tokens, session data, filesystem secrets, or raw user content. + +## Rollback + +List reviewed immutable commit directories, choose the known-good receipt, dry-run, then roll back: + +```bash +python3 scripts/deploy_staging.py --dry-run rollback --commit 40_LOWERCASE_HEX_CHARACTERS +sudo python3 scripts/deploy_staging.py rollback --commit 40_LOWERCASE_HEX_CHARACTERS +``` + +Rollback atomically repoints `current`, restarts only Timmy, and verifies commit-specific loopback health. If rollback verification fails, the tool restores the release that was current when rollback began. Never repoint the symlink manually while the service is running. + +## Backup + +Browser journal data is local to each browser and is not server state. Before host maintenance, back up only operator-owned receipts and any explicitly required `/var/lib/timmy-staging` state with root-only permissions. Release archives should be recoverable from their verified release source; do not back up `/etc/timmy-staging.env` into a general artifact store. Test restore procedures without live credentials. + +## Template validation + +```bash +node --test tests/staging-config.test.js +systemd-analyze verify deploy/timmy-staging.service +HASH="$(caddy hash-password --plaintext validation-only-password)" +TIMMY_STAGING_PASSWORD_HASH="$HASH" caddy validate --config deploy/Caddyfile.staging.example --adapter caddyfile +python3 -m py_compile scripts/deploy_staging.py +``` + +Use only a disposable validation hash. Validation does not authorize installation or reload. + +## Remove staging + +In a separately approved maintenance window: disable and stop only `timmy-staging.service`; remove only the reviewed Caddy staging handle and validate/reload Caddy; verify `/git/`; remove the unit and run `systemctl daemon-reload`; archive required receipts; then remove `/opt/timmy-staging` and `/var/lib/timmy-staging`. Delete the dedicated locked user last. Do not delete shared Caddy, Gitea, TLS, or browser data. diff --git a/package.json b/package.json index 50fc64d..31839e0 100644 --- a/package.json +++ b/package.json @@ -4,11 +4,12 @@ "private": true, "type": "module", "scripts": { - "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js", + "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js", "test:ui": "node tests/ui.acceptance.mjs", "test:photo": "node tests/photo-first.acceptance.mjs", "test:sleek": "node tests/sleek-chat.acceptance.mjs", - "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py", + "test:staging-smoke": "node tests/staging.acceptance.mjs", + "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py", "check:diff": "bash scripts/check_diff.sh", "start": "node server.mjs" }, diff --git a/scripts/build_release.py b/scripts/build_release.py index 0d0c2d6..5d2b391 100755 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -133,7 +133,7 @@ def main() -> int: contact_sheet = release_dir / f"timmy-talking-turd-{version}-demo-contact-sheet.jpg" run(["ffmpeg", "-y", "-v", "error", "-i", str(demo), "-vf", "fps=1/2,scale=180:-1,tile=4x3:padding=4:margin=4", "-frames:v", "1", str(contact_sheet)], tree) run(["npm", "audit", "--audit-level=high"], tree) - for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js"): + for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "tests/staging.acceptance.mjs"): run(["node", "--check", file], tree) run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree) run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree) diff --git a/scripts/deploy_staging.py b/scripts/deploy_staging.py new file mode 100755 index 0000000..5d98212 --- /dev/null +++ b/scripts/deploy_staging.py @@ -0,0 +1,433 @@ +#!/usr/bin/env python3 +"""Promote and roll back immutable Timmy staging release archives safely.""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import secrets +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import time +from typing import Callable, Sequence +import urllib.error +import urllib.request + +COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +TAG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +FORBIDDEN_SUFFIXES = (".pem", ".key", ".p12", ".pfx", ".gguf", ".bin", ".safetensors", ".onnx", ".pyc") +FORBIDDEN_COMPONENTS = {".env", ".git", ".ssh", "secrets", "credentials", "__pycache__"} +FORBIDDEN_PREFIXES = (("video",), ("artifacts",), ("research", "source-pages")) + + +class DeploymentError(RuntimeError): + """A bounded, operator-safe deployment failure.""" + + +@dataclass(frozen=True) +class DeploymentConfig: + root: Path + restart_command: tuple[str, ...] + stop_command: tuple[str, ...] + smoke_command: tuple[str, ...] + health_url: str + command_timeout: float = 180.0 + health_timeout: float = 30.0 + max_members: int = 10_000 + max_member_bytes: int = 50 * 1024 * 1024 + max_total_bytes: int = 250 * 1024 * 1024 + + @property + def releases(self) -> Path: + return self.root / "releases" + + @property + def current(self) -> Path: + return self.root / "current" + + +RunCommand = Callable[..., subprocess.CompletedProcess] +HealthCheck = Callable[[str, str, float], dict] + + +def _validate_identity(tag: str, commit: str, expected_sha256: str) -> None: + if not TAG_RE.fullmatch(tag): + raise DeploymentError("tag must be 1-80 safe release characters") + if not COMMIT_RE.fullmatch(commit): + raise DeploymentError("commit must be exactly 40 lowercase hexadecimal characters") + if not SHA256_RE.fullmatch(expected_sha256): + raise DeploymentError("SHA-256 must be exactly 64 lowercase hexadecimal characters") + + +@contextmanager +def _verified_archive_copy(source_path: Path, expected_sha256: str): + """Copy and hash an untrusted regular archive once, then yield the private copy.""" + source_fd = -1 + flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0) + try: + source_fd = os.open(source_path, flags) + if not stat.S_ISREG(os.fstat(source_fd).st_mode): + raise DeploymentError("archive source must be a regular file, not a symlink or special file") + with tempfile.TemporaryDirectory(prefix="timmy-verified-archive-") as private_dir: + private_path = Path(private_dir) / "archive" + digest = hashlib.sha256() + with os.fdopen(source_fd, "rb", closefd=True) as source: + source_fd = -1 + with private_path.open("xb") as destination: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + destination.write(chunk) + destination.flush() + os.fsync(destination.fileno()) + private_path.chmod(0o400) + if digest.hexdigest() != expected_sha256: + raise DeploymentError("archive SHA-256 mismatch; refusing extraction") + yield private_path + except DeploymentError: + raise + except OSError as error: + raise DeploymentError(f"cannot stage archive: {error.strerror or 'I/O error'}") from error + finally: + if source_fd >= 0: + os.close(source_fd) + + +def _safe_parts(name: str) -> tuple[str, ...]: + if not name or "\\" in name or "\x00" in name or name.startswith("/") or "//" in name: + raise DeploymentError(f"unsafe archive path: {name!r}") + path = PurePosixPath(name) + if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts): + raise DeploymentError(f"unsafe archive path: {name!r}") + return path.parts + + +def _is_forbidden(parts: tuple[str, ...]) -> bool: + lowered = tuple(part.lower() for part in parts) + basename = lowered[-1] + return ( + any(part in FORBIDDEN_COMPONENTS or part.startswith(".env.") for part in lowered) + or basename in {"id_rsa", "id_ed25519", "authorized_keys"} + or basename.endswith(FORBIDDEN_SUFFIXES) + or any(lowered[:len(prefix)] == prefix for prefix in FORBIDDEN_PREFIXES) + ) + + +def inspect_archive(archive_path: Path, config: DeploymentConfig) -> tuple[list[tarfile.TarInfo], str | None]: + """Validate every tar entry and return members plus a common wrapper directory.""" + try: + with tarfile.open(archive_path, mode="r:*") as archive: + members = archive.getmembers() + except (OSError, tarfile.TarError) as error: + raise DeploymentError("archive is not a readable tar file") from error + if not members: + raise DeploymentError("archive is empty") + if len(members) > config.max_members: + raise DeploymentError("archive member count exceeds configured limit") + total = 0 + all_parts: list[tuple[str, ...]] = [] + for member in members: + parts = _safe_parts(member.name) + all_parts.append(parts) + if not (member.isdir() or member.isreg()): + raise DeploymentError(f"unsupported archive member type: {member.name}") + if member.size < 0 or member.size > config.max_member_bytes: + raise DeploymentError(f"archive member size exceeds configured limit: {member.name}") + total += member.size + if total > config.max_total_bytes: + raise DeploymentError("archive expanded size exceeds configured limit") + common_root = all_parts[0][0] if all(parts[0] == all_parts[0][0] for parts in all_parts) else None + if common_root and _is_forbidden((common_root,)): + raise DeploymentError(f"forbidden archive artifact: {common_root}") + for parts in all_parts: + relative = parts[1:] if common_root else parts + if not relative: + continue + if _is_forbidden(relative): + raise DeploymentError(f"forbidden archive artifact: {'/'.join(relative)}") + return members, common_root + + +def _extract_validated(archive_path: Path, destination: Path, members: list[tarfile.TarInfo], common_root: str | None) -> None: + with tarfile.open(archive_path, mode="r:*") as archive: + for member in members: + parts = _safe_parts(member.name) + relative = parts[1:] if common_root else parts + if not relative: + continue + target = destination.joinpath(*relative) + if member.isdir(): + target.mkdir(parents=True, exist_ok=True, mode=0o755) + continue + target.parent.mkdir(parents=True, exist_ok=True, mode=0o755) + source = archive.extractfile(member) + if source is None: + raise DeploymentError(f"could not read archive member: {member.name}") + with source, target.open("xb") as output: + shutil.copyfileobj(source, output, length=1024 * 1024) + target.chmod(0o755 if member.mode & 0o111 else 0o644) + + +def _ensure_deployment_root(config: DeploymentConfig, *, create: bool) -> None: + if config.root.is_symlink(): + raise DeploymentError("deployment root must not be a symlink") + if config.root.exists() and not config.root.is_dir(): + raise DeploymentError("deployment root is not a directory") + if create: + config.root.mkdir(parents=True, exist_ok=True, mode=0o755) + if config.root.is_symlink() or not config.root.is_dir(): + raise DeploymentError("deployment root is not a safe directory") + + +def _ensure_releases_directory(config: DeploymentConfig, *, create: bool) -> None: + _ensure_deployment_root(config, create=create) + if config.releases.is_symlink(): + raise DeploymentError("releases directory must not be a symlink") + if config.releases.exists() and not config.releases.is_dir(): + raise DeploymentError("releases directory is not a directory") + if create: + config.releases.mkdir(parents=True, exist_ok=True, mode=0o755) + + +def _current_commit(config: DeploymentConfig) -> str | None: + _ensure_releases_directory(config, create=False) + if not config.current.is_symlink(): + if config.current.exists(): + raise DeploymentError("current exists but is not a symlink") + return None + target = os.readlink(config.current) + target_path = Path(target) + resolved = (config.current.parent / target_path).resolve() if not target_path.is_absolute() else target_path.resolve() + try: + relative = resolved.relative_to(config.releases.resolve()) + except ValueError as error: + raise DeploymentError("current symlink escapes the releases directory") from error + if len(relative.parts) != 1 or not COMMIT_RE.fullmatch(relative.name) or not resolved.is_dir(): + raise DeploymentError("current symlink does not name a valid immutable release") + return relative.name + + +def _atomic_point(config: DeploymentConfig, commit: str | None) -> None: + _ensure_releases_directory(config, create=False) + if config.current.exists() and not config.current.is_symlink(): + raise DeploymentError("current exists but is not a symlink") + temporary = config.root / f".current-{os.getpid()}-{secrets.token_hex(6)}" + try: + if commit is None: + config.current.unlink(missing_ok=True) + return + temporary.symlink_to(Path("releases") / commit) + os.replace(temporary, config.current) + finally: + temporary.unlink(missing_ok=True) + + +def _run_argv(argv: Sequence[str], **options) -> subprocess.CompletedProcess: + if not argv: + raise DeploymentError("configured command must be a non-empty argv array") + timeout = float(options.get("timeout", 180.0)) + return subprocess.run(list(argv), check=True, text=True, capture_output=True, shell=False, timeout=timeout) + + +def poll_health(url: str, expected_commit: str, timeout: float) -> dict: + deadline = time.monotonic() + timeout + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise DeploymentError("health check deadline expired") + try: + request = urllib.request.Request(url, headers={"accept": "application/json"}) + with urllib.request.urlopen(request, timeout=min(2.0, remaining)) as response: + raw = response.read(65_537) + if len(raw) > 65_536: + raise DeploymentError("health response exceeded 64 KiB") + payload = json.loads(raw) + if response.status == 200 and payload.get("ok") is True and payload.get("commit") == expected_commit: + return payload + except DeploymentError: + raise + except (OSError, urllib.error.URLError, json.JSONDecodeError, AttributeError): + pass + time.sleep(min(0.2, max(0.0, deadline - time.monotonic()))) + + +def _verify(config: DeploymentConfig, commit: str, run_command: RunCommand, health_check: HealthCheck, *, smoke: bool) -> None: + run_command(config.restart_command, check=True, text=True, capture_output=True, shell=False, timeout=config.command_timeout) + health_check(config.health_url, commit, config.health_timeout) + if smoke: + run_command(config.smoke_command, check=True, text=True, capture_output=True, shell=False, timeout=config.command_timeout) + + +def promote(*, config: DeploymentConfig, tag: str, archive: Path, expected_sha256: str, commit: str, + run_command: RunCommand = _run_argv, health_check: HealthCheck = poll_health) -> dict: + archive = Path(archive) + _validate_identity(tag, commit, expected_sha256) + _ensure_deployment_root(config, create=False) + with _verified_archive_copy(archive, expected_sha256) as verified_archive: + members, common_root = inspect_archive(verified_archive, config) + final = config.releases / commit + if final.exists() or final.is_symlink(): + raise DeploymentError(f"immutable release already exists: {commit}") + previous = _current_commit(config) + _ensure_releases_directory(config, create=True) + pending = config.releases / f".pending-{commit}-{secrets.token_hex(6)}" + if pending.exists() or pending.is_symlink(): + raise DeploymentError("pending release boundary already exists") + pending.mkdir(mode=0o755) + if pending.is_symlink() or not pending.is_dir(): + raise DeploymentError("pending release boundary is not a safe directory") + try: + _extract_validated(verified_archive, pending, members, common_root) + metadata = {"schemaVersion": 1, "tag": tag, "commit": commit, "sha256": expected_sha256} + (pending / ".timmy-release.json").write_text(json.dumps(metadata, sort_keys=True) + "\n", encoding="utf-8") + os.replace(pending, final) + except Exception: + shutil.rmtree(pending, ignore_errors=True) + raise + _atomic_point(config, commit) + try: + _verify(config, commit, run_command, health_check, smoke=True) + except Exception as error: + _atomic_point(config, previous) + try: + if previous is not None: + _verify(config, previous, run_command, health_check, smoke=False) + else: + run_command(config.stop_command, check=True, text=True, capture_output=True, shell=False, timeout=config.command_timeout) + except Exception as rollback_error: + raise DeploymentError("promotion verification failed and rollback verification also failed") from rollback_error + if previous is None: + raise DeploymentError("promotion verification failed; no prior release; service stopped") from error + raise DeploymentError("promotion verification failed; prior release restored") from error + return {"ok": True, "action": "promote", **metadata, "previousCommit": previous} + + +def _validate_release_directory(config: DeploymentConfig, commit: str) -> Path: + release = config.releases / commit + if not release.is_dir() or release.is_symlink(): + raise DeploymentError(f"release does not exist: {commit}") + metadata_path = release / ".timmy-release.json" + entrypoint = release / "server.mjs" + if metadata_path.is_symlink() or entrypoint.is_symlink() or not entrypoint.is_file(): + raise DeploymentError("release metadata or entrypoint is invalid") + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise DeploymentError("release metadata is missing or invalid") from error + if metadata.get("commit") != commit or not TAG_RE.fullmatch(str(metadata.get("tag", ""))): + raise DeploymentError("release metadata does not match requested commit") + return release + + +def rollback(*, config: DeploymentConfig, commit: str, run_command: RunCommand = _run_argv, + health_check: HealthCheck = poll_health) -> dict: + if not COMMIT_RE.fullmatch(commit): + raise DeploymentError("commit must be exactly 40 lowercase hexadecimal characters") + _validate_release_directory(config, commit) + previous = _current_commit(config) + if previous == commit: + raise DeploymentError("requested release is already current") + _atomic_point(config, commit) + try: + _verify(config, commit, run_command, health_check, smoke=False) + except Exception as error: + _atomic_point(config, previous) + if previous is not None: + try: + _verify(config, previous, run_command, health_check, smoke=False) + except Exception as rollback_error: + raise DeploymentError("rollback failed and prior release could not be restored") from rollback_error + raise DeploymentError("rollback verification failed; prior release restored") from error + return {"ok": True, "action": "rollback", "commit": commit, "previousCommit": previous} + + +def status(config: DeploymentConfig) -> dict: + commit = _current_commit(config) + if commit is None: + return {"ok": True, "active": False, "commit": None, "release": None} + metadata_path = config.releases / commit / ".timmy-release.json" + metadata = {} + try: + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + pass + return {"ok": True, "active": True, "commit": commit, "release": metadata.get("tag"), "sha256": metadata.get("sha256")} + + +def _json_argv(value: str, option: str) -> tuple[str, ...]: + try: + parsed = json.loads(value) + except json.JSONDecodeError as error: + raise DeploymentError(f"{option} must be a JSON argv array") from error + if not isinstance(parsed, list) or not parsed or not all(isinstance(item, str) and item for item in parsed): + raise DeploymentError(f"{option} must be a non-empty JSON argv array of strings") + return tuple(parsed) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path(os.environ.get("TIMMY_STAGING_ROOT", "/opt/timmy-staging"))) + parser.add_argument("--dry-run", action="store_true", help="validate and print the intended action without mutation or commands") + parser.add_argument("--restart-command", default=os.environ.get("TIMMY_STAGING_RESTART_COMMAND", '["systemctl","restart","timmy-staging.service"]')) + parser.add_argument("--stop-command", default=os.environ.get("TIMMY_STAGING_STOP_COMMAND", '["systemctl","stop","timmy-staging.service"]')) + parser.add_argument("--smoke-command", default=os.environ.get("TIMMY_STAGING_SMOKE_COMMAND", '["npm","run","test:staging-smoke"]')) + parser.add_argument("--health-url", default=os.environ.get("TIMMY_STAGING_HEALTH_URL", "http://127.0.0.1:4174/timmy-staging/api/healthz")) + subparsers = parser.add_subparsers(dest="command", required=True) + promote_parser = subparsers.add_parser("promote") + promote_parser.add_argument("--tag", required=True) + promote_parser.add_argument("--archive", required=True, type=Path) + promote_parser.add_argument("--sha256", required=True) + promote_parser.add_argument("--commit", required=True) + rollback_parser = subparsers.add_parser("rollback") + rollback_parser.add_argument("--commit", required=True) + subparsers.add_parser("status") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + config = DeploymentConfig( + root=args.root, + restart_command=_json_argv(args.restart_command, "restart command"), + stop_command=_json_argv(args.stop_command, "stop command"), + smoke_command=_json_argv(args.smoke_command, "smoke command"), + health_url=args.health_url, + ) + if args.command == "status": + result = status(config) + elif args.command == "promote": + if args.dry_run: + _validate_identity(args.tag, args.commit, args.sha256) + _ensure_deployment_root(config, create=False) + with _verified_archive_copy(args.archive, args.sha256) as verified_archive: + members, wrapper = inspect_archive(verified_archive, config) + result = {"ok": True, "dryRun": True, "action": "promote", "commit": args.commit, "tag": args.tag, "members": len(members), "wrapper": wrapper} + else: + result = promote(config=config, tag=args.tag, archive=args.archive, expected_sha256=args.sha256, commit=args.commit) + else: + if args.dry_run: + if not COMMIT_RE.fullmatch(args.commit): + raise DeploymentError("rollback release does not exist or commit is invalid") + _validate_release_directory(config, args.commit) + result = {"ok": True, "dryRun": True, "action": "rollback", "commit": args.commit} + else: + result = rollback(config=config, commit=args.commit) + print(json.dumps(result, sort_keys=True)) + return 0 + except DeploymentError as error: + print(f"deploy_staging: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/vision-config.js b/src/vision-config.js index e818f26..11d5df2 100644 --- a/src/vision-config.js +++ b/src/vision-config.js @@ -18,7 +18,7 @@ export function resolveVisionConfig(env = process.env) { if (!PROFILES[profile]) throw new Error('TIMMY_VISION_PROFILE must be hosted or selfhost.'); const defaults = PROFILES[profile]; const config = { - enabled: env.TIMMY_VISION_ENABLED !== '0', + enabled: !['0', 'false'].includes(String(env.TIMMY_VISION_ENABLED || '').toLowerCase()), profile, processor: defaults.processor, baseUrl: env.TIMMY_VISION_BASE_URL || defaults.baseUrl, diff --git a/tests/ci-workflow.test.js b/tests/ci-workflow.test.js index f0a6822..07fdb83 100644 --- a/tests/ci-workflow.test.js +++ b/tests/ci-workflow.test.js @@ -38,6 +38,19 @@ test('default unit suite includes the self-host bootstrap contract', async () => assert.match(packageJson.scripts.test, /tests\/selfhost-bootstrap\.test\.js/); }); +test('staging smoke is an explicit syntax-gated promotion command, never a PR network call', async () => { + const [packageJson, workflow] = await Promise.all([ + readFile(packagePath, 'utf8').then(JSON.parse), + readFile(workflowPath, 'utf8'), + ]); + assert.match(workflow, /python3 tests\/staging-deploy\.test\.py -v/); + assert.match(packageJson.scripts.test, /tests\/staging-config\.test\.js/); + assert.equal(packageJson.scripts['test:staging-smoke'], 'node tests/staging.acceptance.mjs'); + assert.match(packageJson.scripts['check:syntax'], /node --check tests\/staging\.acceptance\.mjs/); + assert.match(workflow, /node --check tests\/staging\.acceptance\.mjs/); + assert.doesNotMatch(workflow, /test:staging-smoke|TIMMY_STAGING_URL/); +}); + test('diff hygiene checks only the pull request or latest commit range', async () => { const script = await readFile(diffCheckPath, 'utf8'); diff --git a/tests/staging-config.test.js b/tests/staging-config.test.js new file mode 100644 index 0000000..f4da52e --- /dev/null +++ b/tests/staging-config.test.js @@ -0,0 +1,105 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { spawnSync } from 'node:child_process'; + +const servicePath = new URL('../deploy/timmy-staging.service', import.meta.url); +const envPath = new URL('../deploy/timmy-staging.env.example', import.meta.url); +const caddyPath = new URL('../deploy/Caddyfile.staging.example', import.meta.url); +const runbookPath = new URL('../docs/STAGING-RUNBOOK.md', import.meta.url); + +const literalSecret = /(password|token|secret|api[_-]?key)\s*[=:]\s*(?!\$\{|<|CHANGE_ME|false|$)["']?[A-Za-z0-9/+_.-]{8,}/i; + +test('systemd template runs a dedicated loopback-only agent-disabled service', async () => { + const service = await readFile(servicePath, 'utf8'); + for (const directive of [ + 'User=timmy-staging', 'Group=timmy-staging', + 'WorkingDirectory=/opt/timmy-staging/current', + 'EnvironmentFile=/etc/timmy-staging.env', + 'Environment=HOST=127.0.0.1', 'Environment=PORT=4174', + 'Environment=TIMMY_BASE_PATH=/timmy-staging', + 'Environment=TIMMY_AGENT_ENABLED=false', + ]) assert.match(service, new RegExp(`^${directive.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`, 'm'), directive); + assert.match(service, /^ExecStart=\/usr\/bin\/env TIMMY_AGENT_ENABLED=false TIMMY_VISION_ENABLED=false \/usr\/local\/lib\/timmy-staging\/node server\.mjs$/m); + assert.doesNotMatch(service, /ExecStart=\/usr\/(?:local\/)?bin\/node|0\.0\.0\.0|TIMMY_AGENT_ENABLED=true/); + assert.doesNotMatch(service, literalSecret); +}); + +test('systemd command-level environment keeps agent and vision disabled after EnvironmentFile overrides', async () => { + const service = await readFile(servicePath, 'utf8'); + const environmentFileIndex = service.indexOf('EnvironmentFile='); + const execLine = service.match(/^ExecStart=(.+)$/m)?.[1]; + assert.ok(execLine, 'ExecStart must exist'); + assert.ok(environmentFileIndex < service.indexOf(`ExecStart=${execLine}`), 'EnvironmentFile must be loaded before command overrides'); + const argv = execLine.trim().split(/\s+/); + assert.equal(argv.shift(), '/usr/bin/env'); + const assignments = argv.filter(value => /^TIMMY_(?:AGENT|VISION)_ENABLED=/.test(value)); + assert.deepEqual(assignments, ['TIMMY_AGENT_ENABLED=false', 'TIMMY_VISION_ENABLED=false']); + const probe = spawnSync('/usr/bin/env', [ + ...assignments, process.execPath, '-e', + 'process.stdout.write(`${process.env.TIMMY_AGENT_ENABLED},${process.env.TIMMY_VISION_ENABLED}`)', + ], { env: { ...process.env, TIMMY_AGENT_ENABLED: 'true', TIMMY_VISION_ENABLED: 'true' }, encoding: 'utf8' }); + assert.equal(probe.status, 0, probe.stderr); + assert.equal(probe.stdout, 'false,false'); +}); + +test('systemd template applies filesystem privilege process and resource confinement', async () => { + const service = await readFile(servicePath, 'utf8'); + for (const directive of [ + 'UMask=0077', 'NoNewPrivileges=true', 'PrivateTmp=true', 'PrivateDevices=true', + 'ProtectSystem=strict', 'ProtectHome=true', 'ReadWritePaths=/var/lib/timmy-staging', + 'RestrictSUIDSGID=true', 'LockPersonality=true', 'RestrictNamespaces=true', + 'ProtectKernelTunables=true', 'ProtectKernelModules=true', 'ProtectKernelLogs=true', + 'ProtectControlGroups=true', 'ProtectClock=true', 'ProtectHostname=true', + 'CapabilityBoundingSet=', 'AmbientCapabilities=', 'RestrictRealtime=true', + 'TasksMax=64', 'MemoryMax=512M', 'LimitNOFILE=1024', + ]) assert.match(service, new RegExp(`^${directive.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`, 'm'), directive); + assert.equal((service.match(/^ReadWritePaths=/gm) || []).length, 1); +}); + +test('environment example is least privilege base-path staging configuration without credentials', async () => { + const env = await readFile(envPath, 'utf8'); + for (const setting of [ + 'HOST=127.0.0.1', 'PORT=4174', 'TIMMY_BASE_PATH=/timmy-staging', + 'TIMMY_STAGING_LABEL=true', 'TIMMY_AGENT_ENABLED=false', 'TIMMY_VISION_ENABLED=false', + ]) assert.match(env, new RegExp(`^${setting.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`, 'm')); + assert.match(env, /install -m 600/); + assert.doesNotMatch(env, /TIMMY_AGENT_ACCESS_TOKEN|GITEA_TOKEN|OPENAI_API_KEY|COOKIE/); + assert.doesNotMatch(env, literalSecret); +}); + +test('Caddy snippet isolates git and authenticates staging before a catchall', async () => { + const caddy = await readFile(caddyPath, 'utf8'); + const git = caddy.indexOf('handle @git'); + const staging = caddy.indexOf('handle_path /timmy-staging/*'); + const catchall = caddy.indexOf('handle {'); + assert.ok(git >= 0 && staging > git && catchall > staging, 'route order must be git, staging, catchall'); + assert.match(caddy, /basic_auth/); + assert.match(caddy, /\{\$TIMMY_STAGING_PASSWORD_HASH\}/); + assert.match(caddy, /reverse_proxy 127\.0\.0\.1:4174/); + assert.match(caddy, /max_size 8MB/); + assert.match(caddy, /X-Content-Type-Options "nosniff"/); + assert.match(caddy, /X-Frame-Options "DENY"/); + assert.match(caddy, /Referrer-Policy "no-referrer"/); + assert.match(caddy, /Content-Security-Policy/); + assert.match(caddy, /rewrite \* \/timmy-staging\{uri\}/, 'handle_path strips once, then upstream base path is explicitly reconstructed'); + assert.doesNotMatch(caddy, /\$2[aby]\$[A-Za-z0-9./]{20,}|password\s+[^<{\s]/i); +}); + +test('runbook covers safe installation operation verification rollback backup and removal', async () => { + const runbook = await readFile(runbookPath, 'utf8'); + for (const heading of [ + 'Prerequisites', 'DNS and URL', 'Install', 'Promote', 'Smoke test', 'Status and logs', + 'Rollback', 'Backup', 'Remove staging', 'Template validation', + ]) assert.match(runbook, new RegExp(`^## .*${heading}`, 'mi'), heading); + assert.match(runbook, /sha256/i); + assert.match(runbook, /install -D -o root -g root -m 755[^\n]+\/usr\/local\/lib\/timmy-staging\/node/); + assert.match(runbook, /\/usr\/local\/lib\/timmy-staging\/node --version/); + assert.match(runbook, /chmod 600|install -m 600/); + assert.match(runbook, /systemctl restart timmy-staging\.service/); + assert.match(runbook, /no prior release[^.]*stops[^.]*service/i); + assert.match(runbook, /failed release[^.]*inert[^.]*evidence/i); + assert.match(runbook, /journalctl -u timmy-staging\.service/); + assert.match(runbook, /do not.*live|approval/i); + assert.doesNotMatch(runbook, literalSecret); +}); diff --git a/tests/staging-deploy.test.py b/tests/staging-deploy.test.py new file mode 100644 index 0000000..202a79d --- /dev/null +++ b/tests/staging-deploy.test.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Filesystem and failure-path tests for immutable staging deployment.""" +from __future__ import annotations + +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import sys +import tarfile +import tempfile +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "deploy_staging.py" +COMMIT_A = "a" * 40 +COMMIT_B = "b" * 40 + + +def load_deploy(): + spec = importlib.util.spec_from_file_location("deploy_staging", SCRIPT) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def make_archive(path: Path, members: list[tuple[str, bytes, str]]) -> str: + with tarfile.open(path, "w:gz") as archive: + for name, body, kind in members: + info = tarfile.TarInfo(name) + if kind == "file": + info.size = len(body) + archive.addfile(info, io.BytesIO(body)) + elif kind == "dir": + info.type = tarfile.DIRTYPE + archive.addfile(info) + elif kind == "symlink": + info.type = tarfile.SYMTYPE + info.linkname = "server.mjs" + archive.addfile(info) + elif kind == "hardlink": + info.type = tarfile.LNKTYPE + info.linkname = "server.mjs" + archive.addfile(info) + elif kind == "fifo": + info.type = tarfile.FIFOTYPE + archive.addfile(info) + elif kind == "device": + info.type = tarfile.CHRTYPE + archive.addfile(info) + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class DeployTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = Path(self.tmp.name) / "opt" + self.root.mkdir() + self.deploy = load_deploy() + self.config = self.deploy.DeploymentConfig( + root=self.root, + restart_command=("fixture-restart",), + stop_command=("fixture-stop",), + smoke_command=("fixture-smoke",), + health_url="http://127.0.0.1:4174/api/healthz", + command_timeout=1.0, + health_timeout=1.0, + ) + self.commands = [] + + def tearDown(self): + self.tmp.cleanup() + + def runner(self, argv, **kwargs): + self.commands.append((tuple(argv), kwargs)) + return subprocess.CompletedProcess(argv, 0, "", "") + + def healthy(self, url, commit, timeout): + self.assertEqual(url, self.config.health_url) + self.assertEqual(timeout, self.config.health_timeout) + return {"ok": True, "commit": commit} + + def archive(self, name="release.tar.gz", members=None): + path = Path(self.tmp.name) / name + digest = make_archive(path, members or [ + ("timmy-release/", b"", "dir"), + ("timmy-release/server.mjs", b"console.log('ok')\n", "file"), + ("timmy-release/package.json", b"{}\n", "file"), + ]) + return path, digest + + def seed_release(self, commit): + release = self.root / "releases" / commit + release.mkdir(parents=True) + (release / "server.mjs").write_text("ok", encoding="utf-8") + (release / ".timmy-release.json").write_text(json.dumps({"commit": commit, "tag": "old"}), encoding="utf-8") + return release + + def point_current(self, commit): + (self.root / "current").symlink_to(Path("releases") / commit) + + def promote(self, archive, digest, commit=COMMIT_B, **kwargs): + return self.deploy.promote( + config=self.config, tag="daily-test", archive=archive, + expected_sha256=digest, commit=commit, + run_command=kwargs.get("run_command", self.runner), + health_check=kwargs.get("health_check", self.healthy), + ) + + def test_checksum_mismatch_fails_before_tar_is_opened(self): + archive = Path(self.tmp.name) / "not-even-a-tar" + archive.write_bytes(b"untrusted") + with self.assertRaisesRegex(self.deploy.DeploymentError, "SHA-256 mismatch"): + self.promote(archive, "0" * 64) + self.assertFalse((self.root / "releases").exists()) + + def test_verified_private_archive_copy_is_used_after_caller_archive_mutates(self): + archive, digest = self.archive("mutable.tar.gz") + evil = Path(self.tmp.name) / "evil.tar.gz" + make_archive(evil, [ + ("timmy-release/", b"", "dir"), + ("timmy-release/server.mjs", b"EVIL\n", "file"), + ]) + real_inspect = self.deploy.inspect_archive + inspected_paths = [] + + def mutate_then_inspect(path, config): + inspected_paths.append(Path(path)) + archive.write_bytes(evil.read_bytes()) + return real_inspect(path, config) + + with mock.patch.object(self.deploy, "inspect_archive", side_effect=mutate_then_inspect): + self.promote(archive, digest) + + release = self.root / "releases" / COMMIT_B + self.assertNotEqual(inspected_paths, [archive]) + self.assertEqual((release / "server.mjs").read_text(), "console.log('ok')\n") + self.assertTrue(all(not path.exists() for path in inspected_paths), "verified temp archive must be cleaned") + + def test_archive_source_must_be_a_nonsymlink_regular_file(self): + archive, digest = self.archive("regular.tar.gz") + symlink = Path(self.tmp.name) / "archive-link.tar.gz" + symlink.symlink_to(archive) + for source in (symlink, Path(self.tmp.name)): + with self.subTest(source=source), self.assertRaisesRegex(self.deploy.DeploymentError, "regular file|stage archive"): + self.promote(source, digest) + + def test_absolute_and_traversal_paths_are_rejected(self): + for index, unsafe in enumerate(("/etc/passwd", "root/../../escape", "../escape", "root//double")): + archive, digest = self.archive(f"unsafe-{index}.tar.gz", [(unsafe, b"bad", "file")]) + with self.subTest(unsafe=unsafe), self.assertRaisesRegex(self.deploy.DeploymentError, "unsafe archive path"): + self.promote(archive, digest) + self.assertFalse((self.root / "releases" / COMMIT_B).exists()) + + def test_links_devices_fifos_and_oversized_archives_are_rejected(self): + for index, kind in enumerate(("symlink", "hardlink", "fifo", "device")): + archive, digest = self.archive(f"special-{index}.tar.gz", [(f"root/bad-{kind}", b"", kind)]) + with self.subTest(kind=kind), self.assertRaisesRegex(self.deploy.DeploymentError, "unsupported archive member"): + self.promote(archive, digest) + archive, digest = self.archive("large.tar.gz", [("root/large", b"x" * 17, "file")]) + tiny = self.deploy.DeploymentConfig(**{**self.config.__dict__, "max_member_bytes": 16}) + with self.assertRaisesRegex(self.deploy.DeploymentError, "member size"): + self.deploy.promote(config=tiny, tag="daily-test", archive=archive, expected_sha256=digest, commit=COMMIT_B, run_command=self.runner, health_check=self.healthy) + + def test_member_count_limit_is_enforced(self): + archive, digest = self.archive("many.tar.gz", [(f"root/{i}", b"x", "file") for i in range(3)]) + tiny = self.deploy.DeploymentConfig(**{**self.config.__dict__, "max_members": 2}) + with self.assertRaisesRegex(self.deploy.DeploymentError, "member count"): + self.deploy.promote(config=tiny, tag="daily-test", archive=archive, expected_sha256=digest, commit=COMMIT_B, run_command=self.runner, health_check=self.healthy) + + def test_secrets_env_and_forbidden_artifacts_are_rejected(self): + forbidden = ("root/.env", "root/secrets/token.txt", "root/private.pem", "root/model.gguf", "root/.git/config", "root/video/raw.webm") + for index, name in enumerate(forbidden): + archive, digest = self.archive(f"secret-{index}.tar.gz", [(name, b"secret", "file")]) + with self.subTest(name=name), self.assertRaisesRegex(self.deploy.DeploymentError, "forbidden archive artifact"): + self.promote(archive, digest) + + def test_forbidden_common_roots_are_rejected_case_insensitively_but_release_wrapper_is_allowed(self): + for index, root in enumerate((".git", "ViDeO", "ARTIFACTS", "Credentials")): + archive, _ = self.archive(f"forbidden-root-{index}.tar.gz", [ + (f"{root}/", b"", "dir"), + (f"{root}/server.mjs", b"evil", "file"), + ]) + with self.subTest(root=root), self.assertRaisesRegex(self.deploy.DeploymentError, "forbidden archive artifact"): + self.deploy.inspect_archive(archive, self.config) + archive, _ = self.archive("normal-wrapper.tar.gz") + _, wrapper = self.deploy.inspect_archive(archive, self.config) + self.assertEqual(wrapper, "timmy-release") + + def test_release_root_symlink_is_rejected(self): + outside = Path(self.tmp.name) / "outside" + outside.mkdir() + (self.root / "releases").symlink_to(outside, target_is_directory=True) + archive, digest = self.archive() + with self.assertRaisesRegex(self.deploy.DeploymentError, "releases directory"): + self.promote(archive, digest) + self.assertEqual(list(outside.iterdir()), []) + + def test_deployment_root_symlink_is_rejected_before_writing_outside(self): + outside = Path(self.tmp.name) / "outside-root" + outside.mkdir() + self.root.rmdir() + self.root.symlink_to(outside, target_is_directory=True) + archive, digest = self.archive() + with self.assertRaisesRegex(self.deploy.DeploymentError, "deployment root"): + self.promote(archive, digest) + self.assertEqual(list(outside.iterdir()), []) + + def test_deployment_root_non_directory_is_rejected(self): + self.root.rmdir() + self.root.write_text("not a directory") + archive, digest = self.archive() + with self.assertRaisesRegex(self.deploy.DeploymentError, "deployment root is not a directory"): + self.promote(archive, digest) + + def test_current_and_pending_symlink_boundaries_are_rejected_before_extraction(self): + outside = Path(self.tmp.name) / "outside-boundary" + outside.mkdir() + (self.root / "releases").mkdir() + (self.root / "current").symlink_to(outside, target_is_directory=True) + archive, digest = self.archive("current-boundary.tar.gz") + with self.assertRaisesRegex(self.deploy.DeploymentError, "current symlink escapes"): + self.promote(archive, digest) + self.assertEqual(list(outside.iterdir()), []) + + (self.root / "current").unlink() + pending = self.root / "releases" / f".pending-{COMMIT_B}-fixed" + pending.symlink_to(outside, target_is_directory=True) + with mock.patch.object(self.deploy.secrets, "token_hex", return_value="fixed"): + with self.assertRaisesRegex(self.deploy.DeploymentError, "pending release boundary"): + self.promote(archive, digest) + self.assertEqual(list(outside.iterdir()), []) + + def test_valid_archive_extracts_to_commit_release_and_is_never_overwritten(self): + archive, digest = self.archive() + result = self.promote(archive, digest) + release = self.root / "releases" / COMMIT_B + self.assertEqual(result["commit"], COMMIT_B) + self.assertEqual((release / "server.mjs").read_text(), "console.log('ok')\n") + self.assertEqual(json.loads((release / ".timmy-release.json").read_text())["sha256"], digest) + with self.assertRaisesRegex(self.deploy.DeploymentError, "already exists"): + self.promote(archive, digest) + + def test_success_atomically_swaps_current_and_uses_bounded_argv_commands(self): + self.seed_release(COMMIT_A) + self.point_current(COMMIT_A) + archive, digest = self.archive() + self.promote(archive, digest) + self.assertEqual((self.root / "current").resolve(), self.root / "releases" / COMMIT_B) + self.assertEqual([command for command, _ in self.commands], [("fixture-restart",), ("fixture-smoke",)]) + self.assertTrue(all(options["shell"] is False and options["timeout"] == 1.0 for _, options in self.commands)) + self.assertFalse(any(path.name.startswith(".current-") for path in self.root.iterdir())) + + def test_restart_health_and_smoke_failures_automatically_restore_prior_release(self): + phases = ("restart", "health", "smoke") + for phase in phases: + with self.subTest(phase=phase): + root = Path(self.tmp.name) / phase + config = self.deploy.DeploymentConfig(**{**self.config.__dict__, "root": root}) + release = root / "releases" / COMMIT_A + release.mkdir(parents=True) + (release / "server.mjs").write_text("ok") + (root / "current").symlink_to(Path("releases") / COMMIT_A) + archive, digest = self.archive(f"{phase}.tar.gz") + calls = [] + def run(argv, **kwargs): + calls.append(tuple(argv)) + if (phase == "restart" and len(calls) == 1) or (phase == "smoke" and tuple(argv) == ("fixture-smoke",)): + raise subprocess.CalledProcessError(1, argv) + return subprocess.CompletedProcess(argv, 0, "", "") + def health(url, commit, timeout): + if phase == "health" and commit == COMMIT_B: + raise self.deploy.DeploymentError("health failed") + return {"ok": True, "commit": commit} + with self.assertRaisesRegex(self.deploy.DeploymentError, "promotion verification failed"): + self.deploy.promote(config=config, tag="daily-test", archive=archive, expected_sha256=digest, commit=COMMIT_B, run_command=run, health_check=health) + self.assertEqual((root / "current").resolve(), root / "releases" / COMMIT_A) + self.assertGreaterEqual(calls.count(("fixture-restart",)), 2) + + def test_first_promotion_failure_removes_current_and_stops_service_without_restart(self): + archive, digest = self.archive("first-failure.tar.gz") + calls = [] + + def run(argv, **kwargs): + calls.append(tuple(argv)) + if tuple(argv) == ("fixture-smoke",): + raise subprocess.CalledProcessError(1, argv) + return subprocess.CompletedProcess(argv, 0, "", "") + + with self.assertRaisesRegex(self.deploy.DeploymentError, "no prior release; service stopped"): + self.promote(archive, digest, run_command=run) + + self.assertFalse((self.root / "current").exists()) + self.assertFalse((self.root / "current").is_symlink()) + self.assertEqual(calls, [("fixture-restart",), ("fixture-smoke",), ("fixture-stop",)]) + self.assertTrue((self.root / "releases" / COMMIT_B).is_dir(), "failed release remains inert evidence") + + def test_rollback_requires_valid_immutable_release_and_restarts_and_checks_health(self): + self.seed_release(COMMIT_A) + self.seed_release(COMMIT_B) + self.point_current(COMMIT_B) + result = self.deploy.rollback(config=self.config, commit=COMMIT_A, run_command=self.runner, health_check=self.healthy) + self.assertEqual(result["commit"], COMMIT_A) + self.assertEqual((self.root / "current").resolve(), self.root / "releases" / COMMIT_A) + with self.assertRaisesRegex(self.deploy.DeploymentError, "does not exist"): + self.deploy.rollback(config=self.config, commit="c" * 40, run_command=self.runner, health_check=self.healthy) + tampered = self.seed_release("d" * 40) + (tampered / ".timmy-release.json").write_text(json.dumps({"commit": COMMIT_A, "tag": "wrong"})) + with self.assertRaisesRegex(self.deploy.DeploymentError, "metadata"): + self.deploy.rollback(config=self.config, commit="d" * 40, run_command=self.runner, health_check=self.healthy) + + def test_default_command_adapter_accepts_verifier_kwargs_and_stays_bounded(self): + result = self.deploy._run_argv( + (sys.executable, "-c", "print('ok')"), check=True, text=True, + capture_output=True, shell=False, timeout=1.0, + ) + self.assertEqual(result.stdout.strip(), "ok") + + def test_status_and_cli_dry_run_are_rootless_and_machine_readable(self): + self.seed_release(COMMIT_A) + self.point_current(COMMIT_A) + status = self.deploy.status(self.config) + self.assertEqual(status["commit"], COMMIT_A) + run = subprocess.run([ + sys.executable, str(SCRIPT), "--root", str(self.root), "--dry-run", "status" + ], text=True, capture_output=True, check=False) + self.assertEqual(run.returncode, 0, run.stderr) + self.assertEqual(json.loads(run.stdout)["commit"], COMMIT_A) + + def test_commit_tag_and_checksum_arguments_are_strictly_validated(self): + archive, digest = self.archive() + for commit in ("abc", "A" * 40, "a" * 41, "../" + "a" * 40): + with self.subTest(commit=commit), self.assertRaisesRegex(self.deploy.DeploymentError, "commit"): + self.promote(archive, digest, commit=commit) + with self.assertRaisesRegex(self.deploy.DeploymentError, "tag"): + self.deploy.promote(config=self.config, tag="../bad", archive=archive, expected_sha256=digest, commit=COMMIT_B, run_command=self.runner, health_check=self.healthy) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/staging.acceptance.mjs b/tests/staging.acceptance.mjs new file mode 100644 index 0000000..7c61549 --- /dev/null +++ b/tests/staging.acceptance.mjs @@ -0,0 +1,98 @@ +import { chromium } from 'playwright'; +import assert from 'node:assert/strict'; + +const appUrl = new URL(process.env.TIMMY_STAGING_URL || 'http://127.0.0.1:4173/'); +const expectedTag = process.env.TIMMY_EXPECT_RELEASE_TAG || 'daily-test'; +const expectedCommit = process.env.TIMMY_EXPECT_RELEASE_COMMIT || 'c'.repeat(40); +const username = process.env.TIMMY_STAGING_USER || ''; +const password = process.env.TIMMY_STAGING_PASSWORD || ''; +const loopback = ['127.0.0.1', 'localhost', '::1'].includes(appUrl.hostname); +if (!loopback && (!username || !password)) throw new Error('Live staging smoke requires TIMMY_STAGING_USER and TIMMY_STAGING_PASSWORD.'); +if (!/^[0-9a-f]{40}$/.test(expectedCommit)) throw new Error('TIMMY_EXPECT_RELEASE_COMMIT must be a full lowercase commit.'); + +const browser = await chromium.launch({ headless: true }); +try { + const context = await browser.newContext({ + viewport: { width: 390, height: 844 }, + deviceScaleFactor: 2, + serviceWorkers: 'block', + acceptDownloads: true, + ...(username && password ? { httpCredentials: { username, password } } : {}), + }); + const page = await context.newPage(); + const browserErrors = []; + const secretFindings = []; + let agentChatRequests = 0; + page.on('console', message => { if (message.type() === 'error') browserErrors.push(message.text()); }); + page.on('pageerror', error => browserErrors.push(error.message)); + page.on('request', request => { if (new URL(request.url()).pathname.endsWith('/api/agent/chat')) agentChatRequests += 1; }); + page.on('response', async response => { + const url = new URL(response.url()); + if (!url.pathname.includes('/api/')) return; + for (const name of Object.keys(response.headers())) { + if (/authorization|set-cookie|x-api-key/i.test(name)) secretFindings.push(`sensitive response header ${name}`); + } + try { + const type = response.headers()['content-type'] || ''; + if (/json|text/.test(type)) { + const body = (await response.text()).slice(0, 65_537); + if (body.length > 65_536) secretFindings.push('oversized API response'); + if (/"(?:password|token|cookie|session|api[_-]?key|credential|environment|processEnv)"\s*:/i.test(body)) secretFindings.push(`secret-bearing response ${url.pathname}`); + } + } catch {} + }); + + const navigation = await page.goto(appUrl.toString(), { waitUntil: 'networkidle' }); + assert.equal(navigation?.status(), 200, 'edge authentication and staging navigation must succeed'); + await page.evaluate(() => localStorage.clear()); + await page.reload({ waitUntil: 'networkidle' }); + + const healthUrl = new URL(`${appUrl.pathname.replace(/\/$/, '')}/api/healthz`, appUrl); + const health = await page.evaluate(async url => { + const response = await fetch(url, { headers: { accept: 'application/json' } }); + return { status: response.status, body: await response.json() }; + }, healthUrl.toString()); + assert.equal(health.status, 200); + assert.equal(health.body.ok, true); + assert.equal(health.body.release, expectedTag); + assert.equal(health.body.commit, expectedCommit); + assert.equal(health.body.agentEnabled, false, 'Phase 1 staging must keep Hermes off'); + + assert.match(await page.locator('.staging-label').innerText(), new RegExp(`Staging · ${expectedTag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} · ${expectedCommit.slice(0, 12)}`)); + assert.equal(await page.locator('main .btn-primary:visible').count(), 1, 'exactly one dominant photo action'); + assert.equal(await page.locator('[data-log]:visible').count(), 1, 'manual fallback remains visible'); + assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), true, 'home must not overflow'); + + await page.locator('[data-log]:visible').click(); + await page.locator('[data-type="4"]').click(); + await page.locator('#next').click(); + await page.locator('#note').fill('synthetic staging smoke'); + await page.locator('#next').click(); + await page.locator('#save').click(); + + await page.getByRole('button', { name: 'Journal', exact: true }).click(); + assert.match(await page.locator('main').innerText(), /synthetic staging smoke/); + assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), true, 'journal must not overflow'); + + await page.getByRole('button', { name: 'Timmy', exact: true }).click(); + await page.locator('#chat-message').fill('I barfed'); + await page.locator('#send-chat').click(); + await page.waitForSelector('.bubble.timmy >> text=Pause and get medical help'); + assert.equal(agentChatRequests, 0, 'urgent phrase must be intercepted before Hermes'); + + await page.getByRole('button', { name: 'Journal', exact: true }).click(); + await page.locator('[data-view="privacy"]').click(); + const downloadPromise = page.waitForEvent('download'); + await page.locator('#export').click(); + const download = await downloadPromise; + assert.equal(download.suggestedFilename(), 'timmy-ledger.json'); + assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), true, 'privacy screen must not overflow'); + + await page.waitForTimeout(100); + assert.deepEqual(secretFindings, []); + assert.deepEqual(browserErrors, []); + await context.close(); + console.log(`PASS staging smoke ${expectedTag} ${expectedCommit.slice(0, 12)} at ${appUrl}`); +} finally { + await browser.close(); +} diff --git a/tests/vision-config.test.js b/tests/vision-config.test.js index 14632df..9f4f959 100644 --- a/tests/vision-config.test.js +++ b/tests/vision-config.test.js @@ -2,6 +2,11 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { probeVisionProvider, resolveVisionConfig } from '../src/vision-config.js'; +test('documented boolean false disables staging vision fail closed', () => { + assert.equal(resolveVisionConfig({ TIMMY_VISION_ENABLED: 'false' }).enabled, false); + assert.equal(resolveVisionConfig({ TIMMY_VISION_ENABLED: '0' }).enabled, false); +}); + test('selfhost profile defaults to a loopback OpenAI-compatible server and no remote processor', () => { const config = resolveVisionConfig({ TIMMY_VISION_PROFILE: 'selfhost' }); assert.equal(config.profile, 'selfhost');