Merge pull request 'Hardened atomic private-staging deployment tooling' (#56) from timmy/55-staging-deployment-tooling into main
All checks were successful
Quality gates / quality (push) Successful in 1m41s
All checks were successful
Quality gates / quality (push) Successful in 1m41s
This commit is contained in:
commit
067afaed4c
|
|
@ -30,7 +30,9 @@ jobs:
|
||||||
- name: Install browser
|
- name: Install browser
|
||||||
run: npx playwright install --with-deps chromium
|
run: npx playwright install --with-deps chromium
|
||||||
- name: Unit and security tests
|
- name: Unit and security tests
|
||||||
run: npm test
|
run: |
|
||||||
|
npm test
|
||||||
|
python3 tests/staging-deploy.test.py -v
|
||||||
- name: Mobile browser acceptance
|
- name: Mobile browser acceptance
|
||||||
run: |
|
run: |
|
||||||
npm start > /tmp/timmy-server.log 2>&1 &
|
npm start > /tmp/timmy-server.log 2>&1 &
|
||||||
|
|
@ -52,6 +54,8 @@ jobs:
|
||||||
- name: Dependency audit
|
- name: Dependency audit
|
||||||
run: npm audit --audit-level=high
|
run: npm audit --audit-level=high
|
||||||
- name: Syntax checks
|
- name: Syntax checks
|
||||||
run: npm run check:syntax
|
run: |
|
||||||
|
npm run check:syntax
|
||||||
|
node --check tests/staging.acceptance.mjs
|
||||||
- name: Diff hygiene
|
- name: Diff hygiene
|
||||||
run: npm run check:diff
|
run: npm run check:diff
|
||||||
|
|
|
||||||
13
README.md
13
README.md
|
|
@ -15,6 +15,19 @@ python3 scripts/build_release.py
|
||||||
|
|
||||||
The builder clones the committed `main` tree into an isolated directory, runs unit/security and both mobile acceptance suites, audits dependencies, checks syntax and secrets, excludes model weights and sensitive/generated media, records a vertical feature demonstration from the working app, fully decodes and probes that MP4, and writes checksummed source/video artifacts plus `manifest.json` under `/root/timmy-releases/`. It refuses dirty or non-`main` source trees.
|
The builder clones the committed `main` tree into an isolated directory, runs unit/security and both mobile acceptance suites, audits dependencies, checks syntax and secrets, excludes model weights and sensitive/generated media, records a vertical feature demonstration from the working app, fully decodes and probes that MP4, and writes checksummed source/video artifacts plus `manifest.json` under `/root/timmy-releases/`. It refuses dirty or non-`main` source trees.
|
||||||
|
|
||||||
|
## Promote or roll back private staging
|
||||||
|
|
||||||
|
The standard-library deployment tool verifies the declared archive SHA-256 before tar parsing, rejects unsafe or secret-bearing members, creates one immutable `releases/<40-character-commit>` directory, and atomically repoints `current`. Restart, bounded commit-specific health, or synthetic mobile smoke failures restore and verify the prior release automatically.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/deploy_staging.py --dry-run promote --tag TAG --archive FILE --sha256 HASH --commit FULL_COMMIT
|
||||||
|
sudo python3 scripts/deploy_staging.py promote --tag TAG --archive FILE --sha256 HASH --commit FULL_COMMIT
|
||||||
|
python3 scripts/deploy_staging.py status
|
||||||
|
sudo python3 scripts/deploy_staging.py rollback --commit PRIOR_FULL_COMMIT
|
||||||
|
```
|
||||||
|
|
||||||
|
Commands are fixed argv arrays executed with `shell=False`; JSON argv overrides and `--root` exist for rootless fixtures. See [the staging runbook](docs/STAGING-RUNBOOK.md) for host prerequisites, mode-600 environment injection, Caddy validation, smoke credentials, logs, backups, removal, and the explicit no-live-change boundary.
|
||||||
|
|
||||||
## Run with the self-hosted open-weight path
|
## Run with the self-hosted open-weight path
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|
|
||||||
36
deploy/Caddyfile.staging.example
Normal file
36
deploy/Caddyfile.staging.example
Normal file
|
|
@ -0,0 +1,36 @@
|
||||||
|
# Example complete site used for validation. Merge only the reviewed handles into
|
||||||
|
# the existing forge site. Route order is security-sensitive.
|
||||||
|
forge.example.invalid {
|
||||||
|
encode zstd gzip
|
||||||
|
|
||||||
|
@git path /git /git/*
|
||||||
|
handle @git {
|
||||||
|
reverse_proxy 127.0.0.1:3000
|
||||||
|
}
|
||||||
|
|
||||||
|
handle_path /timmy-staging/* {
|
||||||
|
basic_auth {
|
||||||
|
staging {$TIMMY_STAGING_PASSWORD_HASH}
|
||||||
|
}
|
||||||
|
request_body {
|
||||||
|
max_size 8MB
|
||||||
|
}
|
||||||
|
header {
|
||||||
|
X-Content-Type-Options "nosniff"
|
||||||
|
X-Frame-Options "DENY"
|
||||||
|
Referrer-Policy "no-referrer"
|
||||||
|
Permissions-Policy "camera=(self), microphone=(), geolocation=()"
|
||||||
|
Content-Security-Policy "default-src 'self'; img-src 'self' data: blob:; style-src 'self'; script-src 'self'; connect-src 'self'; object-src 'none'; base-uri 'self'; frame-ancestors 'none'; form-action 'self'"
|
||||||
|
-Server
|
||||||
|
}
|
||||||
|
# handle_path strips the public prefix exactly once. Reconstruct it for
|
||||||
|
# Timmy because its validated base-path router must see that same prefix.
|
||||||
|
rewrite * /timmy-staging{uri}
|
||||||
|
reverse_proxy 127.0.0.1:4174
|
||||||
|
}
|
||||||
|
|
||||||
|
# Existing Gitea catchall belongs after the private staging route.
|
||||||
|
handle {
|
||||||
|
reverse_proxy 127.0.0.1:3000
|
||||||
|
}
|
||||||
|
}
|
||||||
11
deploy/timmy-staging.env.example
Normal file
11
deploy/timmy-staging.env.example
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
# Copy with: sudo install -m 600 deploy/timmy-staging.env.example /etc/timmy-staging.env
|
||||||
|
# Replace release identity for every promotion. Keep this file root-owned and out of archives.
|
||||||
|
HOST=127.0.0.1
|
||||||
|
PORT=4174
|
||||||
|
TIMMY_BASE_PATH=/timmy-staging
|
||||||
|
TIMMY_STAGING_LABEL=true
|
||||||
|
TIMMY_RELEASE_TAG=CHANGE_ME
|
||||||
|
TIMMY_RELEASE_COMMIT=0000000000000000000000000000000000000000
|
||||||
|
TIMMY_AGENT_ENABLED=false
|
||||||
|
TIMMY_VISION_ENABLED=false
|
||||||
|
NODE_ENV=production
|
||||||
61
deploy/timmy-staging.service
Normal file
61
deploy/timmy-staging.service
Normal file
|
|
@ -0,0 +1,61 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Timmy private staging service
|
||||||
|
Documentation=file:/opt/timmy-staging/current/docs/STAGING-RUNBOOK.md
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=simple
|
||||||
|
User=timmy-staging
|
||||||
|
Group=timmy-staging
|
||||||
|
UMask=0077
|
||||||
|
WorkingDirectory=/opt/timmy-staging/current
|
||||||
|
Environment=NODE_ENV=production
|
||||||
|
Environment=HOST=127.0.0.1
|
||||||
|
Environment=PORT=4174
|
||||||
|
Environment=TIMMY_BASE_PATH=/timmy-staging
|
||||||
|
Environment=TIMMY_AGENT_ENABLED=false
|
||||||
|
Environment=TIMMY_VISION_ENABLED=false
|
||||||
|
EnvironmentFile=/etc/timmy-staging.env
|
||||||
|
ExecStart=/usr/bin/env TIMMY_AGENT_ENABLED=false TIMMY_VISION_ENABLED=false /usr/local/lib/timmy-staging/node server.mjs
|
||||||
|
Restart=on-failure
|
||||||
|
RestartSec=5s
|
||||||
|
TimeoutStartSec=30s
|
||||||
|
TimeoutStopSec=15s
|
||||||
|
KillSignal=SIGTERM
|
||||||
|
|
||||||
|
# Filesystem and privilege boundary. Only state is writable.
|
||||||
|
NoNewPrivileges=true
|
||||||
|
PrivateTmp=true
|
||||||
|
PrivateDevices=true
|
||||||
|
ProtectSystem=strict
|
||||||
|
ProtectHome=true
|
||||||
|
ReadWritePaths=/var/lib/timmy-staging
|
||||||
|
StateDirectory=timmy-staging
|
||||||
|
StateDirectoryMode=0700
|
||||||
|
RestrictSUIDSGID=true
|
||||||
|
LockPersonality=true
|
||||||
|
RestrictNamespaces=true
|
||||||
|
ProtectKernelTunables=true
|
||||||
|
ProtectKernelModules=true
|
||||||
|
ProtectKernelLogs=true
|
||||||
|
ProtectControlGroups=true
|
||||||
|
ProtectClock=true
|
||||||
|
ProtectHostname=true
|
||||||
|
CapabilityBoundingSet=
|
||||||
|
AmbientCapabilities=
|
||||||
|
|
||||||
|
# Process, syscall, network, and resource boundary.
|
||||||
|
RestrictRealtime=true
|
||||||
|
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||||
|
SystemCallArchitectures=native
|
||||||
|
SystemCallFilter=@system-service
|
||||||
|
SystemCallFilter=~@privileged @resources
|
||||||
|
ProtectProc=invisible
|
||||||
|
ProcSubset=pid
|
||||||
|
TasksMax=64
|
||||||
|
MemoryMax=512M
|
||||||
|
CPUQuota=100%
|
||||||
|
LimitNOFILE=1024
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
118
docs/STAGING-RUNBOOK.md
Normal file
118
docs/STAGING-RUNBOOK.md
Normal file
|
|
@ -0,0 +1,118 @@
|
||||||
|
# Timmy private staging runbook
|
||||||
|
|
||||||
|
This runbook is a reviewed host template, not approval to change a live host. Apply it only in a separately approved provisioning change. Staging is Phase 1: Hermes Agent and vision are off.
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
- Linux host with Node.js 22, systemd, and Caddy.
|
||||||
|
- Existing HTTPS origin and Gitea route inventory.
|
||||||
|
- A reviewed 40-character commit, source archive, and SHA-256 receipt from the release manifest.
|
||||||
|
- Root only for one-time account/unit/config installation; promotions use the narrowest available sudo policy.
|
||||||
|
- A fresh Caddy password hash delivered outside Git. Never put the password or hash in shell history, release notes, screenshots, or this repository.
|
||||||
|
|
||||||
|
Confirm that `127.0.0.1:4174` is unused. Do not copy `.env`, credentials, model weights, raw media, or a mutable Git checkout into a release.
|
||||||
|
|
||||||
|
## DNS and URL
|
||||||
|
|
||||||
|
No DNS change is required for the approved subpage deployment. The private URL is `https://forge.alexanderwhitestone.com/timmy-staging/`. Caddy remains the only public listener; Node binds only to `127.0.0.1:4174`. Verify `/git/` before and after any separately approved Caddy reload.
|
||||||
|
|
||||||
|
## Install
|
||||||
|
|
||||||
|
These commands are reference commands for an approved maintenance window; they have not been run by this change:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
NODE_SOURCE="$(readlink -f "$(command -v node)")"
|
||||||
|
"$NODE_SOURCE" --version # must report v22.x before installation
|
||||||
|
sudo install -D -o root -g root -m 755 "$NODE_SOURCE" /usr/local/lib/timmy-staging/node
|
||||||
|
/usr/local/lib/timmy-staging/node --version
|
||||||
|
sudo useradd --system --home-dir /nonexistent --no-create-home --shell /usr/sbin/nologin timmy-staging
|
||||||
|
sudo install -d -o root -g root -m 755 /opt/timmy-staging/releases
|
||||||
|
sudo install -d -o timmy-staging -g timmy-staging -m 700 /var/lib/timmy-staging
|
||||||
|
sudo install -o root -g root -m 644 deploy/timmy-staging.service /etc/systemd/system/timmy-staging.service
|
||||||
|
sudo install -o root -g root -m 600 deploy/timmy-staging.env.example /etc/timmy-staging.env
|
||||||
|
sudo chmod 600 /etc/timmy-staging.env
|
||||||
|
sudo systemctl daemon-reload
|
||||||
|
```
|
||||||
|
|
||||||
|
Edit only `TIMMY_RELEASE_TAG` and `TIMMY_RELEASE_COMMIT` for the selected artifact. The unit deliberately applies `TIMMY_AGENT_ENABLED=false` and `TIMMY_VISION_ENABLED=false` on the `ExecStart` command after loading the environment file, so values in that file cannot enable either subsystem. Leave the loopback host, port, and base path unchanged. The environment file must stay root-owned mode 600 and absent from release archives. The copied Node 22 executable is root-owned at `/usr/local/lib/timmy-staging/node`, outside every home directory, so `ProtectHome=true` remains enforceable; replace it only through a separately reviewed Node upgrade.
|
||||||
|
|
||||||
|
Generate Caddy basic-auth material interactively (for example, `caddy hash-password`) and inject it as `TIMMY_STAGING_PASSWORD_HASH`; do not commit the output. Merge the staging handle before the existing catchall, validate a temporary complete config, then use `caddy reload` rather than restarting unrelated services.
|
||||||
|
|
||||||
|
## Promote an immutable release
|
||||||
|
|
||||||
|
First inspect without mutation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/deploy_staging.py --dry-run promote \
|
||||||
|
--tag daily-YYYY-MM-DD.N --archive /secure/inbox/timmy.tar.gz \
|
||||||
|
--sha256 64_LOWERCASE_HEX_CHARACTERS --commit 40_LOWERCASE_HEX_CHARACTERS
|
||||||
|
```
|
||||||
|
|
||||||
|
Then run the same command without `--dry-run`. Promotion opens only a non-symlink regular source, copies it while hashing into a mode-400 file in a private temporary directory, and inspects and extracts only that verified copy before cleaning it. It rejects unsafe members and forbidden artifacts, extracts once to `/opt/timmy-staging/releases/<commit>`, and never overwrites that directory. It atomically swaps `current`, restarts only `timmy-staging.service`, checks bounded health, and invokes `npm run test:staging-smoke` as an argv array with `shell=False`. Restart, health, or smoke failure automatically restores and verifies the prior symlink. If there is no prior release, the tool removes `current` and stops the service instead of restarting it against a missing path; the failed release remains unreferenced as inert immutable evidence for operator review.
|
||||||
|
|
||||||
|
The operator must update `/etc/timmy-staging.env` release identity to the same reviewed tag and commit before promotion. A narrow wrapper/sudo policy may set the smoke environment without granting arbitrary command execution:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export TIMMY_STAGING_URL=https://forge.alexanderwhitestone.com/timmy-staging/
|
||||||
|
export TIMMY_STAGING_USER=staging
|
||||||
|
# Read TIMMY_STAGING_PASSWORD from an approved secret channel; never paste it here.
|
||||||
|
sudo -E python3 scripts/deploy_staging.py promote --tag "$TAG" --archive "$ARCHIVE" --sha256 "$SHA256" --commit "$COMMIT"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Smoke test
|
||||||
|
|
||||||
|
The promotion runs the synthetic 390×844 Playwright suite. It checks edge auth, build identity, one dominant photo action, manual save, urgent-language interception, Journal visibility, export, no horizontal overflow, browser errors, and suspicious secret-bearing responses. It must never use a real stool photo or medical record.
|
||||||
|
|
||||||
|
For an explicit rerun:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
TIMMY_STAGING_URL=https://forge.alexanderwhitestone.com/timmy-staging/ \
|
||||||
|
TIMMY_EXPECT_RELEASE_TAG="$TAG" TIMMY_EXPECT_RELEASE_COMMIT="$COMMIT" \
|
||||||
|
TIMMY_STAGING_USER=staging TIMMY_STAGING_PASSWORD="$(secret-reader)" \
|
||||||
|
npm run test:staging-smoke
|
||||||
|
```
|
||||||
|
|
||||||
|
Also verify `curl --fail http://127.0.0.1:4174/timmy-staging/api/healthz`, authenticated public health, unauthenticated HTTP 401, `/git/`, and that no wildcard/public listener owns port 4174.
|
||||||
|
|
||||||
|
## Status and logs
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/deploy_staging.py status
|
||||||
|
sudo systemctl status timmy-staging.service
|
||||||
|
sudo journalctl -u timmy-staging.service --since today --no-pager
|
||||||
|
# Only after an approved environment-file change:
|
||||||
|
sudo systemctl restart timmy-staging.service
|
||||||
|
```
|
||||||
|
|
||||||
|
Health and logs may show bounded release identity, but must not show environment values, cookies, passwords, tokens, session data, filesystem secrets, or raw user content.
|
||||||
|
|
||||||
|
## Rollback
|
||||||
|
|
||||||
|
List reviewed immutable commit directories, choose the known-good receipt, dry-run, then roll back:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python3 scripts/deploy_staging.py --dry-run rollback --commit 40_LOWERCASE_HEX_CHARACTERS
|
||||||
|
sudo python3 scripts/deploy_staging.py rollback --commit 40_LOWERCASE_HEX_CHARACTERS
|
||||||
|
```
|
||||||
|
|
||||||
|
Rollback atomically repoints `current`, restarts only Timmy, and verifies commit-specific loopback health. If rollback verification fails, the tool restores the release that was current when rollback began. Never repoint the symlink manually while the service is running.
|
||||||
|
|
||||||
|
## Backup
|
||||||
|
|
||||||
|
Browser journal data is local to each browser and is not server state. Before host maintenance, back up only operator-owned receipts and any explicitly required `/var/lib/timmy-staging` state with root-only permissions. Release archives should be recoverable from their verified release source; do not back up `/etc/timmy-staging.env` into a general artifact store. Test restore procedures without live credentials.
|
||||||
|
|
||||||
|
## Template validation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
node --test tests/staging-config.test.js
|
||||||
|
systemd-analyze verify deploy/timmy-staging.service
|
||||||
|
HASH="$(caddy hash-password --plaintext validation-only-password)"
|
||||||
|
TIMMY_STAGING_PASSWORD_HASH="$HASH" caddy validate --config deploy/Caddyfile.staging.example --adapter caddyfile
|
||||||
|
python3 -m py_compile scripts/deploy_staging.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Use only a disposable validation hash. Validation does not authorize installation or reload.
|
||||||
|
|
||||||
|
## Remove staging
|
||||||
|
|
||||||
|
In a separately approved maintenance window: disable and stop only `timmy-staging.service`; remove only the reviewed Caddy staging handle and validate/reload Caddy; verify `/git/`; remove the unit and run `systemctl daemon-reload`; archive required receipts; then remove `/opt/timmy-staging` and `/var/lib/timmy-staging`. Delete the dedicated locked user last. Do not delete shared Caddy, Gitea, TLS, or browser data.
|
||||||
|
|
@ -4,11 +4,12 @@
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js",
|
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js",
|
||||||
"test:ui": "node tests/ui.acceptance.mjs",
|
"test:ui": "node tests/ui.acceptance.mjs",
|
||||||
"test:photo": "node tests/photo-first.acceptance.mjs",
|
"test:photo": "node tests/photo-first.acceptance.mjs",
|
||||||
"test:sleek": "node tests/sleek-chat.acceptance.mjs",
|
"test:sleek": "node tests/sleek-chat.acceptance.mjs",
|
||||||
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py",
|
"test:staging-smoke": "node tests/staging.acceptance.mjs",
|
||||||
|
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py",
|
||||||
"check:diff": "bash scripts/check_diff.sh",
|
"check:diff": "bash scripts/check_diff.sh",
|
||||||
"start": "node server.mjs"
|
"start": "node server.mjs"
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -133,7 +133,7 @@ def main() -> int:
|
||||||
contact_sheet = release_dir / f"timmy-talking-turd-{version}-demo-contact-sheet.jpg"
|
contact_sheet = release_dir / f"timmy-talking-turd-{version}-demo-contact-sheet.jpg"
|
||||||
run(["ffmpeg", "-y", "-v", "error", "-i", str(demo), "-vf", "fps=1/2,scale=180:-1,tile=4x3:padding=4:margin=4", "-frames:v", "1", str(contact_sheet)], tree)
|
run(["ffmpeg", "-y", "-v", "error", "-i", str(demo), "-vf", "fps=1/2,scale=180:-1,tile=4x3:padding=4:margin=4", "-frames:v", "1", str(contact_sheet)], tree)
|
||||||
run(["npm", "audit", "--audit-level=high"], tree)
|
run(["npm", "audit", "--audit-level=high"], tree)
|
||||||
for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js"):
|
for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "tests/staging.acceptance.mjs"):
|
||||||
run(["node", "--check", file], tree)
|
run(["node", "--check", file], tree)
|
||||||
run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree)
|
run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree)
|
||||||
run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree)
|
run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree)
|
||||||
|
|
|
||||||
433
scripts/deploy_staging.py
Executable file
433
scripts/deploy_staging.py
Executable file
|
|
@ -0,0 +1,433 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Promote and roll back immutable Timmy staging release archives safely."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from typing import Callable, Sequence
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
COMMIT_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
TAG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$")
|
||||||
|
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
FORBIDDEN_SUFFIXES = (".pem", ".key", ".p12", ".pfx", ".gguf", ".bin", ".safetensors", ".onnx", ".pyc")
|
||||||
|
FORBIDDEN_COMPONENTS = {".env", ".git", ".ssh", "secrets", "credentials", "__pycache__"}
|
||||||
|
FORBIDDEN_PREFIXES = (("video",), ("artifacts",), ("research", "source-pages"))
|
||||||
|
|
||||||
|
|
||||||
|
class DeploymentError(RuntimeError):
|
||||||
|
"""A bounded, operator-safe deployment failure."""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DeploymentConfig:
|
||||||
|
root: Path
|
||||||
|
restart_command: tuple[str, ...]
|
||||||
|
stop_command: tuple[str, ...]
|
||||||
|
smoke_command: tuple[str, ...]
|
||||||
|
health_url: str
|
||||||
|
command_timeout: float = 180.0
|
||||||
|
health_timeout: float = 30.0
|
||||||
|
max_members: int = 10_000
|
||||||
|
max_member_bytes: int = 50 * 1024 * 1024
|
||||||
|
max_total_bytes: int = 250 * 1024 * 1024
|
||||||
|
|
||||||
|
@property
|
||||||
|
def releases(self) -> Path:
|
||||||
|
return self.root / "releases"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def current(self) -> Path:
|
||||||
|
return self.root / "current"
|
||||||
|
|
||||||
|
|
||||||
|
RunCommand = Callable[..., subprocess.CompletedProcess]
|
||||||
|
HealthCheck = Callable[[str, str, float], dict]
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_identity(tag: str, commit: str, expected_sha256: str) -> None:
|
||||||
|
if not TAG_RE.fullmatch(tag):
|
||||||
|
raise DeploymentError("tag must be 1-80 safe release characters")
|
||||||
|
if not COMMIT_RE.fullmatch(commit):
|
||||||
|
raise DeploymentError("commit must be exactly 40 lowercase hexadecimal characters")
|
||||||
|
if not SHA256_RE.fullmatch(expected_sha256):
|
||||||
|
raise DeploymentError("SHA-256 must be exactly 64 lowercase hexadecimal characters")
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _verified_archive_copy(source_path: Path, expected_sha256: str):
|
||||||
|
"""Copy and hash an untrusted regular archive once, then yield the private copy."""
|
||||||
|
source_fd = -1
|
||||||
|
flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)
|
||||||
|
try:
|
||||||
|
source_fd = os.open(source_path, flags)
|
||||||
|
if not stat.S_ISREG(os.fstat(source_fd).st_mode):
|
||||||
|
raise DeploymentError("archive source must be a regular file, not a symlink or special file")
|
||||||
|
with tempfile.TemporaryDirectory(prefix="timmy-verified-archive-") as private_dir:
|
||||||
|
private_path = Path(private_dir) / "archive"
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with os.fdopen(source_fd, "rb", closefd=True) as source:
|
||||||
|
source_fd = -1
|
||||||
|
with private_path.open("xb") as destination:
|
||||||
|
for chunk in iter(lambda: source.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
destination.write(chunk)
|
||||||
|
destination.flush()
|
||||||
|
os.fsync(destination.fileno())
|
||||||
|
private_path.chmod(0o400)
|
||||||
|
if digest.hexdigest() != expected_sha256:
|
||||||
|
raise DeploymentError("archive SHA-256 mismatch; refusing extraction")
|
||||||
|
yield private_path
|
||||||
|
except DeploymentError:
|
||||||
|
raise
|
||||||
|
except OSError as error:
|
||||||
|
raise DeploymentError(f"cannot stage archive: {error.strerror or 'I/O error'}") from error
|
||||||
|
finally:
|
||||||
|
if source_fd >= 0:
|
||||||
|
os.close(source_fd)
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_parts(name: str) -> tuple[str, ...]:
|
||||||
|
if not name or "\\" in name or "\x00" in name or name.startswith("/") or "//" in name:
|
||||||
|
raise DeploymentError(f"unsafe archive path: {name!r}")
|
||||||
|
path = PurePosixPath(name)
|
||||||
|
if path.is_absolute() or any(part in ("", ".", "..") for part in path.parts):
|
||||||
|
raise DeploymentError(f"unsafe archive path: {name!r}")
|
||||||
|
return path.parts
|
||||||
|
|
||||||
|
|
||||||
|
def _is_forbidden(parts: tuple[str, ...]) -> bool:
|
||||||
|
lowered = tuple(part.lower() for part in parts)
|
||||||
|
basename = lowered[-1]
|
||||||
|
return (
|
||||||
|
any(part in FORBIDDEN_COMPONENTS or part.startswith(".env.") for part in lowered)
|
||||||
|
or basename in {"id_rsa", "id_ed25519", "authorized_keys"}
|
||||||
|
or basename.endswith(FORBIDDEN_SUFFIXES)
|
||||||
|
or any(lowered[:len(prefix)] == prefix for prefix in FORBIDDEN_PREFIXES)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_archive(archive_path: Path, config: DeploymentConfig) -> tuple[list[tarfile.TarInfo], str | None]:
|
||||||
|
"""Validate every tar entry and return members plus a common wrapper directory."""
|
||||||
|
try:
|
||||||
|
with tarfile.open(archive_path, mode="r:*") as archive:
|
||||||
|
members = archive.getmembers()
|
||||||
|
except (OSError, tarfile.TarError) as error:
|
||||||
|
raise DeploymentError("archive is not a readable tar file") from error
|
||||||
|
if not members:
|
||||||
|
raise DeploymentError("archive is empty")
|
||||||
|
if len(members) > config.max_members:
|
||||||
|
raise DeploymentError("archive member count exceeds configured limit")
|
||||||
|
total = 0
|
||||||
|
all_parts: list[tuple[str, ...]] = []
|
||||||
|
for member in members:
|
||||||
|
parts = _safe_parts(member.name)
|
||||||
|
all_parts.append(parts)
|
||||||
|
if not (member.isdir() or member.isreg()):
|
||||||
|
raise DeploymentError(f"unsupported archive member type: {member.name}")
|
||||||
|
if member.size < 0 or member.size > config.max_member_bytes:
|
||||||
|
raise DeploymentError(f"archive member size exceeds configured limit: {member.name}")
|
||||||
|
total += member.size
|
||||||
|
if total > config.max_total_bytes:
|
||||||
|
raise DeploymentError("archive expanded size exceeds configured limit")
|
||||||
|
common_root = all_parts[0][0] if all(parts[0] == all_parts[0][0] for parts in all_parts) else None
|
||||||
|
if common_root and _is_forbidden((common_root,)):
|
||||||
|
raise DeploymentError(f"forbidden archive artifact: {common_root}")
|
||||||
|
for parts in all_parts:
|
||||||
|
relative = parts[1:] if common_root else parts
|
||||||
|
if not relative:
|
||||||
|
continue
|
||||||
|
if _is_forbidden(relative):
|
||||||
|
raise DeploymentError(f"forbidden archive artifact: {'/'.join(relative)}")
|
||||||
|
return members, common_root
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_validated(archive_path: Path, destination: Path, members: list[tarfile.TarInfo], common_root: str | None) -> None:
|
||||||
|
with tarfile.open(archive_path, mode="r:*") as archive:
|
||||||
|
for member in members:
|
||||||
|
parts = _safe_parts(member.name)
|
||||||
|
relative = parts[1:] if common_root else parts
|
||||||
|
if not relative:
|
||||||
|
continue
|
||||||
|
target = destination.joinpath(*relative)
|
||||||
|
if member.isdir():
|
||||||
|
target.mkdir(parents=True, exist_ok=True, mode=0o755)
|
||||||
|
continue
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True, mode=0o755)
|
||||||
|
source = archive.extractfile(member)
|
||||||
|
if source is None:
|
||||||
|
raise DeploymentError(f"could not read archive member: {member.name}")
|
||||||
|
with source, target.open("xb") as output:
|
||||||
|
shutil.copyfileobj(source, output, length=1024 * 1024)
|
||||||
|
target.chmod(0o755 if member.mode & 0o111 else 0o644)
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_deployment_root(config: DeploymentConfig, *, create: bool) -> None:
|
||||||
|
if config.root.is_symlink():
|
||||||
|
raise DeploymentError("deployment root must not be a symlink")
|
||||||
|
if config.root.exists() and not config.root.is_dir():
|
||||||
|
raise DeploymentError("deployment root is not a directory")
|
||||||
|
if create:
|
||||||
|
config.root.mkdir(parents=True, exist_ok=True, mode=0o755)
|
||||||
|
if config.root.is_symlink() or not config.root.is_dir():
|
||||||
|
raise DeploymentError("deployment root is not a safe directory")
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_releases_directory(config: DeploymentConfig, *, create: bool) -> None:
|
||||||
|
_ensure_deployment_root(config, create=create)
|
||||||
|
if config.releases.is_symlink():
|
||||||
|
raise DeploymentError("releases directory must not be a symlink")
|
||||||
|
if config.releases.exists() and not config.releases.is_dir():
|
||||||
|
raise DeploymentError("releases directory is not a directory")
|
||||||
|
if create:
|
||||||
|
config.releases.mkdir(parents=True, exist_ok=True, mode=0o755)
|
||||||
|
|
||||||
|
|
||||||
|
def _current_commit(config: DeploymentConfig) -> str | None:
|
||||||
|
_ensure_releases_directory(config, create=False)
|
||||||
|
if not config.current.is_symlink():
|
||||||
|
if config.current.exists():
|
||||||
|
raise DeploymentError("current exists but is not a symlink")
|
||||||
|
return None
|
||||||
|
target = os.readlink(config.current)
|
||||||
|
target_path = Path(target)
|
||||||
|
resolved = (config.current.parent / target_path).resolve() if not target_path.is_absolute() else target_path.resolve()
|
||||||
|
try:
|
||||||
|
relative = resolved.relative_to(config.releases.resolve())
|
||||||
|
except ValueError as error:
|
||||||
|
raise DeploymentError("current symlink escapes the releases directory") from error
|
||||||
|
if len(relative.parts) != 1 or not COMMIT_RE.fullmatch(relative.name) or not resolved.is_dir():
|
||||||
|
raise DeploymentError("current symlink does not name a valid immutable release")
|
||||||
|
return relative.name
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_point(config: DeploymentConfig, commit: str | None) -> None:
|
||||||
|
_ensure_releases_directory(config, create=False)
|
||||||
|
if config.current.exists() and not config.current.is_symlink():
|
||||||
|
raise DeploymentError("current exists but is not a symlink")
|
||||||
|
temporary = config.root / f".current-{os.getpid()}-{secrets.token_hex(6)}"
|
||||||
|
try:
|
||||||
|
if commit is None:
|
||||||
|
config.current.unlink(missing_ok=True)
|
||||||
|
return
|
||||||
|
temporary.symlink_to(Path("releases") / commit)
|
||||||
|
os.replace(temporary, config.current)
|
||||||
|
finally:
|
||||||
|
temporary.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_argv(argv: Sequence[str], **options) -> subprocess.CompletedProcess:
|
||||||
|
if not argv:
|
||||||
|
raise DeploymentError("configured command must be a non-empty argv array")
|
||||||
|
timeout = float(options.get("timeout", 180.0))
|
||||||
|
return subprocess.run(list(argv), check=True, text=True, capture_output=True, shell=False, timeout=timeout)
|
||||||
|
|
||||||
|
|
||||||
|
def poll_health(url: str, expected_commit: str, timeout: float) -> dict:
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
while True:
|
||||||
|
remaining = deadline - time.monotonic()
|
||||||
|
if remaining <= 0:
|
||||||
|
raise DeploymentError("health check deadline expired")
|
||||||
|
try:
|
||||||
|
request = urllib.request.Request(url, headers={"accept": "application/json"})
|
||||||
|
with urllib.request.urlopen(request, timeout=min(2.0, remaining)) as response:
|
||||||
|
raw = response.read(65_537)
|
||||||
|
if len(raw) > 65_536:
|
||||||
|
raise DeploymentError("health response exceeded 64 KiB")
|
||||||
|
payload = json.loads(raw)
|
||||||
|
if response.status == 200 and payload.get("ok") is True and payload.get("commit") == expected_commit:
|
||||||
|
return payload
|
||||||
|
except DeploymentError:
|
||||||
|
raise
|
||||||
|
except (OSError, urllib.error.URLError, json.JSONDecodeError, AttributeError):
|
||||||
|
pass
|
||||||
|
time.sleep(min(0.2, max(0.0, deadline - time.monotonic())))
|
||||||
|
|
||||||
|
|
||||||
|
def _verify(config: DeploymentConfig, commit: str, run_command: RunCommand, health_check: HealthCheck, *, smoke: bool) -> None:
|
||||||
|
run_command(config.restart_command, check=True, text=True, capture_output=True, shell=False, timeout=config.command_timeout)
|
||||||
|
health_check(config.health_url, commit, config.health_timeout)
|
||||||
|
if smoke:
|
||||||
|
run_command(config.smoke_command, check=True, text=True, capture_output=True, shell=False, timeout=config.command_timeout)
|
||||||
|
|
||||||
|
|
||||||
|
def promote(*, config: DeploymentConfig, tag: str, archive: Path, expected_sha256: str, commit: str,
|
||||||
|
run_command: RunCommand = _run_argv, health_check: HealthCheck = poll_health) -> dict:
|
||||||
|
archive = Path(archive)
|
||||||
|
_validate_identity(tag, commit, expected_sha256)
|
||||||
|
_ensure_deployment_root(config, create=False)
|
||||||
|
with _verified_archive_copy(archive, expected_sha256) as verified_archive:
|
||||||
|
members, common_root = inspect_archive(verified_archive, config)
|
||||||
|
final = config.releases / commit
|
||||||
|
if final.exists() or final.is_symlink():
|
||||||
|
raise DeploymentError(f"immutable release already exists: {commit}")
|
||||||
|
previous = _current_commit(config)
|
||||||
|
_ensure_releases_directory(config, create=True)
|
||||||
|
pending = config.releases / f".pending-{commit}-{secrets.token_hex(6)}"
|
||||||
|
if pending.exists() or pending.is_symlink():
|
||||||
|
raise DeploymentError("pending release boundary already exists")
|
||||||
|
pending.mkdir(mode=0o755)
|
||||||
|
if pending.is_symlink() or not pending.is_dir():
|
||||||
|
raise DeploymentError("pending release boundary is not a safe directory")
|
||||||
|
try:
|
||||||
|
_extract_validated(verified_archive, pending, members, common_root)
|
||||||
|
metadata = {"schemaVersion": 1, "tag": tag, "commit": commit, "sha256": expected_sha256}
|
||||||
|
(pending / ".timmy-release.json").write_text(json.dumps(metadata, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
os.replace(pending, final)
|
||||||
|
except Exception:
|
||||||
|
shutil.rmtree(pending, ignore_errors=True)
|
||||||
|
raise
|
||||||
|
_atomic_point(config, commit)
|
||||||
|
try:
|
||||||
|
_verify(config, commit, run_command, health_check, smoke=True)
|
||||||
|
except Exception as error:
|
||||||
|
_atomic_point(config, previous)
|
||||||
|
try:
|
||||||
|
if previous is not None:
|
||||||
|
_verify(config, previous, run_command, health_check, smoke=False)
|
||||||
|
else:
|
||||||
|
run_command(config.stop_command, check=True, text=True, capture_output=True, shell=False, timeout=config.command_timeout)
|
||||||
|
except Exception as rollback_error:
|
||||||
|
raise DeploymentError("promotion verification failed and rollback verification also failed") from rollback_error
|
||||||
|
if previous is None:
|
||||||
|
raise DeploymentError("promotion verification failed; no prior release; service stopped") from error
|
||||||
|
raise DeploymentError("promotion verification failed; prior release restored") from error
|
||||||
|
return {"ok": True, "action": "promote", **metadata, "previousCommit": previous}
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_release_directory(config: DeploymentConfig, commit: str) -> Path:
|
||||||
|
release = config.releases / commit
|
||||||
|
if not release.is_dir() or release.is_symlink():
|
||||||
|
raise DeploymentError(f"release does not exist: {commit}")
|
||||||
|
metadata_path = release / ".timmy-release.json"
|
||||||
|
entrypoint = release / "server.mjs"
|
||||||
|
if metadata_path.is_symlink() or entrypoint.is_symlink() or not entrypoint.is_file():
|
||||||
|
raise DeploymentError("release metadata or entrypoint is invalid")
|
||||||
|
try:
|
||||||
|
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError) as error:
|
||||||
|
raise DeploymentError("release metadata is missing or invalid") from error
|
||||||
|
if metadata.get("commit") != commit or not TAG_RE.fullmatch(str(metadata.get("tag", ""))):
|
||||||
|
raise DeploymentError("release metadata does not match requested commit")
|
||||||
|
return release
|
||||||
|
|
||||||
|
|
||||||
|
def rollback(*, config: DeploymentConfig, commit: str, run_command: RunCommand = _run_argv,
|
||||||
|
health_check: HealthCheck = poll_health) -> dict:
|
||||||
|
if not COMMIT_RE.fullmatch(commit):
|
||||||
|
raise DeploymentError("commit must be exactly 40 lowercase hexadecimal characters")
|
||||||
|
_validate_release_directory(config, commit)
|
||||||
|
previous = _current_commit(config)
|
||||||
|
if previous == commit:
|
||||||
|
raise DeploymentError("requested release is already current")
|
||||||
|
_atomic_point(config, commit)
|
||||||
|
try:
|
||||||
|
_verify(config, commit, run_command, health_check, smoke=False)
|
||||||
|
except Exception as error:
|
||||||
|
_atomic_point(config, previous)
|
||||||
|
if previous is not None:
|
||||||
|
try:
|
||||||
|
_verify(config, previous, run_command, health_check, smoke=False)
|
||||||
|
except Exception as rollback_error:
|
||||||
|
raise DeploymentError("rollback failed and prior release could not be restored") from rollback_error
|
||||||
|
raise DeploymentError("rollback verification failed; prior release restored") from error
|
||||||
|
return {"ok": True, "action": "rollback", "commit": commit, "previousCommit": previous}
|
||||||
|
|
||||||
|
|
||||||
|
def status(config: DeploymentConfig) -> dict:
|
||||||
|
commit = _current_commit(config)
|
||||||
|
if commit is None:
|
||||||
|
return {"ok": True, "active": False, "commit": None, "release": None}
|
||||||
|
metadata_path = config.releases / commit / ".timmy-release.json"
|
||||||
|
metadata = {}
|
||||||
|
try:
|
||||||
|
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
pass
|
||||||
|
return {"ok": True, "active": True, "commit": commit, "release": metadata.get("tag"), "sha256": metadata.get("sha256")}
|
||||||
|
|
||||||
|
|
||||||
|
def _json_argv(value: str, option: str) -> tuple[str, ...]:
|
||||||
|
try:
|
||||||
|
parsed = json.loads(value)
|
||||||
|
except json.JSONDecodeError as error:
|
||||||
|
raise DeploymentError(f"{option} must be a JSON argv array") from error
|
||||||
|
if not isinstance(parsed, list) or not parsed or not all(isinstance(item, str) and item for item in parsed):
|
||||||
|
raise DeploymentError(f"{option} must be a non-empty JSON argv array of strings")
|
||||||
|
return tuple(parsed)
|
||||||
|
|
||||||
|
|
||||||
|
def build_parser() -> argparse.ArgumentParser:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--root", type=Path, default=Path(os.environ.get("TIMMY_STAGING_ROOT", "/opt/timmy-staging")))
|
||||||
|
parser.add_argument("--dry-run", action="store_true", help="validate and print the intended action without mutation or commands")
|
||||||
|
parser.add_argument("--restart-command", default=os.environ.get("TIMMY_STAGING_RESTART_COMMAND", '["systemctl","restart","timmy-staging.service"]'))
|
||||||
|
parser.add_argument("--stop-command", default=os.environ.get("TIMMY_STAGING_STOP_COMMAND", '["systemctl","stop","timmy-staging.service"]'))
|
||||||
|
parser.add_argument("--smoke-command", default=os.environ.get("TIMMY_STAGING_SMOKE_COMMAND", '["npm","run","test:staging-smoke"]'))
|
||||||
|
parser.add_argument("--health-url", default=os.environ.get("TIMMY_STAGING_HEALTH_URL", "http://127.0.0.1:4174/timmy-staging/api/healthz"))
|
||||||
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||||
|
promote_parser = subparsers.add_parser("promote")
|
||||||
|
promote_parser.add_argument("--tag", required=True)
|
||||||
|
promote_parser.add_argument("--archive", required=True, type=Path)
|
||||||
|
promote_parser.add_argument("--sha256", required=True)
|
||||||
|
promote_parser.add_argument("--commit", required=True)
|
||||||
|
rollback_parser = subparsers.add_parser("rollback")
|
||||||
|
rollback_parser.add_argument("--commit", required=True)
|
||||||
|
subparsers.add_parser("status")
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: Sequence[str] | None = None) -> int:
|
||||||
|
args = build_parser().parse_args(argv)
|
||||||
|
try:
|
||||||
|
config = DeploymentConfig(
|
||||||
|
root=args.root,
|
||||||
|
restart_command=_json_argv(args.restart_command, "restart command"),
|
||||||
|
stop_command=_json_argv(args.stop_command, "stop command"),
|
||||||
|
smoke_command=_json_argv(args.smoke_command, "smoke command"),
|
||||||
|
health_url=args.health_url,
|
||||||
|
)
|
||||||
|
if args.command == "status":
|
||||||
|
result = status(config)
|
||||||
|
elif args.command == "promote":
|
||||||
|
if args.dry_run:
|
||||||
|
_validate_identity(args.tag, args.commit, args.sha256)
|
||||||
|
_ensure_deployment_root(config, create=False)
|
||||||
|
with _verified_archive_copy(args.archive, args.sha256) as verified_archive:
|
||||||
|
members, wrapper = inspect_archive(verified_archive, config)
|
||||||
|
result = {"ok": True, "dryRun": True, "action": "promote", "commit": args.commit, "tag": args.tag, "members": len(members), "wrapper": wrapper}
|
||||||
|
else:
|
||||||
|
result = promote(config=config, tag=args.tag, archive=args.archive, expected_sha256=args.sha256, commit=args.commit)
|
||||||
|
else:
|
||||||
|
if args.dry_run:
|
||||||
|
if not COMMIT_RE.fullmatch(args.commit):
|
||||||
|
raise DeploymentError("rollback release does not exist or commit is invalid")
|
||||||
|
_validate_release_directory(config, args.commit)
|
||||||
|
result = {"ok": True, "dryRun": True, "action": "rollback", "commit": args.commit}
|
||||||
|
else:
|
||||||
|
result = rollback(config=config, commit=args.commit)
|
||||||
|
print(json.dumps(result, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
except DeploymentError as error:
|
||||||
|
print(f"deploy_staging: {error}", file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|
@ -18,7 +18,7 @@ export function resolveVisionConfig(env = process.env) {
|
||||||
if (!PROFILES[profile]) throw new Error('TIMMY_VISION_PROFILE must be hosted or selfhost.');
|
if (!PROFILES[profile]) throw new Error('TIMMY_VISION_PROFILE must be hosted or selfhost.');
|
||||||
const defaults = PROFILES[profile];
|
const defaults = PROFILES[profile];
|
||||||
const config = {
|
const config = {
|
||||||
enabled: env.TIMMY_VISION_ENABLED !== '0',
|
enabled: !['0', 'false'].includes(String(env.TIMMY_VISION_ENABLED || '').toLowerCase()),
|
||||||
profile,
|
profile,
|
||||||
processor: defaults.processor,
|
processor: defaults.processor,
|
||||||
baseUrl: env.TIMMY_VISION_BASE_URL || defaults.baseUrl,
|
baseUrl: env.TIMMY_VISION_BASE_URL || defaults.baseUrl,
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,19 @@ test('default unit suite includes the self-host bootstrap contract', async () =>
|
||||||
assert.match(packageJson.scripts.test, /tests\/selfhost-bootstrap\.test\.js/);
|
assert.match(packageJson.scripts.test, /tests\/selfhost-bootstrap\.test\.js/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('staging smoke is an explicit syntax-gated promotion command, never a PR network call', async () => {
|
||||||
|
const [packageJson, workflow] = await Promise.all([
|
||||||
|
readFile(packagePath, 'utf8').then(JSON.parse),
|
||||||
|
readFile(workflowPath, 'utf8'),
|
||||||
|
]);
|
||||||
|
assert.match(workflow, /python3 tests\/staging-deploy\.test\.py -v/);
|
||||||
|
assert.match(packageJson.scripts.test, /tests\/staging-config\.test\.js/);
|
||||||
|
assert.equal(packageJson.scripts['test:staging-smoke'], 'node tests/staging.acceptance.mjs');
|
||||||
|
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/staging\.acceptance\.mjs/);
|
||||||
|
assert.match(workflow, /node --check tests\/staging\.acceptance\.mjs/);
|
||||||
|
assert.doesNotMatch(workflow, /test:staging-smoke|TIMMY_STAGING_URL/);
|
||||||
|
});
|
||||||
|
|
||||||
test('diff hygiene checks only the pull request or latest commit range', async () => {
|
test('diff hygiene checks only the pull request or latest commit range', async () => {
|
||||||
const script = await readFile(diffCheckPath, 'utf8');
|
const script = await readFile(diffCheckPath, 'utf8');
|
||||||
|
|
||||||
|
|
|
||||||
105
tests/staging-config.test.js
Normal file
105
tests/staging-config.test.js
Normal file
|
|
@ -0,0 +1,105 @@
|
||||||
|
import test from 'node:test';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
import { readFile } from 'node:fs/promises';
|
||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
|
|
||||||
|
const servicePath = new URL('../deploy/timmy-staging.service', import.meta.url);
|
||||||
|
const envPath = new URL('../deploy/timmy-staging.env.example', import.meta.url);
|
||||||
|
const caddyPath = new URL('../deploy/Caddyfile.staging.example', import.meta.url);
|
||||||
|
const runbookPath = new URL('../docs/STAGING-RUNBOOK.md', import.meta.url);
|
||||||
|
|
||||||
|
const literalSecret = /(password|token|secret|api[_-]?key)\s*[=:]\s*(?!\$\{|<|CHANGE_ME|false|$)["']?[A-Za-z0-9/+_.-]{8,}/i;
|
||||||
|
|
||||||
|
test('systemd template runs a dedicated loopback-only agent-disabled service', async () => {
|
||||||
|
const service = await readFile(servicePath, 'utf8');
|
||||||
|
for (const directive of [
|
||||||
|
'User=timmy-staging', 'Group=timmy-staging',
|
||||||
|
'WorkingDirectory=/opt/timmy-staging/current',
|
||||||
|
'EnvironmentFile=/etc/timmy-staging.env',
|
||||||
|
'Environment=HOST=127.0.0.1', 'Environment=PORT=4174',
|
||||||
|
'Environment=TIMMY_BASE_PATH=/timmy-staging',
|
||||||
|
'Environment=TIMMY_AGENT_ENABLED=false',
|
||||||
|
]) assert.match(service, new RegExp(`^${directive.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`, 'm'), directive);
|
||||||
|
assert.match(service, /^ExecStart=\/usr\/bin\/env TIMMY_AGENT_ENABLED=false TIMMY_VISION_ENABLED=false \/usr\/local\/lib\/timmy-staging\/node server\.mjs$/m);
|
||||||
|
assert.doesNotMatch(service, /ExecStart=\/usr\/(?:local\/)?bin\/node|0\.0\.0\.0|TIMMY_AGENT_ENABLED=true/);
|
||||||
|
assert.doesNotMatch(service, literalSecret);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('systemd command-level environment keeps agent and vision disabled after EnvironmentFile overrides', async () => {
|
||||||
|
const service = await readFile(servicePath, 'utf8');
|
||||||
|
const environmentFileIndex = service.indexOf('EnvironmentFile=');
|
||||||
|
const execLine = service.match(/^ExecStart=(.+)$/m)?.[1];
|
||||||
|
assert.ok(execLine, 'ExecStart must exist');
|
||||||
|
assert.ok(environmentFileIndex < service.indexOf(`ExecStart=${execLine}`), 'EnvironmentFile must be loaded before command overrides');
|
||||||
|
const argv = execLine.trim().split(/\s+/);
|
||||||
|
assert.equal(argv.shift(), '/usr/bin/env');
|
||||||
|
const assignments = argv.filter(value => /^TIMMY_(?:AGENT|VISION)_ENABLED=/.test(value));
|
||||||
|
assert.deepEqual(assignments, ['TIMMY_AGENT_ENABLED=false', 'TIMMY_VISION_ENABLED=false']);
|
||||||
|
const probe = spawnSync('/usr/bin/env', [
|
||||||
|
...assignments, process.execPath, '-e',
|
||||||
|
'process.stdout.write(`${process.env.TIMMY_AGENT_ENABLED},${process.env.TIMMY_VISION_ENABLED}`)',
|
||||||
|
], { env: { ...process.env, TIMMY_AGENT_ENABLED: 'true', TIMMY_VISION_ENABLED: 'true' }, encoding: 'utf8' });
|
||||||
|
assert.equal(probe.status, 0, probe.stderr);
|
||||||
|
assert.equal(probe.stdout, 'false,false');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('systemd template applies filesystem privilege process and resource confinement', async () => {
|
||||||
|
const service = await readFile(servicePath, 'utf8');
|
||||||
|
for (const directive of [
|
||||||
|
'UMask=0077', 'NoNewPrivileges=true', 'PrivateTmp=true', 'PrivateDevices=true',
|
||||||
|
'ProtectSystem=strict', 'ProtectHome=true', 'ReadWritePaths=/var/lib/timmy-staging',
|
||||||
|
'RestrictSUIDSGID=true', 'LockPersonality=true', 'RestrictNamespaces=true',
|
||||||
|
'ProtectKernelTunables=true', 'ProtectKernelModules=true', 'ProtectKernelLogs=true',
|
||||||
|
'ProtectControlGroups=true', 'ProtectClock=true', 'ProtectHostname=true',
|
||||||
|
'CapabilityBoundingSet=', 'AmbientCapabilities=', 'RestrictRealtime=true',
|
||||||
|
'TasksMax=64', 'MemoryMax=512M', 'LimitNOFILE=1024',
|
||||||
|
]) assert.match(service, new RegExp(`^${directive.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`, 'm'), directive);
|
||||||
|
assert.equal((service.match(/^ReadWritePaths=/gm) || []).length, 1);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('environment example is least privilege base-path staging configuration without credentials', async () => {
|
||||||
|
const env = await readFile(envPath, 'utf8');
|
||||||
|
for (const setting of [
|
||||||
|
'HOST=127.0.0.1', 'PORT=4174', 'TIMMY_BASE_PATH=/timmy-staging',
|
||||||
|
'TIMMY_STAGING_LABEL=true', 'TIMMY_AGENT_ENABLED=false', 'TIMMY_VISION_ENABLED=false',
|
||||||
|
]) assert.match(env, new RegExp(`^${setting.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`, 'm'));
|
||||||
|
assert.match(env, /install -m 600/);
|
||||||
|
assert.doesNotMatch(env, /TIMMY_AGENT_ACCESS_TOKEN|GITEA_TOKEN|OPENAI_API_KEY|COOKIE/);
|
||||||
|
assert.doesNotMatch(env, literalSecret);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('Caddy snippet isolates git and authenticates staging before a catchall', async () => {
|
||||||
|
const caddy = await readFile(caddyPath, 'utf8');
|
||||||
|
const git = caddy.indexOf('handle @git');
|
||||||
|
const staging = caddy.indexOf('handle_path /timmy-staging/*');
|
||||||
|
const catchall = caddy.indexOf('handle {');
|
||||||
|
assert.ok(git >= 0 && staging > git && catchall > staging, 'route order must be git, staging, catchall');
|
||||||
|
assert.match(caddy, /basic_auth/);
|
||||||
|
assert.match(caddy, /\{\$TIMMY_STAGING_PASSWORD_HASH\}/);
|
||||||
|
assert.match(caddy, /reverse_proxy 127\.0\.0\.1:4174/);
|
||||||
|
assert.match(caddy, /max_size 8MB/);
|
||||||
|
assert.match(caddy, /X-Content-Type-Options "nosniff"/);
|
||||||
|
assert.match(caddy, /X-Frame-Options "DENY"/);
|
||||||
|
assert.match(caddy, /Referrer-Policy "no-referrer"/);
|
||||||
|
assert.match(caddy, /Content-Security-Policy/);
|
||||||
|
assert.match(caddy, /rewrite \* \/timmy-staging\{uri\}/, 'handle_path strips once, then upstream base path is explicitly reconstructed');
|
||||||
|
assert.doesNotMatch(caddy, /\$2[aby]\$[A-Za-z0-9./]{20,}|password\s+[^<{\s]/i);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('runbook covers safe installation operation verification rollback backup and removal', async () => {
|
||||||
|
const runbook = await readFile(runbookPath, 'utf8');
|
||||||
|
for (const heading of [
|
||||||
|
'Prerequisites', 'DNS and URL', 'Install', 'Promote', 'Smoke test', 'Status and logs',
|
||||||
|
'Rollback', 'Backup', 'Remove staging', 'Template validation',
|
||||||
|
]) assert.match(runbook, new RegExp(`^## .*${heading}`, 'mi'), heading);
|
||||||
|
assert.match(runbook, /sha256/i);
|
||||||
|
assert.match(runbook, /install -D -o root -g root -m 755[^\n]+\/usr\/local\/lib\/timmy-staging\/node/);
|
||||||
|
assert.match(runbook, /\/usr\/local\/lib\/timmy-staging\/node --version/);
|
||||||
|
assert.match(runbook, /chmod 600|install -m 600/);
|
||||||
|
assert.match(runbook, /systemctl restart timmy-staging\.service/);
|
||||||
|
assert.match(runbook, /no prior release[^.]*stops[^.]*service/i);
|
||||||
|
assert.match(runbook, /failed release[^.]*inert[^.]*evidence/i);
|
||||||
|
assert.match(runbook, /journalctl -u timmy-staging\.service/);
|
||||||
|
assert.match(runbook, /do not.*live|approval/i);
|
||||||
|
assert.doesNotMatch(runbook, literalSecret);
|
||||||
|
});
|
||||||
344
tests/staging-deploy.test.py
Normal file
344
tests/staging-deploy.test.py
Normal file
|
|
@ -0,0 +1,344 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Filesystem and failure-path tests for immutable staging deployment."""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tarfile
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
SCRIPT = ROOT / "scripts" / "deploy_staging.py"
|
||||||
|
COMMIT_A = "a" * 40
|
||||||
|
COMMIT_B = "b" * 40
|
||||||
|
|
||||||
|
|
||||||
|
def load_deploy():
|
||||||
|
spec = importlib.util.spec_from_file_location("deploy_staging", SCRIPT)
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def make_archive(path: Path, members: list[tuple[str, bytes, str]]) -> str:
|
||||||
|
with tarfile.open(path, "w:gz") as archive:
|
||||||
|
for name, body, kind in members:
|
||||||
|
info = tarfile.TarInfo(name)
|
||||||
|
if kind == "file":
|
||||||
|
info.size = len(body)
|
||||||
|
archive.addfile(info, io.BytesIO(body))
|
||||||
|
elif kind == "dir":
|
||||||
|
info.type = tarfile.DIRTYPE
|
||||||
|
archive.addfile(info)
|
||||||
|
elif kind == "symlink":
|
||||||
|
info.type = tarfile.SYMTYPE
|
||||||
|
info.linkname = "server.mjs"
|
||||||
|
archive.addfile(info)
|
||||||
|
elif kind == "hardlink":
|
||||||
|
info.type = tarfile.LNKTYPE
|
||||||
|
info.linkname = "server.mjs"
|
||||||
|
archive.addfile(info)
|
||||||
|
elif kind == "fifo":
|
||||||
|
info.type = tarfile.FIFOTYPE
|
||||||
|
archive.addfile(info)
|
||||||
|
elif kind == "device":
|
||||||
|
info.type = tarfile.CHRTYPE
|
||||||
|
archive.addfile(info)
|
||||||
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
class DeployTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.root = Path(self.tmp.name) / "opt"
|
||||||
|
self.root.mkdir()
|
||||||
|
self.deploy = load_deploy()
|
||||||
|
self.config = self.deploy.DeploymentConfig(
|
||||||
|
root=self.root,
|
||||||
|
restart_command=("fixture-restart",),
|
||||||
|
stop_command=("fixture-stop",),
|
||||||
|
smoke_command=("fixture-smoke",),
|
||||||
|
health_url="http://127.0.0.1:4174/api/healthz",
|
||||||
|
command_timeout=1.0,
|
||||||
|
health_timeout=1.0,
|
||||||
|
)
|
||||||
|
self.commands = []
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def runner(self, argv, **kwargs):
|
||||||
|
self.commands.append((tuple(argv), kwargs))
|
||||||
|
return subprocess.CompletedProcess(argv, 0, "", "")
|
||||||
|
|
||||||
|
def healthy(self, url, commit, timeout):
|
||||||
|
self.assertEqual(url, self.config.health_url)
|
||||||
|
self.assertEqual(timeout, self.config.health_timeout)
|
||||||
|
return {"ok": True, "commit": commit}
|
||||||
|
|
||||||
|
def archive(self, name="release.tar.gz", members=None):
|
||||||
|
path = Path(self.tmp.name) / name
|
||||||
|
digest = make_archive(path, members or [
|
||||||
|
("timmy-release/", b"", "dir"),
|
||||||
|
("timmy-release/server.mjs", b"console.log('ok')\n", "file"),
|
||||||
|
("timmy-release/package.json", b"{}\n", "file"),
|
||||||
|
])
|
||||||
|
return path, digest
|
||||||
|
|
||||||
|
def seed_release(self, commit):
|
||||||
|
release = self.root / "releases" / commit
|
||||||
|
release.mkdir(parents=True)
|
||||||
|
(release / "server.mjs").write_text("ok", encoding="utf-8")
|
||||||
|
(release / ".timmy-release.json").write_text(json.dumps({"commit": commit, "tag": "old"}), encoding="utf-8")
|
||||||
|
return release
|
||||||
|
|
||||||
|
def point_current(self, commit):
|
||||||
|
(self.root / "current").symlink_to(Path("releases") / commit)
|
||||||
|
|
||||||
|
def promote(self, archive, digest, commit=COMMIT_B, **kwargs):
|
||||||
|
return self.deploy.promote(
|
||||||
|
config=self.config, tag="daily-test", archive=archive,
|
||||||
|
expected_sha256=digest, commit=commit,
|
||||||
|
run_command=kwargs.get("run_command", self.runner),
|
||||||
|
health_check=kwargs.get("health_check", self.healthy),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_checksum_mismatch_fails_before_tar_is_opened(self):
|
||||||
|
archive = Path(self.tmp.name) / "not-even-a-tar"
|
||||||
|
archive.write_bytes(b"untrusted")
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "SHA-256 mismatch"):
|
||||||
|
self.promote(archive, "0" * 64)
|
||||||
|
self.assertFalse((self.root / "releases").exists())
|
||||||
|
|
||||||
|
def test_verified_private_archive_copy_is_used_after_caller_archive_mutates(self):
|
||||||
|
archive, digest = self.archive("mutable.tar.gz")
|
||||||
|
evil = Path(self.tmp.name) / "evil.tar.gz"
|
||||||
|
make_archive(evil, [
|
||||||
|
("timmy-release/", b"", "dir"),
|
||||||
|
("timmy-release/server.mjs", b"EVIL\n", "file"),
|
||||||
|
])
|
||||||
|
real_inspect = self.deploy.inspect_archive
|
||||||
|
inspected_paths = []
|
||||||
|
|
||||||
|
def mutate_then_inspect(path, config):
|
||||||
|
inspected_paths.append(Path(path))
|
||||||
|
archive.write_bytes(evil.read_bytes())
|
||||||
|
return real_inspect(path, config)
|
||||||
|
|
||||||
|
with mock.patch.object(self.deploy, "inspect_archive", side_effect=mutate_then_inspect):
|
||||||
|
self.promote(archive, digest)
|
||||||
|
|
||||||
|
release = self.root / "releases" / COMMIT_B
|
||||||
|
self.assertNotEqual(inspected_paths, [archive])
|
||||||
|
self.assertEqual((release / "server.mjs").read_text(), "console.log('ok')\n")
|
||||||
|
self.assertTrue(all(not path.exists() for path in inspected_paths), "verified temp archive must be cleaned")
|
||||||
|
|
||||||
|
def test_archive_source_must_be_a_nonsymlink_regular_file(self):
|
||||||
|
archive, digest = self.archive("regular.tar.gz")
|
||||||
|
symlink = Path(self.tmp.name) / "archive-link.tar.gz"
|
||||||
|
symlink.symlink_to(archive)
|
||||||
|
for source in (symlink, Path(self.tmp.name)):
|
||||||
|
with self.subTest(source=source), self.assertRaisesRegex(self.deploy.DeploymentError, "regular file|stage archive"):
|
||||||
|
self.promote(source, digest)
|
||||||
|
|
||||||
|
def test_absolute_and_traversal_paths_are_rejected(self):
|
||||||
|
for index, unsafe in enumerate(("/etc/passwd", "root/../../escape", "../escape", "root//double")):
|
||||||
|
archive, digest = self.archive(f"unsafe-{index}.tar.gz", [(unsafe, b"bad", "file")])
|
||||||
|
with self.subTest(unsafe=unsafe), self.assertRaisesRegex(self.deploy.DeploymentError, "unsafe archive path"):
|
||||||
|
self.promote(archive, digest)
|
||||||
|
self.assertFalse((self.root / "releases" / COMMIT_B).exists())
|
||||||
|
|
||||||
|
def test_links_devices_fifos_and_oversized_archives_are_rejected(self):
|
||||||
|
for index, kind in enumerate(("symlink", "hardlink", "fifo", "device")):
|
||||||
|
archive, digest = self.archive(f"special-{index}.tar.gz", [(f"root/bad-{kind}", b"", kind)])
|
||||||
|
with self.subTest(kind=kind), self.assertRaisesRegex(self.deploy.DeploymentError, "unsupported archive member"):
|
||||||
|
self.promote(archive, digest)
|
||||||
|
archive, digest = self.archive("large.tar.gz", [("root/large", b"x" * 17, "file")])
|
||||||
|
tiny = self.deploy.DeploymentConfig(**{**self.config.__dict__, "max_member_bytes": 16})
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "member size"):
|
||||||
|
self.deploy.promote(config=tiny, tag="daily-test", archive=archive, expected_sha256=digest, commit=COMMIT_B, run_command=self.runner, health_check=self.healthy)
|
||||||
|
|
||||||
|
def test_member_count_limit_is_enforced(self):
|
||||||
|
archive, digest = self.archive("many.tar.gz", [(f"root/{i}", b"x", "file") for i in range(3)])
|
||||||
|
tiny = self.deploy.DeploymentConfig(**{**self.config.__dict__, "max_members": 2})
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "member count"):
|
||||||
|
self.deploy.promote(config=tiny, tag="daily-test", archive=archive, expected_sha256=digest, commit=COMMIT_B, run_command=self.runner, health_check=self.healthy)
|
||||||
|
|
||||||
|
def test_secrets_env_and_forbidden_artifacts_are_rejected(self):
|
||||||
|
forbidden = ("root/.env", "root/secrets/token.txt", "root/private.pem", "root/model.gguf", "root/.git/config", "root/video/raw.webm")
|
||||||
|
for index, name in enumerate(forbidden):
|
||||||
|
archive, digest = self.archive(f"secret-{index}.tar.gz", [(name, b"secret", "file")])
|
||||||
|
with self.subTest(name=name), self.assertRaisesRegex(self.deploy.DeploymentError, "forbidden archive artifact"):
|
||||||
|
self.promote(archive, digest)
|
||||||
|
|
||||||
|
def test_forbidden_common_roots_are_rejected_case_insensitively_but_release_wrapper_is_allowed(self):
|
||||||
|
for index, root in enumerate((".git", "ViDeO", "ARTIFACTS", "Credentials")):
|
||||||
|
archive, _ = self.archive(f"forbidden-root-{index}.tar.gz", [
|
||||||
|
(f"{root}/", b"", "dir"),
|
||||||
|
(f"{root}/server.mjs", b"evil", "file"),
|
||||||
|
])
|
||||||
|
with self.subTest(root=root), self.assertRaisesRegex(self.deploy.DeploymentError, "forbidden archive artifact"):
|
||||||
|
self.deploy.inspect_archive(archive, self.config)
|
||||||
|
archive, _ = self.archive("normal-wrapper.tar.gz")
|
||||||
|
_, wrapper = self.deploy.inspect_archive(archive, self.config)
|
||||||
|
self.assertEqual(wrapper, "timmy-release")
|
||||||
|
|
||||||
|
def test_release_root_symlink_is_rejected(self):
|
||||||
|
outside = Path(self.tmp.name) / "outside"
|
||||||
|
outside.mkdir()
|
||||||
|
(self.root / "releases").symlink_to(outside, target_is_directory=True)
|
||||||
|
archive, digest = self.archive()
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "releases directory"):
|
||||||
|
self.promote(archive, digest)
|
||||||
|
self.assertEqual(list(outside.iterdir()), [])
|
||||||
|
|
||||||
|
def test_deployment_root_symlink_is_rejected_before_writing_outside(self):
|
||||||
|
outside = Path(self.tmp.name) / "outside-root"
|
||||||
|
outside.mkdir()
|
||||||
|
self.root.rmdir()
|
||||||
|
self.root.symlink_to(outside, target_is_directory=True)
|
||||||
|
archive, digest = self.archive()
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "deployment root"):
|
||||||
|
self.promote(archive, digest)
|
||||||
|
self.assertEqual(list(outside.iterdir()), [])
|
||||||
|
|
||||||
|
def test_deployment_root_non_directory_is_rejected(self):
|
||||||
|
self.root.rmdir()
|
||||||
|
self.root.write_text("not a directory")
|
||||||
|
archive, digest = self.archive()
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "deployment root is not a directory"):
|
||||||
|
self.promote(archive, digest)
|
||||||
|
|
||||||
|
def test_current_and_pending_symlink_boundaries_are_rejected_before_extraction(self):
|
||||||
|
outside = Path(self.tmp.name) / "outside-boundary"
|
||||||
|
outside.mkdir()
|
||||||
|
(self.root / "releases").mkdir()
|
||||||
|
(self.root / "current").symlink_to(outside, target_is_directory=True)
|
||||||
|
archive, digest = self.archive("current-boundary.tar.gz")
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "current symlink escapes"):
|
||||||
|
self.promote(archive, digest)
|
||||||
|
self.assertEqual(list(outside.iterdir()), [])
|
||||||
|
|
||||||
|
(self.root / "current").unlink()
|
||||||
|
pending = self.root / "releases" / f".pending-{COMMIT_B}-fixed"
|
||||||
|
pending.symlink_to(outside, target_is_directory=True)
|
||||||
|
with mock.patch.object(self.deploy.secrets, "token_hex", return_value="fixed"):
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "pending release boundary"):
|
||||||
|
self.promote(archive, digest)
|
||||||
|
self.assertEqual(list(outside.iterdir()), [])
|
||||||
|
|
||||||
|
def test_valid_archive_extracts_to_commit_release_and_is_never_overwritten(self):
|
||||||
|
archive, digest = self.archive()
|
||||||
|
result = self.promote(archive, digest)
|
||||||
|
release = self.root / "releases" / COMMIT_B
|
||||||
|
self.assertEqual(result["commit"], COMMIT_B)
|
||||||
|
self.assertEqual((release / "server.mjs").read_text(), "console.log('ok')\n")
|
||||||
|
self.assertEqual(json.loads((release / ".timmy-release.json").read_text())["sha256"], digest)
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "already exists"):
|
||||||
|
self.promote(archive, digest)
|
||||||
|
|
||||||
|
def test_success_atomically_swaps_current_and_uses_bounded_argv_commands(self):
|
||||||
|
self.seed_release(COMMIT_A)
|
||||||
|
self.point_current(COMMIT_A)
|
||||||
|
archive, digest = self.archive()
|
||||||
|
self.promote(archive, digest)
|
||||||
|
self.assertEqual((self.root / "current").resolve(), self.root / "releases" / COMMIT_B)
|
||||||
|
self.assertEqual([command for command, _ in self.commands], [("fixture-restart",), ("fixture-smoke",)])
|
||||||
|
self.assertTrue(all(options["shell"] is False and options["timeout"] == 1.0 for _, options in self.commands))
|
||||||
|
self.assertFalse(any(path.name.startswith(".current-") for path in self.root.iterdir()))
|
||||||
|
|
||||||
|
def test_restart_health_and_smoke_failures_automatically_restore_prior_release(self):
|
||||||
|
phases = ("restart", "health", "smoke")
|
||||||
|
for phase in phases:
|
||||||
|
with self.subTest(phase=phase):
|
||||||
|
root = Path(self.tmp.name) / phase
|
||||||
|
config = self.deploy.DeploymentConfig(**{**self.config.__dict__, "root": root})
|
||||||
|
release = root / "releases" / COMMIT_A
|
||||||
|
release.mkdir(parents=True)
|
||||||
|
(release / "server.mjs").write_text("ok")
|
||||||
|
(root / "current").symlink_to(Path("releases") / COMMIT_A)
|
||||||
|
archive, digest = self.archive(f"{phase}.tar.gz")
|
||||||
|
calls = []
|
||||||
|
def run(argv, **kwargs):
|
||||||
|
calls.append(tuple(argv))
|
||||||
|
if (phase == "restart" and len(calls) == 1) or (phase == "smoke" and tuple(argv) == ("fixture-smoke",)):
|
||||||
|
raise subprocess.CalledProcessError(1, argv)
|
||||||
|
return subprocess.CompletedProcess(argv, 0, "", "")
|
||||||
|
def health(url, commit, timeout):
|
||||||
|
if phase == "health" and commit == COMMIT_B:
|
||||||
|
raise self.deploy.DeploymentError("health failed")
|
||||||
|
return {"ok": True, "commit": commit}
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "promotion verification failed"):
|
||||||
|
self.deploy.promote(config=config, tag="daily-test", archive=archive, expected_sha256=digest, commit=COMMIT_B, run_command=run, health_check=health)
|
||||||
|
self.assertEqual((root / "current").resolve(), root / "releases" / COMMIT_A)
|
||||||
|
self.assertGreaterEqual(calls.count(("fixture-restart",)), 2)
|
||||||
|
|
||||||
|
def test_first_promotion_failure_removes_current_and_stops_service_without_restart(self):
|
||||||
|
archive, digest = self.archive("first-failure.tar.gz")
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def run(argv, **kwargs):
|
||||||
|
calls.append(tuple(argv))
|
||||||
|
if tuple(argv) == ("fixture-smoke",):
|
||||||
|
raise subprocess.CalledProcessError(1, argv)
|
||||||
|
return subprocess.CompletedProcess(argv, 0, "", "")
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "no prior release; service stopped"):
|
||||||
|
self.promote(archive, digest, run_command=run)
|
||||||
|
|
||||||
|
self.assertFalse((self.root / "current").exists())
|
||||||
|
self.assertFalse((self.root / "current").is_symlink())
|
||||||
|
self.assertEqual(calls, [("fixture-restart",), ("fixture-smoke",), ("fixture-stop",)])
|
||||||
|
self.assertTrue((self.root / "releases" / COMMIT_B).is_dir(), "failed release remains inert evidence")
|
||||||
|
|
||||||
|
def test_rollback_requires_valid_immutable_release_and_restarts_and_checks_health(self):
|
||||||
|
self.seed_release(COMMIT_A)
|
||||||
|
self.seed_release(COMMIT_B)
|
||||||
|
self.point_current(COMMIT_B)
|
||||||
|
result = self.deploy.rollback(config=self.config, commit=COMMIT_A, run_command=self.runner, health_check=self.healthy)
|
||||||
|
self.assertEqual(result["commit"], COMMIT_A)
|
||||||
|
self.assertEqual((self.root / "current").resolve(), self.root / "releases" / COMMIT_A)
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "does not exist"):
|
||||||
|
self.deploy.rollback(config=self.config, commit="c" * 40, run_command=self.runner, health_check=self.healthy)
|
||||||
|
tampered = self.seed_release("d" * 40)
|
||||||
|
(tampered / ".timmy-release.json").write_text(json.dumps({"commit": COMMIT_A, "tag": "wrong"}))
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "metadata"):
|
||||||
|
self.deploy.rollback(config=self.config, commit="d" * 40, run_command=self.runner, health_check=self.healthy)
|
||||||
|
|
||||||
|
def test_default_command_adapter_accepts_verifier_kwargs_and_stays_bounded(self):
|
||||||
|
result = self.deploy._run_argv(
|
||||||
|
(sys.executable, "-c", "print('ok')"), check=True, text=True,
|
||||||
|
capture_output=True, shell=False, timeout=1.0,
|
||||||
|
)
|
||||||
|
self.assertEqual(result.stdout.strip(), "ok")
|
||||||
|
|
||||||
|
def test_status_and_cli_dry_run_are_rootless_and_machine_readable(self):
|
||||||
|
self.seed_release(COMMIT_A)
|
||||||
|
self.point_current(COMMIT_A)
|
||||||
|
status = self.deploy.status(self.config)
|
||||||
|
self.assertEqual(status["commit"], COMMIT_A)
|
||||||
|
run = subprocess.run([
|
||||||
|
sys.executable, str(SCRIPT), "--root", str(self.root), "--dry-run", "status"
|
||||||
|
], text=True, capture_output=True, check=False)
|
||||||
|
self.assertEqual(run.returncode, 0, run.stderr)
|
||||||
|
self.assertEqual(json.loads(run.stdout)["commit"], COMMIT_A)
|
||||||
|
|
||||||
|
def test_commit_tag_and_checksum_arguments_are_strictly_validated(self):
|
||||||
|
archive, digest = self.archive()
|
||||||
|
for commit in ("abc", "A" * 40, "a" * 41, "../" + "a" * 40):
|
||||||
|
with self.subTest(commit=commit), self.assertRaisesRegex(self.deploy.DeploymentError, "commit"):
|
||||||
|
self.promote(archive, digest, commit=commit)
|
||||||
|
with self.assertRaisesRegex(self.deploy.DeploymentError, "tag"):
|
||||||
|
self.deploy.promote(config=self.config, tag="../bad", archive=archive, expected_sha256=digest, commit=COMMIT_B, run_command=self.runner, health_check=self.healthy)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
98
tests/staging.acceptance.mjs
Normal file
98
tests/staging.acceptance.mjs
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
import { chromium } from 'playwright';
|
||||||
|
import assert from 'node:assert/strict';
|
||||||
|
|
||||||
|
const appUrl = new URL(process.env.TIMMY_STAGING_URL || 'http://127.0.0.1:4173/');
|
||||||
|
const expectedTag = process.env.TIMMY_EXPECT_RELEASE_TAG || 'daily-test';
|
||||||
|
const expectedCommit = process.env.TIMMY_EXPECT_RELEASE_COMMIT || 'c'.repeat(40);
|
||||||
|
const username = process.env.TIMMY_STAGING_USER || '';
|
||||||
|
const password = process.env.TIMMY_STAGING_PASSWORD || '';
|
||||||
|
const loopback = ['127.0.0.1', 'localhost', '::1'].includes(appUrl.hostname);
|
||||||
|
if (!loopback && (!username || !password)) throw new Error('Live staging smoke requires TIMMY_STAGING_USER and TIMMY_STAGING_PASSWORD.');
|
||||||
|
if (!/^[0-9a-f]{40}$/.test(expectedCommit)) throw new Error('TIMMY_EXPECT_RELEASE_COMMIT must be a full lowercase commit.');
|
||||||
|
|
||||||
|
const browser = await chromium.launch({ headless: true });
|
||||||
|
try {
|
||||||
|
const context = await browser.newContext({
|
||||||
|
viewport: { width: 390, height: 844 },
|
||||||
|
deviceScaleFactor: 2,
|
||||||
|
serviceWorkers: 'block',
|
||||||
|
acceptDownloads: true,
|
||||||
|
...(username && password ? { httpCredentials: { username, password } } : {}),
|
||||||
|
});
|
||||||
|
const page = await context.newPage();
|
||||||
|
const browserErrors = [];
|
||||||
|
const secretFindings = [];
|
||||||
|
let agentChatRequests = 0;
|
||||||
|
page.on('console', message => { if (message.type() === 'error') browserErrors.push(message.text()); });
|
||||||
|
page.on('pageerror', error => browserErrors.push(error.message));
|
||||||
|
page.on('request', request => { if (new URL(request.url()).pathname.endsWith('/api/agent/chat')) agentChatRequests += 1; });
|
||||||
|
page.on('response', async response => {
|
||||||
|
const url = new URL(response.url());
|
||||||
|
if (!url.pathname.includes('/api/')) return;
|
||||||
|
for (const name of Object.keys(response.headers())) {
|
||||||
|
if (/authorization|set-cookie|x-api-key/i.test(name)) secretFindings.push(`sensitive response header ${name}`);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const type = response.headers()['content-type'] || '';
|
||||||
|
if (/json|text/.test(type)) {
|
||||||
|
const body = (await response.text()).slice(0, 65_537);
|
||||||
|
if (body.length > 65_536) secretFindings.push('oversized API response');
|
||||||
|
if (/"(?:password|token|cookie|session|api[_-]?key|credential|environment|processEnv)"\s*:/i.test(body)) secretFindings.push(`secret-bearing response ${url.pathname}`);
|
||||||
|
}
|
||||||
|
} catch {}
|
||||||
|
});
|
||||||
|
|
||||||
|
const navigation = await page.goto(appUrl.toString(), { waitUntil: 'networkidle' });
|
||||||
|
assert.equal(navigation?.status(), 200, 'edge authentication and staging navigation must succeed');
|
||||||
|
await page.evaluate(() => localStorage.clear());
|
||||||
|
await page.reload({ waitUntil: 'networkidle' });
|
||||||
|
|
||||||
|
const healthUrl = new URL(`${appUrl.pathname.replace(/\/$/, '')}/api/healthz`, appUrl);
|
||||||
|
const health = await page.evaluate(async url => {
|
||||||
|
const response = await fetch(url, { headers: { accept: 'application/json' } });
|
||||||
|
return { status: response.status, body: await response.json() };
|
||||||
|
}, healthUrl.toString());
|
||||||
|
assert.equal(health.status, 200);
|
||||||
|
assert.equal(health.body.ok, true);
|
||||||
|
assert.equal(health.body.release, expectedTag);
|
||||||
|
assert.equal(health.body.commit, expectedCommit);
|
||||||
|
assert.equal(health.body.agentEnabled, false, 'Phase 1 staging must keep Hermes off');
|
||||||
|
|
||||||
|
assert.match(await page.locator('.staging-label').innerText(), new RegExp(`Staging · ${expectedTag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} · ${expectedCommit.slice(0, 12)}`));
|
||||||
|
assert.equal(await page.locator('main .btn-primary:visible').count(), 1, 'exactly one dominant photo action');
|
||||||
|
assert.equal(await page.locator('[data-log]:visible').count(), 1, 'manual fallback remains visible');
|
||||||
|
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), true, 'home must not overflow');
|
||||||
|
|
||||||
|
await page.locator('[data-log]:visible').click();
|
||||||
|
await page.locator('[data-type="4"]').click();
|
||||||
|
await page.locator('#next').click();
|
||||||
|
await page.locator('#note').fill('synthetic staging smoke');
|
||||||
|
await page.locator('#next').click();
|
||||||
|
await page.locator('#save').click();
|
||||||
|
|
||||||
|
await page.getByRole('button', { name: 'Journal', exact: true }).click();
|
||||||
|
assert.match(await page.locator('main').innerText(), /synthetic staging smoke/);
|
||||||
|
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), true, 'journal must not overflow');
|
||||||
|
|
||||||
|
await page.getByRole('button', { name: 'Timmy', exact: true }).click();
|
||||||
|
await page.locator('#chat-message').fill('I barfed');
|
||||||
|
await page.locator('#send-chat').click();
|
||||||
|
await page.waitForSelector('.bubble.timmy >> text=Pause and get medical help');
|
||||||
|
assert.equal(agentChatRequests, 0, 'urgent phrase must be intercepted before Hermes');
|
||||||
|
|
||||||
|
await page.getByRole('button', { name: 'Journal', exact: true }).click();
|
||||||
|
await page.locator('[data-view="privacy"]').click();
|
||||||
|
const downloadPromise = page.waitForEvent('download');
|
||||||
|
await page.locator('#export').click();
|
||||||
|
const download = await downloadPromise;
|
||||||
|
assert.equal(download.suggestedFilename(), 'timmy-ledger.json');
|
||||||
|
assert.equal(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), true, 'privacy screen must not overflow');
|
||||||
|
|
||||||
|
await page.waitForTimeout(100);
|
||||||
|
assert.deepEqual(secretFindings, []);
|
||||||
|
assert.deepEqual(browserErrors, []);
|
||||||
|
await context.close();
|
||||||
|
console.log(`PASS staging smoke ${expectedTag} ${expectedCommit.slice(0, 12)} at ${appUrl}`);
|
||||||
|
} finally {
|
||||||
|
await browser.close();
|
||||||
|
}
|
||||||
|
|
@ -2,6 +2,11 @@ import test from 'node:test';
|
||||||
import assert from 'node:assert/strict';
|
import assert from 'node:assert/strict';
|
||||||
import { probeVisionProvider, resolveVisionConfig } from '../src/vision-config.js';
|
import { probeVisionProvider, resolveVisionConfig } from '../src/vision-config.js';
|
||||||
|
|
||||||
|
test('documented boolean false disables staging vision fail closed', () => {
|
||||||
|
assert.equal(resolveVisionConfig({ TIMMY_VISION_ENABLED: 'false' }).enabled, false);
|
||||||
|
assert.equal(resolveVisionConfig({ TIMMY_VISION_ENABLED: '0' }).enabled, false);
|
||||||
|
});
|
||||||
|
|
||||||
test('selfhost profile defaults to a loopback OpenAI-compatible server and no remote processor', () => {
|
test('selfhost profile defaults to a loopback OpenAI-compatible server and no remote processor', () => {
|
||||||
const config = resolveVisionConfig({ TIMMY_VISION_PROFILE: 'selfhost' });
|
const config = resolveVisionConfig({ TIMMY_VISION_PROFILE: 'selfhost' });
|
||||||
assert.equal(config.profile, 'selfhost');
|
assert.equal(config.profile, 'selfhost');
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user