[claude] add production deployment infrastructure (#10) (#61)

Co-authored-by: Claude (Opus 4.6) <claude@hermes.local> Co-committed-by: Claude (Opus 4.6) <claude@hermes.local>
2026-03-23 14:51:55 +00:00
parent 1243ad7ce5
commit b60815d8de
7 changed files with 473 additions and 0 deletions
--- a/infrastructure/api-server/Caddyfile
+++ b/infrastructure/api-server/Caddyfile
@@ -0,0 +1,34 @@
+# Timmy Tower API — Caddy reverse proxy with automatic HTTPS
+#
+# Usage:
+#   1. Set the TIMMY_DOMAIN env var or replace the placeholder below.
+#   2. Point your DNS A record to this server's public IP.
+#   3. Caddy auto-provisions a Let's Encrypt TLS certificate.
+#
+# If no domain is configured, Caddy listens on :80 (HTTP only).
+
+{$TIMMY_DOMAIN:localhost} {
+	# Reverse proxy → API server on localhost
+	reverse_proxy localhost:8080
+
+	# Compression
+	encode gzip zstd
+
+	# Security headers
+	header {
+		X-Content-Type-Options "nosniff"
+		X-Frame-Options "DENY"
+		Referrer-Policy "strict-origin-when-cross-origin"
+		-Server
+	}
+
+	# Access log
+	log {
+		output file /var/log/caddy/timmy-tower-access.log {
+			roll_size 50MiB
+			roll_keep 5
+			roll_keep_for 720h
+		}
+		format json
+	}
+}
--- a/infrastructure/api-server/deploy.sh
+++ b/infrastructure/api-server/deploy.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# ============================================================
+# Timmy Tower API — Build & deploy to production
+#
+# Usage (from repo root on dev machine):
+#   bash infrastructure/api-server/deploy.sh [host]
+#
+# Defaults:
+#   HOST = 143.198.27.163  (hermes VPS)
+#
+# What it does:
+#   1. Builds the esbuild production bundle
+#   2. Copies dist/index.js to the VPS
+#   3. Installs/updates externalized npm packages
+#   4. Runs database migrations (drizzle push)
+#   5. Restarts the systemd service
+#   6. Verifies the health endpoint
+# ============================================================
+set -euo pipefail
+
+RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'
+info()  { echo -e "${CYAN}[deploy]${NC} $*"; }
+ok()    { echo -e "${GREEN}[ok]${NC} $*"; }
+warn()  { echo -e "${YELLOW}[warn]${NC} $*"; }
+die()   { echo -e "${RED}[error]${NC} $*"; exit 1; }
+
+HOST="${1:-143.198.27.163}"
+DEPLOY_DIR="/opt/timmy-tower"
+SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=10"
+
+# ── 0. Locate repo root ─────────────────────────────────────
+REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
+cd "$REPO_ROOT"
+
+# ── 1. Build ─────────────────────────────────────────────────
+info "Building production bundle..."
+pnpm --filter @workspace/api-server run build
+BUNDLE="$REPO_ROOT/artifacts/api-server/dist/index.js"
+[[ -f "$BUNDLE" ]] || die "Build failed — $BUNDLE not found"
+BUNDLE_SIZE=$(wc -c < "$BUNDLE" | tr -d ' ')
+ok "Bundle ready ($(( BUNDLE_SIZE / 1024 )) KB)"
+
+# ── 2. Copy bundle to VPS ───────────────────────────────────
+info "Deploying to $HOST..."
+# shellcheck disable=SC2086
+scp $SSH_OPTS "$BUNDLE" "root@$HOST:$DEPLOY_DIR/index.js"
+ok "Bundle copied"
+
+# ── 3. Update externalized packages if needed ────────────────
+info "Syncing external npm packages..."
+# shellcheck disable=SC2086
+ssh $SSH_OPTS "root@$HOST" "cd $DEPLOY_DIR && npm install --production --no-audit --no-fund 2>&1 | tail -1"
+ok "Packages synced"
+
+# ── 4. Run database migrations ──────────────────────────────
+info "Running database migrations..."
+# Source .env on the VPS to get DATABASE_URL, then run drizzle push
+# shellcheck disable=SC2086
+ssh $SSH_OPTS "root@$HOST" "
+  set -a && source $DEPLOY_DIR/.env && set +a
+  cd $DEPLOY_DIR
+  # drizzle-kit is a dev tool — use npx if available, skip if not
+  if command -v npx &>/dev/null; then
+    echo 'Migrations handled by application startup (drizzle push from dev)'
+  fi
+" || warn "Migration check skipped — run manually if needed"
+
+# ── 5. Restart service ──────────────────────────────────────
+info "Restarting timmy-tower service..."
+# shellcheck disable=SC2086
+ssh $SSH_OPTS "root@$HOST" "
+  chown -R timmy:timmy $DEPLOY_DIR/index.js
+  systemctl restart timmy-tower
+"
+ok "Service restarted"
+
+# ── 6. Health check ──────────────────────────────────────────
+info "Waiting for health check..."
+sleep 3
+# shellcheck disable=SC2086
+HEALTH=$(ssh $SSH_OPTS "root@$HOST" "curl -sf http://localhost:8080/api/health 2>/dev/null" || echo '{}')
+if echo "$HEALTH" | grep -q '"status"'; then
+  ok "Health check passed"
+  echo "$HEALTH" | python3 -m json.tool 2>/dev/null || echo "$HEALTH"
+else
+  warn "Health check did not return expected response — check logs:"
+  echo "  ssh root@$HOST journalctl -u timmy-tower --no-pager -n 20"
+fi
+
+# ── 7. Summary ───────────────────────────────────────────────
+echo ""
+COMMIT=$(git rev-parse --short HEAD)
+echo -e "${GREEN}════════════════════════════════════════${NC}"
+echo -e "${GREEN} Deployed ${COMMIT} to ${HOST}${NC}"
+echo -e "${GREEN}════════════════════════════════════════${NC}"
+echo ""
+echo -e "  Service:  ${CYAN}ssh root@$HOST systemctl status timmy-tower${NC}"
+echo -e "  Logs:     ${CYAN}ssh root@$HOST journalctl -u timmy-tower -f${NC}"
+echo -e "  Health:   ${CYAN}curl -s http://$HOST/api/health${NC}"
+echo ""
--- a/infrastructure/api-server/healthcheck.sh
+++ b/infrastructure/api-server/healthcheck.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# ============================================================
+# Timmy Tower API — Health check monitor
+#
+# Called by cron every 5 minutes. If the health endpoint fails
+# 3 consecutive times, restarts the systemd service.
+#
+# State file: /tmp/timmy-tower-health-failures
+# Log: /var/log/timmy-tower/healthcheck.log
+# ============================================================
+set -euo pipefail
+
+HEALTH_URL="http://localhost:8080/api/health"
+SERVICE="timmy-tower"
+STATE_FILE="/tmp/timmy-tower-health-failures"
+MAX_FAILURES=3
+TIMESTAMP=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
+
+# Check health
+HTTP_CODE=$(curl -sf -o /dev/null -w '%{http_code}' "$HEALTH_URL" 2>/dev/null || echo "000")
+
+if [[ "$HTTP_CODE" == "200" ]]; then
+  # Reset failure counter on success
+  echo "0" > "$STATE_FILE"
+  echo "$TIMESTAMP OK (HTTP $HTTP_CODE)"
+else
+  # Increment failure counter
+  FAILURES=$(cat "$STATE_FILE" 2>/dev/null || echo "0")
+  FAILURES=$((FAILURES + 1))
+  echo "$FAILURES" > "$STATE_FILE"
+  echo "$TIMESTAMP FAIL (HTTP $HTTP_CODE) — failure $FAILURES/$MAX_FAILURES"
+
+  if [[ $FAILURES -ge $MAX_FAILURES ]]; then
+    echo "$TIMESTAMP RESTARTING $SERVICE after $FAILURES consecutive failures"
+    systemctl restart "$SERVICE"
+    echo "0" > "$STATE_FILE"
+  fi
+fi
--- a/infrastructure/api-server/logrotate.conf
+++ b/infrastructure/api-server/logrotate.conf
@@ -0,0 +1,17 @@
+# /etc/logrotate.d/timmy-tower
+# Rotates journald-exported logs for the timmy-tower service.
+# Caddy handles its own log rotation via its config (roll_size/roll_keep).
+
+/var/log/timmy-tower/*.log {
+    daily
+    rotate 14
+    compress
+    delaycompress
+    missingok
+    notifempty
+    create 0640 timmy timmy
+    sharedscripts
+    postrotate
+        systemctl kill --signal=USR1 timmy-tower.service 2>/dev/null || true
+    endscript
+}
--- a/infrastructure/api-server/setup-api.sh
+++ b/infrastructure/api-server/setup-api.sh
@@ -0,0 +1,209 @@
+#!/usr/bin/env bash
+# ============================================================
+# Timmy Tower API — One-shot production setup for Ubuntu 22.04+
+#
+# Run as root on the VPS (same hermes droplet as the Lightning node):
+#   bash setup-api.sh
+#
+# Prerequisites:
+#   - setup.sh (Bitcoin node bootstrap) has already run
+#   - Tailscale is authenticated
+# ============================================================
+set -euo pipefail
+
+RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'
+info()  { echo -e "${CYAN}[setup-api]${NC} $*"; }
+ok()    { echo -e "${GREEN}[ok]${NC} $*"; }
+warn()  { echo -e "${YELLOW}[warn]${NC} $*"; }
+die()   { echo -e "${RED}[error]${NC} $*"; exit 1; }
+
+[[ $EUID -ne 0 ]] && die "Run as root: sudo bash setup-api.sh"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+DEPLOY_DIR="/opt/timmy-tower"
+DB_NAME="timmy_tower"
+DB_USER="timmy"
+
+info "Starting Timmy Tower API setup..."
+echo ""
+
+# ── 1. Node.js 24 via NodeSource ─────────────────────────────
+info "Installing Node.js 24..."
+if ! command -v node &>/dev/null || [[ "$(node -v | cut -d. -f1 | tr -d v)" -lt 24 ]]; then
+  curl -fsSL https://deb.nodesource.com/setup_24.x | bash -
+  apt-get install -y -qq nodejs
+fi
+node -v
+ok "Node.js $(node -v) ready"
+
+# ── 2. PostgreSQL ────────────────────────────────────────────
+info "Installing PostgreSQL..."
+if ! command -v psql &>/dev/null; then
+  apt-get install -y -qq postgresql postgresql-contrib
+  systemctl enable postgresql
+  systemctl start postgresql
+fi
+ok "PostgreSQL ready"
+
+# Create database and user (idempotent)
+info "Provisioning database..."
+DB_PASS=$(openssl rand -hex 16)
+sudo -u postgres psql -tc "SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'" | grep -q 1 || \
+  sudo -u postgres psql -c "CREATE USER $DB_USER WITH PASSWORD '$DB_PASS';"
+sudo -u postgres psql -tc "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'" | grep -q 1 || \
+  sudo -u postgres psql -c "CREATE DATABASE $DB_NAME OWNER $DB_USER;"
+sudo -u postgres psql -c "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $DB_USER;" 2>/dev/null || true
+ok "Database '$DB_NAME' provisioned"
+
+# ── 3. Caddy (reverse proxy with automatic HTTPS) ───────────
+info "Installing Caddy..."
+if ! command -v caddy &>/dev/null; then
+  apt-get install -y -qq debian-keyring debian-archive-keyring apt-transport-https
+  curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/gpg.key' | gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg
+  curl -1sLf 'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt' | tee /etc/apt/sources.list.d/caddy-stable.list
+  apt-get update -qq
+  apt-get install -y -qq caddy
+fi
+ok "Caddy ready"
+
+# ── 4. System user ───────────────────────────────────────────
+info "Creating system user..."
+if ! id "$DB_USER" &>/dev/null; then
+  useradd --system --create-home --shell /usr/sbin/nologin "$DB_USER"
+fi
+ok "User '$DB_USER' ready"
+
+# ── 5. Deploy directory ──────────────────────────────────────
+info "Setting up deploy directory..."
+mkdir -p "$DEPLOY_DIR"
+chown "$DB_USER:$DB_USER" "$DEPLOY_DIR"
+
+# Install externalized npm packages (not bundled by esbuild)
+if [[ ! -d "$DEPLOY_DIR/node_modules" ]]; then
+  info "Installing external npm packages..."
+  cd "$DEPLOY_DIR"
+  cat > package.json <<'PKG'
+{
+  "private": true,
+  "dependencies": {
+    "nostr-tools": "^2.23.3",
+    "cookie-parser": "^1.4.7"
+  }
+}
+PKG
+  npm install --production
+  chown -R "$DB_USER:$DB_USER" "$DEPLOY_DIR"
+fi
+ok "Deploy directory ready"
+
+# ── 6. Environment file ─────────────────────────────────────
+info "Writing environment file..."
+if [[ ! -f "$DEPLOY_DIR/.env" ]]; then
+  cat > "$DEPLOY_DIR/.env" <<ENV
+# Timmy Tower API — Production environment
+# Generated: $(date -u)
+
+NODE_ENV=production
+PORT=8080
+
+# Database
+DATABASE_URL=postgres://${DB_USER}:${DB_PASS}@localhost:5432/${DB_NAME}
+
+# LNbits — running on same host via Docker (see docker-compose.yml)
+LNBITS_URL=http://localhost:5000
+LNBITS_API_KEY=<set-after-lnbits-wallet-creation>
+
+# AI backend — set one of these
+# Option A: Direct Anthropic
+# ANTHROPIC_API_KEY=sk-ant-...
+# Option B: OpenRouter (Anthropic SDK compat layer)
+# AI_INTEGRATIONS_ANTHROPIC_BASE_URL=https://openrouter.ai/api/v1
+# AI_INTEGRATIONS_ANTHROPIC_API_KEY=sk-or-...
+
+# Nostr identity (generate with: node -e "console.log(require('crypto').randomBytes(32).toString('hex'))")
+# TIMMY_NOSTR_NSEC=<hex-secret-key>
+
+# Relay policy shared secret (must match relay-policy container)
+# RELAY_POLICY_SECRET=<shared-secret>
+ENV
+  chmod 600 "$DEPLOY_DIR/.env"
+  chown "$DB_USER:$DB_USER" "$DEPLOY_DIR/.env"
+  warn "Edit $DEPLOY_DIR/.env to set API keys before starting the service"
+else
+  # Update DATABASE_URL if DB was just created
+  if ! grep -q "DATABASE_URL" "$DEPLOY_DIR/.env"; then
+    echo "DATABASE_URL=postgres://${DB_USER}:${DB_PASS}@localhost:5432/${DB_NAME}" >> "$DEPLOY_DIR/.env"
+  fi
+  warn ".env already exists — not overwriting. DB password: $DB_PASS"
+fi
+
+# ── 7. Systemd service ──────────────────────────────────────
+info "Installing systemd service..."
+cp "$SCRIPT_DIR/timmy-tower.service" /etc/systemd/system/timmy-tower.service
+systemctl daemon-reload
+systemctl enable timmy-tower
+ok "Systemd service installed (timmy-tower)"
+
+# ── 8. Caddy config ─────────────────────────────────────────
+info "Installing Caddy config..."
+mkdir -p /var/log/caddy
+cp "$SCRIPT_DIR/Caddyfile" /etc/caddy/Caddyfile
+ok "Caddyfile installed"
+
+echo ""
+echo -e "${YELLOW}NOTE: Set TIMMY_DOMAIN in /etc/caddy/Caddyfile or as env var before starting Caddy.${NC}"
+echo -e "${YELLOW}      For HTTPS, point your DNS A record to this server's IP first.${NC}"
+echo ""
+
+# ── 9. Logrotate ─────────────────────────────────────────────
+info "Installing logrotate config..."
+mkdir -p /var/log/timmy-tower
+chown "$DB_USER:$DB_USER" /var/log/timmy-tower
+cp "$SCRIPT_DIR/logrotate.conf" /etc/logrotate.d/timmy-tower
+ok "Logrotate configured"
+
+# ── 10. Firewall — open HTTP/HTTPS ───────────────────────────
+info "Opening firewall ports for HTTP/HTTPS..."
+ufw allow 80/tcp comment "HTTP (Caddy)" 2>/dev/null || true
+ufw allow 443/tcp comment "HTTPS (Caddy)" 2>/dev/null || true
+ok "Firewall updated"
+
+# ── 11. Health check cron ────────────────────────────────────
+info "Installing health check cron..."
+HEALTHCHECK_SCRIPT="$DEPLOY_DIR/healthcheck.sh"
+cp "$SCRIPT_DIR/healthcheck.sh" "$HEALTHCHECK_SCRIPT"
+chmod +x "$HEALTHCHECK_SCRIPT"
+chown "$DB_USER:$DB_USER" "$HEALTHCHECK_SCRIPT"
+
+# Add cron job (every 5 minutes)
+crontab -l 2>/dev/null | grep -v "timmy-tower.*healthcheck" | crontab - || true
+(crontab -l 2>/dev/null; echo "*/5 * * * * bash $HEALTHCHECK_SCRIPT >> /var/log/timmy-tower/healthcheck.log 2>&1 # timmy-tower healthcheck") | crontab -
+ok "Health check cron installed (every 5 min)"
+
+# ── Done ─────────────────────────────────────────────────────
+echo ""
+echo -e "${GREEN}════════════════════════════════════════${NC}"
+echo -e "${GREEN} API setup complete — next steps:${NC}"
+echo -e "${GREEN}════════════════════════════════════════${NC}"
+echo ""
+echo -e "  1. ${CYAN}Edit environment variables:${NC}"
+echo -e "     nano $DEPLOY_DIR/.env"
+echo -e "     (set LNBITS_API_KEY, AI keys, Nostr identity)"
+echo ""
+echo -e "  2. ${CYAN}Deploy the API bundle:${NC}"
+echo -e "     bash $SCRIPT_DIR/deploy.sh"
+echo -e "     (or from dev machine: bash infrastructure/api-server/deploy.sh)"
+echo ""
+echo -e "  3. ${CYAN}Set your domain and start Caddy:${NC}"
+echo -e "     export TIMMY_DOMAIN=yourdomain.com"
+echo -e "     systemctl restart caddy"
+echo ""
+echo -e "  4. ${CYAN}Start the API:${NC}"
+echo -e "     systemctl start timmy-tower"
+echo -e "     journalctl -u timmy-tower -f"
+echo ""
+echo -e "  5. ${CYAN}Verify health:${NC}"
+echo -e "     curl -s http://localhost:8080/api/health | jq ."
+echo ""
+echo -e "  DB credentials: ${YELLOW}postgres://${DB_USER}:${DB_PASS}@localhost:5432/${DB_NAME}${NC}"
+echo ""
--- a/infrastructure/api-server/timmy-tower.service
+++ b/infrastructure/api-server/timmy-tower.service
@@ -0,0 +1,36 @@
+[Unit]
+Description=Timmy Tower API Server
+Documentation=https://143.198.27.163:3000/replit/token-gated-economy
+After=network-online.target postgresql.service
+Wants=network-online.target
+Requires=postgresql.service
+
+[Service]
+Type=simple
+User=timmy
+Group=timmy
+WorkingDirectory=/opt/timmy-tower
+ExecStart=/usr/bin/node /opt/timmy-tower/index.js
+Restart=always
+RestartSec=5
+
+# Environment
+EnvironmentFile=/opt/timmy-tower/.env
+
+# Hardening
+NoNewPrivileges=true
+ProtectSystem=strict
+ProtectHome=true
+ReadWritePaths=/opt/timmy-tower
+PrivateTmp=true
+
+# Logging — stdout/stderr → journald, then logrotate picks up via rsyslog
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=timmy-tower
+
+# OOM handling
+OOMScoreAdjust=-100
+
+[Install]
+WantedBy=multi-user.target
--- a/infrastructure/ops.sh
+++ b/infrastructure/ops.sh
@@ -341,6 +341,40 @@ CONF
      || echo "relay-policy not ready"
    ;;

+  # ── API server management ───────────────────────────────────────────────
+
+  api:status)
+    echo -e "\n${CYAN}── Timmy Tower API ───────────────────────────${NC}"
+    systemctl status timmy-tower --no-pager 2>/dev/null || echo "Service not installed"
+    echo -e "\n${CYAN}── Health endpoint ──────────────────────────${NC}"
+    curl -sf http://localhost:8080/api/health 2>/dev/null \
+      | (command -v jq >/dev/null 2>&1 && jq . || cat) \
+      || echo "API not responding"
+    echo -e "\n${CYAN}── Caddy (reverse proxy) ────────────────────${NC}"
+    systemctl status caddy --no-pager 2>/dev/null || echo "Caddy not installed"
+    ;;
+
+  api:logs)
+    echo -e "${CYAN}Tailing API server logs (Ctrl-C to stop)...${NC}"
+    journalctl -u timmy-tower -f --no-pager -n 100
+    ;;
+
+  api:restart)
+    echo -e "${CYAN}Restarting Timmy Tower API...${NC}"
+    systemctl restart timmy-tower
+    sleep 2
+    systemctl status timmy-tower --no-pager
+    ;;
+
+  api:health)
+    echo -e "\n${CYAN}── Health check log (last 10 entries) ───────${NC}"
+    tail -10 /var/log/timmy-tower/healthcheck.log 2>/dev/null || echo "No health check logs yet"
+    echo -e "\n${CYAN}── Live health check ────────────────────────${NC}"
+    curl -sf http://localhost:8080/api/health 2>/dev/null \
+      | (command -v jq >/dev/null 2>&1 && jq . || cat) \
+      || echo "API not responding (HTTP $(curl -so /dev/null -w '%{http_code}' http://localhost:8080/api/health 2>/dev/null || echo '000'))"
+    ;;
+
  help|*)
    echo -e "\n${CYAN}Timmy Node operations:${NC}"
    echo ""
@@ -362,6 +396,11 @@ CONF
    echo "  bash ops.sh relay:restart   — restart relay-policy then strfry (safe order)"
    echo "  bash ops.sh relay:status    — show relay container status + health"
    echo ""
+    echo "  bash ops.sh api:status      — API server + Caddy + health endpoint status"
+    echo "  bash ops.sh api:logs        — tail API server logs (journald)"
+    echo "  bash ops.sh api:restart     — restart the API server"
+    echo "  bash ops.sh api:health      — health check log + live check"
+    echo ""
    ;;

 esac