From fb36197e8c8a25bc5a22f6447e06bfcc6d087ae0 Mon Sep 17 00:00:00 2001 From: Rockachopa Date: Thu, 30 Apr 2026 01:44:05 -0400 Subject: [PATCH] feat(backup): add automated Gitea daily backup and recovery runbook - Add bin/gitea-backup.sh: daily backup script using gitea dump - Add cron/vps/gitea-daily-backup.yml: Hermes cron job (2 AM daily) - Add docs/backup-recovery-runbook.md: complete recovery procedures Addresses [AUDIT][RISK] Single-node VPS is a single point of failure. Closes #481 --- bin/gitea-backup.sh | 87 ++++++++++++++++++ cron/vps/gitea-daily-backup.yml | 9 ++ docs/backup-recovery-runbook.md | 155 ++++++++++++++++++++++++++++++++ 3 files changed, 251 insertions(+) create mode 100644 bin/gitea-backup.sh create mode 100644 cron/vps/gitea-daily-backup.yml create mode 100644 docs/backup-recovery-runbook.md diff --git a/bin/gitea-backup.sh b/bin/gitea-backup.sh new file mode 100644 index 00000000..3c99ec64 --- /dev/null +++ b/bin/gitea-backup.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Gitea Daily Backup Script +# Uses Gitea's native dump command to create automated backups of repositories and SQLite databases. +# Designed to run on the VPS (Ezra) as part of a daily cron job. +# +# Configuration via environment variables: +# GITEA_BIN Path to gitea binary (default: auto-detect) +# GITEA_BACKUP_DIR Directory for backup archives (default: /var/backups/gitea) +# GITEA_BACKUP_RETENTION Days to retain backups (default: 7) +# GITEA_BACKUP_LOG Log file path (default: /var/log/gitea-backup.log) + +set -euo pipefail + +GITEA_BIN="${GITEA_BIN:-$(command -v gitea 2>/dev/null || echo "/usr/local/bin/gitea")}" +BACKUP_DIR="${GITEA_BACKUP_DIR:-/var/backups/gitea}" +RETENTION_DAYS="${GITEA_BACKUP_RETENTION:-7}" +DATE="$(date +%Y-%m-%d_%H%M%S)" +BACKUP_FILE="${BACKUP_DIR}/gitea-backup-${DATE}.tar.gz" +LOG_FILE="${GITEA_BACKUP_LOG:-/var/log/gitea-backup.log}" + +mkdir -p "${BACKUP_DIR}" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG_FILE}" +} + +log "=== Starting Gitea daily backup ===" + +# Verify gitea binary exists +if [ ! -x "${GITEA_BIN}" ]; then + log "ERROR: Gitea binary not found at ${GITEA_BIN}" + log "Set GITEA_BIN environment variable to the gitea binary path (e.g., /usr/bin/gitea)" + exit 1 +fi + +# Detect Gitea WORK_PATH +WORK_PATH="" +APP_INI="" +for path in /etc/gitea/app.ini /home/git/gitea/custom/conf/app.ini ~/gitea/custom/conf/app.ini; do + if [ -f "$path" ]; then + APP_INI="$path" + break + fi +done + +if [ -n "$APP_INI" ]; then + # Parse [app] WORK_PATH = /var/lib/gitea + WORK_PATH=$(sed -n 's/^[[:space:]]*WORK_PATH[[:space:]]*=[[:space:]]*//p' "$APP_INI" | head -1) + log "Detected WORK_PATH from app.ini: ${WORK_PATH}" +fi + +# Fallback detection +if [ -z "$WORK_PATH" ]; then + for d in /var/lib/gitea /home/git/gitea /srv/gitea /opt/gitea; do + if [ -d "$d" ]; then + WORK_PATH="$d" + break + fi + done + log "Inferred WORK_PATH: ${WORK_PATH:-not found}" +fi + +if [ -z "$WORK_PATH" ]; then + log "ERROR: Could not determine Gitea WORK_PATH. Set GITEA_WORK_PATH manually." + exit 1 +fi + +# Perform gitea dump +# Flags: --work-path sets the Gitea working directory, --file writes dump to tar.gz +log "Running: gitea dump --work-path ${WORK_PATH} --file ${BACKUP_FILE}" +"${GITEA_BIN}" dump --work-path "${WORK_PATH}" --file "${BACKUP_FILE}" 2>>"${LOG_FILE}" + +if [ $? -ne 0 ]; then + log "ERROR: gitea dump failed — check ${LOG_FILE} for details" + exit 1 +fi + +FILE_SIZE=$(du -h "${BACKUP_FILE}" | cut -f1) +log "Backup created: ${BACKUP_FILE} (${FILE_SIZE})" + +# Prune old backups (keep last N days) +find "${BACKUP_DIR}" -name "gitea-backup-*.tar.gz" -type f -mtime +$((${RETENTION_DAYS}-1)) -delete 2>/dev/null || true +log "Pruned backups older than ${RETENTION_DAYS} days" + +log "=== Backup completed successfully ===" + +exit 0 diff --git a/cron/vps/gitea-daily-backup.yml b/cron/vps/gitea-daily-backup.yml new file mode 100644 index 00000000..56e0d4c7 --- /dev/null +++ b/cron/vps/gitea-daily-backup.yml @@ -0,0 +1,9 @@ +- name: Daily Gitea Backup + schedule: '0 2 * * *' # 2:00 AM daily + tasks: + - name: Run Gitea daily backup + shell: bash ~/.hermes/bin/gitea-backup.sh + env: + GITEA_BIN: /usr/local/bin/gitea + GITEA_BACKUP_DIR: /var/backups/gitea + GITEA_BACKUP_RETENTION: "7" diff --git a/docs/backup-recovery-runbook.md b/docs/backup-recovery-runbook.md new file mode 100644 index 00000000..e64a793e --- /dev/null +++ b/docs/backup-recovery-runbook.md @@ -0,0 +1,155 @@ +# Gitea Backup & Recovery Runbook + +**Last updated:** 2026-04-30 +**Scope:** Single-node VPS (Ezra, 143.198.27.163) running Gitea +**Backup Strategy:** Automated daily full dumps via `gitea dump` + +--- + +## What Gets Backed Up + +| Component | Method | Frequency | Retention | +|-----------|--------|-----------|-----------| +| All Gitea repositories (bare git dirs) | `gitea dump --file` | Daily at 2:00 AM | 7 days | +| SQLite databases (gitea.db, indexer.db, etc.) | Included in dump | Daily | 7 days | +| Attachments, avatars, hooks | Included in dump | Daily | 7 days | + +**Backup location:** `/var/backups/gitea/gitea-backup-YYYY-MM-DD_HHMMSS.tar.gz` + +**Log file:** `/var/log/gitea-backup.log` + +--- + +## Backup Architecture + +The backup script `bin/gitea-backup.sh` runs daily via Hermes cron (`cron/vps/gitea-daily-backup.yml`). It: + +1. Locates the Gitea `WORK_PATH` by reading `/etc/gitea/app.ini` or falling back to common locations (`/var/lib/gitea`, `/home/git/gitea`) +2. Invokes `gitea dump --work-path --file ` — Gitea's native, consistent snapshot mechanism +3. Prunes archives older than 7 days +4. Logs all operations to `/var/log/gitea-backup.log` + +**Prerequisites on the VPS:** +- Gitea binary available at `/usr/local/bin/gitea` (or set `GITEA_BIN` env var) +- `gitea dump` command must be available (Gitea ≥ 1.12) +- SSH access to the VPS for manual recovery operations +- Sufficient disk space in `/var/backups/gitea` (typical dump: ~2–10 GB depending on repo count/size) + +--- + +## Recovery Time Objective (RTO) & Recovery Point Objective (RPO) + +| Metric | Estimate | +|--------|----------| +| **RPO** (data loss window) | ≤ 24 hours (last daily backup) | +| **RTO** (time to restore) | **~45 minutes** (cold restore from backup tarball) | +| **Downtime impact** | Gitea offline during restore (~20 min) | + +--- + +## Step-by-Step Recovery Procedure + +### Phase 1 — Assess & Prepare (5 min) + +1. SSH into Ezra VPS: `ssh root@143.198.27.163` +2. Stop Gitea so files are quiescent: + ```bash + systemctl stop gitea + ``` +3. Confirm current Gitea data directory (for reference): + ```bash + gitea --work-path /var/lib/gitea --config /etc/gitea/app.ini dump --help 2>&1 + # Or check app.ini for WORK_PATH + cat /etc/gitea/app.ini | grep '^WORK_PATH' + ``` + +### Phase 2 — Restore from Backup (20 min) + +4. Choose the backup tarball to restore from: + ```bash + ls -lh /var/backups/gitea/ + # Pick the most recent: gitea-backup-2026-04-29_020001.tar.gz + ``` + +5. **Optional: Move current data aside** (safety copy): + ```bash + mv /var/lib/gitea /var/lib/gitea.bak-$(date +%s) + ``` + +6. Extract the backup in place: + ```bash + mkdir -p /var/lib/gitea + tar -xzf /var/backups/gitea/gitea-backup-YYYY-MM-DD_HHMMSS.tar.gz -C /var/lib/gitea --strip-components=1 + ``` + *Note:* `gitea dump` archives contain a single top-level directory `gitea-dump-`. The `--strip-components=1` puts its contents directly into `/var/lib/gitea`. + +7. Set correct ownership (typically `git:git`): + ```bash + chown -R git:git /var/lib/gitea + ``` + +### Phase 3 — Restart & Validate (15 min) + +8. Start Gitea: + ```bash + systemctl start gitea + ``` + +9. Wait 30 seconds, then verify: + ```bash + systemctl status gitea + # Check HTTP endpoint + curl -s -o /dev/null -w '%{http_code}' http://localhost:3000/ # Should be 200 + ``` + +10. Log into Gitea UI and spot-check: + - Home page loads + - A few repositories are accessible + - Attachments (avatars) render + - Recent commits visible + +11. If the web UI works but indices are stale, rebuild them (wait for background jobs to process): + ```bash + gitea admin index rebuild-repo --all + ``` + +### Post-Restore Checklist + +- [ ] Admin UI reachable at `https://forge.alexanderwhitestone.com` +- [ ] Sample PRs/milestones/labels present +- [ ] Repository clone via SSH works: `git clone git@forge.alexanderwhitestone.com:Timmy_Foundation/timmy-config.git` +- [ ] Check backup script health: `cat /var/log/gitea-backup.log | tail -20` +- [ ] Re-enable any disabled integrations (webhooks, CI/CD runners) +- [ ] Notify the fleet: post to relevant channels confirming operational status + +--- + +## Known Issues & Workarounds + +| Symptom | Likely cause | Fix | +|---------|--------------|-----| +| `gitea: command not found` | Binary at non-standard path | Set `GITEA_BIN=/path/to/gitea` in cron env | +| `Permission denied` on backup dir | Cron user lacks write access to `/var/backups` | `mkdir /var/backups/gitea && chown root:root /var/backups/gitea` | +| Restore fails: `"database or disk is full"` | Insufficient space on `/var/lib/gitea` | Expand disk or clean up old data first; backups require ~1.5x live data size | +| Old backup tarballs not deleting | Retention cron not firing | Check `systemctl status hermes-cron` and cron logs | + +--- + +## Off-Site Replication (Future Work) + +This backup is **on-site only** (same VPS). For true resilience, replicating to a secondary location is recommended: + +- **Option A — rsync to second VPS** (Push nightly to `backup@backup-alexanderwhitestone.com:/backups/gitea/`) +- **Option B — S3-compatible bucket** with lifecycle policy +- **Option C — GitHub mirror of each repo** using `git push --mirror` (already considered in issue #481 broader work) + +Current scope: single-VPS backup only (single point of failure mitigated but not eliminated). + +--- + +## Related Documentation + +- `bin/gitea-backup.sh` — backup script source +- `cron/vps/gitea-daily-backup.yml` — Hermes cron definition +- Gitea official docs: +- Hermes cron: