Compare commits
6 Commits
step35/595
...
step35/545
| Author | SHA1 | Date | |
|---|---|---|---|
|
c28d462c4a | ||
|
|
ba4220d5ed | ||
|
|
2451f38bee | ||
|
|
54093991ab | ||
|
|
1ea6bf6e33 | ||
|
|
874ce137b0 |
87
bin/gitea-backup.sh
Normal file
87
bin/gitea-backup.sh
Normal file
@@ -0,0 +1,87 @@
|
||||
#!/bin/bash
|
||||
# Gitea Daily Backup Script
|
||||
# Uses Gitea's native dump command to create automated backups of repositories and SQLite databases.
|
||||
# Designed to run on the VPS (Ezra) as part of a daily cron job.
|
||||
#
|
||||
# Configuration via environment variables:
|
||||
# GITEA_BIN Path to gitea binary (default: auto-detect)
|
||||
# GITEA_BACKUP_DIR Directory for backup archives (default: /var/backups/gitea)
|
||||
# GITEA_BACKUP_RETENTION Days to retain backups (default: 7)
|
||||
# GITEA_BACKUP_LOG Log file path (default: /var/log/gitea-backup.log)
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
GITEA_BIN="${GITEA_BIN:-$(command -v gitea 2>/dev/null || echo "/usr/local/bin/gitea")}"
|
||||
BACKUP_DIR="${GITEA_BACKUP_DIR:-/var/backups/gitea}"
|
||||
RETENTION_DAYS="${GITEA_BACKUP_RETENTION:-7}"
|
||||
DATE="$(date +%Y-%m-%d_%H%M%S)"
|
||||
BACKUP_FILE="${BACKUP_DIR}/gitea-backup-${DATE}.tar.gz"
|
||||
LOG_FILE="${GITEA_BACKUP_LOG:-/var/log/gitea-backup.log}"
|
||||
|
||||
mkdir -p "${BACKUP_DIR}"
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log "=== Starting Gitea daily backup ==="
|
||||
|
||||
# Verify gitea binary exists
|
||||
if [ ! -x "${GITEA_BIN}" ]; then
|
||||
log "ERROR: Gitea binary not found at ${GITEA_BIN}"
|
||||
log "Set GITEA_BIN environment variable to the gitea binary path (e.g., /usr/bin/gitea)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Detect Gitea WORK_PATH
|
||||
WORK_PATH=""
|
||||
APP_INI=""
|
||||
for path in /etc/gitea/app.ini /home/git/gitea/custom/conf/app.ini ~/gitea/custom/conf/app.ini; do
|
||||
if [ -f "$path" ]; then
|
||||
APP_INI="$path"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -n "$APP_INI" ]; then
|
||||
# Parse [app] WORK_PATH = /var/lib/gitea
|
||||
WORK_PATH=$(sed -n 's/^[[:space:]]*WORK_PATH[[:space:]]*=[[:space:]]*//p' "$APP_INI" | head -1)
|
||||
log "Detected WORK_PATH from app.ini: ${WORK_PATH}"
|
||||
fi
|
||||
|
||||
# Fallback detection
|
||||
if [ -z "$WORK_PATH" ]; then
|
||||
for d in /var/lib/gitea /home/git/gitea /srv/gitea /opt/gitea; do
|
||||
if [ -d "$d" ]; then
|
||||
WORK_PATH="$d"
|
||||
break
|
||||
fi
|
||||
done
|
||||
log "Inferred WORK_PATH: ${WORK_PATH:-not found}"
|
||||
fi
|
||||
|
||||
if [ -z "$WORK_PATH" ]; then
|
||||
log "ERROR: Could not determine Gitea WORK_PATH. Set GITEA_WORK_PATH manually."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Perform gitea dump
|
||||
# Flags: --work-path sets the Gitea working directory, --file writes dump to tar.gz
|
||||
log "Running: gitea dump --work-path ${WORK_PATH} --file ${BACKUP_FILE}"
|
||||
"${GITEA_BIN}" dump --work-path "${WORK_PATH}" --file "${BACKUP_FILE}" 2>>"${LOG_FILE}"
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
log "ERROR: gitea dump failed — check ${LOG_FILE} for details"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FILE_SIZE=$(du -h "${BACKUP_FILE}" | cut -f1)
|
||||
log "Backup created: ${BACKUP_FILE} (${FILE_SIZE})"
|
||||
|
||||
# Prune old backups (keep last N days)
|
||||
find "${BACKUP_DIR}" -name "gitea-backup-*.tar.gz" -type f -mtime +$((${RETENTION_DAYS}-1)) -delete 2>/dev/null || true
|
||||
log "Pruned backups older than ${RETENTION_DAYS} days"
|
||||
|
||||
log "=== Backup completed successfully ==="
|
||||
|
||||
exit 0
|
||||
@@ -129,20 +129,42 @@ Preserved by timmy-orchestrator to prevent loss." 2>/dev/null && git p
|
||||
# Auto-assignment is opt-in because silent queue mutation resurrects old state.
|
||||
if [ "$unassigned_count" -gt 0 ]; then
|
||||
if [ "$AUTO_ASSIGN_UNASSIGNED" = "1" ]; then
|
||||
log "Assigning $unassigned_count issues to claude..."
|
||||
log "Assigning $unassigned_count issues via dispatch router..."
|
||||
DISPATCH_LOG="$LOG_DIR/dispatch_decisions.log"
|
||||
while IFS= read -r line; do
|
||||
local repo=$(echo "$line" | sed 's/.*REPO=\([^ ]*\).*/\1/')
|
||||
local num=$(echo "$line" | sed 's/.*NUM=\([^ ]*\).*/\1/')
|
||||
curl -sf -X PATCH "$GITEA_URL/api/v1/repos/$repo/issues/$num" \
|
||||
-H "Authorization: token $GITEA_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"assignees":["claude"]}' >/dev/null 2>&1 && \
|
||||
log " Assigned #$num ($repo) to claude"
|
||||
local repo=$(echo "$line" | sed 's/.*REPO=\([^ ]*\).*//')
|
||||
local num=$(echo "$line" | sed 's/.*NUM=\([^ ]*\).*//')
|
||||
local title=$(echo "$line" | sed 's/.*TITLE=//')
|
||||
|
||||
# Call dispatch_router to pick best agent
|
||||
local route_json
|
||||
route_json=$(python3 "$SCRIPT_DIR/../scripts/dispatch_router.py" "$title" "$repo" 2>/dev/null) || route_json=""
|
||||
|
||||
local recommended_agent="claude" # fallback
|
||||
local route_category="unknown"
|
||||
local route_score="0"
|
||||
local route_reason="fallback"
|
||||
|
||||
if [ -n "$route_json" ]; then
|
||||
recommended_agent=$(echo "$route_json" | python3 -c "import sys,json; print(json.load(sys.stdin).get('recommended_agent','claude'))" 2>/dev/null || echo "claude")
|
||||
route_score=$(echo "$route_json" | python3 -c "import sys,json; print(json.load(sys.stdin).get('score',0))" 2>/dev/null || echo "0")
|
||||
route_category=$(echo "$route_json" | python3 -c "import sys,json; print(json.load(sys.stdin).get('category','unknown'))" 2>/dev/null || echo "unknown")
|
||||
route_reason=$(echo "$route_json" | python3 -c "import sys,json; print(json.load(sys.stdin).get('reason',''))" 2>/dev/null || echo "")
|
||||
fi
|
||||
|
||||
# Assign via API
|
||||
curl -sf -X PATCH "$GITEA_URL/api/v1/repos/$repo/issues/$num" \\
|
||||
-H "Authorization: token $GITEA_TOKEN" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
-d "{\"assignees\":[\"$recommended_agent\"]}" >/dev/null 2>&1 && \\
|
||||
log " Assigned #$num ($repo) to $recommended_agent [score=$route_score cat=$route_category]"
|
||||
|
||||
# Log dispatch decision for audit (RFC3339 timestamp)
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
|
||||
"$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$num" "$repo" "$title" "$recommended_agent" "$route_score" "$route_category|$route_reason" \
|
||||
>> "$DISPATCH_LOG"
|
||||
done < "$state_dir/unassigned.txt"
|
||||
else
|
||||
log "Auto-assign disabled: leaving $unassigned_count unassigned issues untouched"
|
||||
fi
|
||||
fi
|
||||
else fi
|
||||
|
||||
# Phase 2: PR review via Timmy (LLM)
|
||||
if [ "$pr_count" -gt 0 ]; then
|
||||
|
||||
9
cron/vps/gitea-daily-backup.yml
Normal file
9
cron/vps/gitea-daily-backup.yml
Normal file
@@ -0,0 +1,9 @@
|
||||
- name: Daily Gitea Backup
|
||||
schedule: '0 2 * * *' # 2:00 AM daily
|
||||
tasks:
|
||||
- name: Run Gitea daily backup
|
||||
shell: bash ~/.hermes/bin/gitea-backup.sh
|
||||
env:
|
||||
GITEA_BIN: /usr/local/bin/gitea
|
||||
GITEA_BACKUP_DIR: /var/backups/gitea
|
||||
GITEA_BACKUP_RETENTION: "7"
|
||||
@@ -10,6 +10,7 @@ Applied fixes identified by the accessibility audit (#492):
|
||||
| R2 | #552 | 3.3.1 | `aria-required="true"` on required form fields |
|
||||
| R3 | #553 | 4.1.2 | `aria-label` on star/fork count links ("2 stars", "0 forks") |
|
||||
| R4 | #554 | 1.3.1 | `<time datetime="...">` elements for relative timestamps |
|
||||
| R5 | #545 | 2.4.1 | Skip navigation link for keyboard users |
|
||||
|
||||
## Structure
|
||||
|
||||
@@ -20,14 +21,18 @@ deploy/gitea-a11y/
|
||||
└── custom/
|
||||
├── public/
|
||||
│ ├── css/
|
||||
│ │ └── a11y-fixes.css # Global CSS fixes (R4, R5)
|
||||
│ └── js/
|
||||
└── templates/
|
||||
├── custom/
|
||||
│ ├── a11y_head.tmpl # R4: Inject CSS into <head>
|
||||
│ ├── header_banner.tmpl # Bypass: <header role="banner">
|
||||
│ ├── skip_link.tmpl # R5: Skip navigation link (WCAG 2.4.1)
|
||||
│ └── time_relative.tmpl # R4: <time> helper
|
||||
├── repo/
|
||||
│ └── list_a11y.tmpl # R3: aria-label on counts
|
||||
└── user/auth/
|
||||
└── signin_inner.tmpl # R1+R2: password toggle + aria-required
|
||||
├── user/auth/
|
||||
│ └── signin_inner.tmpl # R1+R2: password toggle + aria-required
|
||||
└── repo/
|
||||
└── list_a11y.tmpl # R3: aria-label on star/fork counts
|
||||
```
|
||||
|
||||
## Deploy
|
||||
@@ -60,5 +65,24 @@ readers announce the meaning, not just the number.
|
||||
|
||||
### R4: `<time>` Elements
|
||||
|
||||
Wraps relative timestamps ("2 minutes ago") in `<time datetime="2026-04-13T17:00:00Z">`
|
||||
Wraps relative timestamps ("2 minutes ago") in `<time datetime="...">`
|
||||
providing both human-readable text and machine-readable ISO 8601 dates.
|
||||
|
||||
### R5: Skip Navigation Link (WCAG 2.4.1)
|
||||
|
||||
Adds a "Skip to main content" link as the first focusable element.
|
||||
Keyboard users can press Tab at the top of a page to jump directly
|
||||
to the main content area, bypassing repeated navigation menus.
|
||||
|
||||
**Template:** `custom/skip_link.tmpl`
|
||||
**Insert:** At the very start of `<body>` in your custom `header.tmpl`:
|
||||
|
||||
```handlebars
|
||||
{{template "custom/skip_link" .}}
|
||||
```
|
||||
|
||||
Gitea's default page layout uses `<main id="main">` as the content target.
|
||||
If your instance uses `#content`, adjust the `href` in the template.
|
||||
|
||||
The CSS in `a11y-fixes.css` visually hides the link off-screen until it
|
||||
receives keyboard focus, meeting WCAG 2.4.1 Bypass Blocks (A).
|
||||
|
||||
@@ -9,3 +9,25 @@
|
||||
.markdown-body a:focus {
|
||||
color: #3a5518 !important;
|
||||
}
|
||||
|
||||
/* V5 (#545): Skip navigation link (WCAG 2.4.1) — visible on focus only */
|
||||
.skip-link {
|
||||
position: absolute;
|
||||
top: -100px;
|
||||
left: 8px;
|
||||
background: #1a1a1a;
|
||||
color: #fff;
|
||||
padding: 12px 16px;
|
||||
font-size: 14px;
|
||||
font-weight: 600;
|
||||
z-index: 9999;
|
||||
border-radius: 0 0 4px 4px;
|
||||
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.25);
|
||||
transition: top 0.2s ease-in-out, background 0.2s;
|
||||
text-decoration: none;
|
||||
}
|
||||
.skip-link:focus {
|
||||
top: 0;
|
||||
outline: 3px solid #ffcc00;
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
20
deploy/gitea-a11y/custom/templates/custom/skip_link.tmpl
Normal file
20
deploy/gitea-a11y/custom/templates/custom/skip_link.tmpl
Normal file
@@ -0,0 +1,20 @@
|
||||
{{/*
|
||||
Gitea a11y fix: V1 — Skip navigation link (WCAG 2.4.1)
|
||||
|
||||
Provides a "Skip to main content" link as the first focusable element
|
||||
in the document. Screen reader and keyboard users can activate it
|
||||
to bypass repeated navigation and go straight to the main content.
|
||||
|
||||
Usage: Insert immediately after the opening <body> tag.
|
||||
Include via custom/header.tmpl:
|
||||
|
||||
{{template "custom/skip_link" .}}
|
||||
|
||||
The target should match the id of the main content region. Gitea's
|
||||
main area uses id="main" by default. If your instance uses a
|
||||
different id (e.g. "content"), adjust the href accordingly.
|
||||
|
||||
WCAG 2.4.1 Bypass Blocks — A.
|
||||
*/}}
|
||||
|
||||
<a href="#main" class="skip-link" aria-label="Skip to main content">Skip to main content</a>
|
||||
@@ -6,9 +6,10 @@
|
||||
#
|
||||
# Fixes:
|
||||
# R1: Password visibility toggle on sign-in (#551)
|
||||
# R2: aria-required on required form fields (#552)
|
||||
# R3: aria-label on star/fork count links (#553)
|
||||
# R2: aria-required on required fields (#552)
|
||||
# R3: aria-label on star/fork counts (#553)
|
||||
# R4: <time> elements for relative timestamps (#554)
|
||||
# R5: Skip navigation link for keyboard users (#545)
|
||||
#
|
||||
# Usage:
|
||||
# bash deploy/gitea-a11y/deploy-gitea-a11y.sh [ssh-host]
|
||||
@@ -53,5 +54,6 @@ echo " R1: Password toggle on /user/sign_in"
|
||||
echo " R2: aria-required on required fields"
|
||||
echo " R3: aria-label on star/fork counts"
|
||||
echo " R4: <time> elements on timestamps"
|
||||
echo " R5: Skip navigation link (WCAG 2.4.1)"
|
||||
echo ""
|
||||
echo "Verify at: https://forge.alexanderwhitestone.com/user/sign_in"
|
||||
|
||||
155
docs/backup-recovery-runbook.md
Normal file
155
docs/backup-recovery-runbook.md
Normal file
@@ -0,0 +1,155 @@
|
||||
# Gitea Backup & Recovery Runbook
|
||||
|
||||
**Last updated:** 2026-04-30
|
||||
**Scope:** Single-node VPS (Ezra, 143.198.27.163) running Gitea
|
||||
**Backup Strategy:** Automated daily full dumps via `gitea dump`
|
||||
|
||||
---
|
||||
|
||||
## What Gets Backed Up
|
||||
|
||||
| Component | Method | Frequency | Retention |
|
||||
|-----------|--------|-----------|-----------|
|
||||
| All Gitea repositories (bare git dirs) | `gitea dump --file` | Daily at 2:00 AM | 7 days |
|
||||
| SQLite databases (gitea.db, indexer.db, etc.) | Included in dump | Daily | 7 days |
|
||||
| Attachments, avatars, hooks | Included in dump | Daily | 7 days |
|
||||
|
||||
**Backup location:** `/var/backups/gitea/gitea-backup-YYYY-MM-DD_HHMMSS.tar.gz`
|
||||
|
||||
**Log file:** `/var/log/gitea-backup.log`
|
||||
|
||||
---
|
||||
|
||||
## Backup Architecture
|
||||
|
||||
The backup script `bin/gitea-backup.sh` runs daily via Hermes cron (`cron/vps/gitea-daily-backup.yml`). It:
|
||||
|
||||
1. Locates the Gitea `WORK_PATH` by reading `/etc/gitea/app.ini` or falling back to common locations (`/var/lib/gitea`, `/home/git/gitea`)
|
||||
2. Invokes `gitea dump --work-path <path> --file <backup-tar.gz>` — Gitea's native, consistent snapshot mechanism
|
||||
3. Prunes archives older than 7 days
|
||||
4. Logs all operations to `/var/log/gitea-backup.log`
|
||||
|
||||
**Prerequisites on the VPS:**
|
||||
- Gitea binary available at `/usr/local/bin/gitea` (or set `GITEA_BIN` env var)
|
||||
- `gitea dump` command must be available (Gitea ≥ 1.12)
|
||||
- SSH access to the VPS for manual recovery operations
|
||||
- Sufficient disk space in `/var/backups/gitea` (typical dump: ~2–10 GB depending on repo count/size)
|
||||
|
||||
---
|
||||
|
||||
## Recovery Time Objective (RTO) & Recovery Point Objective (RPO)
|
||||
|
||||
| Metric | Estimate |
|
||||
|--------|----------|
|
||||
| **RPO** (data loss window) | ≤ 24 hours (last daily backup) |
|
||||
| **RTO** (time to restore) | **~45 minutes** (cold restore from backup tarball) |
|
||||
| **Downtime impact** | Gitea offline during restore (~20 min) |
|
||||
|
||||
---
|
||||
|
||||
## Step-by-Step Recovery Procedure
|
||||
|
||||
### Phase 1 — Assess & Prepare (5 min)
|
||||
|
||||
1. SSH into Ezra VPS: `ssh root@143.198.27.163`
|
||||
2. Stop Gitea so files are quiescent:
|
||||
```bash
|
||||
systemctl stop gitea
|
||||
```
|
||||
3. Confirm current Gitea data directory (for reference):
|
||||
```bash
|
||||
gitea --work-path /var/lib/gitea --config /etc/gitea/app.ini dump --help 2>&1
|
||||
# Or check app.ini for WORK_PATH
|
||||
cat /etc/gitea/app.ini | grep '^WORK_PATH'
|
||||
```
|
||||
|
||||
### Phase 2 — Restore from Backup (20 min)
|
||||
|
||||
4. Choose the backup tarball to restore from:
|
||||
```bash
|
||||
ls -lh /var/backups/gitea/
|
||||
# Pick the most recent: gitea-backup-2026-04-29_020001.tar.gz
|
||||
```
|
||||
|
||||
5. **Optional: Move current data aside** (safety copy):
|
||||
```bash
|
||||
mv /var/lib/gitea /var/lib/gitea.bak-$(date +%s)
|
||||
```
|
||||
|
||||
6. Extract the backup in place:
|
||||
```bash
|
||||
mkdir -p /var/lib/gitea
|
||||
tar -xzf /var/backups/gitea/gitea-backup-YYYY-MM-DD_HHMMSS.tar.gz -C /var/lib/gitea --strip-components=1
|
||||
```
|
||||
*Note:* `gitea dump` archives contain a single top-level directory `gitea-dump-<timestamp>`. The `--strip-components=1` puts its contents directly into `/var/lib/gitea`.
|
||||
|
||||
7. Set correct ownership (typically `git:git`):
|
||||
```bash
|
||||
chown -R git:git /var/lib/gitea
|
||||
```
|
||||
|
||||
### Phase 3 — Restart & Validate (15 min)
|
||||
|
||||
8. Start Gitea:
|
||||
```bash
|
||||
systemctl start gitea
|
||||
```
|
||||
|
||||
9. Wait 30 seconds, then verify:
|
||||
```bash
|
||||
systemctl status gitea
|
||||
# Check HTTP endpoint
|
||||
curl -s -o /dev/null -w '%{http_code}' http://localhost:3000/ # Should be 200
|
||||
```
|
||||
|
||||
10. Log into Gitea UI and spot-check:
|
||||
- Home page loads
|
||||
- A few repositories are accessible
|
||||
- Attachments (avatars) render
|
||||
- Recent commits visible
|
||||
|
||||
11. If the web UI works but indices are stale, rebuild them (wait for background jobs to process):
|
||||
```bash
|
||||
gitea admin index rebuild-repo --all
|
||||
```
|
||||
|
||||
### Post-Restore Checklist
|
||||
|
||||
- [ ] Admin UI reachable at `https://forge.alexanderwhitestone.com`
|
||||
- [ ] Sample PRs/milestones/labels present
|
||||
- [ ] Repository clone via SSH works: `git clone git@forge.alexanderwhitestone.com:Timmy_Foundation/timmy-config.git`
|
||||
- [ ] Check backup script health: `cat /var/log/gitea-backup.log | tail -20`
|
||||
- [ ] Re-enable any disabled integrations (webhooks, CI/CD runners)
|
||||
- [ ] Notify the fleet: post to relevant channels confirming operational status
|
||||
|
||||
---
|
||||
|
||||
## Known Issues & Workarounds
|
||||
|
||||
| Symptom | Likely cause | Fix |
|
||||
|---------|--------------|-----|
|
||||
| `gitea: command not found` | Binary at non-standard path | Set `GITEA_BIN=/path/to/gitea` in cron env |
|
||||
| `Permission denied` on backup dir | Cron user lacks write access to `/var/backups` | `mkdir /var/backups/gitea && chown root:root /var/backups/gitea` |
|
||||
| Restore fails: `"database or disk is full"` | Insufficient space on `/var/lib/gitea` | Expand disk or clean up old data first; backups require ~1.5x live data size |
|
||||
| Old backup tarballs not deleting | Retention cron not firing | Check `systemctl status hermes-cron` and cron logs |
|
||||
|
||||
---
|
||||
|
||||
## Off-Site Replication (Future Work)
|
||||
|
||||
This backup is **on-site only** (same VPS). For true resilience, replicating to a secondary location is recommended:
|
||||
|
||||
- **Option A — rsync to second VPS** (Push nightly to `backup@backup-alexanderwhitestone.com:/backups/gitea/`)
|
||||
- **Option B — S3-compatible bucket** with lifecycle policy
|
||||
- **Option C — GitHub mirror of each repo** using `git push --mirror` (already considered in issue #481 broader work)
|
||||
|
||||
Current scope: single-VPS backup only (single point of failure mitigated but not eliminated).
|
||||
|
||||
---
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- `bin/gitea-backup.sh` — backup script source
|
||||
- `cron/vps/gitea-daily-backup.yml` — Hermes cron definition
|
||||
- Gitea official docs: <https://docs.gitea.com/administration/backup-and-restore>
|
||||
- Hermes cron: <https://hermes-agent.nousresearch.com/docs>
|
||||
Reference in New Issue
Block a user