47 lines
1.7 KiB
Bash
Executable File
47 lines
1.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Gitea Runner Health Probe — Poka-yoke for unregistered runners
|
|
set -euo pipefail
|
|
|
|
GITEA_TOKEN="${GITEA_TOKEN:-}"
|
|
GITEA_URL="https://forge.alexanderwhitestone.com"
|
|
ALERT_LOG="/var/log/bezalel_runner_health.log"
|
|
|
|
log() {
|
|
echo "[$(date -Iseconds)] $1" | tee -a "$ALERT_LOG"
|
|
}
|
|
|
|
if [[ -z "$GITEA_TOKEN" ]]; then
|
|
log "ERROR: GITEA_TOKEN not set"
|
|
exit 1
|
|
fi
|
|
|
|
ACTIVE_RUNNERS=$(curl -s -H "Authorization: token ${GITEA_TOKEN}" \
|
|
"${GITEA_URL}/api/v1/repos/Timmy_Foundation/hermes-agent/actions/runners" | \
|
|
python3 -c "import sys,json; d=json.load(sys.stdin); print(len([r for r in d.get('runners',[]) if r.get('status')=='online']))")
|
|
|
|
log "Active runners: ${ACTIVE_RUNNERS}"
|
|
|
|
if [[ "$ACTIVE_RUNNERS" -eq 0 ]]; then
|
|
log "CRITICAL: Zero active runners detected. Attempting self-healing restart."
|
|
pkill -f "act_runner daemon" 2>/dev/null || true
|
|
sleep 2
|
|
cd /opt/gitea-runner && nohup ./act_runner daemon > /var/log/gitea-runner.log 2>&1 &
|
|
sleep 3
|
|
# Re-check
|
|
ACTIVE_RUNNERS_AFTER=$(curl -s -H "Authorization: token ${GITEA_TOKEN}" \
|
|
"${GITEA_URL}/api/v1/repos/Timmy_Foundation/hermes-agent/actions/runners" | \
|
|
python3 -c "import sys,json; d=json.load(sys.stdin); print(len([r for r in d.get('runners',[]) if r.get('status')=='online']))")
|
|
log "Active runners after restart: ${ACTIVE_RUNNERS_AFTER}"
|
|
if [[ "$ACTIVE_RUNNERS_AFTER" -eq 0 ]]; then
|
|
log "CRITICAL: Self-healing failed. Runner still offline."
|
|
touch /var/lib/bezalel/heartbeats/runner_health.last
|
|
exit 1
|
|
else
|
|
log "RECOVERED: Runner back online."
|
|
fi
|
|
else
|
|
log "OK: ${ACTIVE_RUNNERS} runner(s) online."
|
|
fi
|
|
|
|
touch /var/lib/bezalel/heartbeats/runner_health.last
|