feat(ops): deadman switch, model health check, issue filter
Closes #115: bin/deadman-switch.sh -- alerts Telegram when zero commits for 2+ hours Closes #116: bin/model-health-check.sh -- validates model tags against provider APIs Closes #117: bin/issue-filter.json + live loop patches -- excludes DO-NOT-CLOSE, EPIC, META, RETRO, INTEL, MORNING REPORT, Rockachopa-assigned issues from agent pickup All three tested locally: - deadman-switch correctly detected 14h gap and would alert - model-health-check parses config.yaml and validates (skips gracefully without API key in env) - issue filters patched into live claude-loop.sh and gemini-loop.sh
This commit is contained in:
78
bin/deadman-switch.sh
Executable file
78
bin/deadman-switch.sh
Executable file
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
# deadman-switch.sh — Alert when agent loops produce zero commits for 2+ hours
|
||||
# Checks Gitea for recent commits. Sends Telegram alert if threshold exceeded.
|
||||
# Designed to run as a cron job every 30 minutes.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
THRESHOLD_HOURS="${1:-2}"
|
||||
THRESHOLD_SECS=$((THRESHOLD_HOURS * 3600))
|
||||
LOG_DIR="$HOME/.hermes/logs"
|
||||
LOG_FILE="$LOG_DIR/deadman.log"
|
||||
GITEA_URL="http://143.198.27.163:3000"
|
||||
GITEA_TOKEN=$(cat "$HOME/.hermes/gitea_token_vps" 2>/dev/null || echo "")
|
||||
TELEGRAM_TOKEN=$(cat "$HOME/.config/telegram/special_bot" 2>/dev/null || echo "")
|
||||
TELEGRAM_CHAT="-1003664764329"
|
||||
|
||||
REPOS=(
|
||||
"Timmy_Foundation/timmy-config"
|
||||
"Timmy_Foundation/the-nexus"
|
||||
)
|
||||
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE"
|
||||
}
|
||||
|
||||
now=$(date +%s)
|
||||
latest_commit_time=0
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
# Get most recent commit timestamp
|
||||
response=$(curl -sf --max-time 10 \
|
||||
-H "Authorization: token ${GITEA_TOKEN}" \
|
||||
"${GITEA_URL}/api/v1/repos/${repo}/commits?limit=1" 2>/dev/null || echo "[]")
|
||||
|
||||
commit_date=$(echo "$response" | python3 -c "
|
||||
import json, sys, datetime
|
||||
try:
|
||||
commits = json.load(sys.stdin)
|
||||
if commits:
|
||||
ts = commits[0]['created']
|
||||
dt = datetime.datetime.fromisoformat(ts.replace('Z', '+00:00'))
|
||||
print(int(dt.timestamp()))
|
||||
else:
|
||||
print(0)
|
||||
except:
|
||||
print(0)
|
||||
" 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$commit_date" -gt "$latest_commit_time" ]; then
|
||||
latest_commit_time=$commit_date
|
||||
fi
|
||||
done
|
||||
|
||||
gap=$((now - latest_commit_time))
|
||||
gap_hours=$((gap / 3600))
|
||||
gap_mins=$(((gap % 3600) / 60))
|
||||
|
||||
if [ "$latest_commit_time" -eq 0 ]; then
|
||||
log "WARN: Could not fetch any commit timestamps. API may be down."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$gap" -gt "$THRESHOLD_SECS" ]; then
|
||||
msg="DEADMAN ALERT: No commits in ${gap_hours}h${gap_mins}m across all repos. Loops may be dead. Last commit: $(date -r "$latest_commit_time" '+%Y-%m-%d %H:%M' 2>/dev/null || echo 'unknown')"
|
||||
log "ALERT: $msg"
|
||||
|
||||
# Send Telegram alert
|
||||
if [ -n "$TELEGRAM_TOKEN" ]; then
|
||||
curl -sf --max-time 10 -X POST \
|
||||
"https://api.telegram.org/bot${TELEGRAM_TOKEN}/sendMessage" \
|
||||
-d "chat_id=${TELEGRAM_CHAT}" \
|
||||
-d "text=${msg}" >/dev/null 2>&1 || true
|
||||
fi
|
||||
else
|
||||
log "OK: Last commit ${gap_hours}h${gap_mins}m ago (threshold: ${THRESHOLD_HOURS}h)"
|
||||
fi
|
||||
Reference in New Issue
Block a user