95 lines
2.8 KiB
Bash
95 lines
2.8 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
# claudemax-watchdog.sh — keep local Claude/Gemini loops alive without stale tmux assumptions
|
||
|
|
|
||
|
|
set -uo pipefail
|
||
|
|
export PATH="/opt/homebrew/bin:$HOME/.local/bin:$HOME/.hermes/bin:/usr/local/bin:$PATH"
|
||
|
|
|
||
|
|
LOG="$HOME/.hermes/logs/claudemax-watchdog.log"
|
||
|
|
GITEA_URL="http://143.198.27.163:3000"
|
||
|
|
GITEA_TOKEN=$(tr -d '[:space:]' < "$HOME/.hermes/gitea_token_vps" 2>/dev/null || true)
|
||
|
|
REPO_API="$GITEA_URL/api/v1/repos/Timmy_Foundation/the-nexus"
|
||
|
|
MIN_OPEN_ISSUES=10
|
||
|
|
CLAUDE_WORKERS=2
|
||
|
|
GEMINI_WORKERS=1
|
||
|
|
|
||
|
|
log() {
|
||
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] CLAUDEMAX: $*" >> "$LOG"
|
||
|
|
}
|
||
|
|
|
||
|
|
start_loop() {
|
||
|
|
local name="$1"
|
||
|
|
local pattern="$2"
|
||
|
|
local cmd="$3"
|
||
|
|
local pid
|
||
|
|
|
||
|
|
pid=$(pgrep -f "$pattern" 2>/dev/null | head -1 || true)
|
||
|
|
if [ -n "$pid" ]; then
|
||
|
|
log "$name alive (PID $pid)"
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
|
||
|
|
log "$name not running. Restarting..."
|
||
|
|
nohup bash -lc "$cmd" >/dev/null 2>&1 &
|
||
|
|
sleep 2
|
||
|
|
|
||
|
|
pid=$(pgrep -f "$pattern" 2>/dev/null | head -1 || true)
|
||
|
|
if [ -n "$pid" ]; then
|
||
|
|
log "Restarted $name (PID $pid)"
|
||
|
|
else
|
||
|
|
log "ERROR: failed to start $name"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
run_optional_script() {
|
||
|
|
local label="$1"
|
||
|
|
local script_path="$2"
|
||
|
|
|
||
|
|
if [ -x "$script_path" ]; then
|
||
|
|
bash "$script_path" 2>&1 | while read -r line; do
|
||
|
|
log "$line"
|
||
|
|
done
|
||
|
|
else
|
||
|
|
log "$label skipped — missing $script_path"
|
||
|
|
fi
|
||
|
|
}
|
||
|
|
|
||
|
|
claude_quota_blocked() {
|
||
|
|
local cutoff now mtime f
|
||
|
|
now=$(date +%s)
|
||
|
|
cutoff=$((now - 43200))
|
||
|
|
for f in "$HOME"/.hermes/logs/claude-*.log; do
|
||
|
|
[ -f "$f" ] || continue
|
||
|
|
mtime=$(stat -f %m "$f" 2>/dev/null || echo 0)
|
||
|
|
if [ "$mtime" -ge "$cutoff" ] && grep -q "You've hit your limit" "$f" 2>/dev/null; then
|
||
|
|
return 0
|
||
|
|
fi
|
||
|
|
done
|
||
|
|
return 1
|
||
|
|
}
|
||
|
|
|
||
|
|
if [ -z "$GITEA_TOKEN" ]; then
|
||
|
|
log "ERROR: missing Gitea token at ~/.hermes/gitea_token_vps"
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
if claude_quota_blocked; then
|
||
|
|
log "Claude quota exhausted recently — not starting claude-loop until quota resets or logs age out"
|
||
|
|
else
|
||
|
|
start_loop "claude-loop" "bash .*claude-loop.sh" "bash ~/.hermes/bin/claude-loop.sh $CLAUDE_WORKERS >> ~/.hermes/logs/claude-loop.log 2>&1"
|
||
|
|
fi
|
||
|
|
start_loop "gemini-loop" "bash .*gemini-loop.sh" "bash ~/.hermes/bin/gemini-loop.sh $GEMINI_WORKERS >> ~/.hermes/logs/gemini-loop.log 2>&1"
|
||
|
|
|
||
|
|
OPEN_COUNT=$(curl -s --max-time 10 -H "Authorization: token $GITEA_TOKEN" \
|
||
|
|
"$REPO_API/issues?state=open&type=issues&limit=100" 2>/dev/null \
|
||
|
|
| python3 -c "import sys, json; print(len(json.loads(sys.stdin.read() or '[]')))" 2>/dev/null || echo 0)
|
||
|
|
|
||
|
|
log "Open issues: $OPEN_COUNT (minimum: $MIN_OPEN_ISSUES)"
|
||
|
|
|
||
|
|
if [ "$OPEN_COUNT" -lt "$MIN_OPEN_ISSUES" ]; then
|
||
|
|
log "Backlog running low. Checking replenishment helper..."
|
||
|
|
run_optional_script "claudemax-replenish" "$HOME/.hermes/bin/claudemax-replenish.sh"
|
||
|
|
fi
|
||
|
|
|
||
|
|
run_optional_script "autodeploy-matrix" "$HOME/.hermes/bin/autodeploy-matrix.sh"
|
||
|
|
log "Watchdog complete."
|