|
|
|
|
@@ -1,514 +0,0 @@
|
|
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
# pane-watchdog.sh — Detect stuck/dead tmux panes and auto-restart them
|
|
|
|
|
#
|
|
|
|
|
# Tracks output hash per pane across cycles. If a pane's captured output
|
|
|
|
|
# hasn't changed for STUCK_CYCLES consecutive checks, the pane is STUCK.
|
|
|
|
|
# Dead panes (PID gone) are also detected.
|
|
|
|
|
#
|
|
|
|
|
# On STUCK/DEAD:
|
|
|
|
|
# 1. Kill the pane
|
|
|
|
|
# 2. Attempt restart with --resume (session ID from manifest)
|
|
|
|
|
# 3. Fallback: fresh prompt with last known task from logs
|
|
|
|
|
#
|
|
|
|
|
# State file: ~/.hermes/pane-state.json
|
|
|
|
|
# Log: ~/.hermes/logs/pane-watchdog.log
|
|
|
|
|
#
|
|
|
|
|
# Usage:
|
|
|
|
|
# pane-watchdog.sh # One-shot check all sessions
|
|
|
|
|
# pane-watchdog.sh --daemon # Run every CHECK_INTERVAL seconds
|
|
|
|
|
# pane-watchdog.sh --status # Print current pane state
|
|
|
|
|
# pane-watchdog.sh --session NAME # Check only one session
|
|
|
|
|
#
|
|
|
|
|
# Issue: timmy-config #515
|
|
|
|
|
|
|
|
|
|
set -uo pipefail
|
|
|
|
|
export PATH="/opt/homebrew/bin:$HOME/.local/bin:$HOME/.hermes/bin:/usr/local/bin:$PATH"
|
|
|
|
|
|
|
|
|
|
# === CONFIG ===
|
|
|
|
|
STATE_FILE="${PANE_STATE_FILE:-$HOME/.hermes/pane-state.json}"
|
|
|
|
|
LOG_FILE="${PANE_WATCHDOG_LOG:-$HOME/.hermes/logs/pane-watchdog.log}"
|
|
|
|
|
CHECK_INTERVAL="${PANE_CHECK_INTERVAL:-120}" # seconds between cycles
|
|
|
|
|
STUCK_CYCLES=2 # unchanged cycles before STUCK
|
|
|
|
|
MAX_RESTART_ATTEMPTS=3 # per pane per hour
|
|
|
|
|
RESTART_COOLDOWN=3600 # seconds between escalation alerts
|
|
|
|
|
CAPTURE_LINES=40 # lines of output to hash
|
|
|
|
|
|
|
|
|
|
# Sessions to monitor (all if empty)
|
|
|
|
|
MONITOR_SESSIONS="${PANE_WATCHDOG_SESSIONS:-}"
|
|
|
|
|
|
|
|
|
|
mkdir -p "$(dirname "$STATE_FILE")" "$(dirname "$LOG_FILE")"
|
|
|
|
|
|
|
|
|
|
log() {
|
|
|
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG_FILE"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# === HELPERS ===
|
|
|
|
|
|
|
|
|
|
# Capture last N lines of pane output and hash them
|
|
|
|
|
capture_pane_hash() {
|
|
|
|
|
local target="$1"
|
|
|
|
|
local output
|
|
|
|
|
output=$(tmux capture-pane -t "$target" -p -S "-${CAPTURE_LINES}" 2>/dev/null || echo "DEAD")
|
|
|
|
|
echo -n "$output" | shasum -a 256 | cut -d' ' -f1
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Check if pane PID is alive
|
|
|
|
|
pane_pid_alive() {
|
|
|
|
|
local target="$1"
|
|
|
|
|
local pid
|
|
|
|
|
pid=$(tmux list-panes -t "$target" -F '#{pane_pid}' 2>/dev/null | head -1 || echo "")
|
|
|
|
|
if [ -z "$pid" ]; then
|
|
|
|
|
return 1 # pane doesn't exist
|
|
|
|
|
fi
|
|
|
|
|
kill -0 "$pid" 2>/dev/null
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Get pane start command
|
|
|
|
|
pane_start_command() {
|
|
|
|
|
local target="$1"
|
|
|
|
|
tmux list-panes -t "$target" -F '#{pane_start_command}' 2>/dev/null | head -1 || echo "unknown"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Get the pane's current running command (child process)
|
|
|
|
|
pane_current_command() {
|
|
|
|
|
local target="$1"
|
|
|
|
|
tmux list-panes -t "$target" -F '#{pane_current_command}' 2>/dev/null || echo "unknown"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Only restart panes running hermes/agent commands (not zsh, python3 repls, etc.)
|
|
|
|
|
is_restartable() {
|
|
|
|
|
local cmd="$1"
|
|
|
|
|
case "$cmd" in
|
|
|
|
|
hermes|*hermes*|*agent*|*timmy*|*kimi*|*claude-loop*|*gemini-loop*)
|
|
|
|
|
return 0
|
|
|
|
|
;;
|
|
|
|
|
*)
|
|
|
|
|
return 1
|
|
|
|
|
;;
|
|
|
|
|
esac
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Get session ID from hermes manifest if available
|
|
|
|
|
get_hermes_session_id() {
|
|
|
|
|
local session_name="$1"
|
|
|
|
|
local manifest="$HOME/.hermes/sessions/${session_name}/manifest.json"
|
|
|
|
|
if [ -f "$manifest" ]; then
|
|
|
|
|
python3 -c "
|
|
|
|
|
import json, sys
|
|
|
|
|
try:
|
|
|
|
|
m = json.load(open('$manifest'))
|
|
|
|
|
print(m.get('session_id', m.get('id', '')))
|
|
|
|
|
except: pass
|
|
|
|
|
" 2>/dev/null || echo ""
|
|
|
|
|
else
|
|
|
|
|
echo ""
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Get last task from pane logs
|
|
|
|
|
get_last_task() {
|
|
|
|
|
local session_name="$1"
|
|
|
|
|
local log_dir="$HOME/.hermes/logs"
|
|
|
|
|
# Find the most recent log for this session
|
|
|
|
|
local log_file
|
|
|
|
|
log_file=$(find "$log_dir" -name "*${session_name}*" -type f -mtime -1 2>/dev/null | sort -r | head -1)
|
|
|
|
|
if [ -n "$log_file" ] && [ -f "$log_file" ]; then
|
|
|
|
|
# Extract last user prompt or task description
|
|
|
|
|
grep -i "task:\|prompt:\|issue\|working on" "$log_file" 2>/dev/null | tail -1 | sed 's/.*[:>] *//' | head -c 200
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Restart a pane with a fresh shell/command
|
|
|
|
|
restart_pane() {
|
|
|
|
|
local target="$1"
|
|
|
|
|
local session_name="${target%%:*}"
|
|
|
|
|
local session_id last_task cmd
|
|
|
|
|
|
|
|
|
|
log "RESTART: Attempting to restart $target"
|
|
|
|
|
|
|
|
|
|
# Kill existing pane
|
|
|
|
|
tmux kill-pane -t "$target" 2>/dev/null || true
|
|
|
|
|
sleep 1
|
|
|
|
|
|
|
|
|
|
# Try --resume with session ID
|
|
|
|
|
session_id=$(get_hermes_session_id "$session_name")
|
|
|
|
|
if [ -n "$session_id" ]; then
|
|
|
|
|
log "RESTART: Trying --resume with session $session_id"
|
|
|
|
|
tmux split-window -t "$session_name" -d \
|
|
|
|
|
"hermes chat --resume '$session_id' 2>&1 | tee -a '$HOME/.hermes/logs/${session_name}-restart.log'"
|
|
|
|
|
sleep 2
|
|
|
|
|
if pane_pid_alive "${session_name}:1" 2>/dev/null; then
|
|
|
|
|
log "RESTART: Success with --resume"
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Fallback: fresh prompt
|
|
|
|
|
last_task=$(get_last_task "$session_name")
|
|
|
|
|
if [ -n "$last_task" ]; then
|
|
|
|
|
log "RESTART: Fallback — fresh prompt with task: $last_task"
|
|
|
|
|
tmux split-window -t "$session_name" -d \
|
|
|
|
|
"echo 'Watchdog restart — last task: $last_task' && hermes chat 2>&1 | tee -a '$HOME/.hermes/logs/${session_name}-restart.log'"
|
|
|
|
|
else
|
|
|
|
|
log "RESTART: Fallback — fresh hermes chat"
|
|
|
|
|
tmux split-window -t "$session_name" -d \
|
|
|
|
|
"hermes chat 2>&1 | tee -a '$HOME/.hermes/logs/${session_name}-restart.log'"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
sleep 2
|
|
|
|
|
if pane_pid_alive "${session_name}:1" 2>/dev/null; then
|
|
|
|
|
log "RESTART: Fallback restart succeeded"
|
|
|
|
|
return 0
|
|
|
|
|
else
|
|
|
|
|
log "RESTART: FAILED to restart $target"
|
|
|
|
|
return 1
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# === STATE MANAGEMENT ===
|
|
|
|
|
|
|
|
|
|
read_state() {
|
|
|
|
|
if [ -f "$STATE_FILE" ]; then
|
|
|
|
|
cat "$STATE_FILE"
|
|
|
|
|
else
|
|
|
|
|
echo "{}"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
write_state() {
|
|
|
|
|
echo "$1" > "$STATE_FILE"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Update state for a single pane and return JSON status
|
|
|
|
|
update_pane_state() {
|
|
|
|
|
local target="$1"
|
|
|
|
|
local hash="$2"
|
|
|
|
|
local is_alive="$3"
|
|
|
|
|
local now
|
|
|
|
|
now=$(date +%s)
|
|
|
|
|
|
|
|
|
|
python3 - "$STATE_FILE" "$target" "$hash" "$is_alive" "$now" "$STUCK_CYCLES" <<'PYEOF'
|
|
|
|
|
import json, sys, time
|
|
|
|
|
|
|
|
|
|
state_file = sys.argv[1]
|
|
|
|
|
target = sys.argv[2]
|
|
|
|
|
new_hash = sys.argv[3]
|
|
|
|
|
is_alive = sys.argv[4] == "true"
|
|
|
|
|
now = int(sys.argv[5])
|
|
|
|
|
stuck_cycles = int(sys.argv[6])
|
|
|
|
|
|
|
|
|
|
try:
|
|
|
|
|
with open(state_file) as f:
|
|
|
|
|
state = json.load(f)
|
|
|
|
|
except (FileNotFoundError, json.JSONDecodeError):
|
|
|
|
|
state = {}
|
|
|
|
|
|
|
|
|
|
pane = state.get(target, {
|
|
|
|
|
"hash": "",
|
|
|
|
|
"same_count": 0,
|
|
|
|
|
"status": "UNKNOWN",
|
|
|
|
|
"last_change": 0,
|
|
|
|
|
"last_check": 0,
|
|
|
|
|
"restart_attempts": 0,
|
|
|
|
|
"last_restart": 0,
|
|
|
|
|
"current_command": "",
|
|
|
|
|
})
|
|
|
|
|
|
|
|
|
|
if not is_alive:
|
|
|
|
|
pane["status"] = "DEAD"
|
|
|
|
|
pane["same_count"] = 0
|
|
|
|
|
elif new_hash == pane.get("hash", ""):
|
|
|
|
|
pane["same_count"] = pane.get("same_count", 0) + 1
|
|
|
|
|
if pane["same_count"] >= stuck_cycles:
|
|
|
|
|
pane["status"] = "STUCK"
|
|
|
|
|
else:
|
|
|
|
|
pane["status"] = "STALE" if pane["same_count"] > 0 else "OK"
|
|
|
|
|
else:
|
|
|
|
|
pane["hash"] = new_hash
|
|
|
|
|
pane["same_count"] = 0
|
|
|
|
|
pane["status"] = "OK"
|
|
|
|
|
pane["last_change"] = now
|
|
|
|
|
|
|
|
|
|
pane["last_check"] = now
|
|
|
|
|
state[target] = pane
|
|
|
|
|
|
|
|
|
|
with open(state_file, "w") as f:
|
|
|
|
|
json.dump(state, f, indent=2)
|
|
|
|
|
|
|
|
|
|
print(json.dumps(pane))
|
|
|
|
|
PYEOF
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# Reset restart attempt counter if cooldown expired
|
|
|
|
|
maybe_reset_restarts() {
|
|
|
|
|
local target="$1"
|
|
|
|
|
local now
|
|
|
|
|
now=$(date +%s)
|
|
|
|
|
|
|
|
|
|
python3 - "$STATE_FILE" "$target" "$now" "$RESTART_COOLDOWN" <<'PYEOF'
|
|
|
|
|
import json, sys
|
|
|
|
|
|
|
|
|
|
state_file = sys.argv[1]
|
|
|
|
|
target = sys.argv[2]
|
|
|
|
|
now = int(sys.argv[3])
|
|
|
|
|
cooldown = int(sys.argv[4])
|
|
|
|
|
|
|
|
|
|
with open(state_file) as f:
|
|
|
|
|
state = json.load(f)
|
|
|
|
|
|
|
|
|
|
pane = state.get(target, {})
|
|
|
|
|
last_restart = pane.get("last_restart", 0)
|
|
|
|
|
|
|
|
|
|
if now - last_restart > cooldown:
|
|
|
|
|
pane["restart_attempts"] = 0
|
|
|
|
|
|
|
|
|
|
state[target] = pane
|
|
|
|
|
with open(state_file, "w") as f:
|
|
|
|
|
json.dump(state, f, indent=2)
|
|
|
|
|
|
|
|
|
|
print(pane.get("restart_attempts", 0))
|
|
|
|
|
PYEOF
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
increment_restart_attempt() {
|
|
|
|
|
local target="$1"
|
|
|
|
|
local now
|
|
|
|
|
now=$(date +%s)
|
|
|
|
|
|
|
|
|
|
python3 - "$STATE_FILE" "$target" "$now" <<'PYEOF'
|
|
|
|
|
import json, sys
|
|
|
|
|
|
|
|
|
|
state_file = sys.argv[1]
|
|
|
|
|
target = sys.argv[2]
|
|
|
|
|
now = int(sys.argv[3])
|
|
|
|
|
|
|
|
|
|
with open(state_file) as f:
|
|
|
|
|
state = json.load(f)
|
|
|
|
|
|
|
|
|
|
pane = state.get(target, {})
|
|
|
|
|
pane["restart_attempts"] = pane.get("restart_attempts", 0) + 1
|
|
|
|
|
pane["last_restart"] = now
|
|
|
|
|
pane["status"] = "RESTARTING"
|
|
|
|
|
|
|
|
|
|
state[target] = pane
|
|
|
|
|
with open(state_file, "w") as f:
|
|
|
|
|
json.dump(state, f, indent=2)
|
|
|
|
|
|
|
|
|
|
print(pane["restart_attempts"])
|
|
|
|
|
PYEOF
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# === CORE CHECK ===
|
|
|
|
|
|
|
|
|
|
check_pane() {
|
|
|
|
|
local target="$1"
|
|
|
|
|
local hash is_alive status current_cmd
|
|
|
|
|
|
|
|
|
|
# Capture state
|
|
|
|
|
hash=$(capture_pane_hash "$target")
|
|
|
|
|
if pane_pid_alive "$target"; then
|
|
|
|
|
is_alive="true"
|
|
|
|
|
else
|
|
|
|
|
is_alive="false"
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
# Get current command for the pane
|
|
|
|
|
current_cmd=$(pane_current_command "$target")
|
|
|
|
|
|
|
|
|
|
# Update state and get result
|
|
|
|
|
local result
|
|
|
|
|
result=$(update_pane_state "$target" "$hash" "$is_alive")
|
|
|
|
|
status=$(echo "$result" | python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('status','UNKNOWN'))" 2>/dev/null || echo "UNKNOWN")
|
|
|
|
|
|
|
|
|
|
case "$status" in
|
|
|
|
|
OK)
|
|
|
|
|
# Healthy, do nothing
|
|
|
|
|
;;
|
|
|
|
|
DEAD)
|
|
|
|
|
log "DETECTED: $target is DEAD (PID gone) cmd=$current_cmd"
|
|
|
|
|
if is_restartable "$current_cmd"; then
|
|
|
|
|
handle_stuck "$target"
|
|
|
|
|
else
|
|
|
|
|
log "SKIP: $target not a hermes pane (cmd=$current_cmd), not restarting"
|
|
|
|
|
fi
|
|
|
|
|
;;
|
|
|
|
|
STUCK)
|
|
|
|
|
log "DETECTED: $target is STUCK (output unchanged for ${STUCK_CYCLES} cycles) cmd=$current_cmd"
|
|
|
|
|
if is_restartable "$current_cmd"; then
|
|
|
|
|
handle_stuck "$target"
|
|
|
|
|
else
|
|
|
|
|
log "SKIP: $target not a hermes pane (cmd=$current_cmd), not restarting"
|
|
|
|
|
fi
|
|
|
|
|
;;
|
|
|
|
|
STALE)
|
|
|
|
|
# Output unchanged but within threshold — just log
|
|
|
|
|
local count
|
|
|
|
|
count=$(echo "$result" | python3 -c "import json,sys; print(json.loads(sys.stdin.read()).get('same_count',0))" 2>/dev/null || echo "?")
|
|
|
|
|
log "STALE: $target unchanged for $count cycle(s)"
|
|
|
|
|
;;
|
|
|
|
|
esac
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
handle_stuck() {
|
|
|
|
|
local target="$1"
|
|
|
|
|
local session_name="${target%%:*}"
|
|
|
|
|
local attempts
|
|
|
|
|
|
|
|
|
|
# Check restart budget
|
|
|
|
|
attempts=$(maybe_reset_restarts "$target")
|
|
|
|
|
if [ "$attempts" -ge "$MAX_RESTART_ATTEMPTS" ]; then
|
|
|
|
|
log "ESCALATION: $target stuck ${attempts}x — manual intervention needed"
|
|
|
|
|
echo "ALERT: $target stuck after $attempts restart attempts" >&2
|
|
|
|
|
return 1
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
attempts=$(increment_restart_attempt "$target")
|
|
|
|
|
log "ACTION: Restarting $target (attempt $attempts/$MAX_RESTART_ATTEMPTS)"
|
|
|
|
|
|
|
|
|
|
if restart_pane "$target"; then
|
|
|
|
|
log "OK: $target restarted successfully"
|
|
|
|
|
else
|
|
|
|
|
log "FAIL: $target restart failed (attempt $attempts)"
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
check_all_sessions() {
|
|
|
|
|
local sessions
|
|
|
|
|
|
|
|
|
|
if [ -n "$MONITOR_SESSIONS" ]; then
|
|
|
|
|
IFS=',' read -ra sessions <<< "$MONITOR_SESSIONS"
|
|
|
|
|
else
|
|
|
|
|
sessions=()
|
|
|
|
|
while IFS= read -r line; do
|
|
|
|
|
[ -n "$line" ] && sessions+=("$line")
|
|
|
|
|
done < <(tmux list-sessions -F '#{session_name}' 2>/dev/null || true)
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
local total=0 stuck=0 dead=0 ok=0
|
|
|
|
|
for session in "${sessions[@]}"; do
|
|
|
|
|
[ -z "$session" ] && continue
|
|
|
|
|
# Get pane targets
|
|
|
|
|
local panes
|
|
|
|
|
panes=$(tmux list-panes -t "$session" -F "${session}:#{window_index}.#{pane_index}" 2>/dev/null || true)
|
|
|
|
|
for target in $panes; do
|
|
|
|
|
check_pane "$target"
|
|
|
|
|
total=$((total + 1))
|
|
|
|
|
done
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
log "CHECK: Processed $total panes"
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# === STATUS DISPLAY ===
|
|
|
|
|
|
|
|
|
|
show_status() {
|
|
|
|
|
if [ ! -f "$STATE_FILE" ]; then
|
|
|
|
|
echo "No pane state file found at $STATE_FILE"
|
|
|
|
|
echo "Run pane-watchdog.sh once to initialize."
|
|
|
|
|
exit 0
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
python3 - "$STATE_FILE" <<'PYEOF'
|
|
|
|
|
import json, sys, time
|
|
|
|
|
|
|
|
|
|
state_file = sys.argv[1]
|
|
|
|
|
try:
|
|
|
|
|
with open(state_file) as f:
|
|
|
|
|
state = json.load(f)
|
|
|
|
|
except (FileNotFoundError, json.JSONDecodeError):
|
|
|
|
|
print("No state data yet.")
|
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
|
|
|
|
if not state:
|
|
|
|
|
print("No panes tracked.")
|
|
|
|
|
sys.exit(0)
|
|
|
|
|
|
|
|
|
|
now = int(time.time())
|
|
|
|
|
print(f"{'PANE':<35} {'STATUS':<12} {'STALE':<6} {'LAST CHANGE':<15} {'RESTARTS'}")
|
|
|
|
|
print("-" * 90)
|
|
|
|
|
|
|
|
|
|
for target in sorted(state.keys()):
|
|
|
|
|
p = state[target]
|
|
|
|
|
status = p.get("status", "?")
|
|
|
|
|
same = p.get("same_count", 0)
|
|
|
|
|
last_change = p.get("last_change", 0)
|
|
|
|
|
restarts = p.get("restart_attempts", 0)
|
|
|
|
|
|
|
|
|
|
if last_change:
|
|
|
|
|
ago = now - last_change
|
|
|
|
|
if ago < 60:
|
|
|
|
|
change_str = f"{ago}s ago"
|
|
|
|
|
elif ago < 3600:
|
|
|
|
|
change_str = f"{ago//60}m ago"
|
|
|
|
|
else:
|
|
|
|
|
change_str = f"{ago//3600}h ago"
|
|
|
|
|
else:
|
|
|
|
|
change_str = "never"
|
|
|
|
|
|
|
|
|
|
# Color code
|
|
|
|
|
if status == "OK":
|
|
|
|
|
icon = "✓"
|
|
|
|
|
elif status == "STUCK":
|
|
|
|
|
icon = "✖"
|
|
|
|
|
elif status == "DEAD":
|
|
|
|
|
icon = "☠"
|
|
|
|
|
elif status == "STALE":
|
|
|
|
|
icon = "⏳"
|
|
|
|
|
else:
|
|
|
|
|
icon = "?"
|
|
|
|
|
|
|
|
|
|
print(f" {icon} {target:<32} {status:<12} {same:<6} {change_str:<15} {restarts}")
|
|
|
|
|
PYEOF
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# === DAEMON MODE ===
|
|
|
|
|
|
|
|
|
|
run_daemon() {
|
|
|
|
|
log "DAEMON: Starting (interval=${CHECK_INTERVAL}s, stuck_threshold=${STUCK_CYCLES})"
|
|
|
|
|
echo "Pane watchdog started. Checking every ${CHECK_INTERVAL}s. Ctrl+C to stop."
|
|
|
|
|
echo "Log: $LOG_FILE"
|
|
|
|
|
echo "State: $STATE_FILE"
|
|
|
|
|
echo ""
|
|
|
|
|
|
|
|
|
|
while true; do
|
|
|
|
|
check_all_sessions
|
|
|
|
|
sleep "$CHECK_INTERVAL"
|
|
|
|
|
done
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
# === MAIN ===
|
|
|
|
|
|
|
|
|
|
case "${1:-}" in
|
|
|
|
|
--daemon)
|
|
|
|
|
run_daemon
|
|
|
|
|
;;
|
|
|
|
|
--status)
|
|
|
|
|
show_status
|
|
|
|
|
;;
|
|
|
|
|
--session)
|
|
|
|
|
if [ -z "${2:-}" ]; then
|
|
|
|
|
echo "Usage: pane-watchdog.sh --session SESSION_NAME"
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
MONITOR_SESSIONS="$2"
|
|
|
|
|
check_all_sessions
|
|
|
|
|
;;
|
|
|
|
|
--help|-h)
|
|
|
|
|
echo "pane-watchdog.sh — Detect stuck/dead tmux panes and auto-restart"
|
|
|
|
|
echo ""
|
|
|
|
|
echo "Usage:"
|
|
|
|
|
echo " pane-watchdog.sh # One-shot check"
|
|
|
|
|
echo " pane-watchdog.sh --daemon # Continuous monitoring"
|
|
|
|
|
echo " pane-watchdog.sh --status # Show pane state"
|
|
|
|
|
echo " pane-watchdog.sh --session S # Check one session"
|
|
|
|
|
echo ""
|
|
|
|
|
echo "Config (env vars):"
|
|
|
|
|
echo " PANE_CHECK_INTERVAL Seconds between checks (default: 120)"
|
|
|
|
|
echo " PANE_WATCHDOG_SESSIONS Comma-separated session names"
|
|
|
|
|
echo " PANE_STATE_FILE State file path"
|
|
|
|
|
echo " STUCK_CYCLES Unchanged cycles before STUCK (default: 2)"
|
|
|
|
|
;;
|
|
|
|
|
*)
|
|
|
|
|
check_all_sessions
|
|
|
|
|
;;
|
|
|
|
|
esac
|
