92 lines
3.8 KiB
Bash
Executable File
92 lines
3.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ── Tower Watchdog ─────────────────────────────────────────────────────
|
|
# Ensures the tower session stays alive. Restarts dead panes/windows.
|
|
# Run via cron: */5 * * * * ~/hermes-config/bin/tower-watchdog.sh
|
|
#
|
|
# Layout:
|
|
# Window 1, Pane 1: tower-hermes.sh (conversation driver)
|
|
# Window 1, Pane 2: tower-status.sh (status dashboard)
|
|
# Window 2: tower-timmy.sh (Timmy loop, hidden)
|
|
#
|
|
# Source-controlled: gitea/rockachopa/hermes-config
|
|
# ───────────────────────────────────────────────────────────────────────
|
|
|
|
SESSION="tower"
|
|
TOWER_BIN="$HOME/hermes-config/bin"
|
|
LOG="$HOME/.tower/watchdog.log"
|
|
|
|
mkdir -p "$HOME/.tower"
|
|
|
|
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG"; }
|
|
|
|
# If session doesn't exist at all, recreate it
|
|
if ! tmux has-session -t "$SESSION" 2>/dev/null; then
|
|
log "Session '$SESSION' missing. Recreating."
|
|
tmux new-session -d -s "$SESSION" -n "tower" -x 200 -y 50
|
|
tmux split-window -h -p 35 -t "$SESSION:1.1"
|
|
tmux select-pane -t "$SESSION:1.1" -T "⚡ Tower"
|
|
tmux select-pane -t "$SESSION:1.2" -T "📊 Status"
|
|
tmux send-keys -t "$SESSION:1.1" "$TOWER_BIN/tower-hermes.sh" Enter
|
|
tmux send-keys -t "$SESSION:1.2" "$TOWER_BIN/tower-status.sh" Enter
|
|
# Hidden window for Timmy
|
|
tmux new-window -t "$SESSION" -n "timmy-bg"
|
|
tmux send-keys -t "$SESSION:2" "$TOWER_BIN/tower-timmy.sh" Enter
|
|
tmux select-window -t "$SESSION:1"
|
|
log "Session recreated (conversation + status + timmy-bg)."
|
|
exit 0
|
|
fi
|
|
|
|
# Session exists — check window 1 panes
|
|
PANE_COUNT=$(tmux list-panes -t "$SESSION:1" 2>/dev/null | wc -l | tr -d ' ')
|
|
|
|
if [ "$PANE_COUNT" -lt 2 ]; then
|
|
log "Window 1 has only $PANE_COUNT pane(s). Killing and recreating session."
|
|
tmux kill-session -t "$SESSION" 2>/dev/null
|
|
exec "$0" # re-run to recreate
|
|
fi
|
|
|
|
# Check Hermes loop (window 1, pane 1)
|
|
HERMES_PID=$(tmux display-message -p -t "$SESSION:1.1" '#{pane_pid}' 2>/dev/null)
|
|
if [ -n "$HERMES_PID" ]; then
|
|
CHILDREN=$(pgrep -P "$HERMES_PID" 2>/dev/null | wc -l | tr -d ' ')
|
|
if [ "$CHILDREN" -eq 0 ]; then
|
|
log "Hermes pane idle. Restarting tower-hermes.sh"
|
|
rm -f "$HOME/.tower/hermes.lock"
|
|
tmux send-keys -t "$SESSION:1.1" "$TOWER_BIN/tower-hermes.sh" Enter
|
|
fi
|
|
fi
|
|
|
|
# Check status pane (window 1, pane 2) — restart if dead
|
|
STATUS_PID=$(tmux display-message -p -t "$SESSION:1.2" '#{pane_pid}' 2>/dev/null)
|
|
if [ -n "$STATUS_PID" ]; then
|
|
CHILDREN=$(pgrep -P "$STATUS_PID" 2>/dev/null | wc -l | tr -d ' ')
|
|
if [ "$CHILDREN" -eq 0 ]; then
|
|
log "Status pane idle. Restarting tower-status.sh"
|
|
tmux send-keys -t "$SESSION:1.2" "$TOWER_BIN/tower-status.sh" Enter
|
|
fi
|
|
fi
|
|
|
|
# Check Timmy loop (window 2)
|
|
if ! tmux has-window -t "$SESSION:2" 2>/dev/null; then
|
|
log "Timmy window missing. Recreating."
|
|
tmux new-window -t "$SESSION" -n "timmy-bg"
|
|
tmux send-keys -t "$SESSION:2" "$TOWER_BIN/tower-timmy.sh" Enter
|
|
tmux select-window -t "$SESSION:1"
|
|
else
|
|
TIMMY_PID=$(tmux display-message -p -t "$SESSION:2" '#{pane_pid}' 2>/dev/null)
|
|
if [ -n "$TIMMY_PID" ]; then
|
|
CHILDREN=$(pgrep -P "$TIMMY_PID" 2>/dev/null | wc -l | tr -d ' ')
|
|
if [ "$CHILDREN" -eq 0 ]; then
|
|
log "Timmy pane idle. Restarting tower-timmy.sh"
|
|
rm -f "$HOME/.tower/timmy.lock"
|
|
tmux send-keys -t "$SESSION:2" "$TOWER_BIN/tower-timmy.sh" Enter
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Trim log if > 1000 lines
|
|
if [ -f "$LOG" ] && [ "$(wc -l < "$LOG")" -gt 1000 ]; then
|
|
tail -500 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"
|
|
log "Log trimmed to 500 lines."
|
|
fi
|