hermes-config/bin/tower-watchdog.sh

#!/usr/bin/env bash
# ── Tower Watchdog ─────────────────────────────────────────────────────
# Ensures the tower session stays alive. Restarts dead panes/windows.
# Run via cron: */5 * * * * ~/hermes-config/bin/tower-watchdog.sh
#
# Layout:
#   Window 1, Pane 1: tower-hermes.sh (conversation driver)
#   Window 1, Pane 2: tower-status.sh (status dashboard)
#   Window 2:         tower-timmy.sh  (Timmy loop, hidden)
#
# Source-controlled: gitea/rockachopa/hermes-config
# ───────────────────────────────────────────────────────────────────────

SESSION="tower"
TOWER_BIN="$HOME/hermes-config/bin"
LOG="$HOME/.tower/watchdog.log"

mkdir -p "$HOME/.tower"

log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOG"; }

# If session doesn't exist at all, recreate it
if ! tmux has-session -t "$SESSION" 2>/dev/null; then
    log "Session '$SESSION' missing. Recreating."
    tmux new-session -d -s "$SESSION" -n "tower" -x 200 -y 50
    tmux split-window -h -p 35 -t "$SESSION:1.1"
    tmux select-pane -t "$SESSION:1.1" -T "⚡ Tower"
    tmux select-pane -t "$SESSION:1.2" -T "📊 Status"
    tmux send-keys -t "$SESSION:1.1" "$TOWER_BIN/tower-hermes.sh" Enter
    tmux send-keys -t "$SESSION:1.2" "$TOWER_BIN/tower-status.sh" Enter
    # Hidden window for Timmy
    tmux new-window -t "$SESSION" -n "timmy-bg"
    tmux send-keys -t "$SESSION:2" "$TOWER_BIN/tower-timmy.sh" Enter
    tmux select-window -t "$SESSION:1"
    log "Session recreated (conversation + status + timmy-bg)."
    exit 0
fi

# Session exists — check window 1 panes
PANE_COUNT=$(tmux list-panes -t "$SESSION:1" 2>/dev/null | wc -l | tr -d ' ')

if [ "$PANE_COUNT" -lt 2 ]; then
    log "Window 1 has only $PANE_COUNT pane(s). Killing and recreating session."
    tmux kill-session -t "$SESSION" 2>/dev/null
    exec "$0"  # re-run to recreate
fi

# Check Hermes loop (window 1, pane 1)
# Skip if Hermes is gracefully paused (waiting for backend)
if [ -f "$HOME/.tower/hermes.paused" ]; then
    log "Hermes is paused ($(cat "$HOME/.tower/hermes.paused")). Skipping restart."
else
    HERMES_PID=$(tmux display-message -p -t "$SESSION:1.1" '#{pane_pid}' 2>/dev/null)
    if [ -n "$HERMES_PID" ]; then
        CHILDREN=$(pgrep -P "$HERMES_PID" 2>/dev/null | wc -l | tr -d ' ')
        if [ "$CHILDREN" -eq 0 ]; then
            log "Hermes pane idle. Restarting tower-hermes.sh"
            rm -f "$HOME/.tower/hermes.lock"
            tmux send-keys -t "$SESSION:1.1" "$TOWER_BIN/tower-hermes.sh" Enter
        fi
    fi
fi

# Check status pane (window 1, pane 2) — restart if dead
STATUS_PID=$(tmux display-message -p -t "$SESSION:1.2" '#{pane_pid}' 2>/dev/null)
if [ -n "$STATUS_PID" ]; then
    CHILDREN=$(pgrep -P "$STATUS_PID" 2>/dev/null | wc -l | tr -d ' ')
    if [ "$CHILDREN" -eq 0 ]; then
        log "Status pane idle. Restarting tower-status.sh"
        tmux send-keys -t "$SESSION:1.2" "$TOWER_BIN/tower-status.sh" Enter
    fi
fi

# Check Timmy loop (window 2)
if ! tmux has-window -t "$SESSION:2" 2>/dev/null; then
    log "Timmy window missing. Recreating."
    tmux new-window -t "$SESSION" -n "timmy-bg"
    tmux send-keys -t "$SESSION:2" "$TOWER_BIN/tower-timmy.sh" Enter
    tmux select-window -t "$SESSION:1"
else
    # Skip if Timmy is gracefully paused (waiting for backend)
    if [ -f "$HOME/.tower/timmy.paused" ]; then
        log "Timmy is paused ($(cat "$HOME/.tower/timmy.paused")). Skipping restart."
    else
        TIMMY_PID=$(tmux display-message -p -t "$SESSION:2" '#{pane_pid}' 2>/dev/null)
        if [ -n "$TIMMY_PID" ]; then
            CHILDREN=$(pgrep -P "$TIMMY_PID" 2>/dev/null | wc -l | tr -d ' ')
            if [ "$CHILDREN" -eq 0 ]; then
                log "Timmy pane idle. Restarting tower-timmy.sh"
                rm -f "$HOME/.tower/timmy.lock"
                tmux send-keys -t "$SESSION:2" "$TOWER_BIN/tower-timmy.sh" Enter
            fi
        fi
    fi
fi

# Trim log if > 1000 lines
if [ -f "$LOG" ] && [ "$(wc -l < "$LOG")" -gt 1000 ]; then
    tail -500 "$LOG" > "$LOG.tmp" && mv "$LOG.tmp" "$LOG"
    log "Log trimmed to 500 lines."
fi