timmy-config/bin/fleet-status.sh

#!/usr/bin/env bash
# ── fleet-status.sh ───────────────────────────────────────────────────
# One-line-per-wizard health check for all Hermes houses.
# Exit 0 = all healthy, Exit 1 = something down.
# Usage: fleet-status.sh [--no-color] [--json]
# ───────────────────────────────────────────────────────────────────────
set -o pipefail

# ── Options ──
NO_COLOR=false
JSON_OUT=false
for arg in "$@"; do
  case "$arg" in
    --no-color) NO_COLOR=true ;;
    --json)     JSON_OUT=true ;;
  esac
done

# ── Colors ──
if [ "$NO_COLOR" = true ] || [ ! -t 1 ]; then
  G="" ; Y="" ; RD="" ; C="" ; M="" ; B="" ; D="" ; R=""
else
  G='\033[32m' ; Y='\033[33m' ; RD='\033[31m' ; C='\033[36m'
  M='\033[35m' ; B='\033[1m'  ; D='\033[2m'   ; R='\033[0m'
fi

# ── Config ──
GITEA_TOKEN=$(cat ~/.hermes/gitea_token_vps 2>/dev/null)
GITEA_API="http://143.198.27.163:3000/api/v1"
EZRA_HOST="root@143.198.27.163"
BEZALEL_HOST="root@67.205.155.108"
SSH_OPTS="-o ConnectTimeout=4 -o StrictHostKeyChecking=no -o BatchMode=yes"

ANY_DOWN=0

# ── Helpers ──
now_epoch() { date +%s; }

time_ago() {
  local iso="$1"
  [ -z "$iso" ] && echo "unknown" && return
  local ts
  ts=$(python3 -c "
from datetime import datetime, timezone
import sys
t = '$iso'.replace('Z','+00:00')
try:
    dt = datetime.fromisoformat(t)
    print(int(dt.timestamp()))
except:
    print(0)
" 2>/dev/null)
  [ -z "$ts" ] || [ "$ts" = "0" ] && echo "unknown" && return
  local now
  now=$(now_epoch)
  local diff=$(( now - ts ))
  if [ "$diff" -lt 60 ]; then
    echo "${diff}s ago"
  elif [ "$diff" -lt 3600 ]; then
    echo "$(( diff / 60 ))m ago"
  elif [ "$diff" -lt 86400 ]; then
    echo "$(( diff / 3600 ))h $(( (diff % 3600) / 60 ))m ago"
  else
    echo "$(( diff / 86400 ))d ago"
  fi
}

gitea_last_commit() {
  local repo="$1"
  local result
  result=$(curl -sf --max-time 5 \
    "${GITEA_API}/repos/${repo}/commits?limit=1" \
    -H "Authorization: token ${GITEA_TOKEN}" 2>/dev/null)
  [ -z "$result" ] && echo "" && return
  python3 -c "
import json, sys
commits = json.loads('''${result}''')
if commits and len(commits) > 0:
    ts = commits[0].get('created','')
    msg = commits[0]['commit']['message'].split('\n')[0][:40]
    print(ts + '|' + msg)
else:
    print('')
" 2>/dev/null
}

print_line() {
  local name="$1" status="$2" model="$3" activity="$4"
  if [ "$status" = "UP" ]; then
    printf "  ${G}●${R} %-12s ${G}%-4s${R}  %-18s  ${D}%s${R}\n" "$name" "$status" "$model" "$activity"
  elif [ "$status" = "WARN" ]; then
    printf "  ${Y}●${R} %-12s ${Y}%-4s${R}  %-18s  ${D}%s${R}\n" "$name" "$status" "$model" "$activity"
  else
    printf "  ${RD}●${R} %-12s ${RD}%-4s${R}  %-18s  ${D}%s${R}\n" "$name" "$status" "$model" "$activity"
    ANY_DOWN=1
  fi
}

# ── Header ──
echo ""
echo -e "  ${B}${M}⚡ FLEET STATUS${R}  ${D}$(date '+%Y-%m-%d %H:%M:%S')${R}"
echo -e "  ${D}──────────────────────────────────────────────────────────────${R}"
printf "  %-14s %-6s  %-18s  %s\n" "WIZARD" "STATE" "MODEL/SERVICE" "LAST ACTIVITY"
echo -e "  ${D}──────────────────────────────────────────────────────────────${R}"

# ── 1. Timmy (local gateway + loops) ──
TIMMY_STATUS="DOWN"
TIMMY_MODEL=""
TIMMY_ACTIVITY=""

# Check gateway process
GW_PID=$(pgrep -f "hermes.*gateway.*run" 2>/dev/null | head -1)
if [ -z "$GW_PID" ]; then
  GW_PID=$(pgrep -f "gateway run" 2>/dev/null | head -1)
fi

# Check local loops
CLAUDE_LOOPS=$(pgrep -cf "claude-loop" 2>/dev/null || echo 0)
GEMINI_LOOPS=$(pgrep -cf "gemini-loop" 2>/dev/null || echo 0)

if [ -n "$GW_PID" ]; then
  TIMMY_STATUS="UP"
  TIMMY_MODEL="gateway(pid:${GW_PID})"
else
  TIMMY_STATUS="DOWN"
  TIMMY_MODEL="gateway:missing"
fi

# Check local health endpoint
TIMMY_HEALTH=$(curl -sf --max-time 3 "http://localhost:8000/health" 2>/dev/null)
if [ -n "$TIMMY_HEALTH" ]; then
  HEALTH_STATUS=$(python3 -c "import json; print(json.loads('''${TIMMY_HEALTH}''').get('status','?'))" 2>/dev/null)
  if [ "$HEALTH_STATUS" = "healthy" ] || [ "$HEALTH_STATUS" = "ok" ]; then
    TIMMY_STATUS="UP"
  fi
fi

TIMMY_ACTIVITY="loops: claude=${CLAUDE_LOOPS} gemini=${GEMINI_LOOPS}"

# Git activity for timmy-config
TC_COMMIT=$(gitea_last_commit "Timmy_Foundation/timmy-config")
if [ -n "$TC_COMMIT" ]; then
  TC_TIME=$(echo "$TC_COMMIT" | cut -d'|' -f1)
  TC_MSG=$(echo "$TC_COMMIT" | cut -d'|' -f2-)
  TC_AGO=$(time_ago "$TC_TIME")
  TIMMY_ACTIVITY="${TIMMY_ACTIVITY} | cfg:${TC_AGO}"
fi

if [ -z "$GW_PID" ] && [ "$CLAUDE_LOOPS" -eq 0 ] && [ "$GEMINI_LOOPS" -eq 0 ]; then
  TIMMY_STATUS="DOWN"
elif [ -z "$GW_PID" ]; then
  TIMMY_STATUS="WARN"
fi

print_line "Timmy" "$TIMMY_STATUS" "$TIMMY_MODEL" "$TIMMY_ACTIVITY"

# ── 2. Ezra (VPS 143.198.27.163) ──
EZRA_STATUS="DOWN"
EZRA_MODEL="hermes-ezra"
EZRA_ACTIVITY=""

EZRA_SVC=$(ssh $SSH_OPTS "$EZRA_HOST" "systemctl is-active hermes-ezra.service" 2>/dev/null)
if [ "$EZRA_SVC" = "active" ]; then
  EZRA_STATUS="UP"
  # Check health endpoint
  EZRA_HEALTH=$(ssh $SSH_OPTS "$EZRA_HOST" "curl -sf --max-time 3 http://localhost:8080/health 2>/dev/null" 2>/dev/null)
  if [ -n "$EZRA_HEALTH" ]; then
    EZRA_MODEL="hermes-ezra(ok)"
  else
    # Try alternate port
    EZRA_HEALTH=$(ssh $SSH_OPTS "$EZRA_HOST" "curl -sf --max-time 3 http://localhost:8000/health 2>/dev/null" 2>/dev/null)
    if [ -n "$EZRA_HEALTH" ]; then
      EZRA_MODEL="hermes-ezra(ok)"
    else
      EZRA_STATUS="WARN"
      EZRA_MODEL="hermes-ezra(svc:up,http:?)"
    fi
  fi
  # Check uptime
  EZRA_UP=$(ssh $SSH_OPTS "$EZRA_HOST" "systemctl show hermes-ezra.service --property=ActiveEnterTimestamp --value" 2>/dev/null)
  [ -n "$EZRA_UP" ] && EZRA_ACTIVITY="since ${EZRA_UP}"
else
  EZRA_STATUS="DOWN"
  EZRA_MODEL="hermes-ezra(svc:${EZRA_SVC:-unreachable})"
fi

print_line "Ezra" "$EZRA_STATUS" "$EZRA_MODEL" "$EZRA_ACTIVITY"

# ── 3. Bezalel (VPS 67.205.155.108) ──
BEZ_STATUS="DOWN"
BEZ_MODEL="hermes-bezalel"
BEZ_ACTIVITY=""

BEZ_SVC=$(ssh $SSH_OPTS "$BEZALEL_HOST" "systemctl is-active hermes-bezalel.service" 2>/dev/null)
if [ "$BEZ_SVC" = "active" ]; then
  BEZ_STATUS="UP"
  BEZ_HEALTH=$(ssh $SSH_OPTS "$BEZALEL_HOST" "curl -sf --max-time 3 http://localhost:8080/health 2>/dev/null" 2>/dev/null)
  if [ -n "$BEZ_HEALTH" ]; then
    BEZ_MODEL="hermes-bezalel(ok)"
  else
    BEZ_HEALTH=$(ssh $SSH_OPTS "$BEZALEL_HOST" "curl -sf --max-time 3 http://localhost:8000/health 2>/dev/null" 2>/dev/null)
    if [ -n "$BEZ_HEALTH" ]; then
      BEZ_MODEL="hermes-bezalel(ok)"
    else
      BEZ_STATUS="WARN"
      BEZ_MODEL="hermes-bezalel(svc:up,http:?)"
    fi
  fi
  BEZ_UP=$(ssh $SSH_OPTS "$BEZALEL_HOST" "systemctl show hermes-bezalel.service --property=ActiveEnterTimestamp --value" 2>/dev/null)
  [ -n "$BEZ_UP" ] && BEZ_ACTIVITY="since ${BEZ_UP}"
else
  BEZ_STATUS="DOWN"
  BEZ_MODEL="hermes-bezalel(svc:${BEZ_SVC:-unreachable})"
fi

print_line "Bezalel" "$BEZ_STATUS" "$BEZ_MODEL" "$BEZ_ACTIVITY"

# ── 4. the-nexus last commit ──
NEXUS_STATUS="DOWN"
NEXUS_MODEL="the-nexus"
NEXUS_ACTIVITY=""

NX_COMMIT=$(gitea_last_commit "Timmy_Foundation/the-nexus")
if [ -n "$NX_COMMIT" ]; then
  NEXUS_STATUS="UP"
  NX_TIME=$(echo "$NX_COMMIT" | cut -d'|' -f1)
  NX_MSG=$(echo "$NX_COMMIT" | cut -d'|' -f2-)
  NX_AGO=$(time_ago "$NX_TIME")
  NEXUS_MODEL="nexus-repo"
  NEXUS_ACTIVITY="${NX_AGO}: ${NX_MSG}"
else
  NEXUS_STATUS="WARN"
  NEXUS_MODEL="nexus-repo"
  NEXUS_ACTIVITY="(could not fetch)"
fi

print_line "Nexus" "$NEXUS_STATUS" "$NEXUS_MODEL" "$NEXUS_ACTIVITY"

# ── 5. Gitea server itself ──
GITEA_STATUS="DOWN"
GITEA_MODEL="gitea"
GITEA_ACTIVITY=""

GITEA_VER=$(curl -sf --max-time 5 "${GITEA_API}/version" 2>/dev/null)
if [ -n "$GITEA_VER" ]; then
  GITEA_STATUS="UP"
  VER=$(python3 -c "import json; print(json.loads('''${GITEA_VER}''').get('version','?'))" 2>/dev/null)
  GITEA_MODEL="gitea v${VER}"
  GITEA_ACTIVITY="143.198.27.163:3000"
else
  GITEA_STATUS="DOWN"
  GITEA_MODEL="gitea(unreachable)"
fi

print_line "Gitea" "$GITEA_STATUS" "$GITEA_MODEL" "$GITEA_ACTIVITY"

# ── Footer ──
echo -e "  ${D}──────────────────────────────────────────────────────────────${R}"

if [ "$ANY_DOWN" -eq 0 ]; then
  echo -e "  ${G}${B}All systems operational${R}"
  echo ""
  exit 0
else
  echo -e "  ${RD}${B}⚠  One or more systems DOWN${R}"
  echo ""
  exit 1
fi