hermes-config/bin/ops-panel.sh

#!/usr/bin/env bash
# ── Consolidated Ops Panel ─────────────────────────────────────────────
# Everything in one view. Designed for a half-screen pane (~100x45).
# ───────────────────────────────────────────────────────────────────────

B='\033[1m'  ; D='\033[2m'  ; R='\033[0m' ; U='\033[4m'
G='\033[32m' ; Y='\033[33m' ; RD='\033[31m' ; C='\033[36m' ; M='\033[35m' ; W='\033[37m'
OK="${G}●${R}" ; WARN="${Y}●${R}" ; FAIL="${RD}●${R}" ; OFF="${D}○${R}"

TOKEN=$(cat ~/.hermes/gitea_token_vps 2>/dev/null)
API="http://143.198.27.163:3000/api/v1/repos/rockachopa/Timmy-time-dashboard"

# ── HEADER ─────────────────────────────────────────────────────────────
echo ""
echo -e "  ${B}${M}◈ HERMES OPERATIONS${R}                         ${D}$(date '+%a %b %d  %H:%M:%S')${R}"
echo -e "  ${D}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${R}"
echo ""

# ── SERVICES ───────────────────────────────────────────────────────────
echo -e "  ${B}${U}SERVICES${R}"
echo ""

# Gateway
GW_PID=$(pgrep -f "hermes.*gateway.*run" 2>/dev/null | head -1)
[ -n "$GW_PID" ] && echo -e "    ${OK} Gateway          ${D}pid $GW_PID${R}" \
                  || echo -e "    ${FAIL} Gateway          ${RD}DOWN — run: hermes gateway start${R}"

# Kimi Code loop
KIMI_PID=$(pgrep -f "kimi-loop.sh" 2>/dev/null | head -1)
[ -n "$KIMI_PID" ] && echo -e "    ${OK} Kimi Loop        ${D}pid $KIMI_PID${R}" \
                    || echo -e "    ${FAIL} Kimi Loop        ${RD}DOWN — run: ops-wake-kimi${R}"

# Active Kimi Code worker
KIMI_WORK=$(pgrep -f "kimi.*--print" 2>/dev/null | head -1)
if [ -n "$KIMI_WORK" ]; then
    echo -e "    ${OK} Kimi Code        ${D}pid $KIMI_WORK ${G}working${R}"
elif [ -n "$KIMI_PID" ]; then
    echo -e "    ${WARN} Kimi Code        ${Y}between issues${R}"
else
    echo -e "    ${OFF} Kimi Code        ${D}not running${R}"
fi

# Claude Code loop (parallel workers)
CLAUDE_PID=$(pgrep -f "claude-loop.sh" 2>/dev/null | head -1)
CLAUDE_WORKERS=$(pgrep -f "claude.*--print.*--dangerously" 2>/dev/null | wc -l | tr -d ' ')
if [ -n "$CLAUDE_PID" ]; then
    echo -e "    ${OK} Claude Loop      ${D}pid $CLAUDE_PID  ${G}${CLAUDE_WORKERS} workers active${R}"
else
    echo -e "    ${FAIL} Claude Loop      ${RD}DOWN — run: ops-wake-claude${R}"
fi

# Gemini Code loop
GEMINI_PID=$(pgrep -f "gemini-loop.sh" 2>/dev/null | head -1)
GEMINI_WORK=$(pgrep -f "gemini.*--print" 2>/dev/null | head -1)
if [ -n "$GEMINI_PID" ]; then
    if [ -n "$GEMINI_WORK" ]; then
        echo -e "    ${OK} Gemini Loop      ${D}pid $GEMINI_PID ${G}working${R}"
    else
        echo -e "    ${WARN} Gemini Loop      ${D}pid $GEMINI_PID ${Y}between issues${R}"
    fi
else
    echo -e "    ${FAIL} Gemini Loop      ${RD}DOWN — run: ops-wake-gemini${R}"
fi

# Timmy Orchestrator
TIMMY_PID=$(pgrep -f "timmy-orchestrator.sh" 2>/dev/null | head -1)
if [ -n "$TIMMY_PID" ]; then
    TIMMY_LAST=$(tail -1 "$HOME/.hermes/logs/timmy-orchestrator.log" 2>/dev/null | sed 's/.*TIMMY: //')
    echo -e "    ${OK} Timmy (Ollama)   ${D}pid $TIMMY_PID ${G}${TIMMY_LAST:0:30}${R}"
else
    echo -e "    ${FAIL} Timmy            ${RD}DOWN — run: ops-wake-timmy${R}"
fi

# Gitea VPS
if curl -s --max-time 3 "http://143.198.27.163:3000/api/v1/version" >/dev/null 2>&1; then
    echo -e "    ${OK} Gitea VPS        ${D}143.198.27.163:3000${R}"
else
    echo -e "    ${FAIL} Gitea VPS        ${RD}unreachable${R}"
fi

# Matrix staging
HTTP=$(curl -s --max-time 3 -o /dev/null -w "%{http_code}" "http://143.198.27.163/")
[ "$HTTP" = "200" ] && echo -e "    ${OK} Matrix Staging   ${D}143.198.27.163${R}" \
                     || echo -e "    ${FAIL} Matrix Staging   ${RD}HTTP $HTTP${R}"

# Dev cycle cron
CRON_LINE=$(hermes cron list 2>&1 | grep -B1 "consolidated-dev-cycle" | head -1 2>/dev/null)
if echo "$CRON_LINE" | grep -q "active"; then
    NEXT=$(hermes cron list 2>&1 | grep -A4 "consolidated-dev-cycle" | grep "Next" | awk '{print $NF}' | cut -dT -f2 | cut -d. -f1)
    echo -e "    ${OK} Dev Cycle        ${D}every 30m, next ${NEXT:-?}${R}"
else
    echo -e "    ${FAIL} Dev Cycle Cron   ${RD}MISSING${R}"
fi

echo ""

# ── KIMI STATS ─────────────────────────────────────────────────────────
echo -e "  ${B}${U}KIMI${R}"
echo ""
KIMI_LOG="$HOME/.hermes/logs/kimi-loop.log"
if [ -f "$KIMI_LOG" ]; then
    COMPLETED=$(grep -c "SUCCESS:" "$KIMI_LOG" 2>/dev/null || echo 0)
    FAILED=$(grep -c "FAILED:" "$KIMI_LOG" 2>/dev/null || echo 0)
    LAST_ISSUE=$(grep "=== ISSUE" "$KIMI_LOG" | tail -1 | sed 's/.*=== //' | sed 's/ ===//')
    LAST_TIME=$(grep "=== ISSUE\|SUCCESS\|FAILED" "$KIMI_LOG" | tail -1 | cut -d']' -f1 | tr -d '[')
    RATE=""
    if [ "$COMPLETED" -gt 0 ] && [ "$FAILED" -gt 0 ]; then
        TOTAL=$((COMPLETED + FAILED))
        PCT=$((COMPLETED * 100 / TOTAL))
        RATE=" (${PCT}% success)"
    fi
    echo -e "    Completed  ${G}${B}$COMPLETED${R}    Failed  ${RD}$FAILED${R}${D}$RATE${R}"
    echo -e "    Current    ${C}$LAST_ISSUE${R}"
    echo -e "    Last seen  ${D}$LAST_TIME${R}"
fi
echo ""

# ── CLAUDE STATS ──────────────────────────────────────────────────
echo -e "  ${B}${U}CLAUDE${R}"
echo ""
CLAUDE_LOG="$HOME/.hermes/logs/claude-loop.log"
if [ -f "$CLAUDE_LOG" ]; then
    CL_COMPLETED=$(grep -c "SUCCESS" "$CLAUDE_LOG" 2>/dev/null || echo 0)
    CL_FAILED=$(grep -c "FAILED" "$CLAUDE_LOG" 2>/dev/null || echo 0)
    CL_RATE_LIM=$(grep -c "RATE LIMITED" "$CLAUDE_LOG" 2>/dev/null || echo 0)
    CL_RATE=""
    if [ "$CL_COMPLETED" -gt 0 ] || [ "$CL_FAILED" -gt 0 ]; then
        CL_TOTAL=$((CL_COMPLETED + CL_FAILED))
        [ "$CL_TOTAL" -gt 0 ] && CL_PCT=$((CL_COMPLETED * 100 / CL_TOTAL)) && CL_RATE=" (${CL_PCT}%)"
    fi
    echo -e "    ${G}${B}$CL_COMPLETED${R} done  ${RD}$CL_FAILED${R} fail  ${Y}$CL_RATE_LIM${R} rate-limited${D}$CL_RATE${R}"

    # Show active workers
    ACTIVE="$HOME/.hermes/logs/claude-active.json"
    if [ -f "$ACTIVE" ]; then
        python3 -c "
import json
try:
    with open('$ACTIVE') as f: active = json.load(f)
    for wid, info in sorted(active.items()):
        iss = info.get('issue','')
        repo = info.get('repo','').split('/')[-1] if info.get('repo') else ''
        st = info.get('status','')
        if st == 'working':
            print(f'    \033[36mW{wid}\033[0m \033[33m#{iss}\033[0m \033[2m{repo}\033[0m')
        elif st == 'idle':
            print(f'    \033[2mW{wid} idle\033[0m')
except: pass
" 2>/dev/null
    fi
else
    echo -e "    ${D}(no log yet — start with ops-wake-claude)${R}"
fi
echo ""

# ── GEMINI STATS ─────────────────────────────────────────────────────
echo -e "  ${B}${U}GEMINI${R}"
echo ""
GEMINI_LOG="$HOME/.hermes/logs/gemini-loop.log"
if [ -f "$GEMINI_LOG" ]; then
    GM_COMPLETED=$(grep -c "SUCCESS:" "$GEMINI_LOG" 2>/dev/null || echo 0)
    GM_FAILED=$(grep -c "FAILED:" "$GEMINI_LOG" 2>/dev/null || echo 0)
    GM_RATE=""
    if [ "$GM_COMPLETED" -gt 0 ] || [ "$GM_FAILED" -gt 0 ]; then
        GM_TOTAL=$((GM_COMPLETED + GM_FAILED))
        [ "$GM_TOTAL" -gt 0 ] && GM_PCT=$((GM_COMPLETED * 100 / GM_TOTAL)) && GM_RATE=" (${GM_PCT}%)"
    fi
    GM_LAST=$(grep "=== ISSUE" "$GEMINI_LOG" | tail -1 | sed 's/.*=== //' | sed 's/ ===//')
    echo -e "    ${G}${B}$GM_COMPLETED${R} done  ${RD}$GM_FAILED${R} fail${D}$GM_RATE${R}"
    [ -n "$GM_LAST" ] && echo -e "    Current    ${C}$GM_LAST${R}"
else
    echo -e "    ${D}(no log yet — start with ops-wake-gemini)${R}"
fi
echo ""

# ── OPEN PRS ───────────────────────────────────────────────────────────
echo -e "  ${B}${U}PULL REQUESTS${R}"
echo ""
curl -s --max-time 5 -H "Authorization: token $TOKEN" "$API/pulls?state=open&limit=8" 2>/dev/null | python3 -c "
import json,sys
try:
    prs = json.loads(sys.stdin.read())
    if not prs: print('    \033[2m(none open)\033[0m')
    for p in prs[:6]:
        n = p['number']
        t = p['title'][:55]
        u = p['user']['login']
        print(f'    \033[33m#{n:<4d}\033[0m \033[2m{u:8s}\033[0m {t}')
    if len(prs) > 6: print(f'    \033[2m... +{len(prs)-6} more\033[0m')
except: print('    \033[31m(error fetching)\033[0m')
" 2>/dev/null
echo ""

# ── RECENTLY MERGED ────────────────────────────────────────────────────
echo -e "  ${B}${U}RECENTLY MERGED${R}"
echo ""
curl -s --max-time 5 -H "Authorization: token $TOKEN" "$API/pulls?state=closed&sort=updated&limit=5" 2>/dev/null | python3 -c "
import json,sys
try:
    prs = json.loads(sys.stdin.read())
    merged = [p for p in prs if p.get('merged')][:5]
    if not merged: print('    \033[2m(none recent)\033[0m')
    for p in merged:
        n = p['number']
        t = p['title'][:50]
        when = p['merged_at'][11:16]
        print(f'    \033[32m✓ #{n:<4d}\033[0m {t}  \033[2m{when}\033[0m')
except: print('    \033[31m(error)\033[0m')
" 2>/dev/null
echo ""

# ── KIMI QUEUE ─────────────────────────────────────────────────────────
echo -e "  ${B}${U}KIMI QUEUE${R}"
echo ""
curl -s --max-time 5 -H "Authorization: token $TOKEN" "$API/issues?state=open&assignee=kimi&limit=10&type=issues" 2>/dev/null | python3 -c "
import json,sys
try:
    issues = json.loads(sys.stdin.read())
    if not issues: print('    \033[33m⚠ Queue empty — assign more issues to kimi\033[0m')
    for i in issues[:6]:
        n = i['number']
        t = i['title'][:55]
        print(f'    #{n:<4d} {t}')
    if len(issues) > 6: print(f'    \033[2m... +{len(issues)-6} more\033[0m')
except: print('    \033[31m(error)\033[0m')
" 2>/dev/null
echo ""

# ── CLAUDE QUEUE ──────────────────────────────────────────────────
echo -e "  ${B}${U}CLAUDE QUEUE${R}"
echo ""
# Claude works across multiple repos
python3 -c "
import json, sys, urllib.request
token = '$(cat ~/.hermes/claude_token 2>/dev/null)'
base = 'http://143.198.27.163:3000'
repos = ['rockachopa/Timmy-time-dashboard','rockachopa/alexanderwhitestone.com','replit/timmy-tower','replit/token-gated-economy','rockachopa/hermes-agent']
all_issues = []
for repo in repos:
    url = f'{base}/api/v1/repos/{repo}/issues?state=open&assignee=claude&limit=10&type=issues'
    try:
        req = urllib.request.Request(url, headers={'Authorization': f'token {token}'})
        resp = urllib.request.urlopen(req, timeout=5)
        issues = json.loads(resp.read())
        for i in issues:
            i['_repo'] = repo.split('/')[1]
        all_issues.extend(issues)
    except: continue
if not all_issues:
    print('    \033[33m\u26a0 Queue empty \u2014 assign issues to claude\033[0m')
else:
    for i in all_issues[:6]:
        n = i['number']
        t = i['title'][:45]
        r = i['_repo'][:12]
        print(f'    #{n:<4d} \033[2m{r:12s}\033[0m {t}')
    if len(all_issues) > 6:
        print(f'    \033[2m... +{len(all_issues)-6} more\033[0m')
" 2>/dev/null
echo ""

# ── GEMINI QUEUE ─────────────────────────────────────────────────────
echo -e "  ${B}${U}GEMINI QUEUE${R}"
echo ""
curl -s --max-time 5 -H "Authorization: token $TOKEN" "$API/issues?state=open&assignee=gemini&limit=10&type=issues" 2>/dev/null | python3 -c "
import json,sys
try:
    issues = json.loads(sys.stdin.read())
    if not issues: print('    \033[33m⚠ Queue empty — assign issues to gemini\033[0m')
    for i in issues[:6]:
        n = i['number']
        t = i['title'][:55]
        print(f'    #{n:<4d} {t}')
    if len(issues) > 6: print(f'    \033[2m... +{len(issues)-6} more\033[0m')
except: print('    \033[31m(error)\033[0m')
" 2>/dev/null
echo ""

# ── WARNINGS ───────────────────────────────────────────────────────────
HERMES_PROCS=$(ps aux | grep -E "hermes.*python" | grep -v grep | wc -l | tr -d ' ')
STUCK_GIT=$(ps aux | grep "git.*push\|git-remote-http" | grep -v grep | wc -l | tr -d ' ')
ORPHAN_PY=$(ps aux | grep "pytest tests/" | grep -v grep | wc -l | tr -d ' ')
UNASSIGNED=$(curl -s --max-time 3 -H "Authorization: token $TOKEN" "$API/issues?state=open&limit=50&type=issues" 2>/dev/null | python3 -c "import json,sys; issues=json.loads(sys.stdin.read()); print(len([i for i in issues if not i.get('assignees')]))" 2>/dev/null)

WARNS=""
[ "$STUCK_GIT" -gt 0 ] && WARNS+="    ${RD}⚠ $STUCK_GIT stuck git processes${R}\n"
[ "$ORPHAN_PY" -gt 0 ] && WARNS+="    ${Y}⚠ $ORPHAN_PY orphaned pytest runs${R}\n"
[ "${UNASSIGNED:-0}" -gt 10 ] && WARNS+="    ${Y}⚠ $UNASSIGNED unassigned issues — feed the queue${R}\n"

if [ -n "$WARNS" ]; then
    echo -e "  ${B}${U}WARNINGS${R}"
    echo ""
    echo -e "$WARNS"
fi

echo -e "  ${D}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${R}"
echo -e "  ${D}hermes sessions: $HERMES_PROCS    unassigned: ${UNASSIGNED:-?}    ↻ 20s${R}"