d4c79d47a6
- Moved all agent loop scripts into source control (bin/) - claude-loop.sh, gemini-loop.sh, timmy-orchestrator.sh - workforce-manager.py, agent-dispatch.sh, nexus-merge-bot.sh - ops dashboard scripts (ops-panel, ops-helpers, ops-gitea) - monitoring scripts (timmy-status, timmy-loopstat) - deploy.sh: one-command overlay onto ~/.hermes/ - Updated README with sidecar architecture docs - All loops now target the-nexus + autolora only
301 lines
14 KiB
Bash
Executable File
301 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ── Consolidated Ops Panel ─────────────────────────────────────────────
|
|
# Everything in one view. Designed for a half-screen pane (~100x45).
|
|
# ───────────────────────────────────────────────────────────────────────
|
|
|
|
B='\033[1m' ; D='\033[2m' ; R='\033[0m' ; U='\033[4m'
|
|
G='\033[32m' ; Y='\033[33m' ; RD='\033[31m' ; C='\033[36m' ; M='\033[35m' ; W='\033[37m'
|
|
OK="${G}●${R}" ; WARN="${Y}●${R}" ; FAIL="${RD}●${R}" ; OFF="${D}○${R}"
|
|
|
|
TOKEN=$(cat ~/.hermes/gitea_token_vps 2>/dev/null)
|
|
API="http://143.198.27.163:3000/api/v1/repos/rockachopa/Timmy-time-dashboard"
|
|
|
|
# ── HEADER ─────────────────────────────────────────────────────────────
|
|
echo ""
|
|
echo -e " ${B}${M}◈ HERMES OPERATIONS${R} ${D}$(date '+%a %b %d %H:%M:%S')${R}"
|
|
echo -e " ${D}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${R}"
|
|
echo ""
|
|
|
|
# ── SERVICES ───────────────────────────────────────────────────────────
|
|
echo -e " ${B}${U}SERVICES${R}"
|
|
echo ""
|
|
|
|
# Gateway
|
|
GW_PID=$(pgrep -f "hermes.*gateway.*run" 2>/dev/null | head -1)
|
|
[ -n "$GW_PID" ] && echo -e " ${OK} Gateway ${D}pid $GW_PID${R}" \
|
|
|| echo -e " ${FAIL} Gateway ${RD}DOWN — run: hermes gateway start${R}"
|
|
|
|
# Kimi Code loop
|
|
KIMI_PID=$(pgrep -f "kimi-loop.sh" 2>/dev/null | head -1)
|
|
[ -n "$KIMI_PID" ] && echo -e " ${OK} Kimi Loop ${D}pid $KIMI_PID${R}" \
|
|
|| echo -e " ${FAIL} Kimi Loop ${RD}DOWN — run: ops-wake-kimi${R}"
|
|
|
|
# Active Kimi Code worker
|
|
KIMI_WORK=$(pgrep -f "kimi.*--print" 2>/dev/null | head -1)
|
|
if [ -n "$KIMI_WORK" ]; then
|
|
echo -e " ${OK} Kimi Code ${D}pid $KIMI_WORK ${G}working${R}"
|
|
elif [ -n "$KIMI_PID" ]; then
|
|
echo -e " ${WARN} Kimi Code ${Y}between issues${R}"
|
|
else
|
|
echo -e " ${OFF} Kimi Code ${D}not running${R}"
|
|
fi
|
|
|
|
# Claude Code loop (parallel workers)
|
|
CLAUDE_PID=$(pgrep -f "claude-loop.sh" 2>/dev/null | head -1)
|
|
CLAUDE_WORKERS=$(pgrep -f "claude.*--print.*--dangerously" 2>/dev/null | wc -l | tr -d ' ')
|
|
if [ -n "$CLAUDE_PID" ]; then
|
|
echo -e " ${OK} Claude Loop ${D}pid $CLAUDE_PID ${G}${CLAUDE_WORKERS} workers active${R}"
|
|
else
|
|
echo -e " ${FAIL} Claude Loop ${RD}DOWN — run: ops-wake-claude${R}"
|
|
fi
|
|
|
|
# Gemini Code loop
|
|
GEMINI_PID=$(pgrep -f "gemini-loop.sh" 2>/dev/null | head -1)
|
|
GEMINI_WORK=$(pgrep -f "gemini.*--print" 2>/dev/null | head -1)
|
|
if [ -n "$GEMINI_PID" ]; then
|
|
if [ -n "$GEMINI_WORK" ]; then
|
|
echo -e " ${OK} Gemini Loop ${D}pid $GEMINI_PID ${G}working${R}"
|
|
else
|
|
echo -e " ${WARN} Gemini Loop ${D}pid $GEMINI_PID ${Y}between issues${R}"
|
|
fi
|
|
else
|
|
echo -e " ${FAIL} Gemini Loop ${RD}DOWN — run: ops-wake-gemini${R}"
|
|
fi
|
|
|
|
# Timmy Orchestrator
|
|
TIMMY_PID=$(pgrep -f "timmy-orchestrator.sh" 2>/dev/null | head -1)
|
|
if [ -n "$TIMMY_PID" ]; then
|
|
TIMMY_LAST=$(tail -1 "$HOME/.hermes/logs/timmy-orchestrator.log" 2>/dev/null | sed 's/.*TIMMY: //')
|
|
echo -e " ${OK} Timmy (Ollama) ${D}pid $TIMMY_PID ${G}${TIMMY_LAST:0:30}${R}"
|
|
else
|
|
echo -e " ${FAIL} Timmy ${RD}DOWN — run: ops-wake-timmy${R}"
|
|
fi
|
|
|
|
# Gitea VPS
|
|
if curl -s --max-time 3 "http://143.198.27.163:3000/api/v1/version" >/dev/null 2>&1; then
|
|
echo -e " ${OK} Gitea VPS ${D}143.198.27.163:3000${R}"
|
|
else
|
|
echo -e " ${FAIL} Gitea VPS ${RD}unreachable${R}"
|
|
fi
|
|
|
|
# Matrix staging
|
|
HTTP=$(curl -s --max-time 3 -o /dev/null -w "%{http_code}" "http://143.198.27.163/")
|
|
[ "$HTTP" = "200" ] && echo -e " ${OK} Matrix Staging ${D}143.198.27.163${R}" \
|
|
|| echo -e " ${FAIL} Matrix Staging ${RD}HTTP $HTTP${R}"
|
|
|
|
# Dev cycle cron
|
|
CRON_LINE=$(hermes cron list 2>&1 | grep -B1 "consolidated-dev-cycle" | head -1 2>/dev/null)
|
|
if echo "$CRON_LINE" | grep -q "active"; then
|
|
NEXT=$(hermes cron list 2>&1 | grep -A4 "consolidated-dev-cycle" | grep "Next" | awk '{print $NF}' | cut -dT -f2 | cut -d. -f1)
|
|
echo -e " ${OK} Dev Cycle ${D}every 30m, next ${NEXT:-?}${R}"
|
|
else
|
|
echo -e " ${FAIL} Dev Cycle Cron ${RD}MISSING${R}"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# ── KIMI STATS ─────────────────────────────────────────────────────────
|
|
echo -e " ${B}${U}KIMI${R}"
|
|
echo ""
|
|
KIMI_LOG="$HOME/.hermes/logs/kimi-loop.log"
|
|
if [ -f "$KIMI_LOG" ]; then
|
|
COMPLETED=$(grep -c "SUCCESS:" "$KIMI_LOG" 2>/dev/null | tail -1 || echo 0)
|
|
FAILED=$(grep -c "FAILED:" "$KIMI_LOG" 2>/dev/null | tail -1 || echo 0)
|
|
LAST_ISSUE=$(grep "=== ISSUE" "$KIMI_LOG" | tail -1 | sed 's/.*=== //' | sed 's/ ===//')
|
|
LAST_TIME=$(grep "=== ISSUE\|SUCCESS\|FAILED" "$KIMI_LOG" | tail -1 | cut -d']' -f1 | tr -d '[')
|
|
RATE=""
|
|
if [ "$COMPLETED" -gt 0 ] && [ "$FAILED" -gt 0 ]; then
|
|
TOTAL=$((COMPLETED + FAILED))
|
|
PCT=$((COMPLETED * 100 / TOTAL))
|
|
RATE=" (${PCT}% success)"
|
|
fi
|
|
echo -e " Completed ${G}${B}$COMPLETED${R} Failed ${RD}$FAILED${R}${D}$RATE${R}"
|
|
echo -e " Current ${C}$LAST_ISSUE${R}"
|
|
echo -e " Last seen ${D}$LAST_TIME${R}"
|
|
fi
|
|
echo ""
|
|
|
|
# ── CLAUDE STATS ──────────────────────────────────────────────────
|
|
echo -e " ${B}${U}CLAUDE${R}"
|
|
echo ""
|
|
CLAUDE_LOG="$HOME/.hermes/logs/claude-loop.log"
|
|
if [ -f "$CLAUDE_LOG" ]; then
|
|
CL_COMPLETED=$(grep -c "SUCCESS" "$CLAUDE_LOG" 2>/dev/null | tail -1 || echo 0)
|
|
CL_FAILED=$(grep -c "FAILED" "$CLAUDE_LOG" 2>/dev/null | tail -1 || echo 0)
|
|
CL_RATE_LIM=$(grep -c "RATE LIMITED" "$CLAUDE_LOG" 2>/dev/null | tail -1 || echo 0)
|
|
CL_RATE=""
|
|
if [ "$CL_COMPLETED" -gt 0 ] || [ "$CL_FAILED" -gt 0 ]; then
|
|
CL_TOTAL=$((CL_COMPLETED + CL_FAILED))
|
|
[ "$CL_TOTAL" -gt 0 ] && CL_PCT=$((CL_COMPLETED * 100 / CL_TOTAL)) && CL_RATE=" (${CL_PCT}%)"
|
|
fi
|
|
echo -e " ${G}${B}$CL_COMPLETED${R} done ${RD}$CL_FAILED${R} fail ${Y}$CL_RATE_LIM${R} rate-limited${D}$CL_RATE${R}"
|
|
|
|
# Show active workers
|
|
ACTIVE="$HOME/.hermes/logs/claude-active.json"
|
|
if [ -f "$ACTIVE" ]; then
|
|
python3 -c "
|
|
import json
|
|
try:
|
|
with open('$ACTIVE') as f: active = json.load(f)
|
|
for wid, info in sorted(active.items()):
|
|
iss = info.get('issue','')
|
|
repo = info.get('repo','').split('/')[-1] if info.get('repo') else ''
|
|
st = info.get('status','')
|
|
if st == 'working':
|
|
print(f' \033[36mW{wid}\033[0m \033[33m#{iss}\033[0m \033[2m{repo}\033[0m')
|
|
elif st == 'idle':
|
|
print(f' \033[2mW{wid} idle\033[0m')
|
|
except: pass
|
|
" 2>/dev/null
|
|
fi
|
|
else
|
|
echo -e " ${D}(no log yet — start with ops-wake-claude)${R}"
|
|
fi
|
|
echo ""
|
|
|
|
# ── GEMINI STATS ─────────────────────────────────────────────────────
|
|
echo -e " ${B}${U}GEMINI${R}"
|
|
echo ""
|
|
GEMINI_LOG="$HOME/.hermes/logs/gemini-loop.log"
|
|
if [ -f "$GEMINI_LOG" ]; then
|
|
GM_COMPLETED=$(grep -c "SUCCESS:" "$GEMINI_LOG" 2>/dev/null | tail -1 || echo 0)
|
|
GM_FAILED=$(grep -c "FAILED:" "$GEMINI_LOG" 2>/dev/null | tail -1 || echo 0)
|
|
GM_RATE=""
|
|
if [ "$GM_COMPLETED" -gt 0 ] || [ "$GM_FAILED" -gt 0 ]; then
|
|
GM_TOTAL=$((GM_COMPLETED + GM_FAILED))
|
|
[ "$GM_TOTAL" -gt 0 ] && GM_PCT=$((GM_COMPLETED * 100 / GM_TOTAL)) && GM_RATE=" (${GM_PCT}%)"
|
|
fi
|
|
GM_LAST=$(grep "=== ISSUE" "$GEMINI_LOG" | tail -1 | sed 's/.*=== //' | sed 's/ ===//')
|
|
echo -e " ${G}${B}$GM_COMPLETED${R} done ${RD}$GM_FAILED${R} fail${D}$GM_RATE${R}"
|
|
[ -n "$GM_LAST" ] && echo -e " Current ${C}$GM_LAST${R}"
|
|
else
|
|
echo -e " ${D}(no log yet — start with ops-wake-gemini)${R}"
|
|
fi
|
|
echo ""
|
|
|
|
# ── OPEN PRS ───────────────────────────────────────────────────────────
|
|
echo -e " ${B}${U}PULL REQUESTS${R}"
|
|
echo ""
|
|
curl -s --max-time 5 -H "Authorization: token $TOKEN" "$API/pulls?state=open&limit=8" 2>/dev/null | python3 -c "
|
|
import json,sys
|
|
try:
|
|
prs = json.loads(sys.stdin.read())
|
|
if not prs: print(' \033[2m(none open)\033[0m')
|
|
for p in prs[:6]:
|
|
n = p['number']
|
|
t = p['title'][:55]
|
|
u = p['user']['login']
|
|
print(f' \033[33m#{n:<4d}\033[0m \033[2m{u:8s}\033[0m {t}')
|
|
if len(prs) > 6: print(f' \033[2m... +{len(prs)-6} more\033[0m')
|
|
except: print(' \033[31m(error fetching)\033[0m')
|
|
" 2>/dev/null
|
|
echo ""
|
|
|
|
# ── RECENTLY MERGED ────────────────────────────────────────────────────
|
|
echo -e " ${B}${U}RECENTLY MERGED${R}"
|
|
echo ""
|
|
curl -s --max-time 5 -H "Authorization: token $TOKEN" "$API/pulls?state=closed&sort=updated&limit=5" 2>/dev/null | python3 -c "
|
|
import json,sys
|
|
try:
|
|
prs = json.loads(sys.stdin.read())
|
|
merged = [p for p in prs if p.get('merged')][:5]
|
|
if not merged: print(' \033[2m(none recent)\033[0m')
|
|
for p in merged:
|
|
n = p['number']
|
|
t = p['title'][:50]
|
|
when = p['merged_at'][11:16]
|
|
print(f' \033[32m✓ #{n:<4d}\033[0m {t} \033[2m{when}\033[0m')
|
|
except: print(' \033[31m(error)\033[0m')
|
|
" 2>/dev/null
|
|
echo ""
|
|
|
|
# ── KIMI QUEUE ─────────────────────────────────────────────────────────
|
|
echo -e " ${B}${U}KIMI QUEUE${R}"
|
|
echo ""
|
|
curl -s --max-time 5 -H "Authorization: token $TOKEN" "$API/issues?state=open&limit=50&type=issues" 2>/dev/null | python3 -c "
|
|
import json,sys
|
|
try:
|
|
all_issues = json.loads(sys.stdin.read())
|
|
issues = [i for i in all_issues if 'kimi' in [a.get('login','') for a in (i.get('assignees') or [])]]
|
|
if not issues: print(' \033[33m⚠ Queue empty — assign more issues to kimi\033[0m')
|
|
for i in issues[:6]:
|
|
n = i['number']
|
|
t = i['title'][:55]
|
|
print(f' #{n:<4d} {t}')
|
|
if len(issues) > 6: print(f' \033[2m... +{len(issues)-6} more\033[0m')
|
|
except: print(' \033[31m(error)\033[0m')
|
|
" 2>/dev/null
|
|
echo ""
|
|
|
|
# ── CLAUDE QUEUE ──────────────────────────────────────────────────
|
|
echo -e " ${B}${U}CLAUDE QUEUE${R}"
|
|
echo ""
|
|
# Claude works across multiple repos
|
|
python3 -c "
|
|
import json, sys, urllib.request
|
|
token = '$(cat ~/.hermes/claude_token 2>/dev/null)'
|
|
base = 'http://143.198.27.163:3000'
|
|
repos = ['rockachopa/Timmy-time-dashboard','rockachopa/alexanderwhitestone.com','replit/timmy-tower','replit/token-gated-economy','rockachopa/hermes-agent']
|
|
all_issues = []
|
|
for repo in repos:
|
|
url = f'{base}/api/v1/repos/{repo}/issues?state=open&limit=50&type=issues'
|
|
try:
|
|
req = urllib.request.Request(url, headers={'Authorization': f'token {token}'})
|
|
resp = urllib.request.urlopen(req, timeout=5)
|
|
raw = json.loads(resp.read())
|
|
issues = [i for i in raw if 'claude' in [a.get('login','') for a in (i.get('assignees') or [])]]
|
|
for i in issues:
|
|
i['_repo'] = repo.split('/')[1]
|
|
all_issues.extend(issues)
|
|
except: continue
|
|
if not all_issues:
|
|
print(' \033[33m\u26a0 Queue empty \u2014 assign issues to claude\033[0m')
|
|
else:
|
|
for i in all_issues[:6]:
|
|
n = i['number']
|
|
t = i['title'][:45]
|
|
r = i['_repo'][:12]
|
|
print(f' #{n:<4d} \033[2m{r:12s}\033[0m {t}')
|
|
if len(all_issues) > 6:
|
|
print(f' \033[2m... +{len(all_issues)-6} more\033[0m')
|
|
" 2>/dev/null
|
|
echo ""
|
|
|
|
# ── GEMINI QUEUE ─────────────────────────────────────────────────────
|
|
echo -e " ${B}${U}GEMINI QUEUE${R}"
|
|
echo ""
|
|
curl -s --max-time 5 -H "Authorization: token $TOKEN" "$API/issues?state=open&limit=50&type=issues" 2>/dev/null | python3 -c "
|
|
import json,sys
|
|
try:
|
|
all_issues = json.loads(sys.stdin.read())
|
|
issues = [i for i in all_issues if 'gemini' in [a.get('login','') for a in (i.get('assignees') or [])]]
|
|
if not issues: print(' \033[33m⚠ Queue empty — assign issues to gemini\033[0m')
|
|
for i in issues[:6]:
|
|
n = i['number']
|
|
t = i['title'][:55]
|
|
print(f' #{n:<4d} {t}')
|
|
if len(issues) > 6: print(f' \033[2m... +{len(issues)-6} more\033[0m')
|
|
except: print(' \033[31m(error)\033[0m')
|
|
" 2>/dev/null
|
|
echo ""
|
|
|
|
# ── WARNINGS ───────────────────────────────────────────────────────────
|
|
HERMES_PROCS=$(ps aux | grep -E "hermes.*python" | grep -v grep | wc -l | tr -d ' ')
|
|
STUCK_GIT=$(ps aux | grep "git.*push\|git-remote-http" | grep -v grep | wc -l | tr -d ' ')
|
|
ORPHAN_PY=$(ps aux | grep "pytest tests/" | grep -v grep | wc -l | tr -d ' ')
|
|
UNASSIGNED=$(curl -s --max-time 3 -H "Authorization: token $TOKEN" "$API/issues?state=open&limit=50&type=issues" 2>/dev/null | python3 -c "import json,sys; issues=json.loads(sys.stdin.read()); print(len([i for i in issues if not i.get('assignees')]))" 2>/dev/null)
|
|
|
|
WARNS=""
|
|
[ "$STUCK_GIT" -gt 0 ] && WARNS+=" ${RD}⚠ $STUCK_GIT stuck git processes${R}\n"
|
|
[ "$ORPHAN_PY" -gt 0 ] && WARNS+=" ${Y}⚠ $ORPHAN_PY orphaned pytest runs${R}\n"
|
|
[ "${UNASSIGNED:-0}" -gt 10 ] && WARNS+=" ${Y}⚠ $UNASSIGNED unassigned issues — feed the queue${R}\n"
|
|
|
|
if [ -n "$WARNS" ]; then
|
|
echo -e " ${B}${U}WARNINGS${R}"
|
|
echo ""
|
|
echo -e "$WARNS"
|
|
fi
|
|
|
|
echo -e " ${D}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${R}"
|
|
echo -e " ${D}hermes sessions: $HERMES_PROCS unassigned: ${UNASSIGNED:-?} ↻ 20s${R}"
|