280 lines
10 KiB
Python
Executable File
280 lines
10 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Timmy Model Dashboard — where are my models, what are they doing.
|
|
|
|
Usage:
|
|
timmy-dashboard # one-shot
|
|
timmy-dashboard --watch # live refresh every 30s
|
|
timmy-dashboard --hours=48 # look back 48h
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
from datetime import datetime, timezone, timedelta
|
|
from pathlib import Path
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parent.parent
|
|
if str(REPO_ROOT) not in sys.path:
|
|
sys.path.insert(0, str(REPO_ROOT))
|
|
|
|
from metrics_helpers import summarize_local_metrics, summarize_session_rows
|
|
|
|
HERMES_HOME = Path.home() / ".hermes"
|
|
TIMMY_HOME = Path.home() / ".timmy"
|
|
METRICS_DIR = TIMMY_HOME / "metrics"
|
|
|
|
# ── Data Sources ──────────────────────────────────────────────────────
|
|
|
|
def get_ollama_models():
|
|
try:
|
|
req = urllib.request.Request("http://localhost:11434/api/tags")
|
|
with urllib.request.urlopen(req, timeout=5) as resp:
|
|
return json.loads(resp.read()).get("models", [])
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def get_loaded_models():
|
|
try:
|
|
req = urllib.request.Request("http://localhost:11434/api/ps")
|
|
with urllib.request.urlopen(req, timeout=5) as resp:
|
|
return json.loads(resp.read()).get("models", [])
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def get_huey_pid():
|
|
try:
|
|
r = subprocess.run(["pgrep", "-f", "huey_consumer"],
|
|
capture_output=True, text=True, timeout=5)
|
|
return r.stdout.strip().split("\n")[0] if r.returncode == 0 else None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def get_hermes_sessions():
|
|
sessions_file = HERMES_HOME / "sessions" / "sessions.json"
|
|
if not sessions_file.exists():
|
|
return []
|
|
try:
|
|
data = json.loads(sessions_file.read_text())
|
|
return list(data.values())
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def get_session_rows(hours=24):
|
|
state_db = HERMES_HOME / "state.db"
|
|
if not state_db.exists():
|
|
return []
|
|
cutoff = time.time() - (hours * 3600)
|
|
try:
|
|
conn = sqlite3.connect(str(state_db))
|
|
rows = conn.execute(
|
|
"""
|
|
SELECT model, source, COUNT(*) as sessions,
|
|
SUM(message_count) as msgs,
|
|
SUM(tool_call_count) as tools
|
|
FROM sessions
|
|
WHERE started_at > ? AND model IS NOT NULL AND model != ''
|
|
GROUP BY model, source
|
|
""",
|
|
(cutoff,),
|
|
).fetchall()
|
|
conn.close()
|
|
return rows
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def get_heartbeat_ticks(date_str=None):
|
|
if not date_str:
|
|
date_str = datetime.now().strftime("%Y%m%d")
|
|
tick_file = TIMMY_HOME / "heartbeat" / f"ticks_{date_str}.jsonl"
|
|
if not tick_file.exists():
|
|
return []
|
|
ticks = []
|
|
for line in tick_file.read_text().strip().split("\n"):
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
ticks.append(json.loads(line))
|
|
except Exception:
|
|
continue
|
|
return ticks
|
|
|
|
|
|
def get_local_metrics(hours=24):
|
|
"""Read local inference metrics from jsonl files."""
|
|
records = []
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=hours)
|
|
if not METRICS_DIR.exists():
|
|
return records
|
|
for f in sorted(METRICS_DIR.glob("local_*.jsonl")):
|
|
for line in f.read_text().strip().split("\n"):
|
|
if not line.strip():
|
|
continue
|
|
try:
|
|
r = json.loads(line)
|
|
ts = datetime.fromisoformat(r["timestamp"])
|
|
if ts >= cutoff:
|
|
records.append(r)
|
|
except Exception:
|
|
continue
|
|
return records
|
|
|
|
|
|
def get_cron_jobs():
|
|
"""Get Hermes cron job status."""
|
|
try:
|
|
r = subprocess.run(
|
|
["hermes", "cron", "list", "--json"],
|
|
capture_output=True, text=True, timeout=10
|
|
)
|
|
if r.returncode == 0:
|
|
return json.loads(r.stdout).get("jobs", [])
|
|
except Exception:
|
|
pass
|
|
return []
|
|
|
|
|
|
# ── Rendering ─────────────────────────────────────────────────────────
|
|
|
|
DIM = "\033[2m"
|
|
BOLD = "\033[1m"
|
|
GREEN = "\033[32m"
|
|
YELLOW = "\033[33m"
|
|
RED = "\033[31m"
|
|
CYAN = "\033[36m"
|
|
RST = "\033[0m"
|
|
CLR = "\033[2J\033[H"
|
|
|
|
|
|
def render(hours=24):
|
|
models = get_ollama_models()
|
|
loaded = get_loaded_models()
|
|
huey_pid = get_huey_pid()
|
|
ticks = get_heartbeat_ticks()
|
|
metrics = get_local_metrics(hours)
|
|
sessions = get_hermes_sessions()
|
|
session_rows = get_session_rows(hours)
|
|
local_summary = summarize_local_metrics(metrics)
|
|
session_summary = summarize_session_rows(session_rows)
|
|
|
|
loaded_names = {m.get("name", "") for m in loaded}
|
|
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
|
|
print(CLR, end="")
|
|
print(f"{BOLD}{'=' * 70}")
|
|
print(f" TIMMY MODEL DASHBOARD")
|
|
print(f" {now} | Huey: {GREEN}PID {huey_pid}{RST if huey_pid else f'{RED}DOWN{RST}'}")
|
|
print(f"{'=' * 70}{RST}")
|
|
|
|
# ── LOCAL MODELS ──
|
|
print(f"\n {BOLD}LOCAL MODELS (Ollama){RST}")
|
|
print(f" {DIM}{'-' * 55}{RST}")
|
|
if models:
|
|
for m in models:
|
|
name = m.get("name", "?")
|
|
size_gb = m.get("size", 0) / 1e9
|
|
if name in loaded_names:
|
|
status = f"{GREEN}IN VRAM{RST}"
|
|
else:
|
|
status = f"{DIM}on disk{RST}"
|
|
print(f" {name:35s} {size_gb:5.1f}GB {status}")
|
|
else:
|
|
print(f" {RED}(Ollama not responding){RST}")
|
|
|
|
# ── LOCAL INFERENCE ACTIVITY ──
|
|
print(f"\n {BOLD}LOCAL INFERENCE ({len(metrics)} calls, last {hours}h){RST}")
|
|
print(f" {DIM}{'-' * 55}{RST}")
|
|
if metrics:
|
|
print(f" Tokens: {local_summary['input_tokens']} in | {local_summary['output_tokens']} out | {local_summary['total_tokens']} total")
|
|
if local_summary.get('avg_latency_s') is not None:
|
|
print(f" Avg latency: {local_summary['avg_latency_s']:.2f}s")
|
|
if local_summary.get('avg_tokens_per_second') is not None:
|
|
print(f" Avg throughput: {GREEN}{local_summary['avg_tokens_per_second']:.2f} tok/s{RST}")
|
|
for caller, stats in sorted(local_summary['by_caller'].items()):
|
|
err = f" {RED}err:{stats['failed_calls']}{RST}" if stats['failed_calls'] else ""
|
|
print(f" {caller:25s} calls:{stats['calls']:4d} tokens:{stats['total_tokens']:5d} {GREEN}ok:{stats['successful_calls']}{RST}{err}")
|
|
|
|
print(f"\n {DIM}Models used:{RST}")
|
|
for model, stats in sorted(local_summary['by_model'].items(), key=lambda x: -x[1]['calls']):
|
|
print(f" {model:30s} {stats['calls']} calls {stats['total_tokens']} tok")
|
|
else:
|
|
print(f" {DIM}(no local calls recorded yet){RST}")
|
|
|
|
# ── HEARTBEAT STATUS ──
|
|
print(f"\n {BOLD}HEARTBEAT ({len(ticks)} ticks today){RST}")
|
|
print(f" {DIM}{'-' * 55}{RST}")
|
|
if ticks:
|
|
last = ticks[-1]
|
|
decision = last.get("decision", last.get("actions", {}))
|
|
if isinstance(decision, dict):
|
|
severity = decision.get("severity", "unknown")
|
|
reasoning = decision.get("reasoning", "")
|
|
sev_color = GREEN if severity == "ok" else YELLOW if severity == "warning" else RED
|
|
print(f" Last tick: {last.get('tick_id', '?')}")
|
|
print(f" Severity: {sev_color}{severity}{RST}")
|
|
if reasoning:
|
|
print(f" Reasoning: {reasoning[:65]}")
|
|
else:
|
|
print(f" Last tick: {last.get('tick_id', '?')}")
|
|
actions = last.get("actions", [])
|
|
print(f" Actions: {actions if actions else 'none'}")
|
|
|
|
model_decisions = sum(1 for t in ticks
|
|
if isinstance(t.get("decision"), dict)
|
|
and t["decision"].get("severity") != "fallback")
|
|
fallback = len(ticks) - model_decisions
|
|
print(f" {CYAN}Model: {model_decisions}{RST} | {DIM}Fallback: {fallback}{RST}")
|
|
else:
|
|
print(f" {DIM}(no ticks today){RST}")
|
|
|
|
# ── HERMES SESSIONS / SOVEREIGNTY LOAD ──
|
|
local_sessions = [s for s in sessions if "localhost:11434" in str(s.get("base_url", ""))]
|
|
cloud_sessions = [s for s in sessions if s not in local_sessions]
|
|
print(f"\n {BOLD}HERMES SESSIONS / SOVEREIGNTY LOAD{RST}")
|
|
print(f" {DIM}{'-' * 55}{RST}")
|
|
print(f" Session cache: {len(sessions)} total | {GREEN}{len(local_sessions)} local{RST} | {YELLOW}{len(cloud_sessions)} cloud{RST}")
|
|
if session_rows:
|
|
print(f" Session DB: {session_summary['total_sessions']} total | {GREEN}{session_summary['local_sessions']} local{RST} | {YELLOW}{session_summary['cloud_sessions']} cloud{RST}")
|
|
print(f" Token est: {GREEN}{session_summary['local_est_tokens']} local{RST} | {YELLOW}{session_summary['cloud_est_tokens']} cloud{RST}")
|
|
print(f" Est cloud cost: ${session_summary['cloud_est_cost_usd']:.4f}")
|
|
else:
|
|
print(f" {DIM}(no session-db stats available){RST}")
|
|
|
|
# ── ACTIVE LOOPS ──
|
|
print(f"\n {BOLD}ACTIVE LOOPS{RST}")
|
|
print(f" {DIM}{'-' * 55}{RST}")
|
|
print(f" {CYAN}heartbeat_tick{RST} 10m hermes4:14b DECIDE phase")
|
|
print(f" {DIM}model_health{RST} 5m (local check) Ollama ping")
|
|
print(f" {DIM}gemini_worker{RST} 20m gemini-2.5-pro aider")
|
|
print(f" {DIM}grok_worker{RST} 20m grok-3-fast opencode")
|
|
print(f" {DIM}cross_review{RST} 30m gemini+grok PR review")
|
|
|
|
print(f"\n{BOLD}{'=' * 70}{RST}")
|
|
print(f" {DIM}Refresh: timmy-dashboard --watch | History: --hours=N{RST}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
watch = "--watch" in sys.argv
|
|
hours = 24
|
|
for a in sys.argv[1:]:
|
|
if a.startswith("--hours="):
|
|
hours = int(a.split("=")[1])
|
|
|
|
if watch:
|
|
try:
|
|
while True:
|
|
render(hours)
|
|
time.sleep(30)
|
|
except KeyboardInterrupt:
|
|
print(f"\n{DIM}Dashboard stopped.{RST}")
|
|
else:
|
|
render(hours)
|