#!/usr/bin/env python3 """Timmy Model Dashboard — where are my models, what are they doing. Usage: timmy-dashboard # one-shot timmy-dashboard --watch # live refresh every 30s timmy-dashboard --hours=48 # look back 48h """ import json import os import sqlite3 import subprocess import sys import time import urllib.request from datetime import datetime, timezone, timedelta from pathlib import Path REPO_ROOT = Path(__file__).resolve().parent.parent if str(REPO_ROOT) not in sys.path: sys.path.insert(0, str(REPO_ROOT)) from metrics_helpers import summarize_local_metrics, summarize_session_rows HERMES_HOME = Path.home() / ".hermes" TIMMY_HOME = Path.home() / ".timmy" METRICS_DIR = TIMMY_HOME / "metrics" # ── Data Sources ────────────────────────────────────────────────────── def get_ollama_models(): try: req = urllib.request.Request("http://localhost:11434/api/tags") with urllib.request.urlopen(req, timeout=5) as resp: return json.loads(resp.read()).get("models", []) except Exception: return [] def get_loaded_models(): try: req = urllib.request.Request("http://localhost:11434/api/ps") with urllib.request.urlopen(req, timeout=5) as resp: return json.loads(resp.read()).get("models", []) except Exception: return [] def get_huey_pid(): try: r = subprocess.run(["pgrep", "-f", "huey_consumer"], capture_output=True, text=True, timeout=5) return r.stdout.strip().split("\n")[0] if r.returncode == 0 else None except Exception: return None def get_hermes_sessions(): sessions_file = HERMES_HOME / "sessions" / "sessions.json" if not sessions_file.exists(): return [] try: data = json.loads(sessions_file.read_text()) return list(data.values()) except Exception: return [] def get_session_rows(hours=24): state_db = HERMES_HOME / "state.db" if not state_db.exists(): return [] cutoff = time.time() - (hours * 3600) try: conn = sqlite3.connect(str(state_db)) rows = conn.execute( """ SELECT model, source, COUNT(*) as sessions, SUM(message_count) as msgs, SUM(tool_call_count) as tools FROM sessions WHERE started_at > ? AND model IS NOT NULL AND model != '' GROUP BY model, source """, (cutoff,), ).fetchall() conn.close() return rows except Exception: return [] def get_heartbeat_ticks(date_str=None): if not date_str: date_str = datetime.now().strftime("%Y%m%d") tick_file = TIMMY_HOME / "heartbeat" / f"ticks_{date_str}.jsonl" if not tick_file.exists(): return [] ticks = [] for line in tick_file.read_text().strip().split("\n"): if not line.strip(): continue try: ticks.append(json.loads(line)) except Exception: continue return ticks def get_local_metrics(hours=24): """Read local inference metrics from jsonl files.""" records = [] cutoff = datetime.now(timezone.utc) - timedelta(hours=hours) if not METRICS_DIR.exists(): return records for f in sorted(METRICS_DIR.glob("local_*.jsonl")): for line in f.read_text().strip().split("\n"): if not line.strip(): continue try: r = json.loads(line) ts = datetime.fromisoformat(r["timestamp"]) if ts >= cutoff: records.append(r) except Exception: continue return records def get_cron_jobs(): """Get Hermes cron job status.""" try: r = subprocess.run( ["hermes", "cron", "list", "--json"], capture_output=True, text=True, timeout=10 ) if r.returncode == 0: return json.loads(r.stdout).get("jobs", []) except Exception: pass return [] # ── Rendering ───────────────────────────────────────────────────────── DIM = "\033[2m" BOLD = "\033[1m" GREEN = "\033[32m" YELLOW = "\033[33m" RED = "\033[31m" CYAN = "\033[36m" RST = "\033[0m" CLR = "\033[2J\033[H" def render(hours=24): models = get_ollama_models() loaded = get_loaded_models() huey_pid = get_huey_pid() ticks = get_heartbeat_ticks() metrics = get_local_metrics(hours) sessions = get_hermes_sessions() session_rows = get_session_rows(hours) local_summary = summarize_local_metrics(metrics) session_summary = summarize_session_rows(session_rows) loaded_names = {m.get("name", "") for m in loaded} now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") print(CLR, end="") print(f"{BOLD}{'=' * 70}") print(f" TIMMY MODEL DASHBOARD") print(f" {now} | Huey: {GREEN}PID {huey_pid}{RST if huey_pid else f'{RED}DOWN{RST}'}") print(f"{'=' * 70}{RST}") # ── LOCAL MODELS ── print(f"\n {BOLD}LOCAL MODELS (Ollama){RST}") print(f" {DIM}{'-' * 55}{RST}") if models: for m in models: name = m.get("name", "?") size_gb = m.get("size", 0) / 1e9 if name in loaded_names: status = f"{GREEN}IN VRAM{RST}" else: status = f"{DIM}on disk{RST}" print(f" {name:35s} {size_gb:5.1f}GB {status}") else: print(f" {RED}(Ollama not responding){RST}") # ── LOCAL INFERENCE ACTIVITY ── print(f"\n {BOLD}LOCAL INFERENCE ({len(metrics)} calls, last {hours}h){RST}") print(f" {DIM}{'-' * 55}{RST}") if metrics: print(f" Tokens: {local_summary['input_tokens']} in | {local_summary['output_tokens']} out | {local_summary['total_tokens']} total") if local_summary.get('avg_latency_s') is not None: print(f" Avg latency: {local_summary['avg_latency_s']:.2f}s") if local_summary.get('avg_tokens_per_second') is not None: print(f" Avg throughput: {GREEN}{local_summary['avg_tokens_per_second']:.2f} tok/s{RST}") for caller, stats in sorted(local_summary['by_caller'].items()): err = f" {RED}err:{stats['failed_calls']}{RST}" if stats['failed_calls'] else "" print(f" {caller:25s} calls:{stats['calls']:4d} tokens:{stats['total_tokens']:5d} {GREEN}ok:{stats['successful_calls']}{RST}{err}") print(f"\n {DIM}Models used:{RST}") for model, stats in sorted(local_summary['by_model'].items(), key=lambda x: -x[1]['calls']): print(f" {model:30s} {stats['calls']} calls {stats['total_tokens']} tok") else: print(f" {DIM}(no local calls recorded yet){RST}") # ── HEARTBEAT STATUS ── print(f"\n {BOLD}HEARTBEAT ({len(ticks)} ticks today){RST}") print(f" {DIM}{'-' * 55}{RST}") if ticks: last = ticks[-1] decision = last.get("decision", last.get("actions", {})) if isinstance(decision, dict): severity = decision.get("severity", "unknown") reasoning = decision.get("reasoning", "") sev_color = GREEN if severity == "ok" else YELLOW if severity == "warning" else RED print(f" Last tick: {last.get('tick_id', '?')}") print(f" Severity: {sev_color}{severity}{RST}") if reasoning: print(f" Reasoning: {reasoning[:65]}") else: print(f" Last tick: {last.get('tick_id', '?')}") actions = last.get("actions", []) print(f" Actions: {actions if actions else 'none'}") model_decisions = sum(1 for t in ticks if isinstance(t.get("decision"), dict) and t["decision"].get("severity") != "fallback") fallback = len(ticks) - model_decisions print(f" {CYAN}Model: {model_decisions}{RST} | {DIM}Fallback: {fallback}{RST}") else: print(f" {DIM}(no ticks today){RST}") # ── HERMES SESSIONS / SOVEREIGNTY LOAD ── local_sessions = [s for s in sessions if "localhost:11434" in str(s.get("base_url", ""))] cloud_sessions = [s for s in sessions if s not in local_sessions] print(f"\n {BOLD}HERMES SESSIONS / SOVEREIGNTY LOAD{RST}") print(f" {DIM}{'-' * 55}{RST}") print(f" Session cache: {len(sessions)} total | {GREEN}{len(local_sessions)} local{RST} | {YELLOW}{len(cloud_sessions)} cloud{RST}") if session_rows: print(f" Session DB: {session_summary['total_sessions']} total | {GREEN}{session_summary['local_sessions']} local{RST} | {YELLOW}{session_summary['cloud_sessions']} cloud{RST}") print(f" Token est: {GREEN}{session_summary['local_est_tokens']} local{RST} | {YELLOW}{session_summary['cloud_est_tokens']} cloud{RST}") print(f" Est cloud cost: ${session_summary['cloud_est_cost_usd']:.4f}") else: print(f" {DIM}(no session-db stats available){RST}") # ── ACTIVE LOOPS ── print(f"\n {BOLD}ACTIVE LOOPS{RST}") print(f" {DIM}{'-' * 55}{RST}") print(f" {CYAN}heartbeat_tick{RST} 10m hermes4:14b DECIDE phase") print(f" {DIM}model_health{RST} 5m (local check) Ollama ping") print(f" {DIM}gemini_worker{RST} 20m gemini-2.5-pro aider") print(f" {DIM}grok_worker{RST} 20m grok-3-fast opencode") print(f" {DIM}cross_review{RST} 30m gemini+grok PR review") print(f"\n{BOLD}{'=' * 70}{RST}") print(f" {DIM}Refresh: timmy-dashboard --watch | History: --hours=N{RST}") if __name__ == "__main__": watch = "--watch" in sys.argv hours = 24 for a in sys.argv[1:]: if a.startswith("--hours="): hours = int(a.split("=")[1]) if watch: try: while True: render(hours) time.sleep(30) except KeyboardInterrupt: print(f"\n{DIM}Dashboard stopped.{RST}") else: render(hours)