190 lines
6.4 KiB
Python
190 lines
6.4 KiB
Python
"""
|
|
Session Analytics — Per-session token/cost/time tracking
|
|
|
|
Tracks resource consumption per session for transparency.
|
|
|
|
Issue: #753
|
|
"""
|
|
|
|
import json
|
|
import time
|
|
from dataclasses import dataclass, asdict, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional
|
|
|
|
HERMES_HOME = Path.home() / ".hermes"
|
|
ANALYTICS_DIR = HERMES_HOME / "analytics"
|
|
|
|
|
|
# Cost per 1K tokens by provider (input/output)
|
|
COST_TABLE = {
|
|
"anthropic": {"input": 0.015, "output": 0.075},
|
|
"openai": {"input": 0.005, "output": 0.015},
|
|
"nous": {"input": 0.002, "output": 0.006},
|
|
"openrouter": {"input": 0.005, "output": 0.015},
|
|
"ollama": {"input": 0.0, "output": 0.0},
|
|
"local": {"input": 0.0, "output": 0.0},
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class SessionStats:
|
|
"""Statistics for a single session."""
|
|
session_id: str
|
|
start_time: str
|
|
end_time: Optional[str] = None
|
|
|
|
# Token counts
|
|
input_tokens: int = 0
|
|
output_tokens: int = 0
|
|
total_tokens: int = 0
|
|
|
|
# Tool usage
|
|
tool_calls: int = 0
|
|
tool_errors: int = 0
|
|
|
|
# Timing
|
|
wall_time_seconds: float = 0.0
|
|
api_calls: int = 0
|
|
|
|
# Cost
|
|
estimated_cost_usd: float = 0.0
|
|
provider: str = ""
|
|
model: str = ""
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
return asdict(self)
|
|
|
|
|
|
class SessionTracker:
|
|
"""Track per-session analytics."""
|
|
|
|
def __init__(self, session_id: str, provider: str = "", model: str = ""):
|
|
self.session_id = session_id
|
|
self.provider = provider.lower() if provider else ""
|
|
self.model = model
|
|
self.start_time = time.time()
|
|
self.stats = SessionStats(
|
|
session_id=session_id,
|
|
start_time=datetime.now(timezone.utc).isoformat(),
|
|
provider=provider,
|
|
model=model
|
|
)
|
|
|
|
def record_tokens(self, input_tokens: int, output_tokens: int):
|
|
"""Record token usage."""
|
|
self.stats.input_tokens += input_tokens
|
|
self.stats.output_tokens += output_tokens
|
|
self.stats.total_tokens = self.stats.input_tokens + self.stats.output_tokens
|
|
|
|
# Estimate cost
|
|
costs = COST_TABLE.get(self.provider, {"input": 0.01, "output": 0.03})
|
|
cost = (input_tokens / 1000) * costs["input"] + (output_tokens / 1000) * costs["output"]
|
|
self.stats.estimated_cost_usd += cost
|
|
|
|
def record_tool_call(self, success: bool = True):
|
|
"""Record a tool call."""
|
|
self.stats.tool_calls += 1
|
|
if not success:
|
|
self.stats.tool_errors += 1
|
|
|
|
def record_api_call(self):
|
|
"""Record an API call."""
|
|
self.stats.api_calls += 1
|
|
|
|
def finish(self) -> SessionStats:
|
|
"""Finish tracking and return stats."""
|
|
self.stats.end_time = datetime.now(timezone.utc).isoformat()
|
|
self.stats.wall_time_seconds = time.time() - self.start_time
|
|
return self.stats
|
|
|
|
def get_current_stats(self) -> SessionStats:
|
|
"""Get current stats without finishing."""
|
|
self.stats.wall_time_seconds = time.time() - self.start_time
|
|
return self.stats
|
|
|
|
|
|
def format_stats(stats: SessionStats) -> str:
|
|
"""Format stats for display."""
|
|
lines = []
|
|
lines.append(f"Session: {stats.session_id[:20]}...")
|
|
lines.append(f"Provider: {stats.provider or 'unknown'}")
|
|
lines.append(f"Model: {stats.model or 'unknown'}")
|
|
lines.append("")
|
|
lines.append(f"Tokens: {stats.input_tokens:,} in / {stats.output_tokens:,} out ({stats.total_tokens:,} total)")
|
|
lines.append(f"Cost: ${stats.estimated_cost_usd:.4f}")
|
|
lines.append(f"API calls: {stats.api_calls}")
|
|
lines.append(f"Tool calls: {stats.tool_calls} ({stats.tool_errors} errors)")
|
|
lines.append(f"Wall time: {stats.wall_time_seconds:.1f}s")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def save_session_stats(stats: SessionStats):
|
|
"""Save session stats to disk."""
|
|
ANALYTICS_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Daily file
|
|
date_str = datetime.now().strftime("%Y-%m-%d")
|
|
stats_file = ANALYTICS_DIR / f"sessions_{date_str}.jsonl"
|
|
|
|
with open(stats_file, "a") as f:
|
|
f.write(json.dumps(stats.to_dict()) + "\n")
|
|
|
|
|
|
def get_daily_stats(date_str: Optional[str] = None) -> Dict[str, Any]:
|
|
"""Get aggregate stats for a day."""
|
|
if date_str is None:
|
|
date_str = datetime.now().strftime("%Y-%m-%d")
|
|
|
|
stats_file = ANALYTICS_DIR / f"sessions_{date_str}.jsonl"
|
|
if not stats_file.exists():
|
|
return {"date": date_str, "sessions": 0}
|
|
|
|
sessions = []
|
|
with open(stats_file) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
try:
|
|
sessions.append(json.loads(line))
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
if not sessions:
|
|
return {"date": date_str, "sessions": 0}
|
|
|
|
total_tokens = sum(s.get("total_tokens", 0) for s in sessions)
|
|
total_cost = sum(s.get("estimated_cost_usd", 0) for s in sessions)
|
|
total_time = sum(s.get("wall_time_seconds", 0) for s in sessions)
|
|
total_tool_calls = sum(s.get("tool_calls", 0) for s in sessions)
|
|
total_errors = sum(s.get("tool_errors", 0) for s in sessions)
|
|
|
|
return {
|
|
"date": date_str,
|
|
"sessions": len(sessions),
|
|
"total_tokens": total_tokens,
|
|
"total_cost_usd": round(total_cost, 4),
|
|
"total_wall_time_seconds": round(total_time, 1),
|
|
"total_tool_calls": total_tool_calls,
|
|
"total_tool_errors": total_errors,
|
|
"avg_tokens_per_session": total_tokens // len(sessions) if sessions else 0,
|
|
"avg_cost_per_session": round(total_cost / len(sessions), 4) if sessions else 0,
|
|
}
|
|
|
|
|
|
def format_daily_report(stats: Dict[str, Any]) -> str:
|
|
"""Format daily stats as report."""
|
|
lines = []
|
|
lines.append(f"# Session Analytics — {stats['date']}")
|
|
lines.append("")
|
|
lines.append(f"Sessions: {stats['sessions']}")
|
|
lines.append(f"Total tokens: {stats.get('total_tokens', 0):,}")
|
|
lines.append(f"Total cost: ${stats.get('total_cost_usd', 0):.4f}")
|
|
lines.append(f"Total wall time: {stats.get('total_wall_time_seconds', 0):.1f}s")
|
|
lines.append(f"Tool calls: {stats.get('total_tool_calls', 0)} ({stats.get('total_tool_errors', 0)} errors)")
|
|
lines.append("")
|
|
lines.append(f"Avg tokens/session: {stats.get('avg_tokens_per_session', 0):,}")
|
|
lines.append(f"Avg cost/session: ${stats.get('avg_cost_per_session', 0):.4f}")
|
|
return "\n".join(lines)
|