Compare commits
4 Commits
fix/853
...
claude/iss
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
798ca3aa06 | ||
|
|
e8886f10c8 | ||
|
|
d2ce6b8749 | ||
|
|
a8a086548d |
@@ -1,281 +0,0 @@
|
||||
"""
|
||||
Hallucination Metrics — Persistent logging and alerting for tool hallucinations.
|
||||
|
||||
Logs tool hallucination events to a JSONL file and provides aggregated statistics.
|
||||
Integrates with the poka-yoke validation system.
|
||||
|
||||
Usage:
|
||||
from agent.hallucination_metrics import log_hallucination_event, get_hallucination_stats
|
||||
log_hallucination_event("invalid_tool", "unknown_tool", "suggested_correct_name")
|
||||
stats = get_hallucination_stats()
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from threading import Lock
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
from hermes_constants import get_hermes_home
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Constants
|
||||
METRICS_FILE_NAME = "hallucination_metrics.jsonl"
|
||||
ALERT_THRESHOLD = 10 # Alert after this many consecutive failures for a tool
|
||||
SESSION_WINDOW_HOURS = 24 # Consider events within this window as "session"
|
||||
|
||||
# In-memory cache for fast lookups
|
||||
_cache: Dict[str, Any] = {"events": [], "last_flush": 0, "session_counts": defaultdict(int)}
|
||||
_cache_lock = Lock()
|
||||
|
||||
|
||||
def _get_metrics_path() -> Path:
|
||||
"""Return the path to the hallucination metrics file."""
|
||||
return get_hermes_home() / "metrics" / METRICS_FILE_NAME
|
||||
|
||||
|
||||
def _ensure_metrics_dir():
|
||||
"""Ensure the metrics directory exists."""
|
||||
metrics_dir = _get_metrics_path().parent
|
||||
metrics_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def log_hallucination_event(
|
||||
tool_name: str,
|
||||
error_type: str = "unknown_tool",
|
||||
suggested_name: Optional[str] = None,
|
||||
validation_messages: Optional[List[str]] = None,
|
||||
session_id: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Log a hallucination event to the metrics file.
|
||||
|
||||
Args:
|
||||
tool_name: The hallucinated tool name
|
||||
error_type: Type of error (unknown_tool, invalid_params, etc.)
|
||||
suggested_name: Suggested correction if available
|
||||
validation_messages: List of validation error messages
|
||||
session_id: Optional session identifier for grouping
|
||||
|
||||
Returns:
|
||||
The logged event dict with additional metadata
|
||||
"""
|
||||
event = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"tool_name": tool_name,
|
||||
"error_type": error_type,
|
||||
"suggested_name": suggested_name,
|
||||
"validation_messages": validation_messages or [],
|
||||
"session_id": session_id,
|
||||
"unix_timestamp": time.time(),
|
||||
}
|
||||
|
||||
# Write to file
|
||||
_ensure_metrics_dir()
|
||||
metrics_path = _get_metrics_path()
|
||||
|
||||
try:
|
||||
with open(metrics_path, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(event, ensure_ascii=False) + "\n")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to write hallucination event: {e}")
|
||||
|
||||
# Update in-memory cache
|
||||
with _cache_lock:
|
||||
_cache["events"].append(event)
|
||||
_cache["session_counts"][tool_name] += 1
|
||||
session_count = _cache["session_counts"][tool_name]
|
||||
|
||||
# Check alert threshold
|
||||
if session_count >= ALERT_THRESHOLD:
|
||||
logger.warning(
|
||||
f"HALLUCINATION ALERT: Tool '{tool_name}' has failed {session_count} times "
|
||||
f"in this session (threshold: {ALERT_THRESHOLD}). "
|
||||
f"This may indicate a persistent hallucination pattern."
|
||||
)
|
||||
|
||||
return event
|
||||
|
||||
|
||||
def _load_events_from_file() -> List[Dict[str, Any]]:
|
||||
"""Load all events from the metrics file."""
|
||||
metrics_path = _get_metrics_path()
|
||||
if not metrics_path.exists():
|
||||
return []
|
||||
|
||||
events = []
|
||||
try:
|
||||
with open(metrics_path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
try:
|
||||
events.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load hallucination events: {e}")
|
||||
|
||||
return events
|
||||
|
||||
|
||||
def get_hallucination_stats(
|
||||
hours: Optional[int] = None,
|
||||
tool_name: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Get aggregated hallucination statistics.
|
||||
|
||||
Args:
|
||||
hours: Only consider events from the last N hours (None = all time)
|
||||
tool_name: Filter to specific tool name (None = all tools)
|
||||
|
||||
Returns:
|
||||
Dict with aggregated statistics
|
||||
"""
|
||||
events = _load_events_from_file()
|
||||
|
||||
# Filter by time window
|
||||
if hours is not None:
|
||||
cutoff = time.time() - (hours * 3600)
|
||||
events = [e for e in events if e.get("unix_timestamp", 0) >= cutoff]
|
||||
|
||||
# Filter by tool name
|
||||
if tool_name is not None:
|
||||
events = [e for e in events if e.get("tool_name") == tool_name]
|
||||
|
||||
# Aggregate by tool
|
||||
tool_counts: Dict[str, Dict[str, Any]] = defaultdict(
|
||||
lambda: {"count": 0, "suggested_names": [], "error_types": defaultdict(int)}
|
||||
)
|
||||
|
||||
for event in events:
|
||||
name = event.get("tool_name", "unknown")
|
||||
tool_counts[name]["count"] += 1
|
||||
if event.get("suggested_name"):
|
||||
tool_counts[name]["suggested_names"].append(event["suggested_name"])
|
||||
if event.get("error_type"):
|
||||
tool_counts[name]["error_types"][event["error_type"]] += 1
|
||||
|
||||
# Find most common suggestions per tool
|
||||
for name, data in tool_counts.items():
|
||||
suggestions = data["suggested_names"]
|
||||
if suggestions:
|
||||
from collections import Counter
|
||||
most_common = Counter(suggestions).most_common(1)[0]
|
||||
data["most_common_suggestion"] = most_common[0]
|
||||
data["suggestion_count"] = most_common[1]
|
||||
del data["suggested_names"] # Remove raw list from output
|
||||
|
||||
# Calculate time-based stats
|
||||
if events:
|
||||
first_event = min(e.get("unix_timestamp", 0) for e in events)
|
||||
last_event = max(e.get("unix_timestamp", 0) for e in events)
|
||||
time_span_hours = (last_event - first_event) / 3600 if first_event != last_event else 0
|
||||
else:
|
||||
time_span_hours = 0
|
||||
|
||||
# Error type breakdown
|
||||
all_error_types: Dict[str, int] = defaultdict(int)
|
||||
for event in events:
|
||||
et = event.get("error_type", "unknown")
|
||||
all_error_types[et] += 1
|
||||
|
||||
return {
|
||||
"total_events": len(events),
|
||||
"unique_tools": len(tool_counts),
|
||||
"time_span_hours": round(time_span_hours, 1),
|
||||
"top_hallucinated_tools": sorted(
|
||||
[{"tool": k, **v} for k, v in tool_counts.items()],
|
||||
key=lambda x: -x["count"]
|
||||
)[:20],
|
||||
"error_type_breakdown": dict(all_error_types),
|
||||
"alert_threshold": ALERT_THRESHOLD,
|
||||
"session_window_hours": SESSION_WINDOW_HOURS,
|
||||
}
|
||||
|
||||
|
||||
def get_most_hallucinated_tools(n: int = 10) -> List[Tuple[str, int]]:
|
||||
"""Get the top N most frequently hallucinated tool names."""
|
||||
stats = get_hallucination_stats()
|
||||
tools = stats.get("top_hallucinated_tools", [])
|
||||
return [(t["tool"], t["count"]) for t in tools[:n]]
|
||||
|
||||
|
||||
def clear_metrics(older_than_hours: Optional[int] = None) -> int:
|
||||
"""
|
||||
Clear hallucination metrics.
|
||||
|
||||
Args:
|
||||
older_than_hours: Only clear events older than this many hours (None = clear all)
|
||||
|
||||
Returns:
|
||||
Number of events removed
|
||||
"""
|
||||
metrics_path = _get_metrics_path()
|
||||
if not metrics_path.exists():
|
||||
return 0
|
||||
|
||||
if older_than_hours is None:
|
||||
# Clear all
|
||||
count = len(_load_events_from_file())
|
||||
metrics_path.unlink(missing_ok=True)
|
||||
with _cache_lock:
|
||||
_cache["events"].clear()
|
||||
_cache["session_counts"].clear()
|
||||
return count
|
||||
|
||||
# Clear only old events
|
||||
cutoff = time.time() - (older_than_hours * 3600)
|
||||
events = _load_events_from_file()
|
||||
keep = [e for e in events if e.get("unix_timestamp", 0) >= cutoff]
|
||||
removed = len(events) - len(keep)
|
||||
|
||||
# Rewrite file
|
||||
_ensure_metrics_dir()
|
||||
with open(metrics_path, "w", encoding="utf-8") as f:
|
||||
for event in keep:
|
||||
f.write(json.dumps(event, ensure_ascii=False) + "\n")
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def format_stats_for_display(stats: Dict[str, Any]) -> str:
|
||||
"""Format statistics as a human-readable string."""
|
||||
lines = [
|
||||
"=== Hallucination Metrics ===",
|
||||
"",
|
||||
f"Total events: {stats['total_events']}",
|
||||
f"Unique tools hallucinated: {stats['unique_tools']}",
|
||||
f"Time span: {stats['time_span_hours']:.1f} hours",
|
||||
"",
|
||||
"Top Hallucinated Tools:",
|
||||
"-" * 40,
|
||||
]
|
||||
|
||||
for tool in stats.get("top_hallucinated_tools", [])[:10]:
|
||||
lines.append(f" {tool['tool']:<30} {tool['count']:>5} events")
|
||||
if "most_common_suggestion" in tool:
|
||||
lines.append(f" → Suggested: {tool['most_common_suggestion']} ({tool['suggestion_count']}x)")
|
||||
|
||||
if stats.get("error_type_breakdown"):
|
||||
lines.extend([
|
||||
"",
|
||||
"Error Types:",
|
||||
"-" * 40,
|
||||
])
|
||||
for et, count in sorted(stats["error_type_breakdown"].items(), key=lambda x: -x[1]):
|
||||
lines.append(f" {et:<30} {count:>5}")
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
f"Alert threshold: {stats['alert_threshold']} failures per session",
|
||||
f"Session window: {stats['session_window_hours']} hours",
|
||||
])
|
||||
|
||||
return "\n".join(lines)
|
||||
@@ -18,7 +18,6 @@ Usage:
|
||||
hermes cron list # List cron jobs
|
||||
hermes cron status # Check if cron scheduler is running
|
||||
hermes doctor # Check configuration and dependencies
|
||||
hermes hallucination-stats # Show tool hallucination statistics
|
||||
hermes honcho setup # Configure Honcho AI memory integration
|
||||
hermes honcho status # Show Honcho config and connection status
|
||||
hermes honcho sessions # List directory → session name mappings
|
||||
@@ -2805,17 +2804,6 @@ def cmd_doctor(args):
|
||||
run_doctor(args)
|
||||
|
||||
|
||||
def cmd_hallucination_stats(args):
|
||||
"""Show tool hallucination statistics."""
|
||||
from agent.hallucination_metrics import get_hallucination_stats, format_stats_for_display, clear_metrics
|
||||
if getattr(args, 'clear', False):
|
||||
removed = clear_metrics(older_than_hours=getattr(args, 'older_than', None))
|
||||
print(f"Cleared {removed} hallucination events.")
|
||||
return
|
||||
stats = get_hallucination_stats(hours=getattr(args, 'hours', None))
|
||||
print(format_stats_for_display(stats))
|
||||
|
||||
|
||||
def cmd_dump(args):
|
||||
"""Dump setup summary for support/debugging."""
|
||||
from hermes_cli.dump import run_dump
|
||||
@@ -5053,33 +5041,6 @@ For more help on a command:
|
||||
)
|
||||
doctor_parser.set_defaults(func=cmd_doctor)
|
||||
|
||||
# =========================================================================
|
||||
# hallucination-stats command
|
||||
# =========================================================================
|
||||
hallucination_parser = subparsers.add_parser(
|
||||
"hallucination-stats",
|
||||
help="Show tool hallucination statistics",
|
||||
description="View aggregated tool hallucination metrics from poka-yoke validation"
|
||||
)
|
||||
hallucination_parser.add_argument(
|
||||
"--hours",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Only show events from the last N hours"
|
||||
)
|
||||
hallucination_parser.add_argument(
|
||||
"--clear",
|
||||
action="store_true",
|
||||
help="Clear all hallucination metrics"
|
||||
)
|
||||
hallucination_parser.add_argument(
|
||||
"--older-than",
|
||||
type=int,
|
||||
default=None,
|
||||
help="When clearing, only remove events older than N hours"
|
||||
)
|
||||
hallucination_parser.set_defaults(func=cmd_hallucination_stats)
|
||||
|
||||
# =========================================================================
|
||||
# dump command
|
||||
# =========================================================================
|
||||
|
||||
@@ -46,7 +46,6 @@ from hermes_cli.config import (
|
||||
)
|
||||
from gateway.status import get_running_pid, read_runtime_status
|
||||
from agent.agent_card import get_agent_card_json
|
||||
from agent.mtls import is_mtls_configured, MTLSMiddleware, build_server_ssl_context
|
||||
|
||||
try:
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
@@ -88,10 +87,6 @@ app.add_middleware(
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# mTLS: enforce client certificate on A2A endpoints when configured.
|
||||
# Activated by setting HERMES_MTLS_CERT, HERMES_MTLS_KEY, HERMES_MTLS_CA.
|
||||
app.add_middleware(MTLSMiddleware)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Endpoints that do NOT require the session token. Everything else under
|
||||
# /api/ is gated by the auth middleware below. Keep this list minimal —
|
||||
@@ -1986,6 +1981,73 @@ async def update_config_raw(body: RawConfigUpdate):
|
||||
raise HTTPException(status_code=400, detail=f"Invalid YAML: {e}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Action endpoints — restart gateway / update Hermes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class ActionResponse(BaseModel):
|
||||
ok: bool
|
||||
detail: str = ""
|
||||
|
||||
|
||||
@app.post("/api/actions/restart-gateway")
|
||||
async def restart_gateway():
|
||||
"""Send SIGUSR1 to the running gateway so it drains and restarts.
|
||||
|
||||
Falls back to a hard kill+restart if no PID is found or the signal
|
||||
fails (e.g. the gateway is managed by a remote process / container).
|
||||
Returns immediately with ``{"ok": true}`` if the signal was delivered;
|
||||
the caller should poll ``/api/status`` to confirm the new state.
|
||||
"""
|
||||
from gateway.status import get_running_pid
|
||||
|
||||
pid = get_running_pid()
|
||||
if pid is None:
|
||||
raise HTTPException(status_code=409, detail="Gateway is not running")
|
||||
|
||||
import signal as _signal
|
||||
|
||||
try:
|
||||
os.kill(pid, _signal.SIGUSR1)
|
||||
except (ProcessLookupError, PermissionError, OSError, AttributeError) as exc:
|
||||
raise HTTPException(status_code=500, detail=f"Failed to signal gateway: {exc}")
|
||||
|
||||
return {"ok": True, "detail": f"Restart signal sent to PID {pid}"}
|
||||
|
||||
|
||||
@app.post("/api/actions/update-hermes")
|
||||
async def update_hermes():
|
||||
"""Run ``hermes update`` in a subprocess and return the output.
|
||||
|
||||
The update is performed synchronously (in a thread pool executor) so
|
||||
the endpoint blocks until completion. Clients should treat a 200
|
||||
response with ``"ok": true`` as success; ``"ok": false`` means the
|
||||
subprocess exited non-zero.
|
||||
"""
|
||||
import subprocess
|
||||
|
||||
loop = asyncio.get_event_loop()
|
||||
|
||||
def _run_update():
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[sys.executable, "-m", "hermes_cli.main", "update", "--yes"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=300,
|
||||
)
|
||||
combined = (result.stdout + result.stderr).strip()
|
||||
return result.returncode == 0, combined
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "Update timed out after 5 minutes"
|
||||
except Exception as exc:
|
||||
return False, str(exc)
|
||||
|
||||
ok, detail = await loop.run_in_executor(None, _run_update)
|
||||
return {"ok": ok, "detail": detail}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Token / cost analytics endpoint
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -2110,20 +2172,6 @@ def start_server(
|
||||
"authentication. Only use on trusted networks.", host,
|
||||
)
|
||||
|
||||
# mTLS: when configured, pass SSL context to uvicorn so all connections
|
||||
# are TLS with mandatory client certificate verification.
|
||||
ssl_context = None
|
||||
scheme = "http"
|
||||
if is_mtls_configured():
|
||||
try:
|
||||
ssl_context = build_server_ssl_context()
|
||||
scheme = "https"
|
||||
_log.info(
|
||||
"mTLS enabled — server requires client certificates (A2A auth)"
|
||||
)
|
||||
except Exception as exc:
|
||||
_log.error("Failed to build mTLS SSL context: %s — starting without TLS", exc)
|
||||
|
||||
if open_browser:
|
||||
import threading
|
||||
import webbrowser
|
||||
@@ -2131,11 +2179,9 @@ def start_server(
|
||||
def _open():
|
||||
import time as _t
|
||||
_t.sleep(1.0)
|
||||
webbrowser.open(f"{scheme}://{host}:{port}")
|
||||
webbrowser.open(f"http://{host}:{port}")
|
||||
|
||||
threading.Thread(target=_open, daemon=True).start()
|
||||
|
||||
print(f" Hermes Web UI → {scheme}://{host}:{port}")
|
||||
if ssl_context is not None:
|
||||
print(" mTLS enabled — client certificate required for A2A endpoints")
|
||||
uvicorn.run(app, host=host, port=port, log_level="warning", ssl=ssl_context)
|
||||
print(f" Hermes Web UI → http://{host}:{port}")
|
||||
uvicorn.run(app, host=host, port=port, log_level="warning")
|
||||
|
||||
@@ -1176,3 +1176,135 @@ class TestStatusRemoteGateway:
|
||||
assert data["gateway_running"] is True
|
||||
assert data["gateway_pid"] is None
|
||||
assert data["gateway_state"] == "running"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Action endpoint tests — restart-gateway / update-hermes
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestActionEndpoints:
|
||||
"""Test the /api/actions/* endpoints."""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _setup_test_client(self):
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip("fastapi/starlette not installed")
|
||||
|
||||
from hermes_cli.web_server import app, _SESSION_TOKEN
|
||||
self.client = TestClient(app)
|
||||
self.client.headers["Authorization"] = f"Bearer {_SESSION_TOKEN}"
|
||||
|
||||
# ── restart-gateway ────────────────────────────────────────────────────
|
||||
|
||||
def test_restart_gateway_sends_sigusr1(self, monkeypatch):
|
||||
"""POST /api/actions/restart-gateway signals the running PID."""
|
||||
killed = {}
|
||||
|
||||
def _fake_kill(pid, sig):
|
||||
killed["pid"] = pid
|
||||
killed["sig"] = sig
|
||||
|
||||
monkeypatch.setattr("gateway.status.get_running_pid", lambda: 12345)
|
||||
monkeypatch.setattr("hermes_cli.web_server.os.kill", _fake_kill)
|
||||
|
||||
resp = self.client.post("/api/actions/restart-gateway")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["ok"] is True
|
||||
assert "12345" in data["detail"]
|
||||
assert killed["pid"] == 12345
|
||||
|
||||
def test_restart_gateway_409_when_not_running(self, monkeypatch):
|
||||
"""POST /api/actions/restart-gateway returns 409 when gateway is not running."""
|
||||
monkeypatch.setattr("gateway.status.get_running_pid", lambda: None)
|
||||
|
||||
resp = self.client.post("/api/actions/restart-gateway")
|
||||
|
||||
assert resp.status_code == 409
|
||||
|
||||
def test_restart_gateway_500_on_signal_error(self, monkeypatch):
|
||||
"""POST /api/actions/restart-gateway returns 500 when the signal fails."""
|
||||
monkeypatch.setattr("gateway.status.get_running_pid", lambda: 99999)
|
||||
monkeypatch.setattr("hermes_cli.web_server.os.kill", lambda pid, sig: (_ for _ in ()).throw(ProcessLookupError("no such process")))
|
||||
|
||||
resp = self.client.post("/api/actions/restart-gateway")
|
||||
|
||||
assert resp.status_code == 500
|
||||
assert "Failed to signal" in resp.json()["detail"]
|
||||
|
||||
# ── update-hermes ──────────────────────────────────────────────────────
|
||||
|
||||
def test_update_hermes_success(self, monkeypatch):
|
||||
"""POST /api/actions/update-hermes returns ok=true on zero exit."""
|
||||
import hermes_cli.web_server as ws
|
||||
|
||||
class _FakeResult:
|
||||
returncode = 0
|
||||
stdout = "Already up to date.\n"
|
||||
stderr = ""
|
||||
|
||||
def _fake_run(cmd, **kwargs):
|
||||
assert "--yes" in cmd
|
||||
return _FakeResult()
|
||||
|
||||
monkeypatch.setattr("subprocess.run", _fake_run)
|
||||
|
||||
resp = self.client.post("/api/actions/update-hermes")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["ok"] is True
|
||||
assert "Already up to date" in data["detail"]
|
||||
|
||||
def test_update_hermes_failure_on_nonzero_exit(self, monkeypatch):
|
||||
"""POST /api/actions/update-hermes returns ok=false on non-zero exit."""
|
||||
import hermes_cli.web_server as ws
|
||||
|
||||
class _FakeResult:
|
||||
returncode = 1
|
||||
stdout = ""
|
||||
stderr = "error: update failed\n"
|
||||
|
||||
monkeypatch.setattr("subprocess.run", lambda cmd, **kw: _FakeResult())
|
||||
|
||||
resp = self.client.post("/api/actions/update-hermes")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["ok"] is False
|
||||
assert "error: update failed" in data["detail"]
|
||||
|
||||
def test_update_hermes_timeout(self, monkeypatch):
|
||||
"""POST /api/actions/update-hermes returns ok=false on timeout."""
|
||||
import subprocess
|
||||
import hermes_cli.web_server as ws
|
||||
|
||||
def _fake_run(cmd, **kwargs):
|
||||
raise subprocess.TimeoutExpired(cmd, 300)
|
||||
|
||||
monkeypatch.setattr("subprocess.run", _fake_run)
|
||||
|
||||
resp = self.client.post("/api/actions/update-hermes")
|
||||
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["ok"] is False
|
||||
assert "timed out" in data["detail"].lower()
|
||||
|
||||
def test_action_endpoints_require_auth(self):
|
||||
"""Action endpoints reject requests without a valid Bearer token."""
|
||||
try:
|
||||
from starlette.testclient import TestClient
|
||||
except ImportError:
|
||||
pytest.skip("fastapi/starlette not installed")
|
||||
|
||||
from hermes_cli.web_server import app
|
||||
unauthed = TestClient(app)
|
||||
|
||||
for path in ["/api/actions/restart-gateway", "/api/actions/update-hermes"]:
|
||||
resp = unauthed.post(path)
|
||||
assert resp.status_code in (401, 403), f"{path} should require auth"
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
"""Tests for agent/hallucination_metrics.py — #853."""
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agent.hallucination_metrics import (
|
||||
log_hallucination_event,
|
||||
get_hallucination_stats,
|
||||
get_most_hallucinated_tools,
|
||||
clear_metrics,
|
||||
format_stats_for_display,
|
||||
_get_metrics_path,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_metrics(monkeypatch, tmp_path):
|
||||
"""Redirect metrics to a temp file for every test."""
|
||||
metrics_dir = tmp_path / "test_hermes_home" / "metrics"
|
||||
metrics_dir.mkdir(parents=True)
|
||||
metrics_file = metrics_dir / "hallucination_metrics.jsonl"
|
||||
|
||||
# Patch the get_hermes_home function to return our temp path
|
||||
def mock_get_hermes_home():
|
||||
return tmp_path / "test_hermes_home"
|
||||
|
||||
monkeypatch.setattr(
|
||||
"agent.hallucination_metrics.get_hermes_home",
|
||||
mock_get_hermes_home,
|
||||
)
|
||||
|
||||
# Also clear cache
|
||||
from agent.hallucination_metrics import _cache, _cache_lock
|
||||
with _cache_lock:
|
||||
_cache["events"].clear()
|
||||
_cache["session_counts"].clear()
|
||||
yield
|
||||
clear_metrics()
|
||||
|
||||
|
||||
class TestLogEvent:
|
||||
def test_log_event_returns_dict(self):
|
||||
event = log_hallucination_event("fake_tool", "unknown_tool", "real_tool")
|
||||
assert event["tool_name"] == "fake_tool"
|
||||
assert event["error_type"] == "unknown_tool"
|
||||
assert event["suggested_name"] == "real_tool"
|
||||
assert "timestamp" in event
|
||||
assert "unix_timestamp" in event
|
||||
|
||||
def test_log_event_persists_to_file(self):
|
||||
log_hallucination_event("tool_a", "unknown_tool")
|
||||
log_hallucination_event("tool_b", "invalid_params")
|
||||
|
||||
path = _get_metrics_path()
|
||||
assert path.exists()
|
||||
lines = path.read_text().strip().splitlines()
|
||||
assert len(lines) == 2
|
||||
|
||||
data = [json.loads(line) for line in lines]
|
||||
assert data[0]["tool_name"] == "tool_a"
|
||||
assert data[1]["tool_name"] == "tool_b"
|
||||
|
||||
|
||||
class TestGetStats:
|
||||
def test_empty_stats(self):
|
||||
stats = get_hallucination_stats()
|
||||
assert stats["total_events"] == 0
|
||||
assert stats["unique_tools"] == 0
|
||||
|
||||
def test_stats_by_tool(self):
|
||||
log_hallucination_event("tool_x", "unknown_tool", "tool_y")
|
||||
log_hallucination_event("tool_x", "unknown_tool", "tool_y")
|
||||
log_hallucination_event("tool_z", "invalid_params")
|
||||
|
||||
stats = get_hallucination_stats()
|
||||
assert stats["total_events"] == 3
|
||||
assert stats["unique_tools"] == 2
|
||||
|
||||
top = stats["top_hallucinated_tools"]
|
||||
assert len(top) == 2
|
||||
assert top[0]["tool"] == "tool_x"
|
||||
assert top[0]["count"] == 2
|
||||
assert top[1]["tool"] == "tool_z"
|
||||
assert top[1]["count"] == 1
|
||||
|
||||
def test_stats_hours_filter(self):
|
||||
# Log old event by faking timestamp
|
||||
old_event = {
|
||||
"timestamp": "2026-01-01T00:00:00+00:00",
|
||||
"tool_name": "old_tool",
|
||||
"error_type": "unknown_tool",
|
||||
"unix_timestamp": time.time() - 48 * 3600,
|
||||
}
|
||||
path = _get_metrics_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
f.write(json.dumps(old_event) + "\n")
|
||||
|
||||
log_hallucination_event("new_tool", "unknown_tool")
|
||||
|
||||
stats = get_hallucination_stats(hours=24)
|
||||
assert stats["total_events"] == 1
|
||||
assert stats["top_hallucinated_tools"][0]["tool"] == "new_tool"
|
||||
|
||||
def test_error_type_breakdown(self):
|
||||
log_hallucination_event("t1", "unknown_tool")
|
||||
log_hallucination_event("t2", "invalid_params")
|
||||
log_hallucination_event("t3", "unknown_tool")
|
||||
|
||||
stats = get_hallucination_stats()
|
||||
breakdown = stats["error_type_breakdown"]
|
||||
assert breakdown["unknown_tool"] == 2
|
||||
assert breakdown["invalid_params"] == 1
|
||||
|
||||
|
||||
class TestGetMostHallucinated:
|
||||
def test_top_tools(self):
|
||||
for _ in range(5):
|
||||
log_hallucination_event("common_tool", "unknown_tool")
|
||||
for _ in range(2):
|
||||
log_hallucination_event("rare_tool", "unknown_tool")
|
||||
|
||||
tools = get_most_hallucinated_tools(n=2)
|
||||
assert tools[0] == ("common_tool", 5)
|
||||
assert tools[1] == ("rare_tool", 2)
|
||||
|
||||
|
||||
class TestClearMetrics:
|
||||
def test_clear_all(self):
|
||||
log_hallucination_event("t1", "unknown_tool")
|
||||
removed = clear_metrics()
|
||||
assert removed == 1
|
||||
assert _get_metrics_path().exists() is False
|
||||
|
||||
def test_clear_older_than(self):
|
||||
path = _get_metrics_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
old = {"tool_name": "old", "unix_timestamp": time.time() - 48 * 3600}
|
||||
new = {"tool_name": "new", "unix_timestamp": time.time()}
|
||||
with open(path, "w") as f:
|
||||
f.write(json.dumps(old) + "\n")
|
||||
f.write(json.dumps(new) + "\n")
|
||||
|
||||
removed = clear_metrics(older_than_hours=24)
|
||||
assert removed == 1
|
||||
|
||||
remaining = get_hallucination_stats()
|
||||
assert remaining["total_events"] == 1
|
||||
|
||||
|
||||
class TestFormatDisplay:
|
||||
def test_format_includes_headers(self):
|
||||
log_hallucination_event("bad_tool", "unknown_tool", "good_tool")
|
||||
stats = get_hallucination_stats()
|
||||
text = format_stats_for_display(stats)
|
||||
assert "Hallucination Metrics" in text
|
||||
assert "bad_tool" in text
|
||||
assert "Total events: 1" in text
|
||||
|
||||
|
||||
class TestAlertThreshold:
|
||||
def test_alert_after_threshold(self, monkeypatch, caplog):
|
||||
monkeypatch.setattr("agent.hallucination_metrics.ALERT_THRESHOLD", 3)
|
||||
for i in range(4):
|
||||
log_hallucination_event("persistent_tool", "unknown_tool")
|
||||
assert "HALLUCINATION ALERT" in caplog.text
|
||||
assert "persistent_tool" in caplog.text
|
||||
@@ -204,17 +204,6 @@ class ToolCallValidator:
|
||||
self.consecutive_failures[tool_name] = self.consecutive_failures.get(tool_name, 0) + 1
|
||||
count = self.consecutive_failures[tool_name]
|
||||
|
||||
# Log to persistent metrics
|
||||
try:
|
||||
from agent.hallucination_metrics import log_hallucination_event
|
||||
log_hallucination_event(
|
||||
tool_name=tool_name,
|
||||
error_type="unknown_tool",
|
||||
suggested_name=None,
|
||||
)
|
||||
except Exception:
|
||||
pass # Best-effort metrics logging
|
||||
|
||||
if count >= self.failure_threshold:
|
||||
logger.warning(
|
||||
f"Poka-yoke circuit breaker triggered for '{tool_name}': "
|
||||
|
||||
@@ -86,6 +86,15 @@ export const en: Translations = {
|
||||
lastUpdate: "Last update",
|
||||
platformError: "error",
|
||||
platformDisconnected: "disconnected",
|
||||
actions: "Actions",
|
||||
restartGateway: "Restart Gateway",
|
||||
restarting: "Restarting…",
|
||||
restartSuccess: "Gateway restart signal sent",
|
||||
restartFailed: "Restart failed",
|
||||
updateHermes: "Update Hermes",
|
||||
updating: "Updating…",
|
||||
updateSuccess: "Update complete",
|
||||
updateFailed: "Update failed",
|
||||
},
|
||||
|
||||
sessions: {
|
||||
|
||||
@@ -89,6 +89,15 @@ export interface Translations {
|
||||
lastUpdate: string;
|
||||
platformError: string;
|
||||
platformDisconnected: string;
|
||||
actions: string;
|
||||
restartGateway: string;
|
||||
restarting: string;
|
||||
restartSuccess: string;
|
||||
restartFailed: string;
|
||||
updateHermes: string;
|
||||
updating: string;
|
||||
updateSuccess: string;
|
||||
updateFailed: string;
|
||||
};
|
||||
|
||||
// ── Sessions page ──
|
||||
|
||||
@@ -86,6 +86,15 @@ export const zh: Translations = {
|
||||
lastUpdate: "最后更新",
|
||||
platformError: "错误",
|
||||
platformDisconnected: "已断开",
|
||||
actions: "操作",
|
||||
restartGateway: "重启网关",
|
||||
restarting: "重启中…",
|
||||
restartSuccess: "重启信号已发送",
|
||||
restartFailed: "重启失败",
|
||||
updateHermes: "更新 Hermes",
|
||||
updating: "更新中…",
|
||||
updateSuccess: "更新完成",
|
||||
updateFailed: "更新失败",
|
||||
},
|
||||
|
||||
sessions: {
|
||||
|
||||
@@ -182,6 +182,12 @@ export const api = {
|
||||
},
|
||||
);
|
||||
},
|
||||
|
||||
// Dashboard actions
|
||||
restartGateway: () =>
|
||||
fetchJSON<ActionResponse>("/api/actions/restart-gateway", { method: "POST" }),
|
||||
updateHermes: () =>
|
||||
fetchJSON<ActionResponse>("/api/actions/update-hermes", { method: "POST" }),
|
||||
};
|
||||
|
||||
export interface PlatformStatus {
|
||||
@@ -409,9 +415,15 @@ export interface OAuthSubmitResponse {
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export interface ActionResponse {
|
||||
ok: boolean;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface OAuthPollResponse {
|
||||
session_id: string;
|
||||
status: "pending" | "approved" | "denied" | "expired" | "error";
|
||||
error_message?: string | null;
|
||||
expires_at?: number | null;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import {
|
||||
Activity,
|
||||
AlertTriangle,
|
||||
@@ -6,19 +6,30 @@ import {
|
||||
Cpu,
|
||||
Database,
|
||||
Radio,
|
||||
RefreshCw,
|
||||
TriangleAlert,
|
||||
Wifi,
|
||||
WifiOff,
|
||||
Zap,
|
||||
} from "lucide-react";
|
||||
import { api } from "@/lib/api";
|
||||
import type { PlatformStatus, SessionInfo, StatusResponse } from "@/lib/api";
|
||||
import { timeAgo, isoTimeAgo } from "@/lib/utils";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card";
|
||||
import { Badge } from "@/components/ui/badge";
|
||||
import { useI18n } from "@/i18n";
|
||||
|
||||
type ActionState = "idle" | "running" | "success" | "failure";
|
||||
|
||||
export default function StatusPage() {
|
||||
const [status, setStatus] = useState<StatusResponse | null>(null);
|
||||
const [sessions, setSessions] = useState<SessionInfo[]>([]);
|
||||
const [restartState, setRestartState] = useState<ActionState>("idle");
|
||||
const [restartDetail, setRestartDetail] = useState("");
|
||||
const [updateState, setUpdateState] = useState<ActionState>("idle");
|
||||
const [updateDetail, setUpdateDetail] = useState("");
|
||||
const resetTimers = useRef<Record<string, ReturnType<typeof setTimeout>>>({});
|
||||
const { t } = useI18n();
|
||||
|
||||
useEffect(() => {
|
||||
@@ -31,6 +42,39 @@ export default function StatusPage() {
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
|
||||
function scheduleReset(key: string, setter: (s: ActionState) => void) {
|
||||
clearTimeout(resetTimers.current[key]);
|
||||
resetTimers.current[key] = setTimeout(() => setter("idle"), 8000);
|
||||
}
|
||||
|
||||
async function handleRestartGateway() {
|
||||
setRestartState("running");
|
||||
setRestartDetail("");
|
||||
try {
|
||||
const resp = await api.restartGateway();
|
||||
setRestartState(resp.ok ? "success" : "failure");
|
||||
setRestartDetail(resp.detail);
|
||||
} catch (err: unknown) {
|
||||
setRestartState("failure");
|
||||
setRestartDetail(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
scheduleReset("restart", setRestartState);
|
||||
}
|
||||
|
||||
async function handleUpdateHermes() {
|
||||
setUpdateState("running");
|
||||
setUpdateDetail("");
|
||||
try {
|
||||
const resp = await api.updateHermes();
|
||||
setUpdateState(resp.ok ? "success" : "failure");
|
||||
setUpdateDetail(resp.detail);
|
||||
} catch (err: unknown) {
|
||||
setUpdateState("failure");
|
||||
setUpdateDetail(err instanceof Error ? err.message : String(err));
|
||||
}
|
||||
scheduleReset("update", setUpdateState);
|
||||
}
|
||||
|
||||
if (!status) {
|
||||
return (
|
||||
<div className="flex items-center justify-center py-24">
|
||||
@@ -159,6 +203,57 @@ export default function StatusPage() {
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Action buttons — restart gateway / update Hermes */}
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<div className="flex items-center gap-2">
|
||||
<Zap className="h-5 w-5 text-muted-foreground" />
|
||||
<CardTitle className="text-base">{t.status.actions}</CardTitle>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="flex flex-wrap gap-3">
|
||||
{/* Restart Gateway */}
|
||||
<div className="flex flex-col gap-1">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={restartState === "running"}
|
||||
onClick={handleRestartGateway}
|
||||
>
|
||||
<RefreshCw className={`h-3.5 w-3.5 mr-1 ${restartState === "running" ? "animate-spin" : ""}`} />
|
||||
{restartState === "running" ? t.status.restarting : t.status.restartGateway}
|
||||
</Button>
|
||||
{(restartDetail || restartState === "success") && (
|
||||
<p className={`text-xs max-w-xs truncate ${restartState === "failure" ? "text-destructive" : "text-muted-foreground"}`}>
|
||||
{restartState === "failure" && <TriangleAlert className="inline h-3 w-3 mr-1" />}
|
||||
{restartState === "success" ? t.status.restartSuccess : restartState === "failure" ? t.status.restartFailed : ""}
|
||||
{restartDetail && ` — ${restartDetail}`}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Update Hermes */}
|
||||
<div className="flex flex-col gap-1">
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
disabled={updateState === "running"}
|
||||
onClick={handleUpdateHermes}
|
||||
>
|
||||
<RefreshCw className={`h-3.5 w-3.5 mr-1 ${updateState === "running" ? "animate-spin" : ""}`} />
|
||||
{updateState === "running" ? t.status.updating : t.status.updateHermes}
|
||||
</Button>
|
||||
{(updateDetail || updateState === "success" || updateState === "failure") && (
|
||||
<p className={`text-xs max-w-xs ${updateState === "failure" ? "text-destructive" : "text-muted-foreground"}`}>
|
||||
{updateState === "failure" && <TriangleAlert className="inline h-3 w-3 mr-1" />}
|
||||
{updateState === "success" ? t.status.updateSuccess : updateState === "failure" ? t.status.updateFailed : ""}
|
||||
{updateDetail && ` — ${updateDetail}`}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{platforms.length > 0 && (
|
||||
<PlatformsCard platforms={platforms} platformStateBadge={PLATFORM_STATE_BADGE} />
|
||||
)}
|
||||
|
||||
Reference in New Issue
Block a user