Compare commits
1 Commits
fix/747
...
burn/713-1
| Author | SHA1 | Date | |
|---|---|---|---|
|
0d9ff94693 |
@@ -1,353 +0,0 @@
|
||||
"""Multi-Agent Conversation Bridge via Matrix.
|
||||
|
||||
Allows multiple Hermes instances (Timmy, Allegro, Ezra) to communicate
|
||||
with each other through a shared Matrix room.
|
||||
|
||||
Usage:
|
||||
from agent.matrix_bridge import MatrixBridge
|
||||
|
||||
bridge = MatrixBridge(agent_name="Timmy")
|
||||
await bridge.connect()
|
||||
await bridge.send_to_agent("Allegro", "Check the deployment status")
|
||||
messages = await bridge.get_messages_from("Allegro")
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Set
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Configuration
|
||||
MATRIX_BRIDGE_ROOM = os.environ.get("MATRIX_BRIDGE_ROOM", "")
|
||||
MATRIX_BRIDGE_ENABLED = os.environ.get("MATRIX_BRIDGE_ENABLED", "true").lower() == "true"
|
||||
AGENT_NAME = os.environ.get("HERMES_AGENT_NAME", "Hermes")
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentMessage:
|
||||
"""A message from one agent to another."""
|
||||
sender: str
|
||||
recipient: str
|
||||
content: str
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
message_id: str = ""
|
||||
room_id: str = ""
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"sender": self.sender,
|
||||
"recipient": self.recipient,
|
||||
"content": self.content,
|
||||
"timestamp": self.timestamp,
|
||||
"message_id": self.message_id,
|
||||
"room_id": self.room_id,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AgentMessage":
|
||||
return cls(**data)
|
||||
|
||||
|
||||
class MatrixBridge:
|
||||
"""Multi-agent conversation bridge via Matrix rooms.
|
||||
|
||||
Agents communicate by posting messages to a shared Matrix room
|
||||
with a standard format: [@recipient] message content
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent_name: str = None,
|
||||
room_id: str = None,
|
||||
callback: Callable[[AgentMessage], None] = None,
|
||||
):
|
||||
self.agent_name = agent_name or AGENT_NAME
|
||||
self.room_id = room_id or MATRIX_BRIDGE_ROOM
|
||||
self.callback = callback
|
||||
self._matrix_client = None
|
||||
self._running = False
|
||||
self._message_handlers: List[Callable[[AgentMessage], None]] = []
|
||||
self._pending_messages: List[AgentMessage] = []
|
||||
self._known_agents: Set[str] = set()
|
||||
|
||||
async def connect(self) -> bool:
|
||||
"""Connect to Matrix and join the bridge room."""
|
||||
if not MATRIX_BRIDGE_ENABLED:
|
||||
logger.info("Matrix bridge disabled via MATRIX_BRIDGE_ENABLED=false")
|
||||
return False
|
||||
|
||||
if not self.room_id:
|
||||
logger.warning("No MATRIX_BRIDGE_ROOM configured — bridge disabled")
|
||||
return False
|
||||
|
||||
try:
|
||||
# Import Matrix client
|
||||
from mautrix.client import Client
|
||||
from mautrix.types import RoomID, UserID
|
||||
|
||||
# Get credentials
|
||||
homeserver = os.environ.get("MATRIX_HOMESERVER", "")
|
||||
access_token = os.environ.get("MATRIX_ACCESS_TOKEN", "")
|
||||
|
||||
if not homeserver or not access_token:
|
||||
logger.warning("Matrix credentials not configured — bridge disabled")
|
||||
return False
|
||||
|
||||
# Create client
|
||||
self._matrix_client = Client(
|
||||
mxid=UserID(f"@{self.agent_name}:{homeserver.split('//')[1]}"),
|
||||
base_url=homeserver,
|
||||
token=access_token,
|
||||
)
|
||||
|
||||
# Join room
|
||||
await self._matrix_client.join_room(RoomID(self.room_id))
|
||||
logger.info(f"Agent {self.agent_name} joined bridge room {self.room_id}")
|
||||
|
||||
# Register message handler
|
||||
self._matrix_client.add_event_handler(self._on_message)
|
||||
|
||||
# Start sync
|
||||
self._running = True
|
||||
asyncio.create_task(self._sync_loop())
|
||||
|
||||
# Announce presence
|
||||
await self._announce_presence()
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to connect to Matrix bridge: {e}")
|
||||
return False
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
"""Disconnect from the bridge."""
|
||||
self._running = False
|
||||
if self._matrix_client:
|
||||
try:
|
||||
await self._matrix_client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async def send_to_agent(self, recipient: str, content: str) -> bool:
|
||||
"""Send a message to another agent.
|
||||
|
||||
Args:
|
||||
recipient: Agent name (e.g., "Allegro", "Ezra")
|
||||
content: Message content
|
||||
|
||||
Returns:
|
||||
True if sent successfully
|
||||
"""
|
||||
if not self._matrix_client or not self.room_id:
|
||||
logger.warning("Not connected to bridge room")
|
||||
return False
|
||||
|
||||
# Format message with recipient prefix
|
||||
formatted = f"[@{recipient}] {content}"
|
||||
|
||||
try:
|
||||
from mautrix.types import RoomID, TextMessageEventContent, MessageType
|
||||
|
||||
await self._matrix_client.send_message_event(
|
||||
room_id=RoomID(self.room_id),
|
||||
event_type="m.room.message",
|
||||
content=TextMessageEventContent(
|
||||
msgtype=MessageType.TEXT,
|
||||
body=formatted,
|
||||
),
|
||||
)
|
||||
|
||||
logger.info(f"Sent message to {recipient}: {content[:50]}...")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send message: {e}")
|
||||
return False
|
||||
|
||||
async def broadcast(self, content: str) -> bool:
|
||||
"""Broadcast a message to all agents.
|
||||
|
||||
Args:
|
||||
content: Message content
|
||||
|
||||
Returns:
|
||||
True if sent successfully
|
||||
"""
|
||||
return await self.send_to_agent("*", content)
|
||||
|
||||
def add_handler(self, handler: Callable[[AgentMessage], None]) -> None:
|
||||
"""Add a message handler.
|
||||
|
||||
Called when a message is received for this agent.
|
||||
"""
|
||||
self._message_handlers.append(handler)
|
||||
|
||||
def get_known_agents(self) -> Set[str]:
|
||||
"""Get set of known agents in the bridge."""
|
||||
return self._known_agents.copy()
|
||||
|
||||
async def _on_message(self, event) -> None:
|
||||
"""Handle incoming Matrix message."""
|
||||
try:
|
||||
# Extract message content
|
||||
content = event.content
|
||||
if not hasattr(content, 'body'):
|
||||
return
|
||||
|
||||
body = content.body
|
||||
|
||||
# Check if message is for this agent
|
||||
if not self._is_for_me(body):
|
||||
return
|
||||
|
||||
# Parse sender and content
|
||||
sender = self._extract_sender(event)
|
||||
message_content = self._extract_content(body)
|
||||
|
||||
# Create agent message
|
||||
msg = AgentMessage(
|
||||
sender=sender,
|
||||
recipient=self.agent_name,
|
||||
content=message_content,
|
||||
timestamp=time.time(),
|
||||
message_id=str(event.event_id),
|
||||
room_id=str(event.room_id),
|
||||
)
|
||||
|
||||
# Track known agents
|
||||
self._known_agents.add(sender)
|
||||
|
||||
# Call handlers
|
||||
for handler in self._message_handlers:
|
||||
try:
|
||||
handler(msg)
|
||||
except Exception as e:
|
||||
logger.error(f"Message handler error: {e}")
|
||||
|
||||
if self.callback:
|
||||
try:
|
||||
self.callback(msg)
|
||||
except Exception as e:
|
||||
logger.error(f"Callback error: {e}")
|
||||
|
||||
logger.info(f"Received message from {sender}: {message_content[:50]}...")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing message: {e}")
|
||||
|
||||
def _is_for_me(self, body: str) -> bool:
|
||||
"""Check if message is addressed to this agent."""
|
||||
# Direct mention
|
||||
if f"[@{self.agent_name}]" in body:
|
||||
return True
|
||||
|
||||
# Broadcast
|
||||
if "[@*]" in body:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _extract_sender(self, event) -> str:
|
||||
"""Extract sender name from event."""
|
||||
try:
|
||||
sender_id = str(event.sender)
|
||||
# Extract name from @name:server format
|
||||
match = re.match(r"@([^:]+):", sender_id)
|
||||
if match:
|
||||
return match.group(1)
|
||||
return sender_id
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
def _extract_content(self, body: str) -> str:
|
||||
"""Extract message content, removing recipient prefix."""
|
||||
# Remove [@recipient] prefix
|
||||
match = re.match(r"\[@[^\]]+\]\s*(.*)", body, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return body.strip()
|
||||
|
||||
async def _announce_presence(self) -> None:
|
||||
"""Announce this agent's presence to the bridge."""
|
||||
await self.broadcast(f"{self.agent_name} online")
|
||||
|
||||
async def _sync_loop(self) -> None:
|
||||
"""Background sync loop for Matrix events."""
|
||||
while self._running:
|
||||
try:
|
||||
if self._matrix_client:
|
||||
await self._matrix_client.sync(timeout=30000)
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Sync error: {e}")
|
||||
await asyncio.sleep(5)
|
||||
|
||||
|
||||
class AgentRegistry:
|
||||
"""Registry of known agents in the bridge."""
|
||||
|
||||
def __init__(self):
|
||||
self._agents: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
def register(self, name: str, capabilities: List[str] = None) -> None:
|
||||
"""Register an agent with optional capabilities."""
|
||||
self._agents[name] = {
|
||||
"name": name,
|
||||
"capabilities": capabilities or [],
|
||||
"last_seen": time.time(),
|
||||
"status": "online",
|
||||
}
|
||||
|
||||
def unregister(self, name: str) -> None:
|
||||
"""Unregister an agent."""
|
||||
if name in self._agents:
|
||||
self._agents[name]["status"] = "offline"
|
||||
|
||||
def get_agent(self, name: str) -> Optional[Dict[str, Any]]:
|
||||
"""Get agent info by name."""
|
||||
return self._agents.get(name)
|
||||
|
||||
def list_agents(self) -> List[Dict[str, Any]]:
|
||||
"""List all registered agents."""
|
||||
return list(self._agents.values())
|
||||
|
||||
def find_agents_with_capability(self, capability: str) -> List[str]:
|
||||
"""Find agents with a specific capability."""
|
||||
return [
|
||||
name for name, info in self._agents.items()
|
||||
if capability in info.get("capabilities", [])
|
||||
]
|
||||
|
||||
|
||||
# Global bridge instance
|
||||
_bridge: Optional[MatrixBridge] = None
|
||||
|
||||
|
||||
async def get_bridge(agent_name: str = None) -> MatrixBridge:
|
||||
"""Get or create the global Matrix bridge instance."""
|
||||
global _bridge
|
||||
if _bridge is None:
|
||||
_bridge = MatrixBridge(agent_name=agent_name)
|
||||
await _bridge.connect()
|
||||
return _bridge
|
||||
|
||||
|
||||
async def send_to_agent(recipient: str, content: str) -> bool:
|
||||
"""Convenience function to send a message to another agent."""
|
||||
bridge = await get_bridge()
|
||||
return await bridge.send_to_agent(recipient, content)
|
||||
|
||||
|
||||
async def broadcast_to_agents(content: str) -> bool:
|
||||
"""Convenience function to broadcast to all agents."""
|
||||
bridge = await get_bridge()
|
||||
return await bridge.broadcast(content)
|
||||
@@ -1,216 +0,0 @@
|
||||
# Multi-Agent Conversation Bridge
|
||||
|
||||
Allows multiple Hermes instances (Timmy, Allegro, Ezra) to communicate with each other through a shared Matrix room.
|
||||
|
||||
## Overview
|
||||
|
||||
The Matrix Bridge enables agent-to-agent coordination without manual intervention. Agents can:
|
||||
- Send tasks to specific agents
|
||||
- Broadcast to all agents
|
||||
- Respond to requests from other agents
|
||||
- Coordinate on complex workflows
|
||||
|
||||
## Configuration
|
||||
|
||||
### Environment Variables
|
||||
|
||||
```bash
|
||||
# Enable/disable the bridge
|
||||
MATRIX_BRIDGE_ENABLED=true
|
||||
|
||||
# Shared Matrix room ID for agent communication
|
||||
MATRIX_BRIDGE_ROOM=!roomid:matrix.example.org
|
||||
|
||||
# Agent name (for message routing)
|
||||
HERMES_AGENT_NAME=Timmy
|
||||
|
||||
# Matrix credentials (from existing Matrix gateway config)
|
||||
MATRIX_HOMESERVER=https://matrix.example.org
|
||||
MATRIX_ACCESS_TOKEN=syt_...
|
||||
```
|
||||
|
||||
### Matrix Room Setup
|
||||
|
||||
1. Create a Matrix room for agent communication
|
||||
2. Invite all agent accounts to the room
|
||||
3. Set `MATRIX_BRIDGE_ROOM` to the room ID
|
||||
|
||||
## Message Format
|
||||
|
||||
Messages use a simple prefix format for routing:
|
||||
|
||||
```
|
||||
[@Allegro] Check the deployment status on VPS
|
||||
[@Ezra] Can you review PR #456?
|
||||
[@*] System maintenance in 5 minutes
|
||||
```
|
||||
|
||||
- `[@AgentName]` — Message for specific agent
|
||||
- `[@*]` — Broadcast to all agents
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
from agent.matrix_bridge import MatrixBridge, send_to_agent, broadcast_to_agents
|
||||
|
||||
# Create bridge
|
||||
bridge = MatrixBridge(agent_name="Timmy")
|
||||
await bridge.connect()
|
||||
|
||||
# Send to specific agent
|
||||
await bridge.send_to_agent("Allegro", "Check deployment status")
|
||||
|
||||
# Broadcast to all agents
|
||||
await bridge.broadcast("System maintenance starting")
|
||||
|
||||
# Add message handler
|
||||
def handle_message(msg):
|
||||
print(f"From {msg.sender}: {msg.content}")
|
||||
|
||||
bridge.add_handler(handle_message)
|
||||
```
|
||||
|
||||
### Convenience Functions
|
||||
|
||||
```python
|
||||
from agent.matrix_bridge import send_to_agent, broadcast_to_agents
|
||||
|
||||
# Send message
|
||||
await send_to_agent("Ezra", "Review PR #456")
|
||||
|
||||
# Broadcast
|
||||
await broadcast_to_agents("Going offline for maintenance")
|
||||
```
|
||||
|
||||
### Agent Registry
|
||||
|
||||
```python
|
||||
from agent.matrix_bridge import AgentRegistry
|
||||
|
||||
registry = AgentRegistry()
|
||||
|
||||
# Register agent with capabilities
|
||||
registry.register("Timmy", capabilities=["code", "review", "deploy"])
|
||||
registry.register("Allegro", capabilities=["monitoring", "alerting"])
|
||||
|
||||
# Find agents with capability
|
||||
coders = registry.find_agents_with_capability("code")
|
||||
```
|
||||
|
||||
## Message Flow
|
||||
|
||||
```
|
||||
┌─────────┐ ┌─────────┐ ┌─────────┐
|
||||
│ Timmy │────▶│ Matrix │────▶│ Allegro │
|
||||
│ Agent │ │ Room │ │ Agent │
|
||||
└─────────┘ └─────────┘ └─────────┘
|
||||
│ │ │
|
||||
│ [@Allegro] │ │
|
||||
│ Check deps │ │
|
||||
└──────────────▶│ │
|
||||
│ [@Allegro] │
|
||||
│ Check deps │
|
||||
└──────────────▶│
|
||||
│
|
||||
│ [@Timmy] │
|
||||
│ Done ✓ │
|
||||
│◀──────────────┘
|
||||
│ [@Timmy] │
|
||||
│ Done ✓ │
|
||||
│◀──────────────┘
|
||||
```
|
||||
|
||||
## Integration with Hermes
|
||||
|
||||
### In run_agent.py
|
||||
|
||||
```python
|
||||
# Add to conversation loop
|
||||
if self.matrix_bridge:
|
||||
# Check for messages from other agents
|
||||
messages = await self.matrix_bridge.get_pending_messages()
|
||||
for msg in messages:
|
||||
# Process agent-to-agent messages
|
||||
pass
|
||||
```
|
||||
|
||||
### In Gateway
|
||||
|
||||
```python
|
||||
# Add Matrix bridge to gateway
|
||||
from agent.matrix_bridge import MatrixBridge
|
||||
|
||||
bridge = MatrixBridge(agent_name="Timmy")
|
||||
await bridge.connect()
|
||||
gateway.matrix_bridge = bridge
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
### Unit Tests
|
||||
|
||||
```python
|
||||
def test_message_parsing():
|
||||
"""Test message format parsing."""
|
||||
from agent.matrix_bridge import MatrixBridge
|
||||
|
||||
bridge = MatrixBridge(agent_name="Timmy")
|
||||
|
||||
# Test recipient extraction
|
||||
assert bridge._is_for_me("[@Timmy] Hello")
|
||||
assert not bridge._is_for_me("[@Allegro] Hello")
|
||||
assert bridge._is_for_me("[@*] Broadcast")
|
||||
|
||||
# Test content extraction
|
||||
assert bridge._extract_content("[@Timmy] Hello") == "Hello"
|
||||
assert bridge._extract_content("[@*] Test message") == "Test message"
|
||||
```
|
||||
|
||||
### Integration Test
|
||||
|
||||
```bash
|
||||
# Test with two agents
|
||||
MATRIX_BRIDGE_ENABLED=true \
|
||||
MATRIX_BRIDGE_ROOM=!test:matrix.example.org \
|
||||
HERMES_AGENT_NAME=Timmy \
|
||||
python -c "
|
||||
import asyncio
|
||||
from agent.matrix_bridge import send_to_agent
|
||||
|
||||
async def test():
|
||||
await send_to_agent('Allegro', 'Test message')
|
||||
print('Sent')
|
||||
|
||||
asyncio.run(test())
|
||||
"
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Bridge not connecting
|
||||
|
||||
1. Check `MATRIX_BRIDGE_ENABLED=true`
|
||||
2. Verify `MATRIX_BRIDGE_ROOM` is set
|
||||
3. Ensure Matrix credentials are configured
|
||||
4. Check Matrix homeserver is reachable
|
||||
|
||||
### Messages not received
|
||||
|
||||
1. Verify agent is in the Matrix room
|
||||
2. Check message format: `[@AgentName] content`
|
||||
3. Ensure `HERMES_AGENT_NAME` matches agent name
|
||||
4. Check Matrix sync is running
|
||||
|
||||
### Agent not found
|
||||
|
||||
1. Verify agent has joined the bridge room
|
||||
2. Check agent name matches exactly (case-sensitive)
|
||||
3. Ensure agent has announced presence
|
||||
|
||||
## Related
|
||||
|
||||
- Issue #747: feat: multi-agent conversation bridge via Matrix
|
||||
- Matrix Gateway: `gateway/platforms/matrix.py`
|
||||
- Multi-Agent Orchestration: `docs/multi-agent-orchestration.md`
|
||||
272
scripts/inference_health.py
Normal file
272
scripts/inference_health.py
Normal file
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local inference server health check and auto-restart.
|
||||
|
||||
Checks llama-server, Ollama, and other local inference endpoints.
|
||||
Reports status, latency, and can auto-restart dead processes.
|
||||
|
||||
Refs: #713 — llama-server DOWN on port 8081
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional, List, Dict, Any
|
||||
from urllib.request import Request, urlopen
|
||||
from urllib.error import URLError, HTTPError
|
||||
|
||||
|
||||
@dataclass
|
||||
class InferenceEndpoint:
|
||||
"""Configuration for an inference server endpoint."""
|
||||
name: str
|
||||
url: str
|
||||
health_path: str = "/health"
|
||||
port: int = 8080
|
||||
restart_cmd: str = ""
|
||||
process_name: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class HealthResult:
|
||||
"""Result of a health check."""
|
||||
name: str
|
||||
url: str
|
||||
status: str # "ok", "down", "slow", "error"
|
||||
latency_ms: float = 0.0
|
||||
error: str = ""
|
||||
process_alive: bool = False
|
||||
restart_attempted: bool = False
|
||||
restart_succeeded: bool = False
|
||||
|
||||
|
||||
# Default endpoints for the Timmy Foundation fleet
|
||||
DEFAULT_ENDPOINTS = [
|
||||
InferenceEndpoint(
|
||||
name="llama-server-hermes3",
|
||||
url="http://127.0.0.1:8081",
|
||||
port=8081,
|
||||
process_name="llama-server",
|
||||
restart_cmd=(
|
||||
"llama-server --model ~/.ollama/models/blobs/sha256-c8985d "
|
||||
"--port 8081 --host 127.0.0.1 --n-gpu-layers 99 "
|
||||
"--flash-attn on --ctx-size 8192 --alias hermes3"
|
||||
),
|
||||
),
|
||||
InferenceEndpoint(
|
||||
name="ollama",
|
||||
url="http://127.0.0.1:11434",
|
||||
port=11434,
|
||||
process_name="ollama",
|
||||
restart_cmd="ollama serve",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def check_endpoint(ep: InferenceEndpoint, timeout: float = 5.0) -> HealthResult:
|
||||
"""Check a single inference endpoint.
|
||||
|
||||
Args:
|
||||
ep: Endpoint configuration.
|
||||
timeout: HTTP timeout in seconds.
|
||||
|
||||
Returns:
|
||||
HealthResult with status and latency.
|
||||
"""
|
||||
url = ep.url.rstrip("/") + ep.health_path
|
||||
start = time.time()
|
||||
|
||||
# Check if process is alive
|
||||
process_alive = False
|
||||
if ep.process_name:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["pgrep", "-f", ep.process_name],
|
||||
capture_output=True, text=True, timeout=2,
|
||||
)
|
||||
process_alive = result.returncode == 0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# HTTP health check
|
||||
try:
|
||||
req = Request(url, method="GET")
|
||||
resp = urlopen(req, timeout=timeout)
|
||||
latency = (time.time() - start) * 1000
|
||||
|
||||
if resp.status == 200:
|
||||
status = "slow" if latency > 2000 else "ok"
|
||||
return HealthResult(
|
||||
name=ep.name, url=ep.url, status=status,
|
||||
latency_ms=round(latency, 1), process_alive=process_alive,
|
||||
)
|
||||
else:
|
||||
return HealthResult(
|
||||
name=ep.name, url=ep.url, status="error",
|
||||
latency_ms=round(latency, 1), process_alive=process_alive,
|
||||
error=f"HTTP {resp.status}",
|
||||
)
|
||||
|
||||
except URLError as e:
|
||||
latency = (time.time() - start) * 1000
|
||||
error_msg = str(e.reason) if hasattr(e, 'reason') else str(e)
|
||||
return HealthResult(
|
||||
name=ep.name, url=ep.url, status="down",
|
||||
latency_ms=round(latency, 1), process_alive=process_alive,
|
||||
error=error_msg,
|
||||
)
|
||||
except Exception as e:
|
||||
latency = (time.time() - start) * 1000
|
||||
return HealthResult(
|
||||
name=ep.name, url=ep.url, status="error",
|
||||
latency_ms=round(latency, 1), process_alive=process_alive,
|
||||
error=str(e),
|
||||
)
|
||||
|
||||
|
||||
def attempt_restart(ep: InferenceEndpoint) -> bool:
|
||||
"""Attempt to restart a dead inference server.
|
||||
|
||||
Args:
|
||||
ep: Endpoint configuration with restart_cmd.
|
||||
|
||||
Returns:
|
||||
True if restart command executed successfully.
|
||||
"""
|
||||
if not ep.restart_cmd:
|
||||
return False
|
||||
|
||||
try:
|
||||
# Run restart in background
|
||||
subprocess.Popen(
|
||||
ep.restart_cmd,
|
||||
shell=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
# Wait a moment for the process to start
|
||||
time.sleep(3)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Restart failed for {ep.name}: {e}", file=sys.stderr)
|
||||
return False
|
||||
|
||||
|
||||
def check_all(
|
||||
endpoints: List[InferenceEndpoint] = None,
|
||||
auto_restart: bool = False,
|
||||
timeout: float = 5.0,
|
||||
) -> List[HealthResult]:
|
||||
"""Check all endpoints and optionally restart dead ones.
|
||||
|
||||
Args:
|
||||
endpoints: List of endpoints to check. Uses DEFAULT_ENDPOINTS if None.
|
||||
auto_restart: If True, attempt to restart down endpoints.
|
||||
timeout: HTTP timeout per endpoint.
|
||||
|
||||
Returns:
|
||||
List of HealthResult for each endpoint.
|
||||
"""
|
||||
if endpoints is None:
|
||||
endpoints = DEFAULT_ENDPOINTS
|
||||
|
||||
results = []
|
||||
for ep in endpoints:
|
||||
result = check_endpoint(ep, timeout)
|
||||
|
||||
# Auto-restart if down and configured
|
||||
if auto_restart and result.status == "down" and ep.restart_cmd:
|
||||
result.restart_attempted = True
|
||||
result.restart_succeeded = attempt_restart(ep)
|
||||
if result.restart_succeeded:
|
||||
# Re-check after restart
|
||||
time.sleep(2)
|
||||
result2 = check_endpoint(ep, timeout)
|
||||
result.status = result2.status
|
||||
result.latency_ms = result2.latency_ms
|
||||
result.error = result2.error
|
||||
|
||||
results.append(result)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def format_report(results: List[HealthResult]) -> str:
|
||||
"""Format health check results as a human-readable report."""
|
||||
lines = [
|
||||
"# Local Inference Health Check",
|
||||
f"Time: {time.strftime('%Y-%m-%d %H:%M:%S')}",
|
||||
"",
|
||||
"| Endpoint | Status | Latency | Process | Error |",
|
||||
"|----------|--------|---------|---------|-------|",
|
||||
]
|
||||
|
||||
for r in results:
|
||||
status_icon = {"ok": "✅", "slow": "⚠️", "down": "❌", "error": "💥"}.get(r.status, "?")
|
||||
proc = "alive" if r.process_alive else "dead"
|
||||
lat = f"{r.latency_ms}ms" if r.latency_ms > 0 else "-"
|
||||
err = r.error[:40] if r.error else "-"
|
||||
lines.append(f"| {r.name} | {status_icon} {r.status} | {lat} | {proc} | {err} |")
|
||||
|
||||
down = [r for r in results if r.status in ("down", "error")]
|
||||
if down:
|
||||
lines.extend(["", "## DOWN", ""])
|
||||
for r in down:
|
||||
lines.append(f"- **{r.name}** ({r.url}): {r.error}")
|
||||
if r.restart_attempted:
|
||||
status = "✅ restarted" if r.restart_succeeded else "❌ restart failed"
|
||||
lines.append(f" Restart: {status}")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_json(results: List[HealthResult]) -> str:
|
||||
"""Format results as JSON."""
|
||||
data = []
|
||||
for r in results:
|
||||
data.append({
|
||||
"name": r.name,
|
||||
"url": r.url,
|
||||
"status": r.status,
|
||||
"latency_ms": r.latency_ms,
|
||||
"process_alive": r.process_alive,
|
||||
"error": r.error or None,
|
||||
"restart_attempted": r.restart_attempted,
|
||||
"restart_succeeded": r.restart_succeeded,
|
||||
})
|
||||
return json.dumps({"timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"), "endpoints": data}, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(description="Local inference health check")
|
||||
p.add_argument("--json", action="store_true", help="JSON output")
|
||||
p.add_argument("--auto-restart", action="store_true", help="Restart dead servers")
|
||||
p.add_argument("--timeout", type=float, default=5.0, help="HTTP timeout (seconds)")
|
||||
p.add_argument("--port", type=int, help="Check specific port only")
|
||||
a = p.parse_args()
|
||||
|
||||
endpoints = DEFAULT_ENDPOINTS
|
||||
if a.port:
|
||||
endpoints = [ep for ep in DEFAULT_ENDPOINTS if ep.port == a.port]
|
||||
if not endpoints:
|
||||
print(f"No endpoint configured for port {a.port}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
results = check_all(endpoints, auto_restart=a.auto_restart, timeout=a.timeout)
|
||||
|
||||
if a.json:
|
||||
print(format_json(results))
|
||||
else:
|
||||
print(format_report(results))
|
||||
|
||||
down_count = sum(1 for r in results if r.status in ("down", "error"))
|
||||
sys.exit(1 if down_count > 0 else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
96
tests/test_inference_health.py
Normal file
96
tests/test_inference_health.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""Tests for inference health check (#713)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import json
|
||||
|
||||
from scripts.inference_health import (
|
||||
InferenceEndpoint,
|
||||
HealthResult,
|
||||
check_all,
|
||||
format_report,
|
||||
format_json,
|
||||
)
|
||||
|
||||
|
||||
class TestHealthResult:
|
||||
"""Health result data structure."""
|
||||
|
||||
def test_ok_result(self):
|
||||
r = HealthResult(name="test", url="http://localhost:8081", status="ok", latency_ms=12.5)
|
||||
assert r.status == "ok"
|
||||
assert r.latency_ms == 12.5
|
||||
assert not r.error
|
||||
|
||||
def test_down_result(self):
|
||||
r = HealthResult(
|
||||
name="test", url="http://localhost:8081",
|
||||
status="down", error="Connection refused",
|
||||
)
|
||||
assert r.status == "down"
|
||||
assert r.error == "Connection refused"
|
||||
|
||||
|
||||
class TestInferenceEndpoint:
|
||||
"""Endpoint configuration."""
|
||||
|
||||
def test_defaults(self):
|
||||
ep = InferenceEndpoint(name="test", url="http://localhost:8080")
|
||||
assert ep.health_path == "/health"
|
||||
assert ep.port == 8080
|
||||
assert ep.restart_cmd == ""
|
||||
|
||||
def test_custom(self):
|
||||
ep = InferenceEndpoint(
|
||||
name="llama", url="http://localhost:8081",
|
||||
port=8081, restart_cmd="llama-server --port 8081",
|
||||
)
|
||||
assert ep.port == 8081
|
||||
assert "llama-server" in ep.restart_cmd
|
||||
|
||||
|
||||
class TestFormatReport:
|
||||
"""Report formatting."""
|
||||
|
||||
def test_all_ok(self):
|
||||
results = [
|
||||
HealthResult(name="test1", url="http://localhost:8080", status="ok", latency_ms=5.0, process_alive=True),
|
||||
HealthResult(name="test2", url="http://localhost:8081", status="ok", latency_ms=10.0, process_alive=True),
|
||||
]
|
||||
report = format_report(results)
|
||||
assert "Health Check" in report
|
||||
assert "test1" in report
|
||||
assert "test2" in report
|
||||
assert "DOWN" not in report
|
||||
|
||||
def test_with_down(self):
|
||||
results = [
|
||||
HealthResult(name="test1", url="http://localhost:8080", status="ok", latency_ms=5.0),
|
||||
HealthResult(
|
||||
name="test2", url="http://localhost:8081",
|
||||
status="down", error="Connection refused", process_alive=False,
|
||||
),
|
||||
]
|
||||
report = format_report(results)
|
||||
assert "DOWN" in report
|
||||
assert "Connection refused" in report
|
||||
|
||||
|
||||
class TestFormatJson:
|
||||
"""JSON output format."""
|
||||
|
||||
def test_valid_json(self):
|
||||
results = [HealthResult(name="test", url="http://localhost:8080", status="ok", latency_ms=5.0)]
|
||||
output = format_json(results)
|
||||
data = json.loads(output)
|
||||
assert "timestamp" in data
|
||||
assert "endpoints" in data
|
||||
assert len(data["endpoints"]) == 1
|
||||
assert data["endpoints"][0]["name"] == "test"
|
||||
|
||||
def test_none_error_serializes(self):
|
||||
results = [HealthResult(name="test", url="http://localhost:8080", status="ok")]
|
||||
output = format_json(results)
|
||||
data = json.loads(output)
|
||||
assert data["endpoints"][0]["error"] is None
|
||||
@@ -1,114 +0,0 @@
|
||||
"""Tests for Matrix Bridge — Issue #747."""
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||
|
||||
from agent.matrix_bridge import MatrixBridge, AgentMessage, AgentRegistry
|
||||
|
||||
|
||||
class TestMessageParsing:
|
||||
"""Test message format parsing."""
|
||||
|
||||
def test_is_for_me_direct(self):
|
||||
bridge = MatrixBridge(agent_name="Timmy")
|
||||
assert bridge._is_for_me("[@Timmy] Hello") == True
|
||||
|
||||
def test_is_not_for_me(self):
|
||||
bridge = MatrixBridge(agent_name="Timmy")
|
||||
assert bridge._is_for_me("[@Allegro] Hello") == False
|
||||
|
||||
def test_is_broadcast(self):
|
||||
bridge = MatrixBridge(agent_name="Timmy")
|
||||
assert bridge._is_for_me("[@*] Broadcast") == True
|
||||
|
||||
def test_extract_content(self):
|
||||
bridge = MatrixBridge(agent_name="Timmy")
|
||||
assert bridge._extract_content("[@Timmy] Hello world") == "Hello world"
|
||||
|
||||
def test_extract_content_multiline(self):
|
||||
bridge = MatrixBridge(agent_name="Timmy")
|
||||
content = bridge._extract_content("[@Timmy] Line 1\nLine 2")
|
||||
assert content == "Line 1\nLine 2"
|
||||
|
||||
|
||||
class TestAgentMessage:
|
||||
"""Test AgentMessage dataclass."""
|
||||
|
||||
def test_to_dict(self):
|
||||
msg = AgentMessage(
|
||||
sender="Timmy",
|
||||
recipient="Allegro",
|
||||
content="Hello",
|
||||
timestamp=1234567890.0,
|
||||
)
|
||||
d = msg.to_dict()
|
||||
assert d["sender"] == "Timmy"
|
||||
assert d["recipient"] == "Allegro"
|
||||
assert d["content"] == "Hello"
|
||||
|
||||
def test_from_dict(self):
|
||||
d = {
|
||||
"sender": "Timmy",
|
||||
"recipient": "Allegro",
|
||||
"content": "Hello",
|
||||
"timestamp": 1234567890.0,
|
||||
"message_id": "",
|
||||
"room_id": "",
|
||||
}
|
||||
msg = AgentMessage.from_dict(d)
|
||||
assert msg.sender == "Timmy"
|
||||
assert msg.recipient == "Allegro"
|
||||
|
||||
|
||||
class TestAgentRegistry:
|
||||
"""Test AgentRegistry."""
|
||||
|
||||
def test_register(self):
|
||||
registry = AgentRegistry()
|
||||
registry.register("Timmy", capabilities=["code", "review"])
|
||||
agent = registry.get_agent("Timmy")
|
||||
assert agent["name"] == "Timmy"
|
||||
assert "code" in agent["capabilities"]
|
||||
|
||||
def test_list_agents(self):
|
||||
registry = AgentRegistry()
|
||||
registry.register("Timmy")
|
||||
registry.register("Allegro")
|
||||
agents = registry.list_agents()
|
||||
assert len(agents) == 2
|
||||
|
||||
def test_find_with_capability(self):
|
||||
registry = AgentRegistry()
|
||||
registry.register("Timmy", capabilities=["code"])
|
||||
registry.register("Allegro", capabilities=["monitoring"])
|
||||
coders = registry.find_agents_with_capability("code")
|
||||
assert "Timmy" in coders
|
||||
assert "Allegro" not in coders
|
||||
|
||||
def test_unregister(self):
|
||||
registry = AgentRegistry()
|
||||
registry.register("Timmy")
|
||||
registry.unregister("Timmy")
|
||||
agent = registry.get_agent("Timmy")
|
||||
assert agent["status"] == "offline"
|
||||
|
||||
|
||||
class TestBridgeInit:
|
||||
"""Test bridge initialization."""
|
||||
|
||||
def test_default_agent_name(self):
|
||||
bridge = MatrixBridge()
|
||||
assert bridge.agent_name == "Hermes"
|
||||
|
||||
def test_custom_agent_name(self):
|
||||
bridge = MatrixBridge(agent_name="Timmy")
|
||||
assert bridge.agent_name == "Timmy"
|
||||
|
||||
def test_known_agents_empty(self):
|
||||
bridge = MatrixBridge()
|
||||
assert len(bridge.get_known_agents()) == 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user