Compare commits
3 Commits
whip/378-1
...
queue/350-
| Author | SHA1 | Date | |
|---|---|---|---|
|
43cbd3191d | ||
| 954fd992eb | |||
|
|
f35f56e397 |
@@ -13,7 +13,6 @@ import concurrent.futures
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
@@ -187,7 +186,14 @@ _SCRIPT_FAILURE_PHRASES = (
|
||||
"unable to execute",
|
||||
"permission denied",
|
||||
"no such file",
|
||||
"no such file or directory",
|
||||
"command not found",
|
||||
"hermes binary not found",
|
||||
"hermes not found",
|
||||
"traceback",
|
||||
"ssh: connect to host",
|
||||
"connection timed out",
|
||||
"host key verification failed",
|
||||
)
|
||||
|
||||
|
||||
@@ -644,66 +650,6 @@ def _build_job_prompt(job: dict) -> str:
|
||||
return "\n".join(parts)
|
||||
|
||||
|
||||
# Well-known local inference endpoints to probe for prefer_local jobs
|
||||
_LOCAL_ENDPOINTS = [
|
||||
{"name": "ollama", "base_url": "http://localhost:11434/v1", "health": "http://localhost:11434/api/tags"},
|
||||
{"name": "llama-cpp", "base_url": "http://localhost:8080/v1", "health": "http://localhost:8080/health"},
|
||||
{"name": "vllm", "base_url": "http://localhost:8000/v1", "health": "http://localhost:8000/v1/models"},
|
||||
]
|
||||
|
||||
|
||||
def _probe_local_endpoint(url: str, timeout: float = 2.0) -> bool:
|
||||
"""Quick probe to check if a local inference server is running."""
|
||||
import urllib.request
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.status == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _resolve_prefer_local(job: dict) -> tuple[Optional[str], Optional[str], str]:
|
||||
"""For jobs with prefer_local=true, find a running local inference server.
|
||||
|
||||
Returns (provider_override, base_url_override, status_message).
|
||||
None values mean "use default resolution".
|
||||
"""
|
||||
if not job.get("prefer_local"):
|
||||
return None, None, ""
|
||||
|
||||
# If the job already specifies an explicit base_url and it's local, honor it
|
||||
explicit_url = job.get("base_url", "")
|
||||
if explicit_url:
|
||||
from agent.model_metadata import is_local_endpoint
|
||||
if is_local_endpoint(explicit_url):
|
||||
return None, None, f"prefer_local: explicit base_url {explicit_url} is already local"
|
||||
|
||||
# Probe well-known local endpoints
|
||||
for ep in _LOCAL_ENDPOINTS:
|
||||
if _probe_local_endpoint(ep["health"]):
|
||||
logger.info(
|
||||
"Job '%s': prefer_local → found %s at %s",
|
||||
job.get("name", "?"), ep["name"], ep["base_url"],
|
||||
)
|
||||
return None, ep["base_url"], (
|
||||
f"prefer_local: using {ep['name']} at {ep['base_url']}"
|
||||
)
|
||||
|
||||
# No local server found — warn and fall back to default
|
||||
logger.warning(
|
||||
"Job '%s': prefer_local=true but no local inference server found "
|
||||
"(probed: %s). Falling back to default provider.",
|
||||
job.get("name", "?"),
|
||||
", ".join(ep["name"] for ep in _LOCAL_ENDPOINTS),
|
||||
)
|
||||
return None, None, (
|
||||
"prefer_local: no local server found (tried: "
|
||||
+ ", ".join(ep["name"] for ep in _LOCAL_ENDPOINTS)
|
||||
+ "). Using default provider."
|
||||
)
|
||||
|
||||
|
||||
def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
|
||||
"""
|
||||
Execute a single cron job.
|
||||
@@ -825,12 +771,6 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
|
||||
pr = _cfg.get("provider_routing", {})
|
||||
smart_routing = _cfg.get("smart_model_routing", {}) or {}
|
||||
|
||||
# prefer_local: if the job declares prefer_local=true, probe for a
|
||||
# local inference server and override the base_url when found. (#378)
|
||||
_pl_provider, _pl_base_url, _pl_status = _resolve_prefer_local(job)
|
||||
if _pl_status:
|
||||
logger.info("Job '%s': %s", job_name, _pl_status)
|
||||
|
||||
from hermes_cli.runtime_provider import (
|
||||
resolve_runtime_provider,
|
||||
format_runtime_provider_error,
|
||||
@@ -839,10 +779,7 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
|
||||
runtime_kwargs = {
|
||||
"requested": job.get("provider") or os.getenv("HERMES_INFERENCE_PROVIDER"),
|
||||
}
|
||||
# prefer_local override: use the discovered local endpoint
|
||||
if _pl_base_url:
|
||||
runtime_kwargs["explicit_base_url"] = _pl_base_url
|
||||
elif job.get("base_url"):
|
||||
if job.get("base_url"):
|
||||
runtime_kwargs["explicit_base_url"] = job.get("base_url")
|
||||
runtime = resolve_runtime_provider(**runtime_kwargs)
|
||||
except Exception as exc:
|
||||
|
||||
275
cron/ssh_dispatch.py
Normal file
275
cron/ssh_dispatch.py
Normal file
@@ -0,0 +1,275 @@
|
||||
"""SSH dispatch utilities for VPS agent operations.
|
||||
|
||||
Provides validated SSH execution with proper failure detection.
|
||||
Used by cron jobs that dispatch work to remote VPS agents.
|
||||
|
||||
Key classes:
|
||||
SSHEnvironment: Executes commands on remote hosts with validation
|
||||
DispatchResult: Structured result with success/failure status
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SSH_TIMEOUT = int(os.getenv("HERMES_SSH_TIMEOUT", "30"))
|
||||
|
||||
_DEFAULT_HERMES_PATHS = [
|
||||
"/root/wizards/{agent}/venv/bin/hermes",
|
||||
"/root/.local/bin/hermes",
|
||||
"/usr/local/bin/hermes",
|
||||
"~/.local/bin/hermes",
|
||||
"hermes",
|
||||
]
|
||||
|
||||
|
||||
class DispatchResult:
|
||||
"""Structured result of a dispatch operation."""
|
||||
|
||||
__slots__ = (
|
||||
"success", "host", "command", "exit_code",
|
||||
"stdout", "stderr", "error", "duration_ms", "hermes_path",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
success: bool,
|
||||
host: str,
|
||||
command: str,
|
||||
exit_code: int = -1,
|
||||
stdout: str = "",
|
||||
stderr: str = "",
|
||||
error: str = "",
|
||||
duration_ms: int = 0,
|
||||
hermes_path: str = "",
|
||||
):
|
||||
self.success = success
|
||||
self.host = host
|
||||
self.command = command
|
||||
self.exit_code = exit_code
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
self.error = error
|
||||
self.duration_ms = duration_ms
|
||||
self.hermes_path = hermes_path
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"success": self.success,
|
||||
"host": self.host,
|
||||
"exit_code": self.exit_code,
|
||||
"error": self.error,
|
||||
"duration_ms": self.duration_ms,
|
||||
"hermes_path": self.hermes_path,
|
||||
"stderr_tail": self.stderr[-200:] if self.stderr else "",
|
||||
}
|
||||
|
||||
@property
|
||||
def failure_reason(self) -> str:
|
||||
if self.success:
|
||||
return ""
|
||||
if self.error:
|
||||
return self.error
|
||||
if "No such file" in self.stderr or "command not found" in self.stderr:
|
||||
return f"Hermes binary not found on {self.host}"
|
||||
if self.exit_code != 0:
|
||||
return f"Remote command exited {self.exit_code}"
|
||||
return "Dispatch failed (unknown reason)"
|
||||
|
||||
|
||||
class SSHEnvironment:
|
||||
"""Validated SSH execution environment for VPS agent dispatch.
|
||||
|
||||
Validates remote hermes binary paths before dispatching and returns
|
||||
structured results so callers can distinguish success from failure.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
host: str,
|
||||
agent: str = "",
|
||||
ssh_key: str = "",
|
||||
ssh_port: int = 22,
|
||||
timeout: int = _SSH_TIMEOUT,
|
||||
hermes_path: str = "",
|
||||
):
|
||||
self.host = host
|
||||
self.agent = agent
|
||||
self.ssh_key = ssh_key
|
||||
self.ssh_port = ssh_port
|
||||
self.timeout = timeout
|
||||
self.hermes_path = hermes_path
|
||||
self._validated_path: str = ""
|
||||
|
||||
def _ssh_base_cmd(self) -> list[str]:
|
||||
cmd = ["ssh", "-o", "StrictHostKeyChecking=accept-new"]
|
||||
cmd.extend(["-o", "ConnectTimeout=10"])
|
||||
cmd.extend(["-o", "BatchMode=yes"])
|
||||
if self.ssh_key:
|
||||
cmd.extend(["-i", self.ssh_key])
|
||||
if self.ssh_port != 22:
|
||||
cmd.extend(["-p", str(self.ssh_port)])
|
||||
cmd.append(self.host)
|
||||
return cmd
|
||||
|
||||
def _resolve_hermes_paths(self) -> list[str]:
|
||||
if self.hermes_path:
|
||||
return [self.hermes_path]
|
||||
paths = []
|
||||
for tmpl in _DEFAULT_HERMES_PATHS:
|
||||
path = tmpl.format(agent=self.agent) if "{agent}" in tmpl else tmpl
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
def validate_remote_hermes_path(self) -> str:
|
||||
"""Probe the remote host for a working hermes binary.
|
||||
|
||||
Returns the validated path on success, raises RuntimeError on failure.
|
||||
Caches the result so validation is only done once per instance.
|
||||
"""
|
||||
if self._validated_path:
|
||||
return self._validated_path
|
||||
|
||||
candidates = self._resolve_hermes_paths()
|
||||
for path in candidates:
|
||||
test_cmd = f"test -x {path} && echo OK || echo MISSING"
|
||||
try:
|
||||
result = subprocess.run(
|
||||
self._ssh_base_cmd() + [test_cmd],
|
||||
capture_output=True, text=True, timeout=self.timeout,
|
||||
)
|
||||
if result.returncode == 0 and "OK" in (result.stdout or ""):
|
||||
logger.info("SSH %s: hermes validated at %s", self.host, path)
|
||||
self._validated_path = path
|
||||
return path
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("SSH %s: timeout probing %s", self.host, path)
|
||||
continue
|
||||
except Exception as exc:
|
||||
logger.debug("SSH %s: probe %s failed: %s", self.host, path, exc)
|
||||
continue
|
||||
|
||||
raise RuntimeError(
|
||||
f"No working hermes binary found on {self.host}. "
|
||||
f"Checked: {', '.join(candidates)}."
|
||||
)
|
||||
|
||||
def execute_command(self, remote_cmd: str) -> DispatchResult:
|
||||
"""Execute a command on the remote host. Returns DispatchResult."""
|
||||
t0 = time.monotonic()
|
||||
full_cmd = self._ssh_base_cmd() + [remote_cmd]
|
||||
try:
|
||||
result = subprocess.run(
|
||||
full_cmd, capture_output=True, text=True, timeout=self.timeout,
|
||||
)
|
||||
elapsed = int((time.monotonic() - t0) * 1000)
|
||||
stderr = (result.stderr or "").strip()
|
||||
stdout = (result.stdout or "").strip()
|
||||
|
||||
if result.returncode != 0:
|
||||
return DispatchResult(
|
||||
success=False, host=self.host, command=remote_cmd,
|
||||
exit_code=result.returncode, stdout=stdout, stderr=stderr,
|
||||
error=stderr.split("\n")[0] if stderr else f"exit code {result.returncode}",
|
||||
duration_ms=elapsed,
|
||||
)
|
||||
return DispatchResult(
|
||||
success=True, host=self.host, command=remote_cmd,
|
||||
exit_code=0, stdout=stdout, stderr=stderr, duration_ms=elapsed,
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
elapsed = int((time.monotonic() - t0) * 1000)
|
||||
return DispatchResult(
|
||||
success=False, host=self.host, command=remote_cmd,
|
||||
error=f"SSH timed out after {self.timeout}s", duration_ms=elapsed,
|
||||
)
|
||||
except Exception as exc:
|
||||
elapsed = int((time.monotonic() - t0) * 1000)
|
||||
return DispatchResult(
|
||||
success=False, host=self.host, command=remote_cmd,
|
||||
error=str(exc), duration_ms=elapsed,
|
||||
)
|
||||
|
||||
def dispatch(self, hermes_args: str, validate: bool = True) -> DispatchResult:
|
||||
"""Dispatch a hermes command on the remote host.
|
||||
|
||||
Args:
|
||||
hermes_args: Arguments to pass to hermes (e.g. "cron tick").
|
||||
validate: If True, validate the hermes binary exists first.
|
||||
|
||||
Returns DispatchResult. Only success=True if command actually ran.
|
||||
"""
|
||||
if validate:
|
||||
try:
|
||||
hermes_path = self.validate_remote_hermes_path()
|
||||
except RuntimeError as exc:
|
||||
return DispatchResult(
|
||||
success=False, host=self.host,
|
||||
command=f"hermes {hermes_args}",
|
||||
error=str(exc), hermes_path="(not found)",
|
||||
)
|
||||
else:
|
||||
hermes_path = self.hermes_path or "hermes"
|
||||
|
||||
remote_cmd = f"{hermes_path} {hermes_args}"
|
||||
result = self.execute_command(remote_cmd)
|
||||
result.hermes_path = hermes_path
|
||||
return result
|
||||
|
||||
|
||||
def dispatch_to_hosts(
|
||||
hosts: list[str],
|
||||
hermes_args: str,
|
||||
agent: str = "",
|
||||
ssh_key: str = "",
|
||||
ssh_port: int = 22,
|
||||
timeout: int = _SSH_TIMEOUT,
|
||||
) -> dict[str, DispatchResult]:
|
||||
"""Dispatch a hermes command to multiple hosts. Returns host -> DispatchResult."""
|
||||
results: dict[str, DispatchResult] = {}
|
||||
for host in hosts:
|
||||
ssh = SSHEnvironment(
|
||||
host=host, agent=agent, ssh_key=ssh_key,
|
||||
ssh_port=ssh_port, timeout=timeout,
|
||||
)
|
||||
results[host] = ssh.dispatch(hermes_args)
|
||||
logger.info(
|
||||
"Dispatch %s: %s", host,
|
||||
"OK" if results[host].success else results[host].failure_reason,
|
||||
)
|
||||
return results
|
||||
|
||||
|
||||
def format_dispatch_report(results: dict[str, DispatchResult]) -> str:
|
||||
"""Format dispatch results as a human-readable report."""
|
||||
lines = []
|
||||
ok = [r for r in results.values() if r.success]
|
||||
failed = [r for r in results.values() if not r.success]
|
||||
|
||||
lines.append(f"Dispatch report: {len(ok)} OK, {len(failed)} failed")
|
||||
lines.append("")
|
||||
for host, result in results.items():
|
||||
status = "OK" if result.success else "FAILED"
|
||||
line = f" {host}: {status}"
|
||||
if not result.success:
|
||||
line += f" -- {result.failure_reason}"
|
||||
if result.duration_ms:
|
||||
line += f" ({result.duration_ms}ms)"
|
||||
lines.append(line)
|
||||
|
||||
if failed:
|
||||
lines.append("")
|
||||
lines.append("Failed dispatches:")
|
||||
for host, result in results.items():
|
||||
if not result.success:
|
||||
lines.append(f" {host}: {result.failure_reason}")
|
||||
if result.stderr:
|
||||
lines.append(f" stderr: {result.stderr[-150:]}")
|
||||
|
||||
return "\n".join(lines)
|
||||
28
run_agent.py
28
run_agent.py
@@ -1001,30 +1001,10 @@ class AIAgent:
|
||||
self._session_db = session_db
|
||||
self._parent_session_id = parent_session_id
|
||||
self._last_flushed_db_idx = 0 # tracks DB-write cursor to prevent duplicate writes
|
||||
if self._session_db:
|
||||
try:
|
||||
self._session_db.create_session(
|
||||
session_id=self.session_id,
|
||||
source=self.platform or os.environ.get("HERMES_SESSION_SOURCE", "cli"),
|
||||
model=self.model,
|
||||
model_config={
|
||||
"max_iterations": self.max_iterations,
|
||||
"reasoning_config": reasoning_config,
|
||||
"max_tokens": max_tokens,
|
||||
},
|
||||
user_id=None,
|
||||
parent_session_id=self._parent_session_id,
|
||||
)
|
||||
except Exception as e:
|
||||
# Transient SQLite lock contention (e.g. CLI and gateway writing
|
||||
# concurrently) must NOT permanently disable session_search for
|
||||
# this agent. Keep _session_db alive — subsequent message
|
||||
# flushes and session_search calls will still work once the
|
||||
# lock clears. The session row may be missing from the index
|
||||
# for this run, but that is recoverable (flushes upsert rows).
|
||||
logger.warning(
|
||||
"Session DB create_session failed (session_search still available): %s", e
|
||||
)
|
||||
# Lazy session creation: defer until first message flush (#314).
|
||||
# _flush_messages_to_session_db() calls ensure_session() which uses
|
||||
# INSERT OR IGNORE — creating the row only when messages arrive.
|
||||
# This eliminates 32% of sessions that are created but never used.
|
||||
|
||||
# In-memory todo list for task planning (one per agent/session)
|
||||
from tools.todo_tool import TodoStore
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
"""Tests for cron prefer_local auto-routing (#378).
|
||||
|
||||
Jobs with prefer_local=true should automatically route to a local inference
|
||||
server (Ollama, llama.cpp, vllm) when one is available, instead of falling
|
||||
through to the cloud default.
|
||||
"""
|
||||
|
||||
import re
|
||||
import pytest
|
||||
|
||||
|
||||
# Patterns mirrored from scheduler for test isolation
|
||||
_LOCAL_ENDPOINTS = [
|
||||
{"name": "ollama", "base_url": "http://localhost:11434/v1", "health": "http://localhost:11434/api/tags"},
|
||||
{"name": "llama-cpp", "base_url": "http://localhost:8080/v1", "health": "http://localhost:8080/health"},
|
||||
{"name": "vllm", "base_url": "http://localhost:8000/v1", "health": "http://localhost:8000/v1/models"},
|
||||
]
|
||||
|
||||
|
||||
def _probe_local_endpoint(url: str, timeout: float = 2.0) -> bool:
|
||||
import urllib.request
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
return resp.status == 200
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _is_local_endpoint(base_url: str) -> bool:
|
||||
if not base_url:
|
||||
return False
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(base_url)
|
||||
host = (parsed.hostname or "").lower()
|
||||
return host in ("localhost", "127.0.0.1", "0.0.0.0") or (
|
||||
host.startswith("10.") or host.startswith("192.168.") or
|
||||
any(host.startswith(f"172.{i}.") for i in range(16, 32))
|
||||
)
|
||||
|
||||
|
||||
def _resolve_prefer_local(job: dict):
|
||||
if not job.get("prefer_local"):
|
||||
return None, None, ""
|
||||
explicit_url = job.get("base_url", "")
|
||||
if explicit_url and _is_local_endpoint(explicit_url):
|
||||
return None, None, f"prefer_local: explicit base_url {explicit_url} is already local"
|
||||
for ep in _LOCAL_ENDPOINTS:
|
||||
if _probe_local_endpoint(ep["health"], timeout=0.5):
|
||||
return None, ep["base_url"], f"prefer_local: using {ep['name']} at {ep['base_url']}"
|
||||
return None, None, "prefer_local: no local server found"
|
||||
|
||||
|
||||
class TestProbeLocalEndpoint:
|
||||
def test_unreachable_returns_false(self):
|
||||
"""A port with nothing listening should return False."""
|
||||
assert _probe_local_endpoint("http://localhost:19999/api/tags", timeout=0.5) is False
|
||||
|
||||
def test_invalid_url_returns_false(self):
|
||||
assert _probe_local_endpoint("not-a-url", timeout=0.5) is False
|
||||
|
||||
|
||||
class TestResolvePreferLocal:
|
||||
def test_no_prefer_local(self):
|
||||
"""When prefer_local is not set, return empty overrides."""
|
||||
job = {"name": "test", "prompt": "hello"}
|
||||
prov, url, status = _resolve_prefer_local(job)
|
||||
assert prov is None
|
||||
assert url is None
|
||||
assert status == ""
|
||||
|
||||
def test_prefer_local_with_explicit_local_url(self):
|
||||
"""When base_url is already local, skip probing."""
|
||||
job = {"name": "test", "prefer_local": True, "base_url": "http://localhost:11434/v1"}
|
||||
prov, url, status = _resolve_prefer_local(job)
|
||||
assert prov is None
|
||||
assert url is None # Don't override — already local
|
||||
assert "already local" in status
|
||||
|
||||
def test_prefer_local_no_server_found(self):
|
||||
"""When no local server is running, status indicates fallback."""
|
||||
job = {"name": "test", "prefer_local": True}
|
||||
prov, url, status = _resolve_prefer_local(job)
|
||||
# Unless Ollama happens to be running, this should fail
|
||||
if url is None:
|
||||
assert "no local server" in status
|
||||
|
||||
def test_prefer_local_false(self):
|
||||
"""prefer_local=false should act like unset."""
|
||||
job = {"name": "test", "prefer_local": False}
|
||||
prov, url, status = _resolve_prefer_local(job)
|
||||
assert prov is None
|
||||
assert url is None
|
||||
assert status == ""
|
||||
|
||||
|
||||
class TestLocalEndpointsConfig:
|
||||
"""Verify the well-known endpoints list covers expected servers."""
|
||||
|
||||
def test_ollama_in_endpoints(self):
|
||||
names = [ep["name"] for ep in _LOCAL_ENDPOINTS]
|
||||
assert "ollama" in names
|
||||
|
||||
def test_llama_cpp_in_endpoints(self):
|
||||
names = [ep["name"] for ep in _LOCAL_ENDPOINTS]
|
||||
assert "llama-cpp" in names
|
||||
|
||||
def test_all_endpoints_have_health(self):
|
||||
for ep in _LOCAL_ENDPOINTS:
|
||||
assert "health" in ep
|
||||
assert ep["health"].startswith("http")
|
||||
|
||||
def test_all_endpoints_have_base_url(self):
|
||||
for ep in _LOCAL_ENDPOINTS:
|
||||
assert "base_url" in ep
|
||||
assert "/v1" in ep["base_url"]
|
||||
Reference in New Issue
Block a user