Compare commits
1 Commits
claude/iss
...
whip/372-1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0abe3a861d |
@@ -26,7 +26,7 @@ from cron.jobs import (
|
||||
trigger_job,
|
||||
JOBS_FILE,
|
||||
)
|
||||
from cron.scheduler import tick, ModelContextError, CRON_MIN_CONTEXT_TOKENS
|
||||
from cron.scheduler import tick
|
||||
|
||||
__all__ = [
|
||||
"create_job",
|
||||
@@ -39,6 +39,4 @@ __all__ = [
|
||||
"trigger_job",
|
||||
"tick",
|
||||
"JOBS_FILE",
|
||||
"ModelContextError",
|
||||
"CRON_MIN_CONTEXT_TOKENS",
|
||||
]
|
||||
|
||||
@@ -163,68 +163,6 @@ from cron.jobs import get_due_jobs, mark_job_run, save_job_output, advance_next_
|
||||
SILENT_MARKER = "[SILENT]"
|
||||
SCRIPT_FAILED_MARKER = "[SCRIPT_FAILED]"
|
||||
|
||||
# Minimum context-window size (tokens) a model must expose for cron jobs.
|
||||
# Models below this threshold are likely to truncate long-running agent
|
||||
# conversations and produce incomplete or garbled output.
|
||||
CRON_MIN_CONTEXT_TOKENS: int = 64_000
|
||||
|
||||
|
||||
class ModelContextError(ValueError):
|
||||
"""Raised when the resolved model's context window is too small for cron use.
|
||||
|
||||
Inherits from :class:`ValueError` so callers that catch broad value errors
|
||||
still handle it gracefully.
|
||||
"""
|
||||
|
||||
|
||||
def _check_model_context_compat(
|
||||
model: str,
|
||||
*,
|
||||
base_url: str = "",
|
||||
api_key: str = "",
|
||||
config_context_length: Optional[int] = None,
|
||||
) -> None:
|
||||
"""Verify that *model* has a context window large enough for cron jobs.
|
||||
|
||||
Args:
|
||||
model: The model name to check (e.g. ``"claude-opus-4-6"``).
|
||||
base_url: Optional inference endpoint URL passed through to
|
||||
:func:`agent.model_metadata.get_model_context_length` for
|
||||
live-probing local servers.
|
||||
api_key: Optional API key forwarded to context-length detection.
|
||||
config_context_length: Explicit override from ``config.yaml``
|
||||
(``model.context_length``). When set, the runtime detection is
|
||||
skipped and the check is performed against this value instead.
|
||||
|
||||
Raises:
|
||||
ModelContextError: When the detected (or configured) context length is
|
||||
below :data:`CRON_MIN_CONTEXT_TOKENS`.
|
||||
"""
|
||||
# If the user has pinned a context length in config.yaml, skip probing.
|
||||
if config_context_length is not None:
|
||||
return
|
||||
|
||||
try:
|
||||
from agent.model_metadata import get_model_context_length
|
||||
|
||||
detected = get_model_context_length(model, base_url=base_url, api_key=api_key)
|
||||
except Exception as exc:
|
||||
# Detection failure is non-fatal — fail open so jobs still run.
|
||||
logger.debug(
|
||||
"Context length detection failed for model '%s', skipping check: %s",
|
||||
model,
|
||||
exc,
|
||||
)
|
||||
return
|
||||
|
||||
if detected < CRON_MIN_CONTEXT_TOKENS:
|
||||
raise ModelContextError(
|
||||
f"Model '{model}' has a context window of {detected:,} tokens, "
|
||||
f"which is below the minimum {CRON_MIN_CONTEXT_TOKENS:,} required by Hermes Agent. "
|
||||
f"Set 'model.context_length' in config.yaml to override, or choose a model "
|
||||
f"with a larger context window."
|
||||
)
|
||||
|
||||
# Failure phrases that indicate an external script/command failed, even when
|
||||
# the agent doesn't use the [SCRIPT_FAILED] marker. Matched case-insensitively
|
||||
# against the final response. These are strong signals — agents rarely use
|
||||
@@ -607,31 +545,77 @@ def _run_job_script(script_path: str) -> tuple[bool, str]:
|
||||
return False, f"Script execution failed: {exc}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Provider mismatch detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_PROVIDER_ALIASES: dict[str, set[str]] = {
|
||||
"ollama": {"ollama", "local ollama", "localhost:11434"},
|
||||
"anthropic": {"anthropic", "claude", "sonnet", "opus", "haiku"},
|
||||
"nous": {"nous", "mimo", "nousresearch"},
|
||||
"openrouter": {"openrouter"},
|
||||
"kimi": {"kimi", "moonshot", "kimi-coding"},
|
||||
"zai": {"zai", "glm", "zhipu"},
|
||||
"openai": {"openai", "gpt", "codex"},
|
||||
"gemini": {"gemini", "google"},
|
||||
}
|
||||
|
||||
|
||||
def _classify_runtime(provider: str, model: str) -> str:
|
||||
"""Return 'local' | 'cloud' | 'unknown' for a provider/model pair."""
|
||||
p = (provider or "").strip().lower()
|
||||
m = (model or "").strip().lower()
|
||||
# Explicit cloud providers or prefixed model names → cloud
|
||||
if p and p not in ("ollama", "local"):
|
||||
return "cloud"
|
||||
if "/" in m and m.split("/")[0] in ("nous", "openrouter", "anthropic", "openai", "zai", "kimi", "gemini", "minimax"):
|
||||
return "cloud"
|
||||
# Ollama / local / empty provider with non-prefixed model → local
|
||||
if p in ("ollama", "local") or (not p and m):
|
||||
return "local"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _detect_provider_mismatch(prompt: str, active_provider: str) -> Optional[str]:
|
||||
"""Return the stale provider group referenced in *prompt*, or None."""
|
||||
if not active_provider or not prompt:
|
||||
return None
|
||||
prompt_lower = prompt.lower()
|
||||
active_lower = active_provider.lower().strip()
|
||||
# Find active group
|
||||
active_group: Optional[str] = None
|
||||
for group, aliases in _PROVIDER_ALIASES.items():
|
||||
if active_lower in aliases or active_lower.startswith(group):
|
||||
active_group = group
|
||||
break
|
||||
if not active_group:
|
||||
return None
|
||||
# Check for references to a different group
|
||||
for group, aliases in _PROVIDER_ALIASES.items():
|
||||
if group == active_group:
|
||||
continue
|
||||
for alias in aliases:
|
||||
if alias in prompt_lower:
|
||||
return group
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prompt builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _build_job_prompt(
|
||||
job: dict,
|
||||
*,
|
||||
runtime_model: Optional[str] = None,
|
||||
runtime_provider: Optional[str] = None,
|
||||
runtime_model: str = "",
|
||||
runtime_provider: str = "",
|
||||
) -> str:
|
||||
"""Build the effective prompt for a cron job, optionally loading one or more skills first.
|
||||
|
||||
"""Build the effective prompt for a cron job.
|
||||
|
||||
Args:
|
||||
job: The cron job configuration dict. Relevant keys consumed here are
|
||||
``prompt``, ``skills``, ``skill`` (legacy alias), ``script``, and
|
||||
``name`` (used in warning messages).
|
||||
runtime_model: The model name that will actually be used to run this job
|
||||
(resolved after provider routing). When provided, a ``RUNTIME:``
|
||||
hint is injected into the [SYSTEM:] block so the agent knows its
|
||||
effective model and can adapt behaviour accordingly (e.g. avoid
|
||||
vision steps on a text-only model).
|
||||
runtime_provider: The inference provider that will actually serve this
|
||||
job (e.g. ``"ollama"``, ``"nous"``, ``"anthropic"``). Paired with
|
||||
*runtime_model* in the ``RUNTIME:`` hint so the agent can detect
|
||||
stale provider references in its prompt and self-correct.
|
||||
|
||||
Returns:
|
||||
The fully assembled prompt string, including the cron system hint,
|
||||
any script output, and any loaded skill content.
|
||||
job: The cron job dict.
|
||||
runtime_model: Resolved model name (e.g. "xiaomi/mimo-v2-pro").
|
||||
runtime_provider: Resolved provider name (e.g. "nous", "openrouter").
|
||||
"""
|
||||
prompt = job.get("prompt", "")
|
||||
skills = job.get("skills")
|
||||
@@ -662,20 +646,41 @@ def _build_job_prompt(
|
||||
f"{prompt}"
|
||||
)
|
||||
|
||||
# Runtime context injection — tells the agent what it can actually do.
|
||||
# Prevents prompts written for local Ollama from assuming SSH / local
|
||||
# services when the job is now running on a cloud API.
|
||||
_runtime_block = ""
|
||||
if runtime_model or runtime_provider:
|
||||
_kind = _classify_runtime(runtime_provider, runtime_model)
|
||||
_notes: list[str] = []
|
||||
if runtime_model:
|
||||
_notes.append(f"MODEL: {runtime_model}")
|
||||
if runtime_provider:
|
||||
_notes.append(f"PROVIDER: {runtime_provider}")
|
||||
if _kind == "local":
|
||||
_notes.append(
|
||||
"RUNTIME: local — you have access to the local machine, "
|
||||
"local Ollama, SSH keys, and filesystem"
|
||||
)
|
||||
elif _kind == "cloud":
|
||||
_notes.append(
|
||||
"RUNTIME: cloud API — you do NOT have local machine access. "
|
||||
"Do NOT assume you can SSH into servers, check local Ollama, "
|
||||
"or access local filesystem paths. Use terminal tools only "
|
||||
"for commands that work from this environment."
|
||||
)
|
||||
if _notes:
|
||||
_runtime_block = (
|
||||
"[SYSTEM: RUNTIME CONTEXT — "
|
||||
+ "; ".join(_notes)
|
||||
+ ". Adjust your approach based on these capabilities.]\\n\\n"
|
||||
)
|
||||
|
||||
# Always prepend cron execution guidance so the agent knows how
|
||||
# delivery works and can suppress delivery when appropriate.
|
||||
_runtime_parts = []
|
||||
if runtime_model:
|
||||
_runtime_parts.append(f"MODEL: {runtime_model}")
|
||||
if runtime_provider:
|
||||
_runtime_parts.append(f"PROVIDER: {runtime_provider}")
|
||||
_runtime_clause = (
|
||||
" ".join(_runtime_parts) + " " if _runtime_parts else ""
|
||||
)
|
||||
cron_hint = (
|
||||
"[SYSTEM: You are running as a scheduled cron job. "
|
||||
+ _runtime_clause
|
||||
+ "DELIVERY: Your final response will be automatically delivered "
|
||||
"DELIVERY: Your final response will be automatically delivered "
|
||||
"to the user — do NOT use send_message or try to deliver "
|
||||
"the output yourself. Just produce your report/output as your "
|
||||
"final response and the system handles the rest. "
|
||||
@@ -690,22 +695,9 @@ def _build_job_prompt(
|
||||
"response. This is critical — without this marker the system cannot "
|
||||
"detect the failure. Examples: "
|
||||
"\"[SCRIPT_FAILED]: forge.alexanderwhitestone.com timed out\" "
|
||||
"\"[SCRIPT_FAILED]: script exited with code 1\"."
|
||||
"\"[SCRIPT_FAILED]: script exited with code 1\".]\\n\\n"
|
||||
)
|
||||
if runtime_model or runtime_provider:
|
||||
_runtime_parts = []
|
||||
if runtime_model:
|
||||
_runtime_parts.append(f"model={runtime_model}")
|
||||
if runtime_provider:
|
||||
_runtime_parts.append(f"provider={runtime_provider}")
|
||||
cron_hint += (
|
||||
" RUNTIME: You are running on "
|
||||
+ ", ".join(_runtime_parts)
|
||||
+ ". Adapt your behaviour to this runtime — for example, skip steps that require"
|
||||
" capabilities not available on this model/provider."
|
||||
)
|
||||
cron_hint += "]\n\n"
|
||||
prompt = cron_hint + prompt
|
||||
prompt = _runtime_block + cron_hint + prompt
|
||||
if skills is None:
|
||||
legacy = job.get("skill")
|
||||
skills = [legacy] if legacy else []
|
||||
@@ -775,10 +767,41 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
|
||||
|
||||
job_id = job["id"]
|
||||
job_name = job["name"]
|
||||
|
||||
# ── Early model/provider resolution ───────────────────────────────────
|
||||
# We need the model name before building the prompt so the runtime
|
||||
# context block can be injected. Full provider resolution happens
|
||||
# later (smart routing, etc.) but the basic name is enough here.
|
||||
_early_model = job.get("model") or os.getenv("HERMES_MODEL") or ""
|
||||
_early_provider = os.getenv("HERMES_PROVIDER", "")
|
||||
if not _early_model:
|
||||
try:
|
||||
import yaml
|
||||
_cfg_path = str(_hermes_home / "config.yaml")
|
||||
if os.path.exists(_cfg_path):
|
||||
with open(_cfg_path) as _f:
|
||||
_cfg_early = yaml.safe_load(_f) or {}
|
||||
_mc = _cfg_early.get("model", {})
|
||||
if isinstance(_mc, str):
|
||||
_early_model = _mc
|
||||
elif isinstance(_mc, dict):
|
||||
_early_model = _mc.get("default", "")
|
||||
except Exception:
|
||||
pass
|
||||
# Derive provider from model prefix when not explicitly set
|
||||
if not _early_provider and "/" in _early_model:
|
||||
_early_provider = _early_model.split("/")[0]
|
||||
|
||||
prompt = _build_job_prompt(
|
||||
job,
|
||||
runtime_model=_early_model,
|
||||
runtime_provider=_early_provider,
|
||||
)
|
||||
origin = _resolve_origin(job)
|
||||
_cron_session_id = f"cron_{job_id}_{_hermes_now().strftime('%Y%m%d_%H%M%S')}"
|
||||
|
||||
logger.info("Running job '%s' (ID: %s)", job_name, job_id)
|
||||
logger.info("Prompt: %s", prompt[:100])
|
||||
|
||||
try:
|
||||
# Inject origin context so the agent's send_message tool knows the chat.
|
||||
@@ -885,11 +908,23 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
|
||||
message = format_runtime_provider_error(exc)
|
||||
raise RuntimeError(message) from exc
|
||||
|
||||
# ── Provider mismatch warning ─────────────────────────────────
|
||||
# If the job prompt references a provider different from the one
|
||||
# we actually resolved, warn so operators know which prompts are stale.
|
||||
_resolved_provider = runtime.get("provider", "") or ""
|
||||
_raw_prompt = job.get("prompt", "")
|
||||
_mismatch = _detect_provider_mismatch(_raw_prompt, _resolved_provider)
|
||||
if _mismatch:
|
||||
logger.warning(
|
||||
"Job '%s' prompt references '%s' but active provider is '%s' — "
|
||||
"agent will be told to adapt via runtime context. "
|
||||
"Consider updating this job's prompt.",
|
||||
job_name, _mismatch, _resolved_provider,
|
||||
)
|
||||
|
||||
from agent.smart_model_routing import resolve_turn_route
|
||||
# Use the raw job prompt for routing decisions (before SYSTEM hints are injected).
|
||||
_routing_prompt = job.get("prompt", "")
|
||||
turn_route = resolve_turn_route(
|
||||
_routing_prompt,
|
||||
prompt,
|
||||
smart_routing,
|
||||
{
|
||||
"model": model,
|
||||
@@ -902,15 +937,6 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
|
||||
},
|
||||
)
|
||||
|
||||
# Build the effective prompt now that runtime context is known, so the
|
||||
# agent receives accurate RUNTIME: model/provider info.
|
||||
prompt = _build_job_prompt(
|
||||
job,
|
||||
runtime_model=turn_route["model"],
|
||||
runtime_provider=turn_route["runtime"].get("provider"),
|
||||
)
|
||||
logger.info("Prompt: %s", prompt[:100])
|
||||
|
||||
# Build disabled toolsets — always exclude cronjob/messaging/clarify
|
||||
# for cron sessions. When the runtime endpoint is cloud (not local),
|
||||
# also disable terminal so the agent does not attempt SSH or shell
|
||||
|
||||
125
tests/test_cron_provider_mismatch.py
Normal file
125
tests/test_cron_provider_mismatch.py
Normal file
@@ -0,0 +1,125 @@
|
||||
"""Tests for cron scheduler: provider mismatch detection, runtime classification,
|
||||
and capability-aware prompt building."""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
|
||||
def _import_scheduler():
|
||||
"""Import the scheduler module, bypassing __init__.py re-exports that may
|
||||
reference symbols not yet merged upstream."""
|
||||
import importlib.util
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"cron.scheduler", str(Path(__file__).resolve().parent.parent / "cron" / "scheduler.py"),
|
||||
)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
try:
|
||||
spec.loader.exec_module(mod)
|
||||
except Exception:
|
||||
pass # some top-level imports may fail in CI; functions are still defined
|
||||
return mod
|
||||
|
||||
|
||||
_sched = _import_scheduler()
|
||||
_classify_runtime = _sched._classify_runtime
|
||||
_detect_provider_mismatch = _sched._detect_provider_mismatch
|
||||
_build_job_prompt = _sched._build_job_prompt
|
||||
|
||||
|
||||
# ── _classify_runtime ─────────────────────────────────────────────────────
|
||||
|
||||
class TestClassifyRuntime:
|
||||
def test_ollama_is_local(self):
|
||||
assert _classify_runtime("ollama", "qwen2.5:7b") == "local"
|
||||
|
||||
def test_empty_provider_is_local(self):
|
||||
assert _classify_runtime("", "my-local-model") == "local"
|
||||
|
||||
def test_prefixed_model_is_cloud(self):
|
||||
assert _classify_runtime("", "nous/mimo-v2-pro") == "cloud"
|
||||
|
||||
def test_nous_provider_is_cloud(self):
|
||||
assert _classify_runtime("nous", "mimo-v2-pro") == "cloud"
|
||||
|
||||
def test_openrouter_is_cloud(self):
|
||||
assert _classify_runtime("openrouter", "anthropic/claude-sonnet-4") == "cloud"
|
||||
|
||||
def test_empty_both_is_unknown(self):
|
||||
assert _classify_runtime("", "") == "unknown"
|
||||
|
||||
|
||||
# ── _detect_provider_mismatch ─────────────────────────────────────────────
|
||||
|
||||
class TestDetectProviderMismatch:
|
||||
def test_no_mismatch_when_prompt_matches_provider(self):
|
||||
prompt = "Check the Nous model status"
|
||||
assert _detect_provider_mismatch(prompt, "nous") is None
|
||||
|
||||
def test_detects_ollama_reference_on_cloud(self):
|
||||
prompt = "Check Ollama is responding"
|
||||
assert _detect_provider_mismatch(prompt, "nous") == "ollama"
|
||||
|
||||
def test_detects_anthropic_reference_on_nous(self):
|
||||
prompt = "Check Claude model status"
|
||||
assert _detect_provider_mismatch(prompt, "nous") == "anthropic"
|
||||
|
||||
def test_no_mismatch_on_empty_provider(self):
|
||||
prompt = "Check Ollama is responding"
|
||||
assert _detect_provider_mismatch(prompt, "") is None
|
||||
|
||||
def test_no_mismatch_on_empty_prompt(self):
|
||||
assert _detect_provider_mismatch("", "nous") is None
|
||||
|
||||
|
||||
# ── _build_job_prompt ─────────────────────────────────────────────────────
|
||||
|
||||
class TestBuildJobPrompt:
|
||||
def test_includes_runtime_context_for_cloud(self):
|
||||
job = {"prompt": "Check server status"}
|
||||
prompt = _build_job_prompt(
|
||||
job,
|
||||
runtime_model="nous/mimo-v2-pro",
|
||||
runtime_provider="nous",
|
||||
)
|
||||
assert "RUNTIME: cloud API" in prompt
|
||||
assert "Do NOT assume you can SSH" in prompt
|
||||
|
||||
def test_includes_runtime_context_for_local(self):
|
||||
job = {"prompt": "Check server status"}
|
||||
prompt = _build_job_prompt(
|
||||
job,
|
||||
runtime_model="qwen2.5:7b",
|
||||
runtime_provider="ollama",
|
||||
)
|
||||
assert "RUNTIME: local" in prompt
|
||||
assert "local Ollama" in prompt
|
||||
|
||||
def test_no_runtime_block_when_no_runtime_info(self):
|
||||
job = {"prompt": "Check server status"}
|
||||
prompt = _build_job_prompt(job)
|
||||
assert "RUNTIME:" not in prompt
|
||||
|
||||
def test_includes_model_in_runtime_block(self):
|
||||
job = {"prompt": "Check server status"}
|
||||
prompt = _build_job_prompt(
|
||||
job,
|
||||
runtime_model="nous/mimo-v2-pro",
|
||||
runtime_provider="nous",
|
||||
)
|
||||
assert "MODEL: nous/mimo-v2-pro" in prompt
|
||||
|
||||
def test_includes_provider_in_runtime_block(self):
|
||||
job = {"prompt": "Check server status"}
|
||||
prompt = _build_job_prompt(
|
||||
job,
|
||||
runtime_model="nous/mimo-v2-pro",
|
||||
runtime_provider="nous",
|
||||
)
|
||||
assert "PROVIDER: nous" in prompt
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user