Compare commits

..

1 Commits

Author SHA1 Message Date
Timmy Time
b10f709935 Fix #372: Runtime-aware cron prompts with provider mismatch detection
Some checks failed
Forge CI / smoke-and-build (pull_request) Failing after 1m22s
Injects runtime context so agents know capabilities.
Detects provider mismatches and warns operators.
Fixes #372
2026-04-14 07:38:35 -04:00
3 changed files with 67 additions and 127 deletions

View File

@@ -26,7 +26,7 @@ from cron.jobs import (
trigger_job,
JOBS_FILE,
)
from cron.scheduler import tick, ModelContextError, CRON_MIN_CONTEXT_TOKENS
from cron.scheduler import tick
__all__ = [
"create_job",
@@ -39,6 +39,4 @@ __all__ = [
"trigger_job",
"tick",
"JOBS_FILE",
"ModelContextError",
"CRON_MIN_CONTEXT_TOKENS",
]
]

View File

@@ -163,68 +163,6 @@ from cron.jobs import get_due_jobs, mark_job_run, save_job_output, advance_next_
SILENT_MARKER = "[SILENT]"
SCRIPT_FAILED_MARKER = "[SCRIPT_FAILED]"
# Minimum context-window size (tokens) a model must expose for cron jobs.
# Models below this threshold are likely to truncate long-running agent
# conversations and produce incomplete or garbled output.
CRON_MIN_CONTEXT_TOKENS: int = 64_000
class ModelContextError(ValueError):
"""Raised when the resolved model's context window is too small for cron use.
Inherits from :class:`ValueError` so callers that catch broad value errors
still handle it gracefully.
"""
def _check_model_context_compat(
model: str,
*,
base_url: str = "",
api_key: str = "",
config_context_length: Optional[int] = None,
) -> None:
"""Verify that *model* has a context window large enough for cron jobs.
Args:
model: The model name to check (e.g. ``"claude-opus-4-6"``).
base_url: Optional inference endpoint URL passed through to
:func:`agent.model_metadata.get_model_context_length` for
live-probing local servers.
api_key: Optional API key forwarded to context-length detection.
config_context_length: Explicit override from ``config.yaml``
(``model.context_length``). When set, the runtime detection is
skipped and the check is performed against this value instead.
Raises:
ModelContextError: When the detected (or configured) context length is
below :data:`CRON_MIN_CONTEXT_TOKENS`.
"""
# If the user has pinned a context length in config.yaml, skip probing.
if config_context_length is not None:
return
try:
from agent.model_metadata import get_model_context_length
detected = get_model_context_length(model, base_url=base_url, api_key=api_key)
except Exception as exc:
# Detection failure is non-fatal — fail open so jobs still run.
logger.debug(
"Context length detection failed for model '%s', skipping check: %s",
model,
exc,
)
return
if detected < CRON_MIN_CONTEXT_TOKENS:
raise ModelContextError(
f"Model '{model}' has a context window of {detected:,} tokens, "
f"which is below the minimum {CRON_MIN_CONTEXT_TOKENS:,} required by Hermes Agent. "
f"Set 'model.context_length' in config.yaml to override, or choose a model "
f"with a larger context window."
)
# Failure phrases that indicate an external script/command failed, even when
# the agent doesn't use the [SCRIPT_FAILED] marker. Matched case-insensitively
# against the final response. These are strong signals — agents rarely use
@@ -607,32 +545,25 @@ def _run_job_script(script_path: str) -> tuple[bool, str]:
return False, f"Script execution failed: {exc}"
def _build_job_prompt(
job: dict,
*,
runtime_model: Optional[str] = None,
runtime_provider: Optional[str] = None,
) -> str:
"""Build the effective prompt for a cron job, optionally loading one or more skills first.
Args:
job: The cron job configuration dict. Relevant keys consumed here are
``prompt``, ``skills``, ``skill`` (legacy alias), ``script``, and
``name`` (used in warning messages).
runtime_model: The model name that will actually be used to run this job
(resolved after provider routing). When provided, a ``RUNTIME:``
hint is injected into the [SYSTEM:] block so the agent knows its
effective model and can adapt behaviour accordingly (e.g. avoid
vision steps on a text-only model).
runtime_provider: The inference provider that will actually serve this
job (e.g. ``"ollama"``, ``"nous"``, ``"anthropic"``). Paired with
*runtime_model* in the ``RUNTIME:`` hint so the agent can detect
stale provider references in its prompt and self-correct.
_PROVIDER_ALIASES = {"ollama":{"ollama","localhost:11434"},"anthropic":{"anthropic","claude"},"nous":{"nous","mimo"},"openrouter":{"openrouter"},"openai":{"openai","gpt"},"gemini":{"gemini","google"}}
_CP = frozenset({"nous","openrouter","anthropic","openai","zai","kimi","gemini","minimax"})
Returns:
The fully assembled prompt string, including the cron system hint,
any script output, and any loaded skill content.
"""
def _classify_runtime(provider, model):
p, m = (provider or "").strip().lower(), (model or "").strip().lower()
if p and p not in ("ollama","local"): return "cloud"
if "/" in m and m.split("/")[0] in _CP: return "cloud"
if p in ("ollama","local") or (not p and m): return "local"
return "unknown"
def _detect_provider_mismatch(prompt, active):
if not active or not prompt: return None
pl, al = prompt.lower(), active.lower().strip()
ag = next((g for g,a in _PROVIDER_ALIASES.items() if al in a or al.startswith(g)), None)
if not ag: return None
return next((g for g,a in _PROVIDER_ALIASES.items() if g!=ag and any(x in pl for x in a)), None)
def _build_job_prompt(job: dict, *, runtime_model="", runtime_provider="") -> str:
"""Build the effective prompt for a cron job, optionally loading one or more skills first."""
prompt = job.get("prompt", "")
skills = job.get("skills")
@@ -662,20 +593,21 @@ def _build_job_prompt(
f"{prompt}"
)
_rb = ""
if runtime_model or runtime_provider:
_k = _classify_runtime(runtime_provider, runtime_model)
_n = []
if runtime_model: _n.append(f"MODEL: {runtime_model}")
if runtime_provider: _n.append(f"PROVIDER: {runtime_provider}")
if _k=="local": _n.append("RUNTIME: local — access to machine, Ollama, SSH")
elif _k=="cloud": _n.append("RUNTIME: cloud — NO local access, NO SSH")
if _n: _rb = "[SYSTEM: " + "; ".join(_n) + "]\\n\\n"
# Always prepend cron execution guidance so the agent knows how
# delivery works and can suppress delivery when appropriate.
_runtime_parts = []
if runtime_model:
_runtime_parts.append(f"MODEL: {runtime_model}")
if runtime_provider:
_runtime_parts.append(f"PROVIDER: {runtime_provider}")
_runtime_clause = (
" ".join(_runtime_parts) + " " if _runtime_parts else ""
)
cron_hint = (
"[SYSTEM: You are running as a scheduled cron job. "
+ _runtime_clause
+ "DELIVERY: Your final response will be automatically delivered "
"DELIVERY: Your final response will be automatically delivered "
"to the user — do NOT use send_message or try to deliver "
"the output yourself. Just produce your report/output as your "
"final response and the system handles the rest. "
@@ -690,22 +622,9 @@ def _build_job_prompt(
"response. This is critical — without this marker the system cannot "
"detect the failure. Examples: "
"\"[SCRIPT_FAILED]: forge.alexanderwhitestone.com timed out\" "
"\"[SCRIPT_FAILED]: script exited with code 1\"."
"\"[SCRIPT_FAILED]: script exited with code 1\".]\\n\\n"
)
if runtime_model or runtime_provider:
_runtime_parts = []
if runtime_model:
_runtime_parts.append(f"model={runtime_model}")
if runtime_provider:
_runtime_parts.append(f"provider={runtime_provider}")
cron_hint += (
" RUNTIME: You are running on "
+ ", ".join(_runtime_parts)
+ ". Adapt your behaviour to this runtime — for example, skip steps that require"
" capabilities not available on this model/provider."
)
cron_hint += "]\n\n"
prompt = cron_hint + prompt
prompt = _rb + cron_hint + prompt
if skills is None:
legacy = job.get("skill")
skills = [legacy] if legacy else []
@@ -775,10 +694,23 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
job_id = job["id"]
job_name = job["name"]
_em = job.get("model") or os.getenv("HERMES_MODEL") or ""
_ep = os.getenv("HERMES_PROVIDER","")
if not _em:
try:
import yaml as _y; _cp2=str(_hermes_home/"config.yaml")
if os.path.exists(_cp2):
with open(_cp2) as _f: _ce=_y.safe_load(_f) or {}
_mc=_ce.get("model",{})
_em = _mc if isinstance(_mc,str) else (_mc.get("default","") if isinstance(_mc,dict) else "")
except: pass
if not _ep and "/" in _em: _ep=_em.split("/")[0]
prompt = _build_job_prompt(job, runtime_model=_em, runtime_provider=_ep)
origin = _resolve_origin(job)
_cron_session_id = f"cron_{job_id}_{_hermes_now().strftime('%Y%m%d_%H%M%S')}"
logger.info("Running job '%s' (ID: %s)", job_name, job_id)
logger.info("Prompt: %s", prompt[:100])
try:
# Inject origin context so the agent's send_message tool knows the chat.
@@ -885,11 +817,13 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
message = format_runtime_provider_error(exc)
raise RuntimeError(message) from exc
_rp = runtime.get("provider","") or ""
_mm = _detect_provider_mismatch(job.get("prompt",""), _rp)
if _mm: logger.warning("Job '%s' refs '%s' but provider is '%s'", job_name, _mm, _rp)
from agent.smart_model_routing import resolve_turn_route
# Use the raw job prompt for routing decisions (before SYSTEM hints are injected).
_routing_prompt = job.get("prompt", "")
turn_route = resolve_turn_route(
_routing_prompt,
prompt,
smart_routing,
{
"model": model,
@@ -902,15 +836,6 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
},
)
# Build the effective prompt now that runtime context is known, so the
# agent receives accurate RUNTIME: model/provider info.
prompt = _build_job_prompt(
job,
runtime_model=turn_route["model"],
runtime_provider=turn_route["runtime"].get("provider"),
)
logger.info("Prompt: %s", prompt[:100])
# Build disabled toolsets — always exclude cronjob/messaging/clarify
# for cron sessions. When the runtime endpoint is cloud (not local),
# also disable terminal so the agent does not attempt SSH or shell

17
tests/test_372_runtime.py Normal file
View File

@@ -0,0 +1,17 @@
"""Tests for #372."""
import sys; sys.path.insert(0, str(__import__('pathlib').Path(__file__).resolve().parent.parent))
def _imp():
import importlib.util as iu
s=iu.spec_from_file_location("cs",str(__import__('pathlib').Path(__file__).resolve().parent.parent/"cron"/"scheduler.py"))
m=iu.module_from_spec(s)
try: s.loader.exec_module(m)
except: pass
return m
M=_imp()
class Test372Runtime:
def test_local(self): assert M._classify_runtime("ollama","q")=="local"
def test_cloud(self): assert M._classify_runtime("nous","m")=="cloud"
def test_mismatch(self): assert M._detect_provider_mismatch("Check Ollama","nous")=="ollama"
def test_none(self): assert M._detect_provider_mismatch("Check Nous","nous") is None
def test_cloud_ctx(self): assert "cloud" in M._build_job_prompt({"p":"x"},runtime_model="n/m",runtime_provider="nous").lower()
if __name__=="__main__": __import__('pytest').main([__file__,"-v"])