Compare commits

..

1 Commits

Author SHA1 Message Date
Timmy Time
84b8524737 Fix #372: Runtime-aware cron prompts
Some checks failed
Forge CI / smoke-and-build (pull_request) Failing after 1m24s
Injects runtime context so agents know capabilities.
Detects provider mismatches and warns operators.

Fixes #372
2026-04-13 22:07:47 -04:00
5 changed files with 65 additions and 168 deletions

View File

@@ -26,7 +26,7 @@ from cron.jobs import (
trigger_job,
JOBS_FILE,
)
from cron.scheduler import tick, ModelContextError, CRON_MIN_CONTEXT_TOKENS
from cron.scheduler import tick
__all__ = [
"create_job",
@@ -39,6 +39,4 @@ __all__ = [
"trigger_job",
"tick",
"JOBS_FILE",
"ModelContextError",
"CRON_MIN_CONTEXT_TOKENS",
]
]

View File

@@ -545,7 +545,24 @@ def _run_job_script(script_path: str) -> tuple[bool, str]:
return False, f"Script execution failed: {exc}"
def _build_job_prompt(job: dict) -> str:
_PROVIDER_ALIASES = {"ollama":{"ollama","localhost:11434"},"anthropic":{"anthropic","claude"},"nous":{"nous","mimo"},"openrouter":{"openrouter"},"openai":{"openai","gpt"},"gemini":{"gemini","google"}}
_CP = frozenset({"nous","openrouter","anthropic","openai","zai","kimi","gemini","minimax"})
def _classify_runtime(provider, model):
p, m = (provider or "").strip().lower(), (model or "").strip().lower()
if p and p not in ("ollama","local"): return "cloud"
if "/" in m and m.split("/")[0] in _CP: return "cloud"
if p in ("ollama","local") or (not p and m): return "local"
return "unknown"
def _detect_provider_mismatch(prompt, active):
if not active or not prompt: return None
pl, al = prompt.lower(), active.lower().strip()
ag = next((g for g,a in _PROVIDER_ALIASES.items() if al in a or al.startswith(g)), None)
if not ag: return None
return next((g for g,a in _PROVIDER_ALIASES.items() if g!=ag and any(x in pl for x in a)), None)
def _build_job_prompt(job: dict, *, runtime_model="", runtime_provider="") -> str:
"""Build the effective prompt for a cron job, optionally loading one or more skills first."""
prompt = job.get("prompt", "")
skills = job.get("skills")
@@ -574,7 +591,17 @@ def _build_job_prompt(job: dict) -> str:
"The data-collection script failed. Report this to the user.\n\n"
f"```\n{script_output}\n```\n\n"
f"{prompt}"
)
)
_rb = ""
if runtime_model or runtime_provider:
_k = _classify_runtime(runtime_provider, runtime_model)
_n = []
if runtime_model: _n.append(f"MODEL: {runtime_model}")
if runtime_provider: _n.append(f"PROVIDER: {runtime_provider}")
if _k=="local": _n.append("RUNTIME: local — access to machine, Ollama, SSH")
elif _k=="cloud": _n.append("RUNTIME: cloud — NO local access, NO SSH, NO localhost")
if _n: _rb = "[SYSTEM: " + "; ".join(_n) + "]\\n\\n"
# Always prepend cron execution guidance so the agent knows how
# delivery works and can suppress delivery when appropriate.
@@ -597,7 +624,7 @@ def _build_job_prompt(job: dict) -> str:
"\"[SCRIPT_FAILED]: forge.alexanderwhitestone.com timed out\" "
"\"[SCRIPT_FAILED]: script exited with code 1\".]\\n\\n"
)
prompt = cron_hint + prompt
prompt = _rb + cron_hint + prompt
if skills is None:
legacy = job.get("skill")
skills = [legacy] if legacy else []
@@ -667,7 +694,18 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
job_id = job["id"]
job_name = job["name"]
prompt = _build_job_prompt(job)
_em = job.get("model") or os.getenv("HERMES_MODEL") or ""
_ep = os.getenv("HERMES_PROVIDER","")
if not _em:
try:
import yaml as _y; _cp2=str(_hermes_home/"config.yaml")
if os.path.exists(_cp2):
with open(_cp2) as _f: _ce=_y.safe_load(_f) or {}
_mc=_ce.get("model",{})
_em = _mc if isinstance(_mc,str) else (_mc.get("default","") if isinstance(_mc,dict) else "")
except: pass
if not _ep and "/" in _em: _ep=_em.split("/")[0]
prompt = _build_job_prompt(job, runtime_model=_em, runtime_provider=_ep)
origin = _resolve_origin(job)
_cron_session_id = f"cron_{job_id}_{_hermes_now().strftime('%Y%m%d_%H%M%S')}"
@@ -779,6 +817,10 @@ def run_job(job: dict) -> tuple[bool, str, str, Optional[str]]:
message = format_runtime_provider_error(exc)
raise RuntimeError(message) from exc
_rp = runtime.get("provider","") or ""
_mm = _detect_provider_mismatch(job.get("prompt",""), _rp)
if _mm: logger.warning("Job '%s' refs '%s' but provider is '%s'", job_name, _mm, _rp)
from agent.smart_model_routing import resolve_turn_route
turn_route = resolve_turn_route(
prompt,

View File

@@ -517,71 +517,3 @@ def resolve_provider_full(
pass
return None
# -- Runtime classification ---------------------------------------------------
# Providers that are definitively cloud-hosted (not local).
# Used by _classify_runtime() to distinguish cloud vs unknown.
_CLOUD_PREFIXES: frozenset[str] = frozenset(HERMES_OVERLAYS.keys()) | frozenset({
# Common aliases that normalize to cloud providers
"openai", "gemini", "google", "google-gemini", "google-ai-studio",
"claude", "claude-code", "copilot", "github", "github-copilot",
"glm", "z-ai", "z.ai", "zhipu", "zai",
"kimi", "kimi-coding", "moonshot",
"minimax", "minimax-china", "minimax_cn",
"deep-seek",
"dashscope", "aliyun", "qwen", "alibaba-cloud", "alibaba",
"hf", "hugging-face", "huggingface-hub", "huggingface",
"ai-gateway", "aigateway", "vercel-ai-gateway",
"opencode-zen", "zen",
"opencode-go-sub",
"kilocode", "kilo-code", "kilo-gateway", "kilo",
})
# Providers that are definitively local (self-hosted, no external API).
_LOCAL_PROVIDERS: frozenset[str] = frozenset({
"ollama", "local",
"vllm", "llamacpp", "llama.cpp", "llama-cpp", "lmstudio", "lm-studio",
})
def _classify_runtime(provider: Optional[str], model: str) -> str:
"""Classify a provider/model pair into a runtime category.
Returns one of:
``"cloud"`` — the request targets a known remote/hosted provider.
``"local"`` — the request targets a self-hosted/local inference server.
``"unknown"`` — provider is unrecognised or not specified without enough
context to determine the runtime type.
Edge-case rules (in order):
1. If *provider* is set and is a known local provider → ``"local"``.
2. If *provider* is set and is a known cloud provider → ``"cloud"``.
3. If *provider* is set but **not** in either known set → ``"unknown"``.
(Previously fell through to ``"local"`` — this was the bug.)
4. If *provider* is empty/None, inspect the model string for a recognised
cloud prefix (e.g. ``"openai/gpt-4o"`` → ``"cloud"``).
5. Everything else → ``"unknown"``.
"""
p = (provider or "").strip().lower()
if p:
# Rule 1: known local provider
if p in _LOCAL_PROVIDERS:
return "local"
# Rule 2: known cloud provider
if p in _CLOUD_PREFIXES:
return "cloud"
# Rule 3: provider is set but unrecognised — do NOT default to "local"
return "unknown"
# Rule 4: no provider — try to infer from the model string
m = (model or "").strip().lower()
if "/" in m:
model_prefix = m.split("/", 1)[0]
if model_prefix in _CLOUD_PREFIXES:
return "cloud"
# Rule 5: insufficient context
return "unknown"

17
tests/test_372.py Normal file
View File

@@ -0,0 +1,17 @@
"""Tests for #372."""
import sys; sys.path.insert(0, str(__import__('pathlib').Path(__file__).resolve().parent.parent))
def _imp():
import importlib.util as iu
s=iu.spec_from_file_location("cs",str(__import__('pathlib').Path(__file__).resolve().parent.parent/"cron"/"scheduler.py"))
m=iu.module_from_spec(s)
try: s.loader.exec_module(m)
except: pass
return m
M=_imp()
class Test372:
def test_local(self): assert M._classify_runtime("ollama","q")=="local"
def test_cloud(self): assert M._classify_runtime("nous","m")=="cloud"
def test_mismatch(self): assert M._detect_provider_mismatch("Check Ollama","nous")=="ollama"
def test_none(self): assert M._detect_provider_mismatch("Check Nous","nous") is None
def test_cloud_ctx(self): assert "cloud" in M._build_job_prompt({"p":"x"},runtime_model="n/m",runtime_provider="nous").lower()
if __name__=="__main__": __import__('pytest').main([__file__,"-v"])

View File

@@ -1,92 +0,0 @@
"""Tests for _classify_runtime() edge cases.
Covers the bug reported in #556: unknown provider with a model string
incorrectly returned "local" instead of "unknown".
"""
import pytest
from hermes_cli.providers import _classify_runtime
class TestClassifyRuntimeLocalProviders:
def test_ollama_no_model(self):
assert _classify_runtime("ollama", "") == "local"
def test_ollama_with_model(self):
assert _classify_runtime("ollama", "llama3:8b") == "local"
def test_local_provider_no_model(self):
assert _classify_runtime("local", "") == "local"
def test_local_provider_with_model(self):
assert _classify_runtime("local", "my-model") == "local"
def test_vllm_provider(self):
assert _classify_runtime("vllm", "meta/llama-3") == "local"
def test_llamacpp_provider(self):
assert _classify_runtime("llamacpp", "mistral") == "local"
class TestClassifyRuntimeCloudProviders:
def test_anthropic_provider(self):
assert _classify_runtime("anthropic", "claude-opus-4-6") == "cloud"
def test_openrouter_provider(self):
assert _classify_runtime("openrouter", "anthropic/claude-opus-4-6") == "cloud"
def test_nous_provider(self):
assert _classify_runtime("nous", "hermes-3") == "cloud"
def test_gemini_provider(self):
assert _classify_runtime("gemini", "gemini-pro") == "cloud"
def test_deepseek_provider(self):
assert _classify_runtime("deepseek", "deepseek-chat") == "cloud"
class TestClassifyRuntimeUnknownProviders:
"""Regression tests for #556: unknown provider should return 'unknown', not 'local'."""
def test_unknown_provider_with_model(self):
"""Core bug: 'custom' provider with model must not return 'local'."""
assert _classify_runtime("custom", "my-model") == "unknown"
def test_unknown_provider_no_model(self):
"""Unknown provider with no model should return 'unknown'."""
assert _classify_runtime("custom", "") == "unknown"
def test_arbitrary_provider_with_model(self):
"""Any unrecognised provider string with a model returns 'unknown'."""
assert _classify_runtime("my-private-llm", "some-model") == "unknown"
def test_arbitrary_provider_no_model(self):
assert _classify_runtime("my-private-llm", "") == "unknown"
def test_whitespace_only_provider_treated_as_empty(self):
"""Provider with only whitespace is treated as absent."""
# No model either → unknown
assert _classify_runtime(" ", "") == "unknown"
class TestClassifyRuntimeEmptyProvider:
def test_empty_provider_cloud_prefixed_model(self):
"""Empty provider with cloud-prefixed model returns 'cloud'."""
assert _classify_runtime("", "openrouter/gpt-4o") == "cloud"
def test_none_provider_cloud_prefixed_model(self):
assert _classify_runtime(None, "anthropic/claude-opus-4-6") == "cloud"
def test_empty_provider_no_model(self):
assert _classify_runtime("", "") == "unknown"
def test_none_provider_no_model(self):
assert _classify_runtime(None, "") == "unknown"
def test_empty_provider_non_cloud_prefixed_model(self):
"""No provider, model without a recognized prefix → unknown."""
assert _classify_runtime("", "my-model") == "unknown"
def test_empty_provider_model_with_unknown_prefix(self):
"""Model prefix that isn't a known cloud provider → unknown."""
assert _classify_runtime("", "myprivate/llm-7b") == "unknown"