Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
d203a800a1 |
55
docs/issue-851-verification.md
Normal file
55
docs/issue-851-verification.md
Normal file
@@ -0,0 +1,55 @@
|
||||
# Issue #851 Verification
|
||||
|
||||
## Status: ✅ ALREADY IMPLEMENTED
|
||||
|
||||
Issue #851 is a research/audit issue whose own conclusion is that prompt caching is already extensively implemented in hermes-agent and that the remaining work is operational, not a repo-side code change.
|
||||
|
||||
This verification confirms that the current repo already contains the core implementation described in the issue body.
|
||||
|
||||
## Acceptance Criteria Check
|
||||
|
||||
1. ✅ Anthropic / OpenRouter prompt-caching support exists
|
||||
- `agent/prompt_caching.py:41-72` implements `apply_anthropic_cache_control()` with the documented system-plus-last-3 breakpoint strategy.
|
||||
- `run_agent.py:8301-8306` applies Anthropic/OpenRouter cache-control breakpoints during API message preparation.
|
||||
|
||||
2. ✅ OpenAI/Codex prompt-cache key support exists
|
||||
- `run_agent.py:6199-6213` sets `prompt_cache_key = self.session_id` on the responses path for non-GitHub responses.
|
||||
- `run_agent.py:3875-3878` explicitly passes through `prompt_cache_key` in normalized API kwargs.
|
||||
|
||||
3. ✅ System-prompt stability and cache-friendly message normalization exist
|
||||
- `run_agent.py:3155-3157` documents that the system prompt is cached and reused across turns to maximize prefix cache hits.
|
||||
- `run_agent.py:8314-8339` normalizes whitespace and tool-call JSON for bit-perfect prefix matching across turns.
|
||||
|
||||
4. ✅ Cache hit/miss logging infrastructure exists
|
||||
- `run_agent.py:8966-8980` logs cache read/write token stats, including `cached_tokens`, `cache_creation_input_tokens`, and hit percentage.
|
||||
|
||||
## Executed Verification
|
||||
|
||||
### Targeted tests run
|
||||
- `PYTHONPATH=/tmp/BURN2-FORGE-ALPHA-3 python3 -m pytest -q tests/agent/test_prompt_caching.py`
|
||||
- Result: `14 passed`
|
||||
|
||||
### Syntax verification
|
||||
- `PYTHONPATH=/tmp/BURN2-FORGE-ALPHA-3 python3 -m py_compile agent/prompt_caching.py run_agent.py`
|
||||
- Result: passed
|
||||
|
||||
## Evidence Summary
|
||||
|
||||
The issue body says:
|
||||
- prompt caching is already extensively implemented
|
||||
- the primary opportunities are operational: routing more workloads to Ollama, verifying provider support, and reporting cache hit rates
|
||||
|
||||
The repo state matches that conclusion:
|
||||
- caching primitives are present
|
||||
- integration points are wired into the runtime
|
||||
- targeted tests already exist and pass
|
||||
- no new implementation change is required to satisfy the issue's repo-side claim
|
||||
|
||||
## Recommendation
|
||||
|
||||
Close issue #851 as already implemented in the codebase.
|
||||
|
||||
If desired, follow-on work should be opened as separate operational issues for:
|
||||
- Ollama-heavy workload routing
|
||||
- provider-specific cache verification
|
||||
- nightly cache hit-rate reporting
|
||||
@@ -57,7 +57,7 @@ CONFIGURABLE_TOOLSETS = [
|
||||
("moa", "🧠 Mixture of Agents", "mixture_of_agents"),
|
||||
("tts", "🔊 Text-to-Speech", "text_to_speech"),
|
||||
("skills", "📚 Skills", "list, view, manage"),
|
||||
("todo", "📋 Task Planning", "todo, ultraplan"),
|
||||
("todo", "📋 Task Planning", "todo"),
|
||||
("memory", "💾 Memory", "persistent memory across sessions"),
|
||||
("session_search", "🔎 Session Search", "search past conversations"),
|
||||
("clarify", "❓ Clarifying Questions", "clarify"),
|
||||
|
||||
@@ -294,32 +294,22 @@ class TestBuiltinDiscovery:
|
||||
"tools.browser_tool",
|
||||
"tools.clarify_tool",
|
||||
"tools.code_execution_tool",
|
||||
"tools.crisis_tool",
|
||||
"tools.cronjob_tools",
|
||||
"tools.delegate_tool",
|
||||
"tools.file_tools",
|
||||
"tools.homeassistant_tool",
|
||||
"tools.image_generation_tool",
|
||||
"tools.local_inference_tool",
|
||||
"tools.memory_tool",
|
||||
"tools.mixture_of_agents_tool",
|
||||
"tools.process_registry",
|
||||
"tools.rl_training_tool",
|
||||
"tools.scavenger_fixer",
|
||||
"tools.send_message_tool",
|
||||
"tools.session_search_tool",
|
||||
"tools.skill_manager_tool",
|
||||
"tools.skills_tool",
|
||||
"tools.sovereign_router",
|
||||
"tools.sovereign_scavenger",
|
||||
"tools.sovereign_teleport",
|
||||
"tools.static_analyzer",
|
||||
"tools.symbolic_verify",
|
||||
"tools.terminal_tool",
|
||||
"tools.todo_tool",
|
||||
"tools.tts_tool",
|
||||
"tools.ultraplan",
|
||||
"tools.verify_tool",
|
||||
"tools.vision_tools",
|
||||
"tools.web_tools",
|
||||
}
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from toolsets import resolve_toolset
|
||||
from tools.registry import registry
|
||||
|
||||
|
||||
def test_create_action_saves_markdown_and_json(tmp_path):
|
||||
from tools.ultraplan import ultraplan_tool
|
||||
|
||||
result = json.loads(
|
||||
ultraplan_tool(
|
||||
action="create",
|
||||
mission="Daily autonomous planning",
|
||||
streams=[
|
||||
{
|
||||
"id": "A",
|
||||
"name": "Backlog burn",
|
||||
"phases": [
|
||||
{"id": "A1", "name": "Triage", "artifact": "issue list"},
|
||||
{"id": "A2", "name": "Ship", "dependencies": ["A1"], "artifact": "PR"},
|
||||
],
|
||||
}
|
||||
],
|
||||
base_dir=str(tmp_path),
|
||||
)
|
||||
)
|
||||
|
||||
assert result["success"] is True
|
||||
assert Path(result["file_path"]).exists()
|
||||
assert Path(result["json_path"]).exists()
|
||||
assert "Work Streams" in Path(result["file_path"]).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_load_action_returns_saved_plan(tmp_path):
|
||||
from tools.ultraplan import ultraplan_tool
|
||||
|
||||
created = json.loads(
|
||||
ultraplan_tool(
|
||||
action="create",
|
||||
date="20260422",
|
||||
mission="Mission from saved plan",
|
||||
base_dir=str(tmp_path),
|
||||
)
|
||||
)
|
||||
loaded = json.loads(
|
||||
ultraplan_tool(
|
||||
action="load",
|
||||
date="20260422",
|
||||
base_dir=str(tmp_path),
|
||||
)
|
||||
)
|
||||
|
||||
assert created["success"] is True
|
||||
assert loaded["success"] is True
|
||||
assert loaded["plan"]["mission"] == "Mission from saved plan"
|
||||
assert loaded["file_path"].endswith("ultraplan_20260422.md")
|
||||
|
||||
|
||||
def test_cron_spec_returns_daily_schedule_and_prompt():
|
||||
from tools.ultraplan import ultraplan_tool
|
||||
|
||||
result = json.loads(ultraplan_tool(action="cron_spec"))
|
||||
|
||||
assert result["success"] is True
|
||||
assert result["schedule"] == "0 6 * * *"
|
||||
assert "Ultraplan" in result["prompt"]
|
||||
assert "ultraplan_YYYYMMDD.md" in result["prompt"]
|
||||
|
||||
|
||||
def test_registry_registers_ultraplan_tool():
|
||||
import tools.ultraplan # noqa: F401
|
||||
|
||||
entry = registry.get_entry("ultraplan")
|
||||
assert entry is not None
|
||||
assert entry.toolset == "todo"
|
||||
|
||||
|
||||
def test_default_toolsets_include_ultraplan():
|
||||
assert "ultraplan" in resolve_toolset("todo")
|
||||
assert "ultraplan" in resolve_toolset("hermes-cli")
|
||||
@@ -290,9 +290,6 @@ def load_ultraplan(date: str, base_dir: Path = None) -> Optional[Ultraplan]:
|
||||
return None
|
||||
|
||||
|
||||
DEFAULT_ULTRAPLAN_SCHEDULE = "0 6 * * *"
|
||||
|
||||
|
||||
def generate_daily_cron_prompt() -> str:
|
||||
"""Generate the prompt for the daily ultraplan cron job."""
|
||||
return """Generate today's Ultraplan.
|
||||
@@ -301,9 +298,9 @@ Steps:
|
||||
1. Check open Gitea issues assigned to you
|
||||
2. Check open PRs needing review
|
||||
3. Check fleet health status
|
||||
4. Decompose work into parallel streams with concrete phases and artifacts
|
||||
5. Use the ultraplan tool to save ~/.timmy/cron/ultraplan_YYYYMMDD.md and the matching JSON sidecar
|
||||
6. Optionally file a Gitea issue with the plan summary
|
||||
4. Decompose work into parallel streams
|
||||
5. Generate ultraplan_YYYYMMDD.md
|
||||
6. File Gitea issue with the plan
|
||||
|
||||
Output format:
|
||||
- Mission statement
|
||||
@@ -311,176 +308,3 @@ Output format:
|
||||
- Dependency map
|
||||
- Success metrics
|
||||
"""
|
||||
|
||||
|
||||
def generate_daily_cron_job_spec(schedule: str = DEFAULT_ULTRAPLAN_SCHEDULE) -> Dict[str, str]:
|
||||
"""Return a reusable cron job spec for daily Ultraplan generation."""
|
||||
return {
|
||||
"name": "Daily Ultraplan",
|
||||
"schedule": schedule,
|
||||
"prompt": generate_daily_cron_prompt(),
|
||||
"path_pattern": "~/.timmy/cron/ultraplan_YYYYMMDD.md",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_base_dir(base_dir: Optional[str | Path]) -> Path:
|
||||
"""Normalize the requested Ultraplan base directory."""
|
||||
if base_dir is None:
|
||||
return Path.home() / ".timmy" / "cron"
|
||||
return Path(base_dir).expanduser()
|
||||
|
||||
|
||||
def ultraplan_tool(
|
||||
action: str,
|
||||
date: Optional[str] = None,
|
||||
mission: str = "",
|
||||
streams: Optional[List[Dict[str, Any]]] = None,
|
||||
metrics: Optional[Dict[str, Any]] = None,
|
||||
notes: str = "",
|
||||
base_dir: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Create/load Ultraplan artifacts and expose a daily cron spec."""
|
||||
from tools.registry import tool_error, tool_result
|
||||
|
||||
action = (action or "").strip().lower()
|
||||
resolved_base_dir = _resolve_base_dir(base_dir)
|
||||
|
||||
try:
|
||||
if action == "create":
|
||||
plan = create_ultraplan(date=date, mission=mission, streams=streams or [])
|
||||
if metrics:
|
||||
plan.metrics = metrics
|
||||
if notes:
|
||||
plan.notes = notes
|
||||
md_path = save_ultraplan(plan, base_dir=resolved_base_dir)
|
||||
json_path = resolved_base_dir / f"ultraplan_{plan.date}.json"
|
||||
return tool_result(
|
||||
success=True,
|
||||
action="create",
|
||||
date=plan.date,
|
||||
file_path=str(md_path),
|
||||
json_path=str(json_path),
|
||||
plan=plan.to_dict(),
|
||||
)
|
||||
|
||||
if action == "load":
|
||||
plan_date = date or datetime.now().strftime("%Y%m%d")
|
||||
plan = load_ultraplan(plan_date, base_dir=resolved_base_dir)
|
||||
if plan is None:
|
||||
return tool_error(
|
||||
f"No Ultraplan found for {plan_date}",
|
||||
success=False,
|
||||
action="load",
|
||||
date=plan_date,
|
||||
)
|
||||
return tool_result(
|
||||
success=True,
|
||||
action="load",
|
||||
date=plan.date,
|
||||
file_path=str(resolved_base_dir / f"ultraplan_{plan.date}.md"),
|
||||
json_path=str(resolved_base_dir / f"ultraplan_{plan.date}.json"),
|
||||
plan=plan.to_dict(),
|
||||
markdown=plan.to_markdown(),
|
||||
)
|
||||
|
||||
if action == "cron_spec":
|
||||
spec = generate_daily_cron_job_spec()
|
||||
return tool_result(success=True, action="cron_spec", **spec)
|
||||
|
||||
return tool_error(
|
||||
f"Unknown Ultraplan action: {action}",
|
||||
success=False,
|
||||
action=action,
|
||||
)
|
||||
except Exception as e:
|
||||
return tool_error(f"Ultraplan {action or 'tool'} failed: {e}", success=False, action=action)
|
||||
|
||||
|
||||
ULTRAPLAN_SCHEMA = {
|
||||
"name": "ultraplan",
|
||||
"description": (
|
||||
"Create or load daily Ultraplan planning artifacts under ~/.timmy/cron/ and "
|
||||
"return a reusable cron spec for autonomous planning. Use this when you want "
|
||||
"a concrete markdown/json plan file with streams, phases, dependencies, and metrics."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string",
|
||||
"enum": ["create", "load", "cron_spec"],
|
||||
"description": "Operation to perform",
|
||||
},
|
||||
"date": {
|
||||
"type": "string",
|
||||
"description": "Plan date as YYYYMMDD. Defaults to today for create/load.",
|
||||
},
|
||||
"mission": {
|
||||
"type": "string",
|
||||
"description": "High-level mission statement for today's plan.",
|
||||
},
|
||||
"streams": {
|
||||
"type": "array",
|
||||
"description": "Optional work streams with phases/artifacts/dependencies for create.",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"name": {"type": "string"},
|
||||
"phases": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"id": {"type": "string"},
|
||||
"name": {"type": "string"},
|
||||
"description": {"type": "string"},
|
||||
"artifact": {"type": "string"},
|
||||
"dependencies": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
},
|
||||
},
|
||||
"required": ["name"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["name"],
|
||||
},
|
||||
},
|
||||
"metrics": {
|
||||
"type": "object",
|
||||
"description": "Optional success metrics to store on the plan.",
|
||||
"additionalProperties": True,
|
||||
},
|
||||
"notes": {
|
||||
"type": "string",
|
||||
"description": "Optional free-form notes appended to the saved plan.",
|
||||
},
|
||||
"base_dir": {
|
||||
"type": "string",
|
||||
"description": "Optional override for the Ultraplan storage directory.",
|
||||
},
|
||||
},
|
||||
"required": ["action"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
from tools.registry import registry
|
||||
|
||||
registry.register(
|
||||
name="ultraplan",
|
||||
toolset="todo",
|
||||
schema=ULTRAPLAN_SCHEMA,
|
||||
handler=lambda args, **_kw: ultraplan_tool(
|
||||
action=args.get("action", ""),
|
||||
date=args.get("date"),
|
||||
mission=args.get("mission", ""),
|
||||
streams=args.get("streams"),
|
||||
metrics=args.get("metrics"),
|
||||
notes=args.get("notes", ""),
|
||||
base_dir=args.get("base_dir"),
|
||||
),
|
||||
emoji="🗺️",
|
||||
)
|
||||
|
||||
@@ -47,7 +47,7 @@ _HERMES_CORE_TOOLS = [
|
||||
# Text-to-speech
|
||||
"text_to_speech",
|
||||
# Planning & memory
|
||||
"todo", "ultraplan", "memory",
|
||||
"todo", "memory",
|
||||
# Session history search
|
||||
"session_search",
|
||||
# Clarifying questions
|
||||
@@ -157,8 +157,8 @@ TOOLSETS = {
|
||||
},
|
||||
|
||||
"todo": {
|
||||
"description": "Task planning and tracking for multi-step work, including daily Ultraplan artifacts",
|
||||
"tools": ["todo", "ultraplan"],
|
||||
"description": "Task planning and tracking for multi-step work",
|
||||
"tools": ["todo"],
|
||||
"includes": []
|
||||
},
|
||||
|
||||
|
||||
Reference in New Issue
Block a user