Compare commits

...

1 Commits

Author SHA1 Message Date
Hermes Agent
6497e9e7fb feat(skill_manage): add validate action for skill health checks
Some checks failed
Docker Build and Publish / build-and-push (pull_request) Has been skipped
Contributor Attribution Check / check-attribution (pull_request) Failing after 32s
Supply Chain Audit / Scan PR for supply chain risks (pull_request) Successful in 32s
Tests / e2e (pull_request) Successful in 1m18s
Tests / test (pull_request) Failing after 19m55s
Adds a 'validate' action to skill_manage that checks:
- SKILL.md existence and structure (frontmatter, body)
- File size limits before reading (streaming-safe, #623)
- All supporting files under references/, templates/, scripts/, assets/
- Error messages include file path, specific error, and suggested fix (#624)
- Actionable feedback with examples and remediation steps (#626)

Closes #623, #624, #626
2026-04-14 18:11:53 -04:00

View File

@@ -18,6 +18,7 @@ Actions:
delete -- Remove a user skill entirely
write_file -- Add/overwrite a supporting file (reference, template, script, asset)
remove_file-- Remove a supporting file from a user skill
validate -- Validate a skill: check structure, sizes, and all supporting files
Directory layout for user skills:
~/.hermes/skills/
@@ -174,6 +175,162 @@ def _validate_frontmatter(content: str) -> Optional[str]:
return None
def _validate_skill(name: str) -> Dict[str, Any]:
"""
Validate an existing skill: check SKILL.md structure, file sizes,
and all supporting files. Returns detailed results with specific
file paths, error descriptions, and suggested fixes.
Addresses:
- #623: Checks file size before reading; sets reasonable limits
- #624: Error messages include file path, specific error, and suggested fix
- #626: Validation feedback includes actionable suggestions
"""
existing = _find_skill(name)
if not existing:
return {"success": False, "error": f"Skill '{name}' not found. Use skills_list() to see available skills."}
skill_dir = existing["path"]
skill_md = skill_dir / "SKILL.md"
errors = []
warnings = []
# --- Check SKILL.md existence ---
if not skill_md.exists():
errors.append({
"file": "SKILL.md",
"error": "SKILL.md is missing from the skill directory.",
"suggestion": f"Create SKILL.md with: skill_manage(action='create', name='{name}', content='---\\nname: {name}\\ndescription: ...\\n---\\nYour instructions here.')",
})
# Can't continue validation without SKILL.md
return {
"success": False,
"skill": name,
"path": str(skill_dir),
"errors": errors,
"warnings": warnings,
}
# --- Size check BEFORE reading (issue #623) ---
skill_md_bytes = skill_md.stat().st_size
if skill_md_bytes > MAX_SKILL_CONTENT_CHARS:
errors.append({
"file": "SKILL.md",
"error": (
f"SKILL.md is {skill_md_bytes:,} bytes "
f"(limit: {MAX_SKILL_CONTENT_CHARS:,} chars). "
f"Reading was skipped to avoid memory issues."
),
"suggestion": (
"Split large content into supporting files under references/ or templates/. "
"Keep SKILL.md focused on the core instructions."
),
})
# Can't validate frontmatter if file is too large
else:
# --- Validate SKILL.md content ---
try:
content = skill_md.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError) as e:
errors.append({
"file": "SKILL.md",
"error": f"Cannot read SKILL.md: {e}",
"suggestion": "Check file encoding (should be UTF-8) and file permissions.",
})
content = None
if content is not None:
# Frontmatter validation
fm_error = _validate_frontmatter(content)
if fm_error:
errors.append({
"file": "SKILL.md",
"error": fm_error,
"suggestion": "Ensure SKILL.md starts with '---', has valid YAML frontmatter with 'name' and 'description' fields, and closes with '---'.",
})
# Content size warning (80% threshold)
size_pct = len(content) / MAX_SKILL_CONTENT_CHARS * 100
if size_pct > 80:
warnings.append({
"file": "SKILL.md",
"message": (
f"SKILL.md is at {size_pct:.0f}% of size limit "
f"({len(content):,} / {MAX_SKILL_CONTENT_CHARS:,} chars)."
),
"suggestion": "Consider extracting reference material to supporting files.",
})
# --- Validate supporting files ---
for subdir in ALLOWED_SUBDIRS:
subdir_path = skill_dir / subdir
if not subdir_path.exists():
continue
for f in subdir_path.rglob("*"):
if not f.is_file():
continue
rel_path = str(f.relative_to(skill_dir))
file_bytes = f.stat().st_size
# Size check before reading (issue #623)
if file_bytes > MAX_SKILL_FILE_BYTES:
errors.append({
"file": rel_path,
"error": (
f"File is {file_bytes:,} bytes "
f"(limit: {MAX_SKILL_FILE_BYTES:,} bytes / 1 MiB)."
),
"suggestion": "Split into smaller files or compress the content.",
})
continue
# Check for common issues in markdown files
if f.suffix in (".md", ".txt"):
try:
f_content = f.read_text(encoding="utf-8")
if not f_content.strip():
warnings.append({
"file": rel_path,
"message": "File is empty.",
"suggestion": "Add content or remove the file with skill_manage(action='remove_file').",
})
except (OSError, UnicodeDecodeError) as e:
errors.append({
"file": rel_path,
"error": f"Cannot read file: {e}",
"suggestion": "Check file encoding (should be UTF-8) and permissions.",
})
# Size warning for large files (80% threshold)
size_pct = file_bytes / MAX_SKILL_FILE_BYTES * 100
if size_pct > 80:
warnings.append({
"file": rel_path,
"message": f"File is at {size_pct:.0f}% of size limit ({file_bytes:,} / {MAX_SKILL_FILE_BYTES:,} bytes).",
"suggestion": "Consider splitting or compressing.",
})
valid = len(errors) == 0
result = {
"success": True,
"valid": valid,
"skill": name,
"path": str(skill_dir),
"errors": errors,
"warnings": warnings,
"summary": (
f"Skill '{name}' is {'valid' if valid else 'INVALID'}"
f"{len(errors)} error(s), {len(warnings)} warning(s)."
),
}
if valid:
result["hint"] = (
"Tip: Run validate periodically after edits to catch issues early. "
"Use skill_manage(action='edit') or skill_manage(action='patch') to fix problems."
)
return result
def _validate_content_size(content: str, label: str = "SKILL.md") -> Optional[str]:
"""Check that content doesn't exceed the character limit for agent writes.
@@ -633,8 +790,11 @@ def skill_manage(
return tool_error("file_path is required for 'remove_file'.", success=False)
result = _remove_file(name, file_path)
elif action == "validate":
result = _validate_skill(name)
else:
result = {"success": False, "error": f"Unknown action '{action}'. Use: create, edit, patch, delete, write_file, remove_file"}
result = {"success": False, "error": f"Unknown action '{action}'. Use: create, edit, patch, delete, write_file, remove_file, validate"}
if result.get("success"):
try:
@@ -659,7 +819,7 @@ SKILL_MANAGE_SCHEMA = {
"Actions: create (full SKILL.md + optional category), "
"patch (old_string/new_string — preferred for fixes), "
"edit (full SKILL.md rewrite — major overhauls only), "
"delete, write_file, remove_file.\n\n"
"delete, write_file, remove_file, validate (check skill health).\n\n"
"Create when: complex task succeeded (5+ calls), errors overcome, "
"user-corrected approach worked, non-trivial workflow discovered, "
"or user asks you to remember a procedure.\n"
@@ -676,7 +836,7 @@ SKILL_MANAGE_SCHEMA = {
"properties": {
"action": {
"type": "string",
"enum": ["create", "patch", "edit", "delete", "write_file", "remove_file"],
"enum": ["create", "patch", "edit", "delete", "write_file", "remove_file", "validate"],
"description": "The action to perform."
},
"name": {