5148682b43
Skills with scripts/, templates/, and references/ subdirectories need those files available inside sandboxed execution environments. Previously the skills directory was missing entirely from remote backends. Live sync — files stay current as credentials refresh and skills update: - Docker/Singularity: bind mounts are inherently live (host changes visible immediately) - Modal: _sync_files() runs before each command with mtime+size caching, pushing only changed credential and skill files (~13μs no-op overhead) - SSH: rsync --safe-links before each command (naturally incremental) - Daytona: _upload_if_changed() with mtime+size caching before each command Security — symlink filtering: - Docker/Singularity: sanitized temp copy when symlinks detected - Modal/Daytona: iter_skills_files() skips symlinks - SSH: rsync --safe-links skips symlinks pointing outside source tree - Temp dir cleanup via atexit + reuse across calls Non-root user support: - SSH: detects remote home via echo $HOME, syncs to $HOME/.hermes/ - Daytona: detects sandbox home before sync, uploads to $HOME/.hermes/ - Docker/Modal/Singularity: run as root, /root/.hermes/ is correct Also: - credential_files.py: fix name/path key fallback in required_credential_files - Singularity, SSH, Daytona: gained credential file support - 14 tests covering symlink filtering, name/path fallback, iter_skills_files
536 lines
20 KiB
Python
536 lines
20 KiB
Python
"""Docker execution environment for sandboxed command execution.
|
|
|
|
Security hardened (cap-drop ALL, no-new-privileges, PID limits),
|
|
configurable resource limits (CPU, memory, disk), and optional filesystem
|
|
persistence via bind mounts.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from typing import Optional
|
|
|
|
from tools.environments.base import BaseEnvironment
|
|
from tools.interrupt import is_interrupted
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Common Docker Desktop install paths checked when 'docker' is not in PATH.
|
|
# macOS Intel: /usr/local/bin, macOS Apple Silicon (Homebrew): /opt/homebrew/bin,
|
|
# Docker Desktop app bundle: /Applications/Docker.app/Contents/Resources/bin
|
|
_DOCKER_SEARCH_PATHS = [
|
|
"/usr/local/bin/docker",
|
|
"/opt/homebrew/bin/docker",
|
|
"/Applications/Docker.app/Contents/Resources/bin/docker",
|
|
]
|
|
|
|
_docker_executable: Optional[str] = None # resolved once, cached
|
|
_ENV_VAR_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
|
|
|
|
|
def _normalize_forward_env_names(forward_env: list[str] | None) -> list[str]:
|
|
"""Return a deduplicated list of valid environment variable names."""
|
|
normalized: list[str] = []
|
|
seen: set[str] = set()
|
|
|
|
for item in forward_env or []:
|
|
if not isinstance(item, str):
|
|
logger.warning("Ignoring non-string docker_forward_env entry: %r", item)
|
|
continue
|
|
|
|
key = item.strip()
|
|
if not key:
|
|
continue
|
|
if not _ENV_VAR_NAME_RE.match(key):
|
|
logger.warning("Ignoring invalid docker_forward_env entry: %r", item)
|
|
continue
|
|
if key in seen:
|
|
continue
|
|
|
|
seen.add(key)
|
|
normalized.append(key)
|
|
|
|
return normalized
|
|
|
|
|
|
def _load_hermes_env_vars() -> dict[str, str]:
|
|
"""Load ~/.hermes/.env values without failing Docker command execution."""
|
|
try:
|
|
from hermes_cli.config import load_env
|
|
|
|
return load_env() or {}
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def find_docker() -> Optional[str]:
|
|
"""Locate the docker CLI binary.
|
|
|
|
Checks ``shutil.which`` first (respects PATH), then probes well-known
|
|
install locations on macOS where Docker Desktop may not be in PATH
|
|
(e.g. when running as a gateway service via launchd).
|
|
|
|
Returns the absolute path, or ``None`` if docker cannot be found.
|
|
"""
|
|
global _docker_executable
|
|
if _docker_executable is not None:
|
|
return _docker_executable
|
|
|
|
found = shutil.which("docker")
|
|
if found:
|
|
_docker_executable = found
|
|
return found
|
|
|
|
for path in _DOCKER_SEARCH_PATHS:
|
|
if os.path.isfile(path) and os.access(path, os.X_OK):
|
|
_docker_executable = path
|
|
logger.info("Found docker at non-PATH location: %s", path)
|
|
return path
|
|
|
|
return None
|
|
|
|
|
|
# Security flags applied to every container.
|
|
# The container itself is the security boundary (isolated from host).
|
|
# We drop all capabilities then add back the minimum needed:
|
|
# DAC_OVERRIDE - root can write to bind-mounted dirs owned by host user
|
|
# CHOWN/FOWNER - package managers (pip, npm, apt) need to set file ownership
|
|
# Block privilege escalation and limit PIDs.
|
|
# /tmp is size-limited and nosuid but allows exec (needed by pip/npm builds).
|
|
_SECURITY_ARGS = [
|
|
"--cap-drop", "ALL",
|
|
"--cap-add", "DAC_OVERRIDE",
|
|
"--cap-add", "CHOWN",
|
|
"--cap-add", "FOWNER",
|
|
"--security-opt", "no-new-privileges",
|
|
"--pids-limit", "256",
|
|
"--tmpfs", "/tmp:rw,nosuid,size=512m",
|
|
"--tmpfs", "/var/tmp:rw,noexec,nosuid,size=256m",
|
|
"--tmpfs", "/run:rw,noexec,nosuid,size=64m",
|
|
]
|
|
|
|
|
|
_storage_opt_ok: Optional[bool] = None # cached result across instances
|
|
|
|
|
|
def _ensure_docker_available() -> None:
|
|
"""Best-effort check that the docker CLI is available before use.
|
|
|
|
Reuses ``find_docker()`` so this preflight stays consistent with the rest of
|
|
the Docker backend, including known non-PATH Docker Desktop locations.
|
|
"""
|
|
docker_exe = find_docker()
|
|
if not docker_exe:
|
|
logger.error(
|
|
"Docker backend selected but no docker executable was found in PATH "
|
|
"or known install locations. Install Docker Desktop and ensure the "
|
|
"CLI is available."
|
|
)
|
|
raise RuntimeError(
|
|
"Docker executable not found in PATH or known install locations. "
|
|
"Install Docker and ensure the 'docker' command is available."
|
|
)
|
|
|
|
try:
|
|
result = subprocess.run(
|
|
[docker_exe, "version"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
except FileNotFoundError:
|
|
logger.error(
|
|
"Docker backend selected but the resolved docker executable '%s' could "
|
|
"not be executed.",
|
|
docker_exe,
|
|
exc_info=True,
|
|
)
|
|
raise RuntimeError(
|
|
"Docker executable could not be executed. Check your Docker installation."
|
|
)
|
|
except subprocess.TimeoutExpired:
|
|
logger.error(
|
|
"Docker backend selected but '%s version' timed out. "
|
|
"The Docker daemon may not be running.",
|
|
docker_exe,
|
|
exc_info=True,
|
|
)
|
|
raise RuntimeError(
|
|
"Docker daemon is not responding. Ensure Docker is running and try again."
|
|
)
|
|
except Exception:
|
|
logger.error(
|
|
"Unexpected error while checking Docker availability.",
|
|
exc_info=True,
|
|
)
|
|
raise
|
|
else:
|
|
if result.returncode != 0:
|
|
logger.error(
|
|
"Docker backend selected but '%s version' failed "
|
|
"(exit code %d, stderr=%s)",
|
|
docker_exe,
|
|
result.returncode,
|
|
result.stderr.strip(),
|
|
)
|
|
raise RuntimeError(
|
|
"Docker command is available but 'docker version' failed. "
|
|
"Check your Docker installation."
|
|
)
|
|
|
|
|
|
class DockerEnvironment(BaseEnvironment):
|
|
"""Hardened Docker container execution with resource limits and persistence.
|
|
|
|
Security: all capabilities dropped, no privilege escalation, PID limits,
|
|
size-limited tmpfs for scratch dirs. The container itself is the security
|
|
boundary — the filesystem inside is writable so agents can install packages
|
|
(pip, npm, apt) as needed. Writable workspace via tmpfs or bind mounts.
|
|
|
|
Persistence: when enabled, bind mounts preserve /workspace and /root
|
|
across container restarts.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
image: str,
|
|
cwd: str = "/root",
|
|
timeout: int = 60,
|
|
cpu: float = 0,
|
|
memory: int = 0,
|
|
disk: int = 0,
|
|
persistent_filesystem: bool = False,
|
|
task_id: str = "default",
|
|
volumes: list = None,
|
|
forward_env: list[str] | None = None,
|
|
network: bool = True,
|
|
host_cwd: str = None,
|
|
auto_mount_cwd: bool = False,
|
|
):
|
|
if cwd == "~":
|
|
cwd = "/root"
|
|
super().__init__(cwd=cwd, timeout=timeout)
|
|
self._base_image = image
|
|
self._persistent = persistent_filesystem
|
|
self._task_id = task_id
|
|
self._forward_env = _normalize_forward_env_names(forward_env)
|
|
self._container_id: Optional[str] = None
|
|
logger.info(f"DockerEnvironment volumes: {volumes}")
|
|
# Ensure volumes is a list (config.yaml could be malformed)
|
|
if volumes is not None and not isinstance(volumes, list):
|
|
logger.warning(f"docker_volumes config is not a list: {volumes!r}")
|
|
volumes = []
|
|
|
|
# Fail fast if Docker is not available.
|
|
_ensure_docker_available()
|
|
|
|
# Build resource limit args
|
|
resource_args = []
|
|
if cpu > 0:
|
|
resource_args.extend(["--cpus", str(cpu)])
|
|
if memory > 0:
|
|
resource_args.extend(["--memory", f"{memory}m"])
|
|
if disk > 0 and sys.platform != "darwin":
|
|
if self._storage_opt_supported():
|
|
resource_args.extend(["--storage-opt", f"size={disk}m"])
|
|
else:
|
|
logger.warning(
|
|
"Docker storage driver does not support per-container disk limits "
|
|
"(requires overlay2 on XFS with pquota). Container will run without disk quota."
|
|
)
|
|
if not network:
|
|
resource_args.append("--network=none")
|
|
|
|
# Persistent workspace via bind mounts from a configurable host directory
|
|
# (TERMINAL_SANDBOX_DIR, default ~/.hermes/sandboxes/). Non-persistent
|
|
# mode uses tmpfs (ephemeral, fast, gone on cleanup).
|
|
from tools.environments.base import get_sandbox_dir
|
|
|
|
# User-configured volume mounts (from config.yaml docker_volumes)
|
|
volume_args = []
|
|
workspace_explicitly_mounted = False
|
|
for vol in (volumes or []):
|
|
if not isinstance(vol, str):
|
|
logger.warning(f"Docker volume entry is not a string: {vol!r}")
|
|
continue
|
|
vol = vol.strip()
|
|
if not vol:
|
|
continue
|
|
if ":" in vol:
|
|
volume_args.extend(["-v", vol])
|
|
if ":/workspace" in vol:
|
|
workspace_explicitly_mounted = True
|
|
else:
|
|
logger.warning(f"Docker volume '{vol}' missing colon, skipping")
|
|
|
|
host_cwd_abs = os.path.abspath(os.path.expanduser(host_cwd)) if host_cwd else ""
|
|
bind_host_cwd = (
|
|
auto_mount_cwd
|
|
and bool(host_cwd_abs)
|
|
and os.path.isdir(host_cwd_abs)
|
|
and not workspace_explicitly_mounted
|
|
)
|
|
if auto_mount_cwd and host_cwd and not os.path.isdir(host_cwd_abs):
|
|
logger.debug(f"Skipping docker cwd mount: host_cwd is not a valid directory: {host_cwd}")
|
|
|
|
self._workspace_dir: Optional[str] = None
|
|
self._home_dir: Optional[str] = None
|
|
writable_args = []
|
|
if self._persistent:
|
|
sandbox = get_sandbox_dir() / "docker" / task_id
|
|
self._home_dir = str(sandbox / "home")
|
|
os.makedirs(self._home_dir, exist_ok=True)
|
|
writable_args.extend([
|
|
"-v", f"{self._home_dir}:/root",
|
|
])
|
|
if not bind_host_cwd and not workspace_explicitly_mounted:
|
|
self._workspace_dir = str(sandbox / "workspace")
|
|
os.makedirs(self._workspace_dir, exist_ok=True)
|
|
writable_args.extend([
|
|
"-v", f"{self._workspace_dir}:/workspace",
|
|
])
|
|
else:
|
|
if not bind_host_cwd and not workspace_explicitly_mounted:
|
|
writable_args.extend([
|
|
"--tmpfs", "/workspace:rw,exec,size=10g",
|
|
])
|
|
writable_args.extend([
|
|
"--tmpfs", "/home:rw,exec,size=1g",
|
|
"--tmpfs", "/root:rw,exec,size=1g",
|
|
])
|
|
|
|
if bind_host_cwd:
|
|
logger.info(f"Mounting configured host cwd to /workspace: {host_cwd_abs}")
|
|
volume_args = ["-v", f"{host_cwd_abs}:/workspace", *volume_args]
|
|
elif workspace_explicitly_mounted:
|
|
logger.debug("Skipping docker cwd mount: /workspace already mounted by user config")
|
|
|
|
# Mount credential files (OAuth tokens, etc.) declared by skills.
|
|
# Read-only so the container can authenticate but not modify host creds.
|
|
try:
|
|
from tools.credential_files import get_credential_file_mounts, get_skills_directory_mount
|
|
|
|
for mount_entry in get_credential_file_mounts():
|
|
volume_args.extend([
|
|
"-v",
|
|
f"{mount_entry['host_path']}:{mount_entry['container_path']}:ro",
|
|
])
|
|
logger.info(
|
|
"Docker: mounting credential %s -> %s",
|
|
mount_entry["host_path"],
|
|
mount_entry["container_path"],
|
|
)
|
|
|
|
# Mount the skills directory so skill scripts/templates are
|
|
# available inside the container at the same relative path.
|
|
skills_mount = get_skills_directory_mount()
|
|
if skills_mount:
|
|
volume_args.extend([
|
|
"-v",
|
|
f"{skills_mount['host_path']}:{skills_mount['container_path']}:ro",
|
|
])
|
|
logger.info(
|
|
"Docker: mounting skills dir %s -> %s",
|
|
skills_mount["host_path"],
|
|
skills_mount["container_path"],
|
|
)
|
|
except Exception as e:
|
|
logger.debug("Docker: could not load credential file mounts: %s", e)
|
|
|
|
logger.info(f"Docker volume_args: {volume_args}")
|
|
all_run_args = list(_SECURITY_ARGS) + writable_args + resource_args + volume_args
|
|
logger.info(f"Docker run_args: {all_run_args}")
|
|
|
|
# Resolve the docker executable once so it works even when
|
|
# /usr/local/bin is not in PATH (common on macOS gateway/service).
|
|
self._docker_exe = find_docker() or "docker"
|
|
|
|
# Start the container directly via `docker run -d`.
|
|
container_name = f"hermes-{uuid.uuid4().hex[:8]}"
|
|
run_cmd = [
|
|
self._docker_exe, "run", "-d",
|
|
"--name", container_name,
|
|
"-w", cwd,
|
|
*all_run_args,
|
|
image,
|
|
"sleep", "2h",
|
|
]
|
|
logger.debug(f"Starting container: {' '.join(run_cmd)}")
|
|
result = subprocess.run(
|
|
run_cmd,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120, # image pull may take a while
|
|
check=True,
|
|
)
|
|
self._container_id = result.stdout.strip()
|
|
logger.info(f"Started container {container_name} ({self._container_id[:12]})")
|
|
|
|
@staticmethod
|
|
def _storage_opt_supported() -> bool:
|
|
"""Check if Docker's storage driver supports --storage-opt size=.
|
|
|
|
Only overlay2 on XFS with pquota supports per-container disk quotas.
|
|
Ubuntu (and most distros) default to ext4, where this flag errors out.
|
|
"""
|
|
global _storage_opt_ok
|
|
if _storage_opt_ok is not None:
|
|
return _storage_opt_ok
|
|
try:
|
|
docker = find_docker() or "docker"
|
|
result = subprocess.run(
|
|
[docker, "info", "--format", "{{.Driver}}"],
|
|
capture_output=True, text=True, timeout=10,
|
|
)
|
|
driver = result.stdout.strip().lower()
|
|
if driver != "overlay2":
|
|
_storage_opt_ok = False
|
|
return False
|
|
# overlay2 only supports storage-opt on XFS with pquota.
|
|
# Probe by attempting a dry-ish run — the fastest reliable check.
|
|
probe = subprocess.run(
|
|
[docker, "create", "--storage-opt", "size=1m", "hello-world"],
|
|
capture_output=True, text=True, timeout=15,
|
|
)
|
|
if probe.returncode == 0:
|
|
# Clean up the created container
|
|
container_id = probe.stdout.strip()
|
|
if container_id:
|
|
subprocess.run([docker, "rm", container_id],
|
|
capture_output=True, timeout=5)
|
|
_storage_opt_ok = True
|
|
else:
|
|
_storage_opt_ok = False
|
|
except Exception:
|
|
_storage_opt_ok = False
|
|
logger.debug("Docker --storage-opt support: %s", _storage_opt_ok)
|
|
return _storage_opt_ok
|
|
|
|
def execute(self, command: str, cwd: str = "", *,
|
|
timeout: int | None = None,
|
|
stdin_data: str | None = None) -> dict:
|
|
exec_command, sudo_stdin = self._prepare_command(command)
|
|
work_dir = cwd or self.cwd
|
|
effective_timeout = timeout or self.timeout
|
|
|
|
# Merge sudo password (if any) with caller-supplied stdin_data.
|
|
if sudo_stdin is not None and stdin_data is not None:
|
|
effective_stdin = sudo_stdin + stdin_data
|
|
elif sudo_stdin is not None:
|
|
effective_stdin = sudo_stdin
|
|
else:
|
|
effective_stdin = stdin_data
|
|
|
|
# docker exec -w doesn't expand ~, so prepend a cd into the command
|
|
if work_dir == "~" or work_dir.startswith("~/"):
|
|
exec_command = f"cd {work_dir} && {exec_command}"
|
|
work_dir = "/"
|
|
|
|
assert self._container_id, "Container not started"
|
|
cmd = [self._docker_exe, "exec"]
|
|
if effective_stdin is not None:
|
|
cmd.append("-i")
|
|
cmd.extend(["-w", work_dir])
|
|
# Combine explicit docker_forward_env with skill-declared env_passthrough
|
|
# vars so skills that declare required_environment_variables (e.g. Notion)
|
|
# have their keys forwarded into the container automatically.
|
|
forward_keys = set(self._forward_env)
|
|
try:
|
|
from tools.env_passthrough import get_all_passthrough
|
|
forward_keys |= get_all_passthrough()
|
|
except Exception:
|
|
pass
|
|
hermes_env = _load_hermes_env_vars() if forward_keys else {}
|
|
for key in sorted(forward_keys):
|
|
value = os.getenv(key)
|
|
if value is None:
|
|
value = hermes_env.get(key)
|
|
if value is not None:
|
|
cmd.extend(["-e", f"{key}={value}"])
|
|
cmd.extend([self._container_id, "bash", "-lc", exec_command])
|
|
|
|
try:
|
|
_output_chunks = []
|
|
proc = subprocess.Popen(
|
|
cmd,
|
|
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
|
stdin=subprocess.PIPE if effective_stdin else subprocess.DEVNULL,
|
|
text=True,
|
|
)
|
|
if effective_stdin:
|
|
try:
|
|
proc.stdin.write(effective_stdin)
|
|
proc.stdin.close()
|
|
except Exception:
|
|
pass
|
|
|
|
def _drain():
|
|
try:
|
|
for line in proc.stdout:
|
|
_output_chunks.append(line)
|
|
except Exception:
|
|
pass
|
|
|
|
reader = threading.Thread(target=_drain, daemon=True)
|
|
reader.start()
|
|
deadline = time.monotonic() + effective_timeout
|
|
|
|
while proc.poll() is None:
|
|
if is_interrupted():
|
|
proc.terminate()
|
|
try:
|
|
proc.wait(timeout=1)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
reader.join(timeout=2)
|
|
return {
|
|
"output": "".join(_output_chunks) + "\n[Command interrupted]",
|
|
"returncode": 130,
|
|
}
|
|
if time.monotonic() > deadline:
|
|
proc.kill()
|
|
reader.join(timeout=2)
|
|
return self._timeout_result(effective_timeout)
|
|
time.sleep(0.2)
|
|
|
|
reader.join(timeout=5)
|
|
return {"output": "".join(_output_chunks), "returncode": proc.returncode}
|
|
except Exception as e:
|
|
return {"output": f"Docker execution error: {e}", "returncode": 1}
|
|
|
|
def cleanup(self):
|
|
"""Stop and remove the container. Bind-mount dirs persist if persistent=True."""
|
|
if self._container_id:
|
|
try:
|
|
# Stop in background so cleanup doesn't block
|
|
stop_cmd = (
|
|
f"(timeout 60 {self._docker_exe} stop {self._container_id} || "
|
|
f"{self._docker_exe} rm -f {self._container_id}) >/dev/null 2>&1 &"
|
|
)
|
|
subprocess.Popen(stop_cmd, shell=True)
|
|
except Exception as e:
|
|
logger.warning("Failed to stop container %s: %s", self._container_id, e)
|
|
|
|
if not self._persistent:
|
|
# Also schedule removal (stop only leaves it as stopped)
|
|
try:
|
|
subprocess.Popen(
|
|
f"sleep 3 && {self._docker_exe} rm -f {self._container_id} >/dev/null 2>&1 &",
|
|
shell=True,
|
|
)
|
|
except Exception:
|
|
pass
|
|
self._container_id = None
|
|
|
|
if not self._persistent:
|
|
for d in (self._workspace_dir, self._home_dir):
|
|
if d:
|
|
shutil.rmtree(d, ignore_errors=True)
|