a3a28aa4c2
Implements Phase 1 & 2 of the [COMPUTER_USE] epic:
- nexus/computer_use.py — four Hermes tools with safety guards and
JSONL action logging:
computer_screenshot(), computer_click(), computer_type(), computer_scroll()
Poka-yoke: right/middle clicks require confirm=True; text containing
password/token/key keywords is refused without confirm=True.
pyautogui.FAILSAFE=True enabled globally (corner-abort).
- nexus/computer_use_demo.py — end-to-end Phase 1 demo: baseline
screenshot → open browser → navigate to Gitea → evidence screenshot.
- tests/test_computer_use.py — 29 unit tests, fully headless (pyautogui
mocked); all pass.
- docs/computer-use.md — full Phase 1–3 documentation including API
reference, safety table, action-log format, and pilot recipes.
- docker-compose.desktop.yml — sandboxed Xvfb + noVNC container for
safe headless desktop automation.
The existing mcp_servers/desktop_control_server.py is unchanged; it
remains available for external/MCP callers (Bannerlord harness etc).
Fixes #1125
52 lines
1.5 KiB
YAML
52 lines
1.5 KiB
YAML
---
|
|
# docker-compose.desktop.yml — Sandboxed desktop environment for Hermes computer-use
|
|
#
|
|
# Provides a virtual desktop (Xvfb + noVNC) so agents can run computer_use
|
|
# primitives safely inside a container.
|
|
#
|
|
# Usage:
|
|
# docker-compose -f docker-compose.desktop.yml up
|
|
# # Open noVNC at http://localhost:6080
|
|
# # Run demo: docker exec -it nexus-desktop python nexus/computer_use_demo.py
|
|
#
|
|
# Refs: #1125
|
|
|
|
version: "3.8"
|
|
|
|
services:
|
|
desktop:
|
|
image: python:3.11-slim
|
|
container_name: nexus-desktop
|
|
working_dir: /workspace
|
|
volumes:
|
|
- .:/workspace:ro # mount repo read-only
|
|
- nexus_home:/root/.nexus # persistent screenshot/log store
|
|
ports:
|
|
- "6080:6080" # noVNC web viewer
|
|
- "5900:5900" # VNC (optional, for native VNC clients)
|
|
environment:
|
|
- DISPLAY=:99
|
|
- GITEA_URL=${GITEA_URL:-https://forge.alexanderwhitestone.com}
|
|
command: >
|
|
bash -c "
|
|
apt-get update -qq &&
|
|
apt-get install -y -qq
|
|
xvfb x11vnc novnc websockify
|
|
chromium chromium-driver
|
|
python3-tk python3-dev scrot &&
|
|
pip install -q pyautogui pillow &&
|
|
Xvfb :99 -screen 0 1280x800x24 &
|
|
x11vnc -display :99 -forever -nopw -quiet &
|
|
websockify --web /usr/share/novnc 6080 localhost:5900 &
|
|
echo 'Desktop ready — noVNC at http://localhost:6080' &&
|
|
tail -f /dev/null
|
|
"
|
|
healthcheck:
|
|
test: ["CMD", "pgrep", "Xvfb"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 5
|
|
|
|
volumes:
|
|
nexus_home:
|