Merge pull request #1062 from NousResearch/feat/optional-rl-training

feat: make tinker-atropos RL training fully optional

README.md

@@ -124,8 +124,9 @@ We welcome contributions! See the [Contributing Guide](https://hermes-agent.nous
 Quick start for contributors:
 
 ```bash
-git clone --recurse-submodules https://github.com/NousResearch/hermes-agent.git
+git clone https://github.com/NousResearch/hermes-agent.git
 cd hermes-agent
+git submodule update --init mini-swe-agent   # required terminal backend
 curl -LsSf https://astral.sh/uv/install.sh | sh
 uv venv .venv --python 3.11
 source .venv/bin/activate
@@ -134,6 +135,12 @@ uv pip install -e "./mini-swe-agent"
 python -m pytest tests/ -q
 ```
 
+> **RL Training (optional):** To work on the RL/Tinker-Atropos integration, also run:
+> ```bash
+> git submodule update --init tinker-atropos
+> uv pip install -e "./tinker-atropos"
+> ```
+
 ---
 
 ## Community

pyproject.toml

@@ -54,6 +54,13 @@ pty = [
 honcho = ["honcho-ai>=2.0.1"]
 mcp = ["mcp>=1.2.0"]
 homeassistant = ["aiohttp>=3.9.0"]
+rl = [
+  "atroposlib @ git+https://github.com/NousResearch/atropos.git",
+  "tinker @ git+https://github.com/thinking-machines-lab/tinker.git",
+  "fastapi>=0.104.0",
+  "uvicorn[standard]>=0.24.0",
+  "wandb>=0.15.0",
+]
 yc-bench = ["yc-bench @ git+https://github.com/collinear-ai/yc-bench.git"]
 all = [
   "hermes-agent[modal]",

scripts/install.sh

@@ -572,17 +572,16 @@ clone_repo() {
         fi
     else
         # Try SSH first (for private repo access), fall back to HTTPS
-        # Use --recurse-submodules to also clone mini-swe-agent and tinker-atropos
         # GIT_SSH_COMMAND disables interactive prompts and sets a short timeout
         # so SSH fails fast instead of hanging when no key is configured.
         log_info "Trying SSH clone..."
         if GIT_SSH_COMMAND="ssh -o BatchMode=yes -o ConnectTimeout=5" \
-           git clone --branch "$BRANCH" --recurse-submodules "$REPO_URL_SSH" "$INSTALL_DIR" 2>/dev/null; then
+           git clone --branch "$BRANCH" "$REPO_URL_SSH" "$INSTALL_DIR" 2>/dev/null; then
             log_success "Cloned via SSH"
         else
             rm -rf "$INSTALL_DIR" 2>/dev/null  # Clean up partial SSH clone
             log_info "SSH failed, trying HTTPS..."
-            if git clone --branch "$BRANCH" --recurse-submodules "$REPO_URL_HTTPS" "$INSTALL_DIR"; then
+            if git clone --branch "$BRANCH" "$REPO_URL_HTTPS" "$INSTALL_DIR"; then
                 log_success "Cloned via HTTPS"
             else
                 log_error "Failed to clone repository"
@@ -593,10 +592,12 @@ clone_repo() {
 
     cd "$INSTALL_DIR"
 
-    # Ensure submodules are initialized and updated (for existing installs or if --recurse failed)
-    log_info "Initializing submodules (mini-swe-agent, tinker-atropos)..."
-    git submodule update --init --recursive
-    log_success "Submodules ready"
+    # Only init mini-swe-agent (terminal tool backend — required).
+    # tinker-atropos (RL training) is optional and heavy — users can opt in later
+    # with: git submodule update --init tinker-atropos && uv pip install -e ./tinker-atropos
+    log_info "Initializing mini-swe-agent submodule (terminal backend)..."
+    git submodule update --init mini-swe-agent
+    log_success "Submodule ready"
 
     log_success "Repository ready"
 }
@@ -679,12 +680,11 @@ install_deps() {
         log_warn "mini-swe-agent not found (run: git submodule update --init)"
     fi
 
-    log_info "Installing tinker-atropos (RL training backend)..."
+    # tinker-atropos (RL training) is optional — skip by default.
+    # To enable RL tools: git submodule update --init tinker-atropos && uv pip install -e "./tinker-atropos"
     if [ -d "tinker-atropos" ] && [ -f "tinker-atropos/pyproject.toml" ]; then
-        $UV_CMD pip install -e "./tinker-atropos" || log_warn "tinker-atropos install failed (RL tools may not work)"
-        log_success "tinker-atropos installed"
-    else
-        log_warn "tinker-atropos not found (run: git submodule update --init)"
+        log_info "tinker-atropos submodule found — skipping install (optional, for RL training)"
+        log_info "  To install: $UV_CMD pip install -e \"./tinker-atropos\""
     fi
 
     log_success "All dependencies installed"

tools/rl_training_tool.py

@@ -54,8 +54,11 @@ ENVIRONMENTS_DIR = TINKER_ATROPOS_ROOT / "tinker_atropos" / "environments"
 CONFIGS_DIR = TINKER_ATROPOS_ROOT / "configs"
 LOGS_DIR = TINKER_ATROPOS_ROOT / "logs"
 
-# Ensure logs directory exists
-LOGS_DIR.mkdir(exist_ok=True)
+
+def _ensure_logs_dir():
+    """Lazily create logs directory on first use (avoid side effects at import time)."""
+    if TINKER_ATROPOS_ROOT.exists():
+        LOGS_DIR.mkdir(exist_ok=True)
 
 
 # ============================================================================
@@ -314,6 +317,8 @@ async def _spawn_training_run(run_state: RunState, config_path: Path):
     """
     run_id = run_state.run_id
     
+    _ensure_logs_dir()
+
     # Log file paths
     api_log = LOGS_DIR / f"api_{run_id}.log"
     trainer_log = LOGS_DIR / f"trainer_{run_id}.log"
@@ -1092,6 +1097,7 @@ async def rl_test_inference(
     }
     
     # Create output directory for test results
+    _ensure_logs_dir()
     test_output_dir = LOGS_DIR / "inference_tests"
     test_output_dir.mkdir(exist_ok=True)