hermes-agent/tests/test_parallel_tool_calling.py

#!/usr/bin/env python3
"""
test_parallel_tool_calling.py — Tests for parallel tool calling (2+ tools per response).

Verifies that hermes-agent correctly handles multiple tool calls in a single
response, including ordering, dependency resolution, and parallel safety.

Issue #798: Gemma 4 Tool Calling Hardening
"""

import json
import os
import sys
import pytest
from dataclasses import dataclass
from pathlib import Path
from unittest.mock import MagicMock, patch, call

# Add project root to path
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))

from run_agent import (
    _should_parallelize_tool_batch,
    _extract_parallel_scope_path,
    _is_destructive_command,
    _PARALLEL_SAFE_TOOLS,
    _NEVER_PARALLEL_TOOLS,
    _PATH_SCOPED_TOOLS,
)


# ── Mock Tool Call Structure ──────────────────────────────────────────────────

@dataclass
class MockFunction:
    name: str
    arguments: str

@dataclass
class MockToolCall:
    id: str
    function: MockFunction

    @classmethod
    def make(cls, name: str, args: dict, idx: int = 0):
        return cls(
            id=f"call_{idx}",
            function=MockFunction(name=name, arguments=json.dumps(args)),
        )


# ── Test: _should_parallelize_tool_batch ──────────────────────────────────────

class TestParallelizationDecision:
    """Test whether tool batches are correctly identified as parallel-safe."""

    def test_single_tool_not_parallel(self):
        """A single tool call should never be parallelized."""
        calls = [MockToolCall.make("read_file", {"path": "a.txt"})]
        assert _should_parallelize_tool_batch(calls) is False

    def test_two_read_files_different_paths(self):
        """Two read_file calls on different paths should parallelize."""
        calls = [
            MockToolCall.make("read_file", {"path": "a.txt"}, 0),
            MockToolCall.make("read_file", {"path": "b.txt"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is True

    def test_two_read_files_same_path(self):
        """Two read_file calls on the same path should NOT parallelize."""
        calls = [
            MockToolCall.make("read_file", {"path": "a.txt"}, 0),
            MockToolCall.make("read_file", {"path": "a.txt"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is False

    def test_read_plus_search_parallel(self):
        """read_file + search_files should parallelize (both safe, different scopes)."""
        calls = [
            MockToolCall.make("read_file", {"path": "a.txt"}, 0),
            MockToolCall.make("search_files", {"pattern": "foo"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is True

    def test_clarify_never_parallel(self):
        """clarify tool should block parallelization."""
        calls = [
            MockToolCall.make("read_file", {"path": "a.txt"}, 0),
            MockToolCall.make("clarify", {"question": "what?"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is False

    def test_three_read_files_all_different(self):
        """Three read_file calls on different paths should parallelize."""
        calls = [
            MockToolCall.make("read_file", {"path": f"file{i}.txt"}, i)
            for i in range(3)
        ]
        assert _should_parallelize_tool_batch(calls) is True

    def test_write_plus_read_same_path(self):
        """write_file + read_file on same path should NOT parallelize."""
        calls = [
            MockToolCall.make("read_file", {"path": "a.txt"}, 0),
            MockToolCall.make("write_file", {"path": "a.txt", "content": "new"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is False

    def test_write_plus_read_different_paths(self):
        """write_file + read_file on different paths should parallelize."""
        calls = [
            MockToolCall.make("read_file", {"path": "a.txt"}, 0),
            MockToolCall.make("write_file", {"path": "b.txt", "content": "new"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is True

    def test_unsafe_tool_blocks_parallel(self):
        """A tool not in _PARALLEL_SAFE_TOOLS or _PATH_SCOPED_TOOLS blocks parallel."""
        calls = [
            MockToolCall.make("read_file", {"path": "a.txt"}, 0),
            MockToolCall.make("some_unknown_tool", {"param": "value"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is False

    def test_all_safe_tools(self):
        """All tools in _PARALLEL_SAFE_TOOLS should parallelize together."""
        calls = [
            MockToolCall.make("web_search", {"query": "test"}, 0),
            MockToolCall.make("session_search", {"query": "test"}, 1),
            MockToolCall.make("skills_list", {}, 2),
        ]
        assert _should_parallelize_tool_batch(calls) is True

    def test_malformed_json_args(self):
        """Malformed JSON arguments should block parallelization."""
        tc = MockToolCall(id="call_0", function=MockFunction(
            name="read_file", arguments="not json"
        ))
        calls = [MockToolCall.make("read_file", {"path": "a.txt"}, 1), tc]
        assert _should_parallelize_tool_batch(calls) is False

    def test_non_dict_args(self):
        """Non-dict arguments should block parallelization."""
        tc = MockToolCall(id="call_0", function=MockFunction(
            name="read_file", arguments='"just a string"'
        ))
        calls = [MockToolCall.make("read_file", {"path": "a.txt"}, 1), tc]
        assert _should_parallelize_tool_batch(calls) is False


# ── Test: Path Scope Extraction ──────────────────────────────────────────────

class TestPathScopeExtraction:
    """Test path extraction for scoped parallel tools."""

    def test_relative_path(self):
        result = _extract_parallel_scope_path("read_file", {"path": "foo/bar.txt"})
        assert result is not None
        assert "bar.txt" in str(result)

    def test_absolute_path(self):
        result = _extract_parallel_scope_path("read_file", {"path": "/tmp/test.txt"})
        assert result == Path("/tmp/test.txt")

    def test_home_expansion(self):
        result = _extract_parallel_scope_path("read_file", {"path": "~/test.txt"})
        assert result is not None
        assert str(result).endswith("test.txt")

    def test_missing_path(self):
        result = _extract_parallel_scope_path("read_file", {})
        assert result is None

    def test_empty_path(self):
        result = _extract_parallel_scope_path("read_file", {"path": "   "})
        assert result is None

    def test_non_scoped_tool(self):
        result = _extract_parallel_scope_path("web_search", {"path": "foo"})
        assert result is None


# ── Test: Destructive Command Detection ───────────────────────────────────────

class TestDestructiveCommands:
    """Test detection of destructive terminal commands."""

    def test_rm_is_destructive(self):
        assert _is_destructive_command("rm -rf /tmp/foo") is True

    def test_mv_is_destructive(self):
        assert _is_destructive_command("mv old.txt new.txt") is True

    def test_sed_inplace(self):
        assert _is_destructive_command("sed -i 's/foo/bar/g' file.txt") is True

    def test_cat_is_safe(self):
        assert _is_destructive_command("cat file.txt") is False

    def test_echo_redirect_overwrite(self):
        assert _is_destructive_command("echo hello > file.txt") is True

    def test_echo_redirect_append(self):
        assert _is_destructive_command("echo hello >> file.txt") is False

    def test_git_reset(self):
        assert _is_destructive_command("git reset --hard HEAD") is True

    def test_git_status_safe(self):
        assert _is_destructive_command("git status") is False

    def test_piped_rm(self):
        assert _is_destructive_command("echo foo | rm file.txt") is True

    def test_chained_safe(self):
        assert _is_destructive_command("ls && echo done") is False


# ── Test: Parallel Safe Tools Registry ────────────────────────────────────────

class TestParallelSafeRegistry:
    """Test the tool classification sets."""

    def test_clarify_in_never_parallel(self):
        assert "clarify" in _NEVER_PARALLEL_TOOLS

    def test_read_file_in_safe(self):
        assert "read_file" in _PARALLEL_SAFE_TOOLS

    def test_read_file_in_path_scoped(self):
        assert "read_file" in _PATH_SCOPED_TOOLS

    def test_write_file_in_path_scoped(self):
        assert "write_file" in _PATH_SCOPED_TOOLS

    def test_web_search_in_safe(self):
        assert "web_search" in _PARALLEL_SAFE_TOOLS

    def test_no_overlap_between_never_and_safe(self):
        assert not (_NEVER_PARALLEL_TOOLS & _PARALLEL_SAFE_TOOLS)


# ── Test: Batch Sizes (2, 3, 4 tools) ───────────────────────────────────────

class TestBatchSizes:
    """Test parallelization with different batch sizes (2, 3, 4 tools)."""

    def test_two_tool_batch(self):
        calls = [
            MockToolCall.make("read_file", {"path": "a.txt"}, 0),
            MockToolCall.make("read_file", {"path": "b.txt"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is True

    def test_three_tool_batch(self):
        calls = [
            MockToolCall.make("read_file", {"path": f"f{i}.txt"}, i)
            for i in range(3)
        ]
        assert _should_parallelize_tool_batch(calls) is True

    def test_four_tool_batch(self):
        calls = [
            MockToolCall.make("web_search", {"query": f"q{i}"}, i)
            for i in range(4)
        ]
        assert _should_parallelize_tool_batch(calls) is True

    def test_four_tool_batch_with_one_collision(self):
        """4 tools where 2 collide on the same path."""
        calls = [
            MockToolCall.make("read_file", {"path": "a.txt"}, 0),
            MockToolCall.make("read_file", {"path": "b.txt"}, 1),
            MockToolCall.make("read_file", {"path": "a.txt"}, 2),  # collision
            MockToolCall.make("read_file", {"path": "c.txt"}, 3),
        ]
        assert _should_parallelize_tool_batch(calls) is False


# ── Test: Gemma 4 Specific Patterns ──────────────────────────────────────────

class TestGemma4Patterns:
    """
    Test patterns specific to Gemma 4 tool calling behavior.

    Gemma 4 may issue tool calls in specific ordering patterns that
    need to be handled correctly by the parallel execution layer.
    """

    def test_gemma4_typical_2tool_pattern(self):
        """Gemma 4 typically issues read+search as a pair."""
        calls = [
            MockToolCall.make("read_file", {"path": "config.yaml"}, 0),
            MockToolCall.make("search_files", {"pattern": "provider"}, 1),
        ]
        # These should parallelize — different tools, no path conflict
        assert _should_parallelize_tool_batch(calls) is True

    def test_gemma4_typical_3tool_pattern(self):
        """Gemma 4 may issue 3 reads for different files."""
        calls = [
            MockToolCall.make("read_file", {"path": "a.py"}, 0),
            MockToolCall.make("read_file", {"path": "b.py"}, 1),
            MockToolCall.make("read_file", {"path": "c.py"}, 2),
        ]
        assert _should_parallelize_tool_batch(calls) is True

    def test_gemma4_sequential_dependency(self):
        """
        Gemma 4 may issue: search_files then read_file on search result.
        These have implicit dependency but are issued as a batch.
        The agent should handle this — search first, then read.
        This test verifies the batch IS marked as parallel-safe
        (ordering is the agent loop's responsibility, not this function's).
        """
        calls = [
            MockToolCall.make("search_files", {"pattern": "import"}, 0),
            MockToolCall.make("read_file", {"path": "main.py"}, 1),
        ]
        # Both tools are in safe/scoped sets with no path conflict
        assert _should_parallelize_tool_batch(calls) is True

    def test_gemma4_mixed_safe_unsafe(self):
        """Gemma 4 may mix read (safe) with write (path-scoped)."""
        calls = [
            MockToolCall.make("read_file", {"path": "input.txt"}, 0),
            MockToolCall.make("write_file", {"path": "output.txt", "content": "x"}, 1),
            MockToolCall.make("read_file", {"path": "config.txt"}, 2),
        ]
        # All path-scoped on different paths, no unsafe tools
        assert _should_parallelize_tool_batch(calls) is True

    def test_gemma4_terminal_parallel(self):
        """
        Terminal commands are NOT in _PARALLEL_SAFE_TOOLS.
        If Gemma 4 issues 2 terminal calls, they should NOT parallelize.
        """
        calls = [
            MockToolCall.make("terminal", {"command": "ls"}, 0),
            MockToolCall.make("terminal", {"command": "pwd"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is False


# ── Test: Integration-style (mocked) ─────────────────────────────────────────

class TestParallelExecutionMocked:
    """Test the parallel execution path with mocked tool handlers."""

    def test_parallel_results_collected(self):
        """Simulate parallel execution and verify results are collected."""
        # Mock two tool calls returning different results
        results = {}

        def mock_handler(name, args):
            return f"result_{name}_{args.get('path', 'x')}"

        calls = [
            MockToolCall.make("read_file", {"path": "a.txt"}, 0),
            MockToolCall.make("read_file", {"path": "b.txt"}, 1),
        ]

        # Simulate parallel execution
        for tc in calls:
            results[tc.id] = mock_handler(tc.function.name,
                                          json.loads(tc.function.arguments))

        assert results["call_0"] == "result_read_file_a.txt"
        assert results["call_1"] == "result_read_file_b.txt"

    def test_parallel_results_order_preserved(self):
        """Results should be ordered by tool call ID, not completion time."""
        import time
        results = {}

        calls = [
            MockToolCall.make("read_file", {"path": "slow.txt"}, 0),
            MockToolCall.make("read_file", {"path": "fast.txt"}, 1),
        ]

        # Simulate out-of-order completion
        results["call_1"] = "fast_result"
        results["call_0"] = "slow_result"

        # Verify we can reconstruct in order
        ordered = [results[tc.id] for tc in calls]
        assert ordered == ["slow_result", "fast_result"]


# ── Test: Edge Cases ──────────────────────────────────────────────────────────

class TestEdgeCases:
    """Edge cases for parallel tool calling."""

    def test_empty_batch(self):
        assert _should_parallelize_tool_batch([]) is False

    def test_patch_with_same_path(self):
        """Two patch calls on the same file should NOT parallelize."""
        calls = [
            MockToolCall.make("patch", {"path": "a.py", "old_string": "x", "new_string": "y"}, 0),
            MockToolCall.make("patch", {"path": "a.py", "old_string": "a", "new_string": "b"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is False

    def test_patch_different_paths(self):
        """patch on different files should parallelize."""
        calls = [
            MockToolCall.make("patch", {"path": "a.py", "old_string": "x", "new_string": "y"}, 0),
            MockToolCall.make("patch", {"path": "b.py", "old_string": "a", "new_string": "b"}, 1),
        ]
        assert _should_parallelize_tool_batch(calls) is True

    def test_max_workers_defined(self):
        """Verify max workers constant exists and is reasonable."""
        from run_agent import _MAX_TOOL_WORKERS
        assert 1 <= _MAX_TOOL_WORKERS <= 32