2026-03-09 15:32:02 -07:00
|
|
|
"""Tests for tools/vision_tools.py — URL validation, type hints, error logging."""
|
|
|
|
|
|
|
|
|
|
import asyncio
|
|
|
|
|
import json
|
|
|
|
|
import logging
|
|
|
|
|
import os
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
from typing import Awaitable
|
|
|
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
|
|
|
|
|
|
from tools.vision_tools import (
|
|
|
|
|
_validate_image_url,
|
|
|
|
|
_handle_vision_analyze,
|
|
|
|
|
_determine_mime_type,
|
|
|
|
|
_image_to_base64_data_url,
|
2026-04-11 11:07:18 -07:00
|
|
|
_resize_image_for_vision,
|
|
|
|
|
_is_image_size_error,
|
|
|
|
|
_MAX_BASE64_BYTES,
|
|
|
|
|
_RESIZE_TARGET_BYTES,
|
2026-03-09 15:32:02 -07:00
|
|
|
vision_analyze_tool,
|
|
|
|
|
check_vision_requirements,
|
|
|
|
|
get_debug_session_info,
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# _validate_image_url — urlparse-based validation
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
class TestValidateImageUrl:
|
|
|
|
|
"""Tests for URL validation, including urlparse-based netloc check."""
|
|
|
|
|
|
|
|
|
|
def test_valid_https_url(self):
|
fix(tests): fix several failing/flaky tests on main (#6777)
* fix(tests): mock is_safe_url in tests that use example.com
Tests using example.com URLs were failing because is_safe_url does a real DNS lookup which fails in environments where example.com doesn't resolve, causing the request to be blocked before reaching the already-mocked HTTP client. This should fix around 17 failing tests.
These tests test logic, caching, etc. so mocking this method should not modify them in any way. TestMattermostSendUrlAsFile was already doing this so we follow the same pattern.
* fix(test): use case-insensitive lookup for model context length check
DEFAULT_CONTEXT_LENGTHS uses inconsistent casing (MiniMax keys are lowercase, Qwen keys are mixed-case) so the test was broken in some cases since it couldn't find the model.
* fix(test): patch is_linux in systemd gateway restart test
The test only patched is_macos to False but didn't patch is_linux to True. On macOS hosts, is_linux() returns False and the systemd restart code path is skipped entirely, making the assertion fail.
* fix(test): use non-blocklisted env var in docker forward_env tests
GITHUB_TOKEN is in api_key_env_vars and thus in _HERMES_PROVIDER_ENV_BLOCKLIST so the env var is silently dropped, we replace it with a non-blocked one like DATABASE_URL so the tests actually work.
* fix(test): fully isolate _has_any_provider_configured from host env
_has_any_provider_configured() checks all env vars from PROVIDER_REGISTRY (not just the 5 the tests were clearing) and also calls get_auth_status() which detects gh auth token for Copilot. On machines with any of these set, the function returns True before reaching the code path under test.
Clear all registry vars and mock get_auth_status so host credentials don't interfere.
* fix(test): correct path to hermes_base_env.py in tool parser tests
Path(__file__).parent.parent resolved to tests/, not the project root.
The file lives at environments/hermes_base_env.py so we need one more parent level.
* fix(test): accept optional HTML fields in Matrix send payload
_send_matrix sometimes adds format and formatted_body when the markdown library is installed. The test was doing an exact dict equality check which broke. Check required fields instead.
* fix(test): add config.yaml to codex vision requirements test
The test only wrote auth.json but not config.yaml, so _read_main_provider() returned empty and vision auto-detect never tried the codex provider. Add a config.yaml pointing at openai-codex so the fallback path actually resolves the client.
* fix(test): clear OPENROUTER_API_KEY in _isolate_hermes_home
run_agent.py calls load_hermes_dotenv() at import time, which injects API keys from ~/.hermes/.env into os.environ before any test fixture runs. This caused test_agent_loop_tool_calling to make real API calls instead of skipping, which ends up making some tests fail.
* fix(test): add get_rate_limit_state to agent mock in usage report tests
_show_usage now calls agent.get_rate_limit_state() for rate limit
display. The SimpleNamespace mock was missing this method.
* fix(test): update expected Camofox config version from 12 to 13
* fix(test): mock _get_enabled_platforms in nous managed defaults test
Importing gateway.run leaks DISCORD_BOT_TOKEN into os.environ, which makes _get_enabled_platforms() return ["cli", "discord"] instead of just ["cli"]. tools_command loops per platform, so apply_nous_managed_defaults
runs twice: the first call sets config values, the second sees them as
already configured and returns an empty set, causing the assertion to
fail.
2026-04-09 17:17:06 -03:00
|
|
|
with patch("tools.url_safety.socket.getaddrinfo", return_value=[
|
|
|
|
|
(2, 1, 6, "", ("93.184.216.34", 0)),
|
|
|
|
|
]):
|
|
|
|
|
assert _validate_image_url("https://example.com/image.jpg") is True
|
2026-03-09 15:32:02 -07:00
|
|
|
|
|
|
|
|
def test_valid_http_url(self):
|
fix(security): add SSRF protection to vision_tools and web_tools (hardened)
* fix(security): add SSRF protection to vision_tools and web_tools
Both vision_analyze and web_extract/web_crawl accept arbitrary URLs
without checking if they target private/internal network addresses.
A prompt-injected or malicious skill could use this to access cloud
metadata endpoints (169.254.169.254), localhost services, or private
network hosts.
Adds a shared url_safety.is_safe_url() that resolves hostnames and
blocks private, loopback, link-local, and reserved IP ranges. Also
blocks known internal hostnames (metadata.google.internal).
Integrated at the URL validation layer in vision_tools and before
each website_policy check in web_tools (extract, crawl).
* test(vision): update localhost test to reflect SSRF protection
The existing test_valid_url_with_port asserted localhost URLs pass
validation. With SSRF protection, localhost is now correctly blocked.
Update the test to verify the block, and add a separate test for
valid URLs with ports using a public hostname.
* fix(security): harden SSRF protection — fail-closed, CGNAT, multicast, redirect guard
Follow-up hardening on top of dieutx's SSRF protection (PR #2630):
- Change fail-open to fail-closed: DNS errors and unexpected exceptions
now block the request instead of allowing it (OWASP best practice)
- Block CGNAT range (100.64.0.0/10): Python's ipaddress.is_private
does NOT cover this range (returns False for both is_private and
is_global). Used by Tailscale/WireGuard and carrier infrastructure.
- Add is_multicast and is_unspecified checks: multicast (224.0.0.0/4)
and unspecified (0.0.0.0) addresses were not caught by the original
four-check chain
- Add redirect guard for vision_tools: httpx event hook re-validates
each redirect target against SSRF checks, preventing the classic
redirect-based SSRF bypass (302 to internal IP)
- Move SSRF filtering before backend dispatch in web_extract: now
covers Parallel and Tavily backends, not just Firecrawl
- Extract _is_blocked_ip() helper for cleaner IP range checking
- Add 24 new tests (CGNAT, multicast, IPv4-mapped IPv6, fail-closed
behavior, parametrized blocked/allowed IP lists)
- Fix existing tests to mock DNS resolution for test hostnames
---------
Co-authored-by: dieutx <dangtc94@gmail.com>
2026-03-23 15:40:42 -07:00
|
|
|
with patch("tools.url_safety.socket.getaddrinfo", return_value=[
|
|
|
|
|
(2, 1, 6, "", ("93.184.216.34", 0)),
|
|
|
|
|
]):
|
|
|
|
|
assert _validate_image_url("http://cdn.example.org/photo.png") is True
|
2026-03-09 15:32:02 -07:00
|
|
|
|
|
|
|
|
def test_valid_url_without_extension(self):
|
|
|
|
|
"""CDN endpoints that redirect to images should still pass."""
|
fix(security): add SSRF protection to vision_tools and web_tools (hardened)
* fix(security): add SSRF protection to vision_tools and web_tools
Both vision_analyze and web_extract/web_crawl accept arbitrary URLs
without checking if they target private/internal network addresses.
A prompt-injected or malicious skill could use this to access cloud
metadata endpoints (169.254.169.254), localhost services, or private
network hosts.
Adds a shared url_safety.is_safe_url() that resolves hostnames and
blocks private, loopback, link-local, and reserved IP ranges. Also
blocks known internal hostnames (metadata.google.internal).
Integrated at the URL validation layer in vision_tools and before
each website_policy check in web_tools (extract, crawl).
* test(vision): update localhost test to reflect SSRF protection
The existing test_valid_url_with_port asserted localhost URLs pass
validation. With SSRF protection, localhost is now correctly blocked.
Update the test to verify the block, and add a separate test for
valid URLs with ports using a public hostname.
* fix(security): harden SSRF protection — fail-closed, CGNAT, multicast, redirect guard
Follow-up hardening on top of dieutx's SSRF protection (PR #2630):
- Change fail-open to fail-closed: DNS errors and unexpected exceptions
now block the request instead of allowing it (OWASP best practice)
- Block CGNAT range (100.64.0.0/10): Python's ipaddress.is_private
does NOT cover this range (returns False for both is_private and
is_global). Used by Tailscale/WireGuard and carrier infrastructure.
- Add is_multicast and is_unspecified checks: multicast (224.0.0.0/4)
and unspecified (0.0.0.0) addresses were not caught by the original
four-check chain
- Add redirect guard for vision_tools: httpx event hook re-validates
each redirect target against SSRF checks, preventing the classic
redirect-based SSRF bypass (302 to internal IP)
- Move SSRF filtering before backend dispatch in web_extract: now
covers Parallel and Tavily backends, not just Firecrawl
- Extract _is_blocked_ip() helper for cleaner IP range checking
- Add 24 new tests (CGNAT, multicast, IPv4-mapped IPv6, fail-closed
behavior, parametrized blocked/allowed IP lists)
- Fix existing tests to mock DNS resolution for test hostnames
---------
Co-authored-by: dieutx <dangtc94@gmail.com>
2026-03-23 15:40:42 -07:00
|
|
|
with patch("tools.url_safety.socket.getaddrinfo", return_value=[
|
|
|
|
|
(2, 1, 6, "", ("93.184.216.34", 0)),
|
|
|
|
|
]):
|
|
|
|
|
assert _validate_image_url("https://cdn.example.com/abcdef123") is True
|
2026-03-09 15:32:02 -07:00
|
|
|
|
|
|
|
|
def test_valid_url_with_query_params(self):
|
fix(security): add SSRF protection to vision_tools and web_tools (hardened)
* fix(security): add SSRF protection to vision_tools and web_tools
Both vision_analyze and web_extract/web_crawl accept arbitrary URLs
without checking if they target private/internal network addresses.
A prompt-injected or malicious skill could use this to access cloud
metadata endpoints (169.254.169.254), localhost services, or private
network hosts.
Adds a shared url_safety.is_safe_url() that resolves hostnames and
blocks private, loopback, link-local, and reserved IP ranges. Also
blocks known internal hostnames (metadata.google.internal).
Integrated at the URL validation layer in vision_tools and before
each website_policy check in web_tools (extract, crawl).
* test(vision): update localhost test to reflect SSRF protection
The existing test_valid_url_with_port asserted localhost URLs pass
validation. With SSRF protection, localhost is now correctly blocked.
Update the test to verify the block, and add a separate test for
valid URLs with ports using a public hostname.
* fix(security): harden SSRF protection — fail-closed, CGNAT, multicast, redirect guard
Follow-up hardening on top of dieutx's SSRF protection (PR #2630):
- Change fail-open to fail-closed: DNS errors and unexpected exceptions
now block the request instead of allowing it (OWASP best practice)
- Block CGNAT range (100.64.0.0/10): Python's ipaddress.is_private
does NOT cover this range (returns False for both is_private and
is_global). Used by Tailscale/WireGuard and carrier infrastructure.
- Add is_multicast and is_unspecified checks: multicast (224.0.0.0/4)
and unspecified (0.0.0.0) addresses were not caught by the original
four-check chain
- Add redirect guard for vision_tools: httpx event hook re-validates
each redirect target against SSRF checks, preventing the classic
redirect-based SSRF bypass (302 to internal IP)
- Move SSRF filtering before backend dispatch in web_extract: now
covers Parallel and Tavily backends, not just Firecrawl
- Extract _is_blocked_ip() helper for cleaner IP range checking
- Add 24 new tests (CGNAT, multicast, IPv4-mapped IPv6, fail-closed
behavior, parametrized blocked/allowed IP lists)
- Fix existing tests to mock DNS resolution for test hostnames
---------
Co-authored-by: dieutx <dangtc94@gmail.com>
2026-03-23 15:40:42 -07:00
|
|
|
with patch("tools.url_safety.socket.getaddrinfo", return_value=[
|
|
|
|
|
(2, 1, 6, "", ("93.184.216.34", 0)),
|
|
|
|
|
]):
|
|
|
|
|
assert _validate_image_url("https://img.example.com/pic?w=200&h=200") is True
|
|
|
|
|
|
|
|
|
|
def test_localhost_url_blocked_by_ssrf(self):
|
|
|
|
|
"""localhost URLs are now blocked by SSRF protection."""
|
|
|
|
|
assert _validate_image_url("http://localhost:8080/image.png") is False
|
2026-03-09 15:32:02 -07:00
|
|
|
|
|
|
|
|
def test_valid_url_with_port(self):
|
fix(tests): fix several failing/flaky tests on main (#6777)
* fix(tests): mock is_safe_url in tests that use example.com
Tests using example.com URLs were failing because is_safe_url does a real DNS lookup which fails in environments where example.com doesn't resolve, causing the request to be blocked before reaching the already-mocked HTTP client. This should fix around 17 failing tests.
These tests test logic, caching, etc. so mocking this method should not modify them in any way. TestMattermostSendUrlAsFile was already doing this so we follow the same pattern.
* fix(test): use case-insensitive lookup for model context length check
DEFAULT_CONTEXT_LENGTHS uses inconsistent casing (MiniMax keys are lowercase, Qwen keys are mixed-case) so the test was broken in some cases since it couldn't find the model.
* fix(test): patch is_linux in systemd gateway restart test
The test only patched is_macos to False but didn't patch is_linux to True. On macOS hosts, is_linux() returns False and the systemd restart code path is skipped entirely, making the assertion fail.
* fix(test): use non-blocklisted env var in docker forward_env tests
GITHUB_TOKEN is in api_key_env_vars and thus in _HERMES_PROVIDER_ENV_BLOCKLIST so the env var is silently dropped, we replace it with a non-blocked one like DATABASE_URL so the tests actually work.
* fix(test): fully isolate _has_any_provider_configured from host env
_has_any_provider_configured() checks all env vars from PROVIDER_REGISTRY (not just the 5 the tests were clearing) and also calls get_auth_status() which detects gh auth token for Copilot. On machines with any of these set, the function returns True before reaching the code path under test.
Clear all registry vars and mock get_auth_status so host credentials don't interfere.
* fix(test): correct path to hermes_base_env.py in tool parser tests
Path(__file__).parent.parent resolved to tests/, not the project root.
The file lives at environments/hermes_base_env.py so we need one more parent level.
* fix(test): accept optional HTML fields in Matrix send payload
_send_matrix sometimes adds format and formatted_body when the markdown library is installed. The test was doing an exact dict equality check which broke. Check required fields instead.
* fix(test): add config.yaml to codex vision requirements test
The test only wrote auth.json but not config.yaml, so _read_main_provider() returned empty and vision auto-detect never tried the codex provider. Add a config.yaml pointing at openai-codex so the fallback path actually resolves the client.
* fix(test): clear OPENROUTER_API_KEY in _isolate_hermes_home
run_agent.py calls load_hermes_dotenv() at import time, which injects API keys from ~/.hermes/.env into os.environ before any test fixture runs. This caused test_agent_loop_tool_calling to make real API calls instead of skipping, which ends up making some tests fail.
* fix(test): add get_rate_limit_state to agent mock in usage report tests
_show_usage now calls agent.get_rate_limit_state() for rate limit
display. The SimpleNamespace mock was missing this method.
* fix(test): update expected Camofox config version from 12 to 13
* fix(test): mock _get_enabled_platforms in nous managed defaults test
Importing gateway.run leaks DISCORD_BOT_TOKEN into os.environ, which makes _get_enabled_platforms() return ["cli", "discord"] instead of just ["cli"]. tools_command loops per platform, so apply_nous_managed_defaults
runs twice: the first call sets config values, the second sees them as
already configured and returns an empty set, causing the assertion to
fail.
2026-04-09 17:17:06 -03:00
|
|
|
with patch("tools.url_safety.socket.getaddrinfo", return_value=[
|
|
|
|
|
(2, 1, 6, "", ("93.184.216.34", 0)),
|
|
|
|
|
]):
|
|
|
|
|
assert _validate_image_url("http://example.com:8080/image.png") is True
|
2026-03-09 15:32:02 -07:00
|
|
|
|
|
|
|
|
def test_valid_url_with_path_only(self):
|
fix(tests): fix several failing/flaky tests on main (#6777)
* fix(tests): mock is_safe_url in tests that use example.com
Tests using example.com URLs were failing because is_safe_url does a real DNS lookup which fails in environments where example.com doesn't resolve, causing the request to be blocked before reaching the already-mocked HTTP client. This should fix around 17 failing tests.
These tests test logic, caching, etc. so mocking this method should not modify them in any way. TestMattermostSendUrlAsFile was already doing this so we follow the same pattern.
* fix(test): use case-insensitive lookup for model context length check
DEFAULT_CONTEXT_LENGTHS uses inconsistent casing (MiniMax keys are lowercase, Qwen keys are mixed-case) so the test was broken in some cases since it couldn't find the model.
* fix(test): patch is_linux in systemd gateway restart test
The test only patched is_macos to False but didn't patch is_linux to True. On macOS hosts, is_linux() returns False and the systemd restart code path is skipped entirely, making the assertion fail.
* fix(test): use non-blocklisted env var in docker forward_env tests
GITHUB_TOKEN is in api_key_env_vars and thus in _HERMES_PROVIDER_ENV_BLOCKLIST so the env var is silently dropped, we replace it with a non-blocked one like DATABASE_URL so the tests actually work.
* fix(test): fully isolate _has_any_provider_configured from host env
_has_any_provider_configured() checks all env vars from PROVIDER_REGISTRY (not just the 5 the tests were clearing) and also calls get_auth_status() which detects gh auth token for Copilot. On machines with any of these set, the function returns True before reaching the code path under test.
Clear all registry vars and mock get_auth_status so host credentials don't interfere.
* fix(test): correct path to hermes_base_env.py in tool parser tests
Path(__file__).parent.parent resolved to tests/, not the project root.
The file lives at environments/hermes_base_env.py so we need one more parent level.
* fix(test): accept optional HTML fields in Matrix send payload
_send_matrix sometimes adds format and formatted_body when the markdown library is installed. The test was doing an exact dict equality check which broke. Check required fields instead.
* fix(test): add config.yaml to codex vision requirements test
The test only wrote auth.json but not config.yaml, so _read_main_provider() returned empty and vision auto-detect never tried the codex provider. Add a config.yaml pointing at openai-codex so the fallback path actually resolves the client.
* fix(test): clear OPENROUTER_API_KEY in _isolate_hermes_home
run_agent.py calls load_hermes_dotenv() at import time, which injects API keys from ~/.hermes/.env into os.environ before any test fixture runs. This caused test_agent_loop_tool_calling to make real API calls instead of skipping, which ends up making some tests fail.
* fix(test): add get_rate_limit_state to agent mock in usage report tests
_show_usage now calls agent.get_rate_limit_state() for rate limit
display. The SimpleNamespace mock was missing this method.
* fix(test): update expected Camofox config version from 12 to 13
* fix(test): mock _get_enabled_platforms in nous managed defaults test
Importing gateway.run leaks DISCORD_BOT_TOKEN into os.environ, which makes _get_enabled_platforms() return ["cli", "discord"] instead of just ["cli"]. tools_command loops per platform, so apply_nous_managed_defaults
runs twice: the first call sets config values, the second sees them as
already configured and returns an empty set, causing the assertion to
fail.
2026-04-09 17:17:06 -03:00
|
|
|
with patch("tools.url_safety.socket.getaddrinfo", return_value=[
|
|
|
|
|
(2, 1, 6, "", ("93.184.216.34", 0)),
|
|
|
|
|
]):
|
|
|
|
|
assert _validate_image_url("https://example.com/") is True
|
2026-03-09 15:32:02 -07:00
|
|
|
|
|
|
|
|
def test_rejects_empty_string(self):
|
|
|
|
|
assert _validate_image_url("") is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_none(self):
|
|
|
|
|
assert _validate_image_url(None) is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_non_string(self):
|
|
|
|
|
assert _validate_image_url(12345) is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_ftp_scheme(self):
|
|
|
|
|
assert _validate_image_url("ftp://files.example.com/image.jpg") is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_file_scheme(self):
|
|
|
|
|
assert _validate_image_url("file:///etc/passwd") is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_no_scheme(self):
|
|
|
|
|
assert _validate_image_url("example.com/image.jpg") is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_javascript_scheme(self):
|
|
|
|
|
assert _validate_image_url("javascript:alert(1)") is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_http_without_netloc(self):
|
|
|
|
|
"""http:// alone has no network location — urlparse catches this."""
|
|
|
|
|
assert _validate_image_url("http://") is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_https_without_netloc(self):
|
|
|
|
|
assert _validate_image_url("https://") is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_http_colon_only(self):
|
|
|
|
|
assert _validate_image_url("http:") is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_data_url(self):
|
|
|
|
|
assert _validate_image_url("data:image/png;base64,iVBOR") is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_whitespace_only(self):
|
|
|
|
|
assert _validate_image_url(" ") is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_boolean(self):
|
|
|
|
|
assert _validate_image_url(True) is False
|
|
|
|
|
|
|
|
|
|
def test_rejects_list(self):
|
|
|
|
|
assert _validate_image_url(["https://example.com"]) is False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# _determine_mime_type
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
class TestDetermineMimeType:
|
|
|
|
|
def test_jpg(self):
|
|
|
|
|
assert _determine_mime_type(Path("photo.jpg")) == "image/jpeg"
|
|
|
|
|
|
|
|
|
|
def test_jpeg(self):
|
|
|
|
|
assert _determine_mime_type(Path("photo.jpeg")) == "image/jpeg"
|
|
|
|
|
|
|
|
|
|
def test_png(self):
|
|
|
|
|
assert _determine_mime_type(Path("screenshot.png")) == "image/png"
|
|
|
|
|
|
|
|
|
|
def test_gif(self):
|
|
|
|
|
assert _determine_mime_type(Path("anim.gif")) == "image/gif"
|
|
|
|
|
|
|
|
|
|
def test_webp(self):
|
|
|
|
|
assert _determine_mime_type(Path("modern.webp")) == "image/webp"
|
|
|
|
|
|
|
|
|
|
def test_unknown_extension_defaults_to_jpeg(self):
|
|
|
|
|
assert _determine_mime_type(Path("file.xyz")) == "image/jpeg"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# _image_to_base64_data_url
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
class TestImageToBase64DataUrl:
|
|
|
|
|
def test_returns_data_url(self, tmp_path):
|
|
|
|
|
img = tmp_path / "test.png"
|
|
|
|
|
img.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 8)
|
|
|
|
|
result = _image_to_base64_data_url(img)
|
|
|
|
|
assert result.startswith("data:image/png;base64,")
|
|
|
|
|
|
|
|
|
|
def test_custom_mime_type(self, tmp_path):
|
|
|
|
|
img = tmp_path / "test.bin"
|
|
|
|
|
img.write_bytes(b"\x00" * 16)
|
|
|
|
|
result = _image_to_base64_data_url(img, mime_type="image/webp")
|
|
|
|
|
assert result.startswith("data:image/webp;base64,")
|
|
|
|
|
|
|
|
|
|
def test_file_not_found_raises(self, tmp_path):
|
|
|
|
|
with pytest.raises(FileNotFoundError):
|
|
|
|
|
_image_to_base64_data_url(tmp_path / "nonexistent.png")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# _handle_vision_analyze — type signature & behavior
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
class TestHandleVisionAnalyze:
|
|
|
|
|
"""Verify _handle_vision_analyze returns an Awaitable and builds correct prompt."""
|
|
|
|
|
|
|
|
|
|
def test_returns_awaitable(self):
|
|
|
|
|
"""The handler must return an Awaitable (coroutine) since it's registered as async."""
|
2026-03-10 16:03:19 -04:00
|
|
|
with patch(
|
|
|
|
|
"tools.vision_tools.vision_analyze_tool", new_callable=AsyncMock
|
|
|
|
|
) as mock_tool:
|
2026-03-09 15:32:02 -07:00
|
|
|
mock_tool.return_value = json.dumps({"result": "ok"})
|
|
|
|
|
result = _handle_vision_analyze(
|
2026-03-10 16:03:19 -04:00
|
|
|
{
|
|
|
|
|
"image_url": "https://example.com/img.png",
|
|
|
|
|
"question": "What is this?",
|
|
|
|
|
}
|
2026-03-09 15:32:02 -07:00
|
|
|
)
|
|
|
|
|
# It should be an Awaitable (coroutine)
|
|
|
|
|
assert isinstance(result, Awaitable)
|
|
|
|
|
# Clean up the coroutine to avoid RuntimeWarning
|
|
|
|
|
result.close()
|
|
|
|
|
|
|
|
|
|
def test_prompt_contains_question(self):
|
|
|
|
|
"""The full prompt should incorporate the user's question."""
|
2026-03-10 16:03:19 -04:00
|
|
|
with patch(
|
|
|
|
|
"tools.vision_tools.vision_analyze_tool", new_callable=AsyncMock
|
|
|
|
|
) as mock_tool:
|
2026-03-09 15:32:02 -07:00
|
|
|
mock_tool.return_value = json.dumps({"result": "ok"})
|
|
|
|
|
coro = _handle_vision_analyze(
|
2026-03-10 16:03:19 -04:00
|
|
|
{
|
|
|
|
|
"image_url": "https://example.com/img.png",
|
|
|
|
|
"question": "Describe the cat",
|
|
|
|
|
}
|
2026-03-09 15:32:02 -07:00
|
|
|
)
|
|
|
|
|
# Clean up coroutine
|
|
|
|
|
coro.close()
|
|
|
|
|
call_args = mock_tool.call_args
|
|
|
|
|
full_prompt = call_args[0][1] # second positional arg
|
|
|
|
|
assert "Describe the cat" in full_prompt
|
|
|
|
|
assert "Fully describe and explain" in full_prompt
|
|
|
|
|
|
|
|
|
|
def test_uses_auxiliary_vision_model_env(self):
|
|
|
|
|
"""AUXILIARY_VISION_MODEL env var should override DEFAULT_VISION_MODEL."""
|
2026-03-10 16:03:19 -04:00
|
|
|
with (
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools.vision_analyze_tool", new_callable=AsyncMock
|
|
|
|
|
) as mock_tool,
|
|
|
|
|
patch.dict(os.environ, {"AUXILIARY_VISION_MODEL": "custom/model-v1"}),
|
|
|
|
|
):
|
2026-03-09 15:32:02 -07:00
|
|
|
mock_tool.return_value = json.dumps({"result": "ok"})
|
|
|
|
|
coro = _handle_vision_analyze(
|
|
|
|
|
{"image_url": "https://example.com/img.png", "question": "test"}
|
|
|
|
|
)
|
|
|
|
|
coro.close()
|
|
|
|
|
call_args = mock_tool.call_args
|
|
|
|
|
model = call_args[0][2] # third positional arg
|
|
|
|
|
assert model == "custom/model-v1"
|
|
|
|
|
|
|
|
|
|
def test_falls_back_to_default_model(self):
|
2026-03-11 21:06:54 -07:00
|
|
|
"""Without AUXILIARY_VISION_MODEL, model should be None (let call_llm resolve default)."""
|
2026-03-10 16:03:19 -04:00
|
|
|
with (
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools.vision_analyze_tool", new_callable=AsyncMock
|
|
|
|
|
) as mock_tool,
|
|
|
|
|
patch.dict(os.environ, {}, clear=False),
|
|
|
|
|
):
|
2026-03-09 15:32:02 -07:00
|
|
|
# Ensure AUXILIARY_VISION_MODEL is not set
|
|
|
|
|
os.environ.pop("AUXILIARY_VISION_MODEL", None)
|
|
|
|
|
mock_tool.return_value = json.dumps({"result": "ok"})
|
|
|
|
|
coro = _handle_vision_analyze(
|
|
|
|
|
{"image_url": "https://example.com/img.png", "question": "test"}
|
|
|
|
|
)
|
|
|
|
|
coro.close()
|
|
|
|
|
call_args = mock_tool.call_args
|
|
|
|
|
model = call_args[0][2]
|
2026-03-11 21:06:54 -07:00
|
|
|
# With no AUXILIARY_VISION_MODEL set, model should be None
|
|
|
|
|
# (the centralized call_llm router picks the default)
|
|
|
|
|
assert model is None
|
2026-03-09 15:32:02 -07:00
|
|
|
|
|
|
|
|
def test_empty_args_graceful(self):
|
|
|
|
|
"""Missing keys should default to empty strings, not raise."""
|
2026-03-10 16:03:19 -04:00
|
|
|
with patch(
|
|
|
|
|
"tools.vision_tools.vision_analyze_tool", new_callable=AsyncMock
|
|
|
|
|
) as mock_tool:
|
2026-03-09 15:32:02 -07:00
|
|
|
mock_tool.return_value = json.dumps({"result": "ok"})
|
|
|
|
|
result = _handle_vision_analyze({})
|
|
|
|
|
assert isinstance(result, Awaitable)
|
|
|
|
|
result.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# Error logging with exc_info — verify tracebacks are logged
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
class TestErrorLoggingExcInfo:
|
|
|
|
|
"""Verify that exc_info=True is used in error/warning log calls."""
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_download_failure_logs_exc_info(self, tmp_path, caplog):
|
|
|
|
|
"""After max retries, the download error should include exc_info."""
|
|
|
|
|
from tools.vision_tools import _download_image
|
|
|
|
|
|
|
|
|
|
with patch("tools.vision_tools.httpx.AsyncClient") as mock_client_cls:
|
|
|
|
|
mock_client = AsyncMock()
|
|
|
|
|
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
|
|
|
|
mock_client.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
|
mock_client.get = AsyncMock(side_effect=ConnectionError("network down"))
|
|
|
|
|
mock_client_cls.return_value = mock_client
|
|
|
|
|
|
|
|
|
|
dest = tmp_path / "image.jpg"
|
2026-03-10 16:03:19 -04:00
|
|
|
with (
|
|
|
|
|
caplog.at_level(logging.ERROR, logger="tools.vision_tools"),
|
|
|
|
|
pytest.raises(ConnectionError),
|
|
|
|
|
):
|
|
|
|
|
await _download_image(
|
|
|
|
|
"https://example.com/img.jpg", dest, max_retries=1
|
|
|
|
|
)
|
2026-03-09 15:32:02 -07:00
|
|
|
|
|
|
|
|
# Should have logged with exc_info (traceback present)
|
|
|
|
|
error_records = [r for r in caplog.records if r.levelno >= logging.ERROR]
|
|
|
|
|
assert len(error_records) >= 1
|
|
|
|
|
assert error_records[0].exc_info is not None
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_analysis_error_logs_exc_info(self, caplog):
|
|
|
|
|
"""When vision_analyze_tool encounters an error, it should log with exc_info."""
|
2026-03-10 16:03:19 -04:00
|
|
|
with (
|
|
|
|
|
patch("tools.vision_tools._validate_image_url", return_value=True),
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools._download_image",
|
|
|
|
|
new_callable=AsyncMock,
|
|
|
|
|
side_effect=Exception("download boom"),
|
|
|
|
|
),
|
|
|
|
|
caplog.at_level(logging.ERROR, logger="tools.vision_tools"),
|
|
|
|
|
):
|
2026-03-09 15:32:02 -07:00
|
|
|
result = await vision_analyze_tool(
|
|
|
|
|
"https://example.com/img.jpg", "describe this", "test/model"
|
|
|
|
|
)
|
|
|
|
|
result_data = json.loads(result)
|
|
|
|
|
# Error response uses "success": False, not an "error" key
|
|
|
|
|
assert result_data["success"] is False
|
|
|
|
|
|
|
|
|
|
error_records = [r for r in caplog.records if r.levelno >= logging.ERROR]
|
2026-03-11 06:31:56 -07:00
|
|
|
assert any(r.exc_info and r.exc_info[0] is not None for r in error_records)
|
2026-03-09 15:32:02 -07:00
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_cleanup_error_logs_exc_info(self, tmp_path, caplog):
|
|
|
|
|
"""Temp file cleanup failure should log warning with exc_info."""
|
|
|
|
|
# Create a real temp file that will be "downloaded"
|
|
|
|
|
temp_dir = tmp_path / "temp_vision_images"
|
|
|
|
|
temp_dir.mkdir()
|
|
|
|
|
|
|
|
|
|
async def fake_download(url, dest, max_retries=3):
|
|
|
|
|
"""Simulate download by writing file to the expected destination."""
|
|
|
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
dest.write_bytes(b"\xff\xd8\xff" + b"\x00" * 16)
|
|
|
|
|
return dest
|
|
|
|
|
|
2026-03-10 16:03:19 -04:00
|
|
|
with (
|
|
|
|
|
patch("tools.vision_tools._validate_image_url", return_value=True),
|
|
|
|
|
patch("tools.vision_tools._download_image", side_effect=fake_download),
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools._image_to_base64_data_url",
|
|
|
|
|
return_value="data:image/jpeg;base64,abc",
|
|
|
|
|
),
|
|
|
|
|
caplog.at_level(logging.WARNING, logger="tools.vision_tools"),
|
|
|
|
|
):
|
2026-03-11 21:06:54 -07:00
|
|
|
# Mock the async_call_llm function to return a mock response
|
2026-03-09 15:32:02 -07:00
|
|
|
mock_response = MagicMock()
|
|
|
|
|
mock_choice = MagicMock()
|
|
|
|
|
mock_choice.message.content = "A test image description"
|
|
|
|
|
mock_response.choices = [mock_choice]
|
|
|
|
|
|
2026-03-10 16:03:19 -04:00
|
|
|
with (
|
2026-03-11 21:06:54 -07:00
|
|
|
patch("tools.vision_tools.async_call_llm", new_callable=AsyncMock, return_value=mock_response),
|
2026-03-10 16:03:19 -04:00
|
|
|
):
|
2026-03-09 15:32:02 -07:00
|
|
|
# Make unlink fail to trigger cleanup warning
|
|
|
|
|
original_unlink = Path.unlink
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
def failing_unlink(self, *args, **kwargs):
|
|
|
|
|
raise PermissionError("no permission")
|
|
|
|
|
|
|
|
|
|
with patch.object(Path, "unlink", failing_unlink):
|
|
|
|
|
result = await vision_analyze_tool(
|
|
|
|
|
"https://example.com/tempimg.jpg", "describe", "test/model"
|
|
|
|
|
)
|
|
|
|
|
|
2026-03-10 16:03:19 -04:00
|
|
|
warning_records = [
|
|
|
|
|
r
|
|
|
|
|
for r in caplog.records
|
|
|
|
|
if r.levelno == logging.WARNING
|
|
|
|
|
and "temporary file" in r.getMessage().lower()
|
|
|
|
|
]
|
2026-03-09 15:32:02 -07:00
|
|
|
assert len(warning_records) >= 1
|
|
|
|
|
assert warning_records[0].exc_info is not None
|
|
|
|
|
|
|
|
|
|
|
2026-03-29 20:55:04 -07:00
|
|
|
class TestVisionSafetyGuards:
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_local_non_image_file_rejected_before_llm_call(self, tmp_path):
|
|
|
|
|
secret = tmp_path / "secret.txt"
|
|
|
|
|
secret.write_text("TOP-SECRET=1\n", encoding="utf-8")
|
|
|
|
|
|
|
|
|
|
with patch("tools.vision_tools.async_call_llm", new_callable=AsyncMock) as mock_llm:
|
|
|
|
|
result = json.loads(await vision_analyze_tool(str(secret), "extract text"))
|
|
|
|
|
|
|
|
|
|
assert result["success"] is False
|
|
|
|
|
assert "Only real image files are supported" in result["error"]
|
|
|
|
|
mock_llm.assert_not_awaited()
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_blocked_remote_url_short_circuits_before_download(self):
|
|
|
|
|
blocked = {
|
|
|
|
|
"host": "blocked.test",
|
|
|
|
|
"rule": "blocked.test",
|
|
|
|
|
"source": "config",
|
|
|
|
|
"message": "Blocked by website policy",
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
with (
|
|
|
|
|
patch("tools.vision_tools.check_website_access", return_value=blocked),
|
|
|
|
|
patch("tools.vision_tools._validate_image_url", return_value=True),
|
|
|
|
|
patch("tools.vision_tools._download_image", new_callable=AsyncMock) as mock_download,
|
|
|
|
|
):
|
|
|
|
|
result = json.loads(await vision_analyze_tool("https://blocked.test/cat.png", "describe"))
|
|
|
|
|
|
|
|
|
|
assert result["success"] is False
|
|
|
|
|
assert "Blocked by website policy" in result["error"]
|
|
|
|
|
mock_download.assert_not_awaited()
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_download_blocks_redirected_final_url(self, tmp_path):
|
|
|
|
|
from tools.vision_tools import _download_image
|
|
|
|
|
|
|
|
|
|
def fake_check(url):
|
|
|
|
|
if url == "https://allowed.test/cat.png":
|
|
|
|
|
return None
|
|
|
|
|
if url == "https://blocked.test/final.png":
|
|
|
|
|
return {
|
|
|
|
|
"host": "blocked.test",
|
|
|
|
|
"rule": "blocked.test",
|
|
|
|
|
"source": "config",
|
|
|
|
|
"message": "Blocked by website policy",
|
|
|
|
|
}
|
|
|
|
|
raise AssertionError(f"unexpected URL checked: {url}")
|
|
|
|
|
|
|
|
|
|
class FakeResponse:
|
|
|
|
|
url = "https://blocked.test/final.png"
|
2026-04-11 01:51:11 -07:00
|
|
|
headers = {"content-length": "24"}
|
2026-03-29 20:55:04 -07:00
|
|
|
content = b"\x89PNG\r\n\x1a\n" + b"\x00" * 16
|
|
|
|
|
|
|
|
|
|
def raise_for_status(self):
|
|
|
|
|
return None
|
|
|
|
|
|
|
|
|
|
with (
|
|
|
|
|
patch("tools.vision_tools.check_website_access", side_effect=fake_check),
|
|
|
|
|
patch("tools.vision_tools.httpx.AsyncClient") as mock_client_cls,
|
|
|
|
|
pytest.raises(PermissionError, match="Blocked by website policy"),
|
|
|
|
|
):
|
|
|
|
|
mock_client = AsyncMock()
|
|
|
|
|
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
|
|
|
|
|
mock_client.__aexit__ = AsyncMock(return_value=False)
|
|
|
|
|
mock_client.get = AsyncMock(return_value=FakeResponse())
|
|
|
|
|
mock_client_cls.return_value = mock_client
|
|
|
|
|
|
|
|
|
|
await _download_image("https://allowed.test/cat.png", tmp_path / "cat.png", max_retries=1)
|
|
|
|
|
|
|
|
|
|
assert not (tmp_path / "cat.png").exists()
|
|
|
|
|
|
|
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# check_vision_requirements & get_debug_session_info
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
class TestVisionRequirements:
|
|
|
|
|
def test_check_requirements_returns_bool(self):
|
|
|
|
|
result = check_vision_requirements()
|
|
|
|
|
assert isinstance(result, bool)
|
|
|
|
|
|
2026-03-14 20:22:13 -07:00
|
|
|
def test_check_requirements_accepts_codex_auth(self, monkeypatch, tmp_path):
|
|
|
|
|
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
|
|
|
|
|
(tmp_path / "auth.json").write_text(
|
|
|
|
|
'{"active_provider":"openai-codex","providers":{"openai-codex":{"tokens":{"access_token":"codex-access-token","refresh_token":"codex-refresh-token"}}}}'
|
|
|
|
|
)
|
fix(tests): fix several failing/flaky tests on main (#6777)
* fix(tests): mock is_safe_url in tests that use example.com
Tests using example.com URLs were failing because is_safe_url does a real DNS lookup which fails in environments where example.com doesn't resolve, causing the request to be blocked before reaching the already-mocked HTTP client. This should fix around 17 failing tests.
These tests test logic, caching, etc. so mocking this method should not modify them in any way. TestMattermostSendUrlAsFile was already doing this so we follow the same pattern.
* fix(test): use case-insensitive lookup for model context length check
DEFAULT_CONTEXT_LENGTHS uses inconsistent casing (MiniMax keys are lowercase, Qwen keys are mixed-case) so the test was broken in some cases since it couldn't find the model.
* fix(test): patch is_linux in systemd gateway restart test
The test only patched is_macos to False but didn't patch is_linux to True. On macOS hosts, is_linux() returns False and the systemd restart code path is skipped entirely, making the assertion fail.
* fix(test): use non-blocklisted env var in docker forward_env tests
GITHUB_TOKEN is in api_key_env_vars and thus in _HERMES_PROVIDER_ENV_BLOCKLIST so the env var is silently dropped, we replace it with a non-blocked one like DATABASE_URL so the tests actually work.
* fix(test): fully isolate _has_any_provider_configured from host env
_has_any_provider_configured() checks all env vars from PROVIDER_REGISTRY (not just the 5 the tests were clearing) and also calls get_auth_status() which detects gh auth token for Copilot. On machines with any of these set, the function returns True before reaching the code path under test.
Clear all registry vars and mock get_auth_status so host credentials don't interfere.
* fix(test): correct path to hermes_base_env.py in tool parser tests
Path(__file__).parent.parent resolved to tests/, not the project root.
The file lives at environments/hermes_base_env.py so we need one more parent level.
* fix(test): accept optional HTML fields in Matrix send payload
_send_matrix sometimes adds format and formatted_body when the markdown library is installed. The test was doing an exact dict equality check which broke. Check required fields instead.
* fix(test): add config.yaml to codex vision requirements test
The test only wrote auth.json but not config.yaml, so _read_main_provider() returned empty and vision auto-detect never tried the codex provider. Add a config.yaml pointing at openai-codex so the fallback path actually resolves the client.
* fix(test): clear OPENROUTER_API_KEY in _isolate_hermes_home
run_agent.py calls load_hermes_dotenv() at import time, which injects API keys from ~/.hermes/.env into os.environ before any test fixture runs. This caused test_agent_loop_tool_calling to make real API calls instead of skipping, which ends up making some tests fail.
* fix(test): add get_rate_limit_state to agent mock in usage report tests
_show_usage now calls agent.get_rate_limit_state() for rate limit
display. The SimpleNamespace mock was missing this method.
* fix(test): update expected Camofox config version from 12 to 13
* fix(test): mock _get_enabled_platforms in nous managed defaults test
Importing gateway.run leaks DISCORD_BOT_TOKEN into os.environ, which makes _get_enabled_platforms() return ["cli", "discord"] instead of just ["cli"]. tools_command loops per platform, so apply_nous_managed_defaults
runs twice: the first call sets config values, the second sees them as
already configured and returns an empty set, causing the assertion to
fail.
2026-04-09 17:17:06 -03:00
|
|
|
# config.yaml must reference the codex provider so vision auto-detect
|
|
|
|
|
# falls back to the active provider via _read_main_provider().
|
|
|
|
|
(tmp_path / "config.yaml").write_text(
|
|
|
|
|
'model:\n default: gpt-4o\n provider: openai-codex\n'
|
|
|
|
|
)
|
2026-03-14 20:22:13 -07:00
|
|
|
monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
|
|
|
|
|
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
|
|
|
|
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
|
|
|
|
|
monkeypatch.delenv("AUXILIARY_VISION_PROVIDER", raising=False)
|
|
|
|
|
monkeypatch.delenv("CONTEXT_VISION_PROVIDER", raising=False)
|
|
|
|
|
|
|
|
|
|
assert check_vision_requirements() is True
|
|
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
def test_debug_session_info_returns_dict(self):
|
|
|
|
|
info = get_debug_session_info()
|
|
|
|
|
assert isinstance(info, dict)
|
|
|
|
|
# DebugSession.get_session_info() returns these keys
|
|
|
|
|
assert "enabled" in info
|
|
|
|
|
assert "session_id" in info
|
|
|
|
|
assert "total_calls" in info
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# Integration: registry entry
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-22 23:48:32 -07:00
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# Tilde expansion in local file paths
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestTildeExpansion:
|
|
|
|
|
"""Verify that ~/path style paths are expanded correctly."""
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_tilde_path_expanded_to_local_file(self, tmp_path, monkeypatch):
|
|
|
|
|
"""vision_analyze_tool should expand ~ in file paths."""
|
|
|
|
|
# Create a fake image file under a fake home directory
|
|
|
|
|
fake_home = tmp_path / "fakehome"
|
|
|
|
|
fake_home.mkdir()
|
|
|
|
|
img = fake_home / "test_image.png"
|
|
|
|
|
img.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 8)
|
|
|
|
|
|
|
|
|
|
monkeypatch.setenv("HOME", str(fake_home))
|
|
|
|
|
|
|
|
|
|
mock_response = MagicMock()
|
|
|
|
|
mock_choice = MagicMock()
|
|
|
|
|
mock_choice.message.content = "A test image"
|
|
|
|
|
mock_response.choices = [mock_choice]
|
|
|
|
|
|
|
|
|
|
with (
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools._image_to_base64_data_url",
|
|
|
|
|
return_value="data:image/png;base64,abc",
|
|
|
|
|
),
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools.async_call_llm",
|
|
|
|
|
new_callable=AsyncMock,
|
|
|
|
|
return_value=mock_response,
|
|
|
|
|
),
|
|
|
|
|
):
|
|
|
|
|
result = await vision_analyze_tool(
|
|
|
|
|
"~/test_image.png", "describe this", "test/model"
|
|
|
|
|
)
|
|
|
|
|
data = json.loads(result)
|
|
|
|
|
assert data["success"] is True
|
|
|
|
|
assert data["analysis"] == "A test image"
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_tilde_path_nonexistent_file_gives_error(self, tmp_path, monkeypatch):
|
|
|
|
|
"""A tilde path that doesn't resolve to a real file should fail gracefully."""
|
|
|
|
|
fake_home = tmp_path / "fakehome"
|
|
|
|
|
fake_home.mkdir()
|
|
|
|
|
monkeypatch.setenv("HOME", str(fake_home))
|
|
|
|
|
|
|
|
|
|
result = await vision_analyze_tool(
|
|
|
|
|
"~/nonexistent.png", "describe this", "test/model"
|
|
|
|
|
)
|
|
|
|
|
data = json.loads(result)
|
|
|
|
|
assert data["success"] is False
|
|
|
|
|
|
|
|
|
|
|
2026-04-10 15:11:14 +10:00
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# file:// URI support
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestFileUriSupport:
|
|
|
|
|
"""Verify that file:// URIs resolve as local file paths."""
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_file_uri_resolved_as_local_path(self, tmp_path):
|
|
|
|
|
"""file:///absolute/path should be treated as a local file."""
|
|
|
|
|
img = tmp_path / "photo.png"
|
|
|
|
|
img.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 8)
|
|
|
|
|
|
|
|
|
|
mock_response = MagicMock()
|
|
|
|
|
mock_choice = MagicMock()
|
|
|
|
|
mock_choice.message.content = "A test image"
|
|
|
|
|
mock_response.choices = [mock_choice]
|
|
|
|
|
|
|
|
|
|
with (
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools._image_to_base64_data_url",
|
|
|
|
|
return_value="data:image/png;base64,abc",
|
|
|
|
|
),
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools.async_call_llm",
|
|
|
|
|
new_callable=AsyncMock,
|
|
|
|
|
return_value=mock_response,
|
|
|
|
|
),
|
|
|
|
|
):
|
|
|
|
|
result = await vision_analyze_tool(
|
|
|
|
|
f"file://{img}", "describe this", "test/model"
|
|
|
|
|
)
|
|
|
|
|
data = json.loads(result)
|
|
|
|
|
assert data["success"] is True
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_file_uri_nonexistent_gives_error(self, tmp_path):
|
|
|
|
|
"""file:// pointing to a missing file should fail gracefully."""
|
|
|
|
|
result = await vision_analyze_tool(
|
|
|
|
|
f"file://{tmp_path}/nonexistent.png", "describe this", "test/model"
|
|
|
|
|
)
|
|
|
|
|
data = json.loads(result)
|
|
|
|
|
assert data["success"] is False
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# Base64 size pre-flight check
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestBase64SizeLimit:
|
|
|
|
|
"""Verify that oversized images are rejected before hitting the API."""
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_oversized_image_rejected_before_api_call(self, tmp_path):
|
2026-04-11 11:07:18 -07:00
|
|
|
"""Images exceeding the 20 MB hard limit should fail with a clear error."""
|
2026-04-10 15:11:14 +10:00
|
|
|
img = tmp_path / "huge.png"
|
|
|
|
|
img.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * (4 * 1024 * 1024))
|
|
|
|
|
|
2026-04-11 11:07:18 -07:00
|
|
|
# Patch the hard limit to a small value so the test runs fast.
|
|
|
|
|
with patch("tools.vision_tools._MAX_BASE64_BYTES", 1000), \
|
|
|
|
|
patch("tools.vision_tools.async_call_llm", new_callable=AsyncMock) as mock_llm:
|
2026-04-10 15:11:14 +10:00
|
|
|
result = json.loads(await vision_analyze_tool(str(img), "describe this"))
|
|
|
|
|
|
|
|
|
|
assert result["success"] is False
|
|
|
|
|
assert "too large" in result["error"].lower()
|
|
|
|
|
mock_llm.assert_not_awaited()
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_small_image_not_rejected(self, tmp_path):
|
|
|
|
|
"""Images well under the limit should pass the size check."""
|
|
|
|
|
img = tmp_path / "small.png"
|
|
|
|
|
img.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 64)
|
|
|
|
|
|
|
|
|
|
mock_response = MagicMock()
|
|
|
|
|
mock_choice = MagicMock()
|
|
|
|
|
mock_choice.message.content = "Small image"
|
|
|
|
|
mock_response.choices = [mock_choice]
|
|
|
|
|
|
|
|
|
|
with (
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools.async_call_llm",
|
|
|
|
|
new_callable=AsyncMock,
|
|
|
|
|
return_value=mock_response,
|
|
|
|
|
),
|
|
|
|
|
):
|
|
|
|
|
result = json.loads(await vision_analyze_tool(str(img), "describe this", "test/model"))
|
|
|
|
|
|
|
|
|
|
assert result["success"] is True
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# Error classification for 400 responses
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestErrorClassification:
|
|
|
|
|
"""Verify that API 400 errors produce actionable guidance."""
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_invalid_request_error_gives_image_guidance(self, tmp_path):
|
|
|
|
|
"""An invalid_request_error from the API should mention image size/format."""
|
|
|
|
|
img = tmp_path / "test.png"
|
|
|
|
|
img.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 8)
|
|
|
|
|
|
|
|
|
|
api_error = Exception(
|
|
|
|
|
"Error code: 400 - {'type': 'error', 'error': "
|
|
|
|
|
"{'type': 'invalid_request_error', 'message': 'Invalid request data'}}"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
with (
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools._image_to_base64_data_url",
|
|
|
|
|
return_value="data:image/png;base64,abc",
|
|
|
|
|
),
|
|
|
|
|
patch(
|
|
|
|
|
"tools.vision_tools.async_call_llm",
|
|
|
|
|
new_callable=AsyncMock,
|
|
|
|
|
side_effect=api_error,
|
|
|
|
|
),
|
|
|
|
|
):
|
|
|
|
|
result = json.loads(await vision_analyze_tool(str(img), "describe", "test/model"))
|
|
|
|
|
|
|
|
|
|
assert result["success"] is False
|
|
|
|
|
assert "rejected the image" in result["analysis"].lower()
|
|
|
|
|
assert "smaller" in result["analysis"].lower()
|
|
|
|
|
|
|
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
class TestVisionRegistration:
|
|
|
|
|
def test_vision_analyze_registered(self):
|
|
|
|
|
from tools.registry import registry
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
entry = registry._tools.get("vision_analyze")
|
|
|
|
|
assert entry is not None
|
|
|
|
|
assert entry.toolset == "vision"
|
|
|
|
|
assert entry.is_async is True
|
|
|
|
|
|
|
|
|
|
def test_schema_has_required_fields(self):
|
|
|
|
|
from tools.registry import registry
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
entry = registry._tools.get("vision_analyze")
|
|
|
|
|
schema = entry.schema
|
|
|
|
|
assert schema["name"] == "vision_analyze"
|
|
|
|
|
params = schema.get("parameters", {})
|
|
|
|
|
props = params.get("properties", {})
|
|
|
|
|
assert "image_url" in props
|
|
|
|
|
assert "question" in props
|
|
|
|
|
|
|
|
|
|
def test_handler_is_callable(self):
|
|
|
|
|
from tools.registry import registry
|
2026-03-10 16:03:19 -04:00
|
|
|
|
2026-03-09 15:32:02 -07:00
|
|
|
entry = registry._tools.get("vision_analyze")
|
|
|
|
|
assert callable(entry.handler)
|
2026-04-11 11:07:18 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# _resize_image_for_vision — auto-resize oversized images
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestResizeImageForVision:
|
|
|
|
|
"""Tests for the auto-resize function."""
|
|
|
|
|
|
|
|
|
|
def test_small_image_returned_as_is(self, tmp_path):
|
|
|
|
|
"""Images under the limit should be returned unchanged."""
|
|
|
|
|
# Create a small 10x10 red PNG
|
|
|
|
|
try:
|
|
|
|
|
from PIL import Image
|
|
|
|
|
except ImportError:
|
|
|
|
|
pytest.skip("Pillow not installed")
|
|
|
|
|
img = Image.new("RGB", (10, 10), (255, 0, 0))
|
|
|
|
|
path = tmp_path / "small.png"
|
|
|
|
|
img.save(path, "PNG")
|
|
|
|
|
|
|
|
|
|
result = _resize_image_for_vision(path, mime_type="image/png")
|
|
|
|
|
assert result.startswith("data:image/png;base64,")
|
|
|
|
|
assert len(result) < _MAX_BASE64_BYTES
|
|
|
|
|
|
|
|
|
|
def test_large_image_is_resized(self, tmp_path):
|
|
|
|
|
"""Images over the default target should be auto-resized to fit."""
|
|
|
|
|
try:
|
|
|
|
|
from PIL import Image
|
|
|
|
|
except ImportError:
|
|
|
|
|
pytest.skip("Pillow not installed")
|
|
|
|
|
# Create a large image that will exceed 5 MB in base64
|
|
|
|
|
# A 4000x4000 uncompressed PNG will be large
|
|
|
|
|
img = Image.new("RGB", (4000, 4000), (128, 200, 50))
|
|
|
|
|
path = tmp_path / "large.png"
|
|
|
|
|
img.save(path, "PNG")
|
|
|
|
|
|
|
|
|
|
result = _resize_image_for_vision(path, mime_type="image/png")
|
|
|
|
|
assert result.startswith("data:image/png;base64,")
|
|
|
|
|
# Default target is _RESIZE_TARGET_BYTES (5 MB), not _MAX_BASE64_BYTES (20 MB)
|
|
|
|
|
assert len(result) <= _RESIZE_TARGET_BYTES
|
|
|
|
|
|
|
|
|
|
def test_custom_max_bytes(self, tmp_path):
|
|
|
|
|
"""The max_base64_bytes parameter should be respected."""
|
|
|
|
|
try:
|
|
|
|
|
from PIL import Image
|
|
|
|
|
except ImportError:
|
|
|
|
|
pytest.skip("Pillow not installed")
|
|
|
|
|
img = Image.new("RGB", (200, 200), (0, 128, 255))
|
|
|
|
|
path = tmp_path / "medium.png"
|
|
|
|
|
img.save(path, "PNG")
|
|
|
|
|
|
|
|
|
|
# Set a very low limit to force resizing
|
|
|
|
|
result = _resize_image_for_vision(path, max_base64_bytes=500)
|
|
|
|
|
# Should still return a valid data URL
|
|
|
|
|
assert result.startswith("data:image/")
|
|
|
|
|
|
|
|
|
|
def test_jpeg_output_for_non_png(self, tmp_path):
|
|
|
|
|
"""Non-PNG images should be resized as JPEG."""
|
|
|
|
|
try:
|
|
|
|
|
from PIL import Image
|
|
|
|
|
except ImportError:
|
|
|
|
|
pytest.skip("Pillow not installed")
|
|
|
|
|
img = Image.new("RGB", (2000, 2000), (255, 128, 0))
|
|
|
|
|
path = tmp_path / "photo.jpg"
|
|
|
|
|
img.save(path, "JPEG", quality=95)
|
|
|
|
|
|
|
|
|
|
result = _resize_image_for_vision(path, mime_type="image/jpeg",
|
|
|
|
|
max_base64_bytes=50_000)
|
|
|
|
|
assert result.startswith("data:image/jpeg;base64,")
|
|
|
|
|
|
|
|
|
|
def test_constants_sane(self):
|
|
|
|
|
"""Hard limit should be larger than resize target."""
|
|
|
|
|
assert _MAX_BASE64_BYTES == 20 * 1024 * 1024
|
|
|
|
|
assert _RESIZE_TARGET_BYTES == 5 * 1024 * 1024
|
|
|
|
|
assert _MAX_BASE64_BYTES > _RESIZE_TARGET_BYTES
|
|
|
|
|
|
|
|
|
|
def test_no_pillow_returns_original(self, tmp_path):
|
|
|
|
|
"""Without Pillow, oversized images should be returned as-is."""
|
|
|
|
|
# Create a dummy file
|
|
|
|
|
path = tmp_path / "test.png"
|
|
|
|
|
# Write enough bytes to exceed a tiny limit
|
|
|
|
|
path.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 1000)
|
|
|
|
|
|
|
|
|
|
with patch("tools.vision_tools._image_to_base64_data_url") as mock_b64:
|
|
|
|
|
# Simulate a large base64 result
|
|
|
|
|
mock_b64.return_value = "data:image/png;base64," + "A" * 200
|
|
|
|
|
with patch.dict("sys.modules", {"PIL": None, "PIL.Image": None}):
|
|
|
|
|
result = _resize_image_for_vision(path, max_base64_bytes=100)
|
|
|
|
|
# Should return the original (oversized) data url
|
|
|
|
|
assert len(result) > 100
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
# _is_image_size_error — detect size-related API errors
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class TestIsImageSizeError:
|
|
|
|
|
"""Tests for the size-error detection helper."""
|
|
|
|
|
|
|
|
|
|
def test_too_large_message(self):
|
|
|
|
|
assert _is_image_size_error(Exception("Request payload too large"))
|
|
|
|
|
|
|
|
|
|
def test_413_status(self):
|
|
|
|
|
assert _is_image_size_error(Exception("HTTP 413 Payload Too Large"))
|
|
|
|
|
|
|
|
|
|
def test_invalid_request(self):
|
|
|
|
|
assert _is_image_size_error(Exception("invalid_request_error: image too big"))
|
|
|
|
|
|
|
|
|
|
def test_exceeds_limit(self):
|
|
|
|
|
assert _is_image_size_error(Exception("Image exceeds maximum size"))
|
|
|
|
|
|
|
|
|
|
def test_unrelated_error(self):
|
|
|
|
|
assert not _is_image_size_error(Exception("Connection refused"))
|
|
|
|
|
|
|
|
|
|
def test_auth_error(self):
|
|
|
|
|
assert not _is_image_size_error(Exception("401 Unauthorized"))
|
|
|
|
|
|
|
|
|
|
def test_empty_message(self):
|
|
|
|
|
assert not _is_image_size_error(Exception(""))
|