diff --git a/.gitignore b/.gitignore index a57ed39e..949cc5db 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ /venv/ -/_pycache/ \ No newline at end of file +/_pycache/ +hecate/ +hecate-lib/ +*.pyc* \ No newline at end of file diff --git a/model_tools.py b/model_tools.py index 6a6c59f6..65580d7c 100644 --- a/model_tools.py +++ b/model_tools.py @@ -62,7 +62,7 @@ def get_web_tool_definitions() -> List[Dict[str, Any]]: "type": "function", "function": { "name": "web_extract", - "description": "Extract and read the full content from specific web page URLs. Useful for getting detailed information from webpages found through search.", + "description": "Extract and read the full content from specific web page URLs. Useful for getting detailed information from webpages found through search. The content returned will be excerpts and key points summarized with an LLM to reduce impact on the context window.", "parameters": { "type": "object", "properties": { @@ -86,7 +86,7 @@ def get_web_tool_definitions() -> List[Dict[str, Any]]: "type": "function", "function": { "name": "web_crawl", - "description": "Crawl a website with specific instructions to find and extract targeted content. Uses AI to intelligently navigate and extract relevant information from across the site.", + "description": "Crawl a website with specific instructions to find and extract targeted content. Uses AI to intelligently navigate and extract relevant information from across the site. The content returned will be excerpts and key points summarized with an LLM to reduce impact on the context window.", "parameters": { "type": "object", "properties": { diff --git a/test_run.sh b/test_run.sh new file mode 100644 index 00000000..424333f7 --- /dev/null +++ b/test_run.sh @@ -0,0 +1,6 @@ +python run_agent.py \ + --query "search up the latest docs on huggingface datasets in python 3.13 and write me basic example that's not in their docs. profile its performance" \ + --max_turns 30 \ + --model claude-sonnet-4-20250514 \ + --base_url https://api.anthropic.com/v1/ \ + --api_key $ANTHROPIC_API_KEY \ No newline at end of file diff --git a/web_tools.py b/web_tools.py index 07d11b79..71b2bb22 100644 --- a/web_tools.py +++ b/web_tools.py @@ -15,6 +15,10 @@ Backend compatibility: - Tavily: https://docs.tavily.com/ - Firecrawl: https://docs.firecrawl.dev/features/search +LLM Processing: +- Uses Nous Research API with Gemini 2.5 Flash for intelligent content extraction +- Extracts key excerpts and creates markdown summaries to reduce token usage + Usage: from web_tools import web_search_tool, web_extract_tool, web_crawl_tool @@ -35,12 +39,110 @@ Usage: import json import os import re -from typing import List +import asyncio +from typing import List, Dict, Any, Optional from tavily import TavilyClient +from openai import AsyncOpenAI # Initialize Tavily client once at module level tavily_client = TavilyClient(api_key=os.getenv("TAVILY_API_KEY")) +# Initialize Nous Research API client for LLM processing (async) +nous_client = AsyncOpenAI( + api_key=os.getenv("NOUS_API_KEY"), + base_url="https://inference-api.nousresearch.com/v1" +) + +# Configuration for LLM processing +DEFAULT_SUMMARIZER_MODEL = "gemini-2.5-flash" +DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION = 5000 + + +async def process_content_with_llm( + content: str, + url: str = "", + title: str = "", + model: str = DEFAULT_SUMMARIZER_MODEL, + min_length: int = DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION +) -> Optional[str]: + """ + Process web content using LLM to create intelligent summaries with key excerpts. + + This function uses Gemini 2.5 Flash (or specified model) via Nous Research API + to intelligently extract key information and create markdown summaries, + significantly reducing token usage while preserving all important information. + + Args: + content (str): The raw content to process + url (str): The source URL (for context, optional) + title (str): The page title (for context, optional) + model (str): The model to use for processing (default: gemini-2.5-flash) + min_length (int): Minimum content length to trigger processing (default: 5000) + + Returns: + Optional[str]: Processed markdown content, or None if content too short or processing fails + """ + try: + # Skip processing if content is too short + if len(content) < min_length: + print(f"šŸ“ Content too short ({len(content)} < {min_length} chars), skipping LLM processing") + return None + + print(f"šŸ§  Processing content with LLM ({len(content)} characters)") + + # Create context information + context_info = [] + if title: + context_info.append(f"Title: {title}") + if url: + context_info.append(f"Source: {url}") + + context_str = "\n".join(context_info) + "\n\n" if context_info else "" + + # Simplified prompt for better quality markdown output + system_prompt = """You are an expert content analyst. Your job is to process web content and create a comprehensive yet concise summary that preserves all important information while dramatically reducing bulk. + +Create a well-structured markdown summary that includes: +1. Key excerpts (quotes, code snippets, important facts) in their original format +2. Comprehensive summary of all other important information +3. Proper markdown formatting with headers, bullets, and emphasis + +Your goal is to preserve ALL important information while reducing length. Never lose key facts, figures, insights, or actionable information. Make it scannable and well-organized.""" + + user_prompt = f"""Please process this web content and create a comprehensive markdown summary: + +{context_str}CONTENT TO PROCESS: +{content} + +Create a markdown summary that captures all key information in a well-organized, scannable format. Include important quotes and code snippets in their original formatting. Focus on actionable information, specific details, and unique insights.""" + + # Call the LLM asynchronously + response = await nous_client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt} + ], + temperature=0.1, # Low temperature for consistent extraction + max_tokens=4000 # Generous limit for comprehensive processing + ) + + # Get the markdown response directly + processed_content = response.choices[0].message.content.strip() + + # Calculate compression metrics for logging + original_length = len(content) + processed_length = len(processed_content) + compression_ratio = processed_length / original_length if original_length > 0 else 1.0 + + print(f"āœ… Content processed: {original_length} ā†’ {processed_length} chars ({compression_ratio:.1%})") + + return processed_content + + except Exception as e: + print(f"āŒ Error processing content with LLM: {str(e)}") + return None + def clean_base64_images(text: str) -> str: """ @@ -82,6 +184,8 @@ def web_search_tool(query: str, limit: int = 5) -> str: This function provides a generic interface for web search that can work with multiple backends. Currently uses Tavily but can be easily swapped. + Note: Search results are already concise snippets, so no LLM processing is applied. + Args: query (str): The search query to look up limit (int): Maximum number of results to return (default: 5) @@ -111,6 +215,7 @@ def web_search_tool(query: str, limit: int = 5) -> str: response = tavily_client.search(query=query, max_results=limit, search_depth="advanced") print(f"āœ… Found {len(response.get('results', []))} results") + result_json = json.dumps(response, indent=2) # Clean base64 images from search results return clean_base64_images(result_json) @@ -121,7 +226,13 @@ def web_search_tool(query: str, limit: int = 5) -> str: return json.dumps({"error": error_msg}) -def web_extract_tool(urls: List[str], format: str = None) -> str: +async def web_extract_tool( + urls: List[str], + format: str = None, + use_llm_processing: bool = True, + model: str = DEFAULT_SUMMARIZER_MODEL, + min_length: int = DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION +) -> str: """ Extract content from specific web pages using available extraction API backend. @@ -131,20 +242,13 @@ def web_extract_tool(urls: List[str], format: str = None) -> str: Args: urls (List[str]): List of URLs to extract content from format (str): Desired output format ("markdown" or "html", optional) + use_llm_processing (bool): Whether to process content with LLM for summarization (default: True) + model (str): The model to use for LLM processing (default: gemini-2.5-flash) + min_length (int): Minimum content length to trigger LLM processing (default: 5000) Returns: - str: JSON string containing extracted content with the following structure: - { - "results": [ - { - "url": str, - "title": str, - "raw_content": str, - "content": str - }, - ... - ] - } + str: JSON string containing extracted content. If LLM processing is enabled and successful, + the 'content' field will contain the processed markdown summary instead of raw content. Raises: Exception: If extraction fails or API key is not set @@ -157,11 +261,40 @@ def web_extract_tool(urls: List[str], format: str = None) -> str: print(f"āœ… Extracted content from {len(response.get('results', []))} pages") - # Print summary of extracted pages for debugging - for result in response.get('results', []): - url = result.get('url', 'Unknown URL') - content_length = len(result.get('raw_content', '')) - print(f" šŸ“ {url} ({content_length} characters)") + # Process each result with LLM if enabled + if use_llm_processing and os.getenv("NOUS_API_KEY"): + print("šŸ§  Processing extracted content with LLM...") + + for result in response.get('results', []): + url = result.get('url', 'Unknown URL') + title = result.get('title', '') + raw_content = result.get('raw_content', '') or result.get('content', '') + + if raw_content: + # Process content with LLM + processed = await process_content_with_llm( + raw_content, url, title, model, min_length + ) + + if processed: + # Replace content with processed version + result['content'] = processed + # Keep raw content in separate field for reference + result['raw_content'] = raw_content + print(f" šŸ“ {url} (processed)") + else: + print(f" šŸ“ {url} (no processing - content too short)") + else: + print(f" āš ļø {url} (no content to process)") + else: + if use_llm_processing and not os.getenv("NOUS_API_KEY"): + print("āš ļø LLM processing requested but NOUS_API_KEY not set, returning raw content") + + # Print summary of extracted pages for debugging (original behavior) + for result in response.get('results', []): + url = result.get('url', 'Unknown URL') + content_length = len(result.get('raw_content', '')) + print(f" šŸ“ {url} ({content_length} characters)") result_json = json.dumps(response, indent=2) # Clean base64 images from extracted content @@ -173,7 +306,14 @@ def web_extract_tool(urls: List[str], format: str = None) -> str: return json.dumps({"error": error_msg}) -def web_crawl_tool(url: str, instructions: str = None, depth: str = "basic") -> str: +async def web_crawl_tool( + url: str, + instructions: str = None, + depth: str = "basic", + use_llm_processing: bool = True, + model: str = DEFAULT_SUMMARIZER_MODEL, + min_length: int = DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION +) -> str: """ Crawl a website with specific instructions using available crawling API backend. @@ -184,19 +324,14 @@ def web_crawl_tool(url: str, instructions: str = None, depth: str = "basic") -> url (str): The base URL to crawl (can include or exclude https://) instructions (str): Instructions for what to crawl/extract using LLM intelligence (optional) depth (str): Depth of extraction ("basic" or "advanced", default: "basic") + use_llm_processing (bool): Whether to process content with LLM for summarization (default: True) + model (str): The model to use for LLM processing (default: gemini-2.5-flash) + min_length (int): Minimum content length to trigger LLM processing (default: 5000) Returns: - str: JSON string containing crawled content with the following structure: - { - "results": [ - { - "url": str, - "title": str, - "content": str - }, - ... - ] - } + str: JSON string containing crawled content. If LLM processing is enabled and successful, + the 'content' field will contain the processed markdown summary instead of raw content. + Each page is processed individually. Raises: Exception: If crawling fails or API key is not set @@ -215,11 +350,40 @@ def web_crawl_tool(url: str, instructions: str = None, depth: str = "basic") -> print(f"āœ… Crawled {len(response.get('results', []))} pages") - # Print summary of crawled pages for debugging - for result in response.get('results', []): - page_url = result.get('url', 'Unknown URL') - content_length = len(result.get('content', '')) - print(f" šŸŒ {page_url} ({content_length} characters)") + # Process each result with LLM if enabled + if use_llm_processing and os.getenv("NOUS_API_KEY"): + print("šŸ§  Processing crawled content with LLM...") + + for result in response.get('results', []): + page_url = result.get('url', 'Unknown URL') + title = result.get('title', '') + content = result.get('content', '') + + if content: + # Process content with LLM + processed = await process_content_with_llm( + content, page_url, title, model, min_length + ) + + if processed: + # Keep original content in raw_content field + result['raw_content'] = content + # Replace content with processed version + result['content'] = processed + print(f" šŸŒ {page_url} (processed)") + else: + print(f" šŸŒ {page_url} (no processing - content too short)") + else: + print(f" āš ļø {page_url} (no content to process)") + else: + if use_llm_processing and not os.getenv("NOUS_API_KEY"): + print("āš ļø LLM processing requested but NOUS_API_KEY not set, returning raw content") + + # Print summary of crawled pages for debugging (original behavior) + for result in response.get('results', []): + page_url = result.get('url', 'Unknown URL') + content_length = len(result.get('content', '')) + print(f" šŸŒ {page_url} ({content_length} characters)") result_json = json.dumps(response, indent=2) # Clean base64 images from crawled content @@ -242,6 +406,16 @@ def check_tavily_api_key() -> bool: return bool(os.getenv("TAVILY_API_KEY")) +def check_nous_api_key() -> bool: + """ + Check if the Nous Research API key is available in environment variables. + + Returns: + bool: True if API key is set, False otherwise + """ + return bool(os.getenv("NOUS_API_KEY")) + + if __name__ == "__main__": """ Simple test/demo when run directly @@ -249,17 +423,61 @@ if __name__ == "__main__": print("šŸŒ Standalone Web Tools Module") print("=" * 40) - # Check if API key is available - if not check_tavily_api_key(): + # Check if API keys are available + tavily_available = check_tavily_api_key() + nous_available = check_nous_api_key() + + if not tavily_available: print("āŒ TAVILY_API_KEY environment variable not set") print("Please set your API key: export TAVILY_API_KEY='your-key-here'") print("Get API key at: https://tavily.com/") + else: + print("āœ… Tavily API key found") + + if not nous_available: + print("āŒ NOUS_API_KEY environment variable not set") + print("Please set your API key: export NOUS_API_KEY='your-key-here'") + print("Get API key at: https://inference-api.nousresearch.com/") + print("āš ļø Without Nous API key, LLM content processing will be disabled") + else: + print("āœ… Nous Research API key found") + + if not tavily_available: exit(1) - print("āœ… Tavily API key found") print("šŸ› ļø Web tools ready for use!") - print("\nExample usage:") + + if nous_available: + print("šŸ§  LLM content processing available with Gemini 2.5 Flash") + print(f" Default min length for processing: {DEFAULT_MIN_LENGTH_FOR_SUMMARIZATION} chars") + + print("\nBasic usage:") print(" from web_tools import web_search_tool, web_extract_tool, web_crawl_tool") + print(" import asyncio") + print("") + print(" # Search (synchronous)") print(" results = web_search_tool('Python tutorials')") - print(" content = web_extract_tool(['https://example.com'])") - print(" crawl_data = web_crawl_tool('example.com', 'Find documentation')") + print("") + print(" # Extract and crawl (asynchronous)") + print(" async def main():") + print(" content = await web_extract_tool(['https://example.com'])") + print(" crawl_data = await web_crawl_tool('example.com', 'Find docs')") + print(" asyncio.run(main())") + + if nous_available: + print("\nLLM-enhanced usage:") + print(" # Content automatically processed for pages >5000 chars (default)") + print(" content = await web_extract_tool(['https://python.org/about/'])") + print("") + print(" # Customize processing parameters") + print(" crawl_data = await web_crawl_tool(") + print(" 'docs.python.org',") + print(" 'Find key concepts',") + print(" model='gemini-2.5-flash',") + print(" min_length=3000") + print(" )") + print("") + print(" # Disable LLM processing") + print(" raw_content = await web_extract_tool(['https://example.com'], use_llm_processing=False)") + + print(f"\nšŸ“ Run 'python test_web_tools_llm.py' to test LLM processing capabilities")