implement first pass of scrape/crawl content compression

This commit is contained in:
Teknium
2025-07-31 10:11:27 -07:00
parent 1dacd941f6
commit bf4223f381
4 changed files with 272 additions and 45 deletions

View File

@@ -62,7 +62,7 @@ def get_web_tool_definitions() -> List[Dict[str, Any]]:
"type": "function",
"function": {
"name": "web_extract",
"description": "Extract and read the full content from specific web page URLs. Useful for getting detailed information from webpages found through search.",
"description": "Extract and read the full content from specific web page URLs. Useful for getting detailed information from webpages found through search. The content returned will be excerpts and key points summarized with an LLM to reduce impact on the context window.",
"parameters": {
"type": "object",
"properties": {
@@ -86,7 +86,7 @@ def get_web_tool_definitions() -> List[Dict[str, Any]]:
"type": "function",
"function": {
"name": "web_crawl",
"description": "Crawl a website with specific instructions to find and extract targeted content. Uses AI to intelligently navigate and extract relevant information from across the site.",
"description": "Crawl a website with specific instructions to find and extract targeted content. Uses AI to intelligently navigate and extract relevant information from across the site. The content returned will be excerpts and key points summarized with an LLM to reduce impact on the context window.",
"parameters": {
"type": "object",
"properties": {