spark: url_title_extractor - Python script to extract HTML page titles from URLs with caching support

wizard-work/spark/test_url_title_extractor.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""Test the url_title_extractor module."""
+
+import sys
+sys.path.insert(0, "/root/.hermes/wizard-outputs/spark")
+
+from url_title_extractor import extract_title, TitleParser
+import json
+from pathlib import Path
+
+def test_title_parser():
+    """Test basic HTML title extraction."""
+    parser = TitleParser()
+    parser.feed("<html><head><title>Python Testing</title></head><body>Content</body></html>")
+    assert parser.title == "Python Testing", f"Expected 'Python Testing', got '{parser.title}'"
+    return True
+
+def test_cache_file_creation():
+    """Test that cache file location is writable."""
+    cache_dir = Path("/root/.hermes/wizard-outputs/spark")
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    return cache_dir.exists() and cache_dir.is_dir()
+
+if __name__ == "__main__":
+    tests = [
+        ("TitleParser", test_title_parser),
+        ("Cache directory", test_cache_file_creation),
+    ]
+    
+    results = []
+    for name, test_fn in tests:
+        try:
+            passed = test_fn()
+            results.append({"test": name, "passed": passed, "error": None})
+            print(f"PASSED: {name}")
+        except Exception as e:
+            results.append({"test": name, "passed": False, "error": str(e)})
+            print(f"FAILED: {name} - {e}")
+    
+    # Print summary
+    passed_count = sum(1 for r in results if r["passed"])
+    print(f"\nSummary: {passed_count}/{len(results)} tests passed")
+    sys.exit(0 if passed_count == len(results) else 1)

wizard-work/spark/url_title_extractor.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+"""Extract page titles from URLs with optional caching."""
+
+import sys
+import json
+import argparse
+from pathlib import Path
+from urllib.parse import urlparse
+from html.parser import HTMLParser
+
+try:
+    import requests
+    HAS_REQUESTS = True
+except ImportError:
+    HAS_REQUESTS = False
+
+CACHE_FILE = Path("/root/.hermes/wizard-outputs/spark/title_cache.json")
+
+class TitleParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.in_title = False
+        self.title = ""
+    
+    def handle_starttag(self, tag, attrs):
+        if tag.lower() == "title":
+            self.in_title = True
+    
+    def handle_endtag(self, tag):
+        if tag.lower() == "title":
+            self.in_title = False
+    
+    def handle_data(self, data):
+        if self.in_title:
+            self.title += data
+            self.in_title = False  # Only take text directly in title tag
+
+def extract_title(url: str, use_cache: bool = True) -> dict:
+    """Extract page title from URL, with optional caching."""
+    cache_key = url
+    
+    # Check cache
+    if use_cache and CACHE_FILE.exists():
+        cache = json.loads(CACHE_FILE.read_text())
+        if cache_key in cache:
+            return {"url": url, "title": cache[cache_key], "cached": True}
+    
+    if not HAS_REQUESTS:
+        return {"url": url, "error": "requests library not available", "cached": False}
+    
+    try:
+        resp = requests.get(url, timeout=10, headers={"User-Agent": "Hermes-Spark/1.0"})
+        resp.raise_for_status()
+        parser = TitleParser()
+        parser.feed(resp.text)
+        title = parser.title.strip() or "(no title found)"
+        
+        # Update cache
+        if use_cache:
+            cache = json.loads(CACHE_FILE.read_text()) if CACHE_FILE.exists() else {}
+            cache[cache_key] = title
+            CACHE_FILE.write_text(json.dumps(cache, indent=2))
+        
+        return {"url": url, "title": title, "cached": False}
+    except Exception as e:
+        return {"url": url, "error": str(e), "cached": False}
+
+def main():
+    parser = argparse.ArgumentParser(description="Extract page titles from URLs")
+    parser.add_argument("urls", nargs="*", help="URLs to extract titles from")
+    parser.add_argument("--clear-cache", action="store_true", help="Clear the title cache")
+    parser.add_argument("--no-cache", action="store_true", help="Disable caching")
+    args = parser.parse_args()
+    
+    if args.clear_cache and CACHE_FILE.exists():
+        CACHE_FILE.unlink()
+        print(json.dumps({"cleared_cache": str(CACHE_FILE)}, indent=2))
+        return
+    
+    results = []
+    for url in args.urls:
+        result = extract_title(url, use_cache=not args.no_cache)
+        results.append(result)
+        print(json.dumps(result, indent=2))
+    
+    if not results:
+        parser.print_help()
+
+if __name__ == "__main__":
+    main()