90 lines
2.8 KiB
Python
90 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract page titles from URLs with optional caching."""
|
|
|
|
import sys
|
|
import json
|
|
import argparse
|
|
from pathlib import Path
|
|
from urllib.parse import urlparse
|
|
from html.parser import HTMLParser
|
|
|
|
try:
|
|
import requests
|
|
HAS_REQUESTS = True
|
|
except ImportError:
|
|
HAS_REQUESTS = False
|
|
|
|
CACHE_FILE = Path("/root/.hermes/wizard-outputs/spark/title_cache.json")
|
|
|
|
class TitleParser(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.in_title = False
|
|
self.title = ""
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag.lower() == "title":
|
|
self.in_title = True
|
|
|
|
def handle_endtag(self, tag):
|
|
if tag.lower() == "title":
|
|
self.in_title = False
|
|
|
|
def handle_data(self, data):
|
|
if self.in_title:
|
|
self.title += data
|
|
self.in_title = False # Only take text directly in title tag
|
|
|
|
def extract_title(url: str, use_cache: bool = True) -> dict:
|
|
"""Extract page title from URL, with optional caching."""
|
|
cache_key = url
|
|
|
|
# Check cache
|
|
if use_cache and CACHE_FILE.exists():
|
|
cache = json.loads(CACHE_FILE.read_text())
|
|
if cache_key in cache:
|
|
return {"url": url, "title": cache[cache_key], "cached": True}
|
|
|
|
if not HAS_REQUESTS:
|
|
return {"url": url, "error": "requests library not available", "cached": False}
|
|
|
|
try:
|
|
resp = requests.get(url, timeout=10, headers={"User-Agent": "Hermes-Spark/1.0"})
|
|
resp.raise_for_status()
|
|
parser = TitleParser()
|
|
parser.feed(resp.text)
|
|
title = parser.title.strip() or "(no title found)"
|
|
|
|
# Update cache
|
|
if use_cache:
|
|
cache = json.loads(CACHE_FILE.read_text()) if CACHE_FILE.exists() else {}
|
|
cache[cache_key] = title
|
|
CACHE_FILE.write_text(json.dumps(cache, indent=2))
|
|
|
|
return {"url": url, "title": title, "cached": False}
|
|
except Exception as e:
|
|
return {"url": url, "error": str(e), "cached": False}
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Extract page titles from URLs")
|
|
parser.add_argument("urls", nargs="*", help="URLs to extract titles from")
|
|
parser.add_argument("--clear-cache", action="store_true", help="Clear the title cache")
|
|
parser.add_argument("--no-cache", action="store_true", help="Disable caching")
|
|
args = parser.parse_args()
|
|
|
|
if args.clear_cache and CACHE_FILE.exists():
|
|
CACHE_FILE.unlink()
|
|
print(json.dumps({"cleared_cache": str(CACHE_FILE)}, indent=2))
|
|
return
|
|
|
|
results = []
|
|
for url in args.urls:
|
|
result = extract_title(url, use_cache=not args.no_cache)
|
|
results.append(result)
|
|
print(json.dumps(result, indent=2))
|
|
|
|
if not results:
|
|
parser.print_help()
|
|
|
|
if __name__ == "__main__":
|
|
main() |