spark: add URL extraction and validation utility

wizard-work/spark/url_extractor.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""URL extraction and validation utility."""
+
+import re
+from urllib.parse import urlparse
+from typing import List
+
+
+def extract_and_validate_urls(text: str) -> List[str]:
+    """
+    Extract all valid HTTP/HTTPS URLs from a text string.
+    Returns a list of URLs with valid http/https schemes and proper structure.
+    
+    Args:
+        text: Input text to search for URLs
+        
+    Returns:
+        List of validated URL strings
+    """
+    # Regex pattern to match URLs
+    url_pattern = r'https?://[^\s<>"{}|\\^`\[\]]+'
+    
+    matches = re.findall(url_pattern, text)
+    
+    valid_urls = []
+    for match in matches:
+        try:
+            parsed = urlparse(match)
+            # Require http/https scheme AND a netloc (hostname)
+            if parsed.scheme in ('http', 'https') and parsed.netloc:
+                valid_urls.append(match.rstrip('.,;:!?)'))
+        except Exception:
+            continue
+    
+    return valid_urls
+
+
+if __name__ == "__main__":
+    test_text = '''
+    Check out https://example.com and http://test.org for info.
+    Invalid: ftp://bad.com and malformed https://  and no-scheme://test.com
+    Also visit https://docs.python.org/3/library/re.html#module-re
+    '''
+    
+    urls = extract_and_validate_urls(test_text)
+    print("Extracted URLs:", urls)
+    
+    # Verify expected results - only valid http/https URLs
+    expected = ['https://example.com', 'http://test.org', 'https://docs.python.org/3/library/re.html#module-re']
+    assert urls == expected, f"Mismatch: got {urls}, expected {expected}"
+    print("VERIFICATION PASSED")