spark: add word frequency counter utility with verification tests

wizard-work/spark/test_wordfreq_counter.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+"""Verification test for wordfreq_counter.py"""
+import json
+import sys
+from pathlib import Path
+
+# Import the module
+sys.path.insert(0, '/root/.hermes/wizard-outputs/spark')
+from wordfreq_counter import count_words, process_file
+
+def test_count_words():
+    """Test basic word counting."""
+    result = count_words("hello hello world")
+    assert result == {"hello": 2, "world": 1}, f"Expected hello:2, world:1, got {result}"
+    print("test_count_words: PASS")
+
+def test_process_file():
+    """Test file processing."""
+    result = process_file('/root/.hermes/wizard-outputs/spark/test_input.txt')
+    assert result['total_words'] == 24, f"Expected 24 words, got {result['total_words']}"
+    assert result['unique_words'] == 18, f"Expected 18 unique words, got {result['unique_words']}"
+    assert result['word_counts']['the'] == 4, f"Expected 'the': 4, got {result['word_counts'].get('the')}"
+    print("test_process_file: PASS")
+
+def test_output_format():
+    """Test JSON output format."""
+    result = process_file('/root/.hermes/wizard-outputs/spark/test_input.txt')
+    required_keys = {'file', 'total_words', 'unique_words', 'word_counts'}
+    assert required_keys.issubset(result.keys()), f"Missing keys: {required_keys - result.keys()}"
+    print("test_output_format: PASS")
+
+if __name__ == "__main__":
+    test_count_words()
+    test_process_file()
+    test_output_format()
+    print("\nAll tests passed!")

wizard-work/spark/wordfreq_counter.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""
+Word Frequency Counter - counts words in a file and outputs JSON report.
+Verifiable: deterministic output for given input.
+"""
+import json
+import sys
+from collections import Counter
+from pathlib import Path
+
+
+def count_words(text: str) -> dict:
+    """Count word frequencies in text."""
+    words = text.lower().split()
+    return dict(Counter(words))
+
+
+def process_file(input_path: str, output_path: str = None) -> dict:
+    """Process a file and return word frequency counts."""
+    path = Path(input_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Input file not found: {input_path}")
+    
+    content = path.read_text()
+    counts = count_words(content)
+    
+    result = {
+        "file": str(path),
+        "total_words": sum(counts.values()),
+        "unique_words": len(counts),
+        "word_counts": counts
+    }
+    
+    if output_path:
+        Path(output_path).write_text(json.dumps(result, indent=2))
+    
+    return result
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: wordfreq_counter.py <input_file> [output_file]", file=sys.stderr)
+        sys.exit(1)
+    
+    output = process_file(sys.argv[1], sys.argv[2] if len(sys.argv) > 2 else None)
+    print(json.dumps(output, indent=2))
+    return output
+
+
+if __name__ == "__main__":
+    main()