spark: add verified JSON-to-CSV converter utility with test data

wizard-work/spark/json_to_csv.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""
+JSON to CSV Converter - A simple utility to convert JSON files to CSV format.
+Usage: python3 json_to_csv.py <input.json> <output.csv>
+"""
+
+import json
+import csv
+import sys
+from pathlib import Path
+
+
+def flatten_dict(d, parent_key='', sep='_'):
+    """Flatten nested dictionary for CSV compatibility."""
+    items = []
+    for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.extend(flatten_dict(v, new_key, sep).items())
+        elif isinstance(v, list):
+            items.append((new_key, json.dumps(v)))
+        else:
+            items.append((new_key, v))
+    return dict(items)
+
+
+def convert_json_to_csv(json_path, csv_path):
+    """Convert JSON file to CSV."""
+    with open(json_path, 'r') as f:
+        data = json.load(f)
+
+    if isinstance(data, list):
+        records = [flatten_dict(record) for record in data] if data else []
+    elif isinstance(data, dict):
+        records = [flatten_dict(data)]
+    else:
+        raise ValueError("JSON must contain objects or array of objects")
+
+    if not records:
+        with open(csv_path, 'w') as f:
+            pass
+        return
+
+    fieldnames = sorted(set(key for record in records for key in record.keys()))
+
+    with open(csv_path, 'w', newline='') as f:
+        writer = csv.DictWriter(f, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(records)
+
+    return {
+        'input_path': str(json_path),
+        'output_path': str(csv_path),
+        'records_processed': len(records),
+        'columns': fieldnames
+    }
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 3:
+        print("Usage: python3 json_to_csv.py <input.json> <output.csv>")
+        sys.exit(1)
+
+    result = convert_json_to_csv(sys.argv[1], sys.argv[2])
+    print(json.dumps(result, indent=2))

wizard-work/spark/output.csv

@@ -0,0 +1,3 @@
+active,email,id,metadata_department,metadata_role,name,tags
+True,alice@example.com,1,Engineering,Senior Developer,Alice Smith,"[""python"", ""rust"", ""ai""]"
+False,bob@example.com,2,Marketing,Manager,Bob Jones,"[""social"", ""analytics""]"

wizard-work/spark/sample_data.json

@@ -0,0 +1,24 @@
+[
+  {
+    "id": 1,
+    "name": "Alice Smith",
+    "email": "alice@example.com",
+    "active": true,
+    "metadata": {
+      "department": "Engineering",
+      "role": "Senior Developer"
+    },
+    "tags": ["python", "rust", "ai"]
+  },
+  {
+    "id": 2,
+    "name": "Bob Jones",
+    "email": "bob@example.com",
+    "active": false,
+    "metadata": {
+      "department": "Marketing",
+      "role": "Manager"
+    },
+    "tags": ["social", "analytics"]
+  }
+]

wizard-work/spark/verify_csv.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""Verify CSV output is valid."""
+
+import csv
+import json
+
+# Verify CSV is valid
+with open('/root/.hermes/wizard-outputs/spark/output.csv', 'r') as f:
+    reader = csv.DictReader(f)
+    rows = list(reader)
+
+assert len(rows) == 2, f"Expected 2 rows, got {len(rows)}"
+assert 'name' in rows[0], "Missing 'name' column"
+assert rows[0]['name'] == 'Alice Smith', f"Wrong name: {rows[0]['name']}"
+
+# Verify the JSON source was correctly processed
+with open('/root/.hermes/wizard-outputs/spark/sample_data.json', 'r') as f:
+    source = json.load(f)
+
+assert len(source) == 2, "Source JSON should have 2 records"
+
+print("PASS: CSV is valid with correct data structure")
+print(f"Records: {len(rows)}")
+print(f"Columns: {sorted(rows[0].keys())}")