spark: add verification script for json_to_csv converter

wizard-work/spark/check_conversion.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Check that json_to_csv.py correctly converts sample_data.json
+# Returns 0 on success, non-zero on failure
+
+python3 -c 'import sys,json; d=json.load(sys.stdin); exit(0 if d["records_processed"]==2 else 1)' < <(python3 /root/.hermes/wizard-outputs/spark/json_to_csv.py /root/.hermes/wizard-outputs/spark/sample_data.json /tmp/test_out.csv 2>&1)

wizard-work/spark/json_to_csv_verify.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+"""Verification script for json_to_csv.py converter."""
+
+import subprocess
+import json
+import sys
+import os
+
+def verify_conversion():
+    input_file = "/root/.hermes/wizard-outputs/spark/sample_data.json"
+    output_file = "/tmp/test_out.csv"
+    
+    # Run the converter
+    result = subprocess.run(
+        ["python3", "/root/.hermes/wizard-outputs/spark/json_to_csv.py", input_file, output_file],
+        capture_output=True,
+        text=True
+    )
+    
+    if result.returncode != 0:
+        print(f"Conversion failed: {result.stderr}")
+        return False
+    
+    # Parse the output
+    try:
+        output_data = json.loads(result.stdout)
+    except json.JSONDecodeError as e:
+        print(f"Failed to parse output JSON: {e}")
+        return False
+    
+    # Verify records_processed is 2
+    if output_data.get("records_processed") != 2:
+        print(f"Expected 2 records, got {output_data.get('records_processed')}")
+        return False
+    
+    # Verify output file exists and has content
+    if not os.path.exists(output_file):
+        print(f"Output file {output_file} was not created")
+        return False
+    
+    with open(output_file, 'r') as f:
+        content = f.read()
+        if len(content) == 0:
+            print("Output CSV file is empty")
+            return False
+    
+    print("Verification PASSED")
+    return True
+
+if __name__ == "__main__":
+    success = verify_conversion()
+    sys.exit(0 if success else 1)