diff --git a/wizard-work/spark/check_conversion.sh b/wizard-work/spark/check_conversion.sh new file mode 100644 index 0000000..7ab9fe4 --- /dev/null +++ b/wizard-work/spark/check_conversion.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# Check that json_to_csv.py correctly converts sample_data.json +# Returns 0 on success, non-zero on failure + +python3 -c 'import sys,json; d=json.load(sys.stdin); exit(0 if d["records_processed"]==2 else 1)' < <(python3 /root/.hermes/wizard-outputs/spark/json_to_csv.py /root/.hermes/wizard-outputs/spark/sample_data.json /tmp/test_out.csv 2>&1) \ No newline at end of file diff --git a/wizard-work/spark/json_to_csv_verify.py b/wizard-work/spark/json_to_csv_verify.py new file mode 100644 index 0000000..5e576f3 --- /dev/null +++ b/wizard-work/spark/json_to_csv_verify.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Verification script for json_to_csv.py converter.""" + +import subprocess +import json +import sys +import os + +def verify_conversion(): + input_file = "/root/.hermes/wizard-outputs/spark/sample_data.json" + output_file = "/tmp/test_out.csv" + + # Run the converter + result = subprocess.run( + ["python3", "/root/.hermes/wizard-outputs/spark/json_to_csv.py", input_file, output_file], + capture_output=True, + text=True + ) + + if result.returncode != 0: + print(f"Conversion failed: {result.stderr}") + return False + + # Parse the output + try: + output_data = json.loads(result.stdout) + except json.JSONDecodeError as e: + print(f"Failed to parse output JSON: {e}") + return False + + # Verify records_processed is 2 + if output_data.get("records_processed") != 2: + print(f"Expected 2 records, got {output_data.get('records_processed')}") + return False + + # Verify output file exists and has content + if not os.path.exists(output_file): + print(f"Output file {output_file} was not created") + return False + + with open(output_file, 'r') as f: + content = f.read() + if len(content) == 0: + print("Output CSV file is empty") + return False + + print("Verification PASSED") + return True + +if __name__ == "__main__": + success = verify_conversion() + sys.exit(0 if success else 1) \ No newline at end of file