diff --git a/wizard-work/spark/json_to_csv.py b/wizard-work/spark/json_to_csv.py new file mode 100644 index 0000000..550f9d7 --- /dev/null +++ b/wizard-work/spark/json_to_csv.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +""" +JSON to CSV Converter - A simple utility to convert JSON files to CSV format. +Usage: python3 json_to_csv.py +""" + +import json +import csv +import sys +from pathlib import Path + + +def flatten_dict(d, parent_key='', sep='_'): + """Flatten nested dictionary for CSV compatibility.""" + items = [] + for k, v in d.items(): + new_key = f"{parent_key}{sep}{k}" if parent_key else k + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, sep).items()) + elif isinstance(v, list): + items.append((new_key, json.dumps(v))) + else: + items.append((new_key, v)) + return dict(items) + + +def convert_json_to_csv(json_path, csv_path): + """Convert JSON file to CSV.""" + with open(json_path, 'r') as f: + data = json.load(f) + + if isinstance(data, list): + records = [flatten_dict(record) for record in data] if data else [] + elif isinstance(data, dict): + records = [flatten_dict(data)] + else: + raise ValueError("JSON must contain objects or array of objects") + + if not records: + with open(csv_path, 'w') as f: + pass + return + + fieldnames = sorted(set(key for record in records for key in record.keys())) + + with open(csv_path, 'w', newline='') as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(records) + + return { + 'input_path': str(json_path), + 'output_path': str(csv_path), + 'records_processed': len(records), + 'columns': fieldnames + } + + +if __name__ == '__main__': + if len(sys.argv) != 3: + print("Usage: python3 json_to_csv.py ") + sys.exit(1) + + result = convert_json_to_csv(sys.argv[1], sys.argv[2]) + print(json.dumps(result, indent=2)) \ No newline at end of file diff --git a/wizard-work/spark/output.csv b/wizard-work/spark/output.csv new file mode 100644 index 0000000..e156105 --- /dev/null +++ b/wizard-work/spark/output.csv @@ -0,0 +1,3 @@ +active,email,id,metadata_department,metadata_role,name,tags +True,alice@example.com,1,Engineering,Senior Developer,Alice Smith,"[""python"", ""rust"", ""ai""]" +False,bob@example.com,2,Marketing,Manager,Bob Jones,"[""social"", ""analytics""]" diff --git a/wizard-work/spark/sample_data.json b/wizard-work/spark/sample_data.json new file mode 100644 index 0000000..7ff36f4 --- /dev/null +++ b/wizard-work/spark/sample_data.json @@ -0,0 +1,24 @@ +[ + { + "id": 1, + "name": "Alice Smith", + "email": "alice@example.com", + "active": true, + "metadata": { + "department": "Engineering", + "role": "Senior Developer" + }, + "tags": ["python", "rust", "ai"] + }, + { + "id": 2, + "name": "Bob Jones", + "email": "bob@example.com", + "active": false, + "metadata": { + "department": "Marketing", + "role": "Manager" + }, + "tags": ["social", "analytics"] + } +] \ No newline at end of file diff --git a/wizard-work/spark/verify_csv.py b/wizard-work/spark/verify_csv.py new file mode 100644 index 0000000..277c099 --- /dev/null +++ b/wizard-work/spark/verify_csv.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Verify CSV output is valid.""" + +import csv +import json + +# Verify CSV is valid +with open('/root/.hermes/wizard-outputs/spark/output.csv', 'r') as f: + reader = csv.DictReader(f) + rows = list(reader) + +assert len(rows) == 2, f"Expected 2 rows, got {len(rows)}" +assert 'name' in rows[0], "Missing 'name' column" +assert rows[0]['name'] == 'Alice Smith', f"Wrong name: {rows[0]['name']}" + +# Verify the JSON source was correctly processed +with open('/root/.hermes/wizard-outputs/spark/sample_data.json', 'r') as f: + source = json.load(f) + +assert len(source) == 2, "Source JSON should have 2 records" + +print("PASS: CSV is valid with correct data structure") +print(f"Records: {len(rows)}") +print(f"Columns: {sorted(rows[0].keys())}") \ No newline at end of file