142 lines
4.6 KiB
Python
142 lines
4.6 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
JSON to CSV Converter for Troupe Standup Data
|
|
Converts nested JSON structures (like inbox seeds, triage reports) to flat CSV format.
|
|
|
|
Usage: python3 json_to_csv_converter.py <input.json> <output.csv>
|
|
"""
|
|
|
|
import json
|
|
import csv
|
|
import sys
|
|
import argparse
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional, Set
|
|
|
|
|
|
def flatten_dict(d: Any, parent_key: str = '', sep: str = '_') -> Dict[str, Any]:
|
|
"""Flatten nested dictionary for CSV compatibility."""
|
|
items: List[tuple] = []
|
|
|
|
if isinstance(d, dict):
|
|
for k, v in d.items():
|
|
new_key = f"{parent_key}{sep}{k}" if parent_key else str(k)
|
|
if isinstance(v, dict):
|
|
items.extend(flatten_dict(v, new_key, sep).items())
|
|
elif isinstance(v, list):
|
|
# For lists, serialize to JSON string for CSV
|
|
items.append((new_key, json.dumps(v) if v else ''))
|
|
else:
|
|
items.append((new_key, v if v is not None else ''))
|
|
else:
|
|
# Non-dict at root, treat as scalar
|
|
items.append((parent_key or 'value', d))
|
|
|
|
return dict(items)
|
|
|
|
|
|
def json_path_value(data: Any, path: str) -> Any:
|
|
"""Get value from JSON using dot-notation path."""
|
|
keys = path.split('.')
|
|
current = data
|
|
for key in keys:
|
|
if isinstance(current, dict):
|
|
current = current.get(key)
|
|
elif isinstance(current, list) and key.isdigit():
|
|
current = current[int(key)] if int(key) < len(current) else None
|
|
else:
|
|
return None
|
|
if current is None:
|
|
break
|
|
return current
|
|
|
|
|
|
def convert_json_to_csv(json_path: str, csv_path: str, flatten_nested: bool = True) -> Dict[str, Any]:
|
|
"""
|
|
Convert JSON file to CSV.
|
|
|
|
Args:
|
|
json_path: Path to input JSON file
|
|
csv_path: Path to output CSV file
|
|
flatten_nested: Whether to flatten nested structures (default True)
|
|
|
|
Returns:
|
|
Dictionary with conversion metadata
|
|
"""
|
|
# Read JSON file
|
|
with open(json_path, 'r', encoding='utf-8') as f:
|
|
data = json.load(f)
|
|
|
|
# Normalize to list of records
|
|
if isinstance(data, list):
|
|
records = data if data else []
|
|
elif isinstance(data, dict):
|
|
# Check if it's a record with 'cutover', 'decided', 'pending' etc.
|
|
if any(k in data for k in ['cutover', 'cutoff', 'created_at', 'items', 'records']):
|
|
# Single record object
|
|
records = [data] if data else []
|
|
else:
|
|
# Dict of items - treat keys as record IDs or values as records
|
|
records = [data]
|
|
else:
|
|
records = [{'value': data}]
|
|
|
|
# Flatten records if needed
|
|
if flatten_nested:
|
|
flattened_records = []
|
|
for record in records:
|
|
flat_record = flatten_dict(record)
|
|
# Convert any non-string values to strings for CSV
|
|
for k, v in flat_record.items():
|
|
if not isinstance(v, (str, int, float, bool)) and v is not None:
|
|
flat_record[k] = json.dumps(v, ensure_ascii=False)
|
|
flattened_records.append(flat_record)
|
|
records = flattened_records
|
|
|
|
if not records:
|
|
# Write empty CSV
|
|
with open(csv_path, 'w', newline='', encoding='utf-8') as f:
|
|
pass
|
|
return {
|
|
'input_path': json_path,
|
|
'output_path': csv_path,
|
|
'records_processed': 0,
|
|
'columns': [],
|
|
'status': 'empty_file_written'
|
|
}
|
|
|
|
# Get all column names (union of all keys)
|
|
all_keys: Set[str] = set()
|
|
for record in records:
|
|
all_keys.update(record.keys())
|
|
fieldnames = sorted(all_keys)
|
|
|
|
# Write CSV
|
|
with open(csv_path, 'w', newline='', encoding='utf-8') as f:
|
|
writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction='ignore')
|
|
writer.writeheader()
|
|
writer.writerows(records)
|
|
|
|
return {
|
|
'input_path': json_path,
|
|
'output_path': csv_path,
|
|
'records_processed': len(records),
|
|
'columns': len(fieldnames),
|
|
'status': 'success'
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description='Convert JSON to CSV')
|
|
parser.add_argument('input', help='Input JSON file path')
|
|
parser.add_argument('output', help='Output CSV file path')
|
|
parser.add_argument('--raw', action='store_true', help='Do not flatten nested structures')
|
|
|
|
args = parser.parse_args()
|
|
|
|
result = convert_json_to_csv(args.input, args.output, flatten_nested=not args.raw)
|
|
print(json.dumps(result, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main() |