85 lines
2.6 KiB
Python
85 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Independent cross-check: extracted CSV vs the raw source JSON.
|
|
|
|
Usage:
|
|
python3 verify_extraction.py <source_json> <csv_path>
|
|
|
|
Verifies that every source record appears exactly once in the CSV:
|
|
- row count equals the source record count
|
|
- alpha-2 code sets match exactly
|
|
- name_common values match for every row
|
|
|
|
Exits 0 only if all checks pass. Evidence, not the formal gate.
|
|
"""
|
|
import csv
|
|
import json
|
|
import sys
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) != 3:
|
|
sys.stderr.write('usage: verify_extraction.py <source_json> <csv_path>\n')
|
|
return 2
|
|
source_path, csv_path = sys.argv[1], sys.argv[2]
|
|
try:
|
|
with open(source_path, encoding='utf-8') as fh:
|
|
records = json.load(fh)
|
|
except (OSError, ValueError) as exc:
|
|
sys.stderr.write('cannot read source %s: %s\n' % (source_path, exc))
|
|
return 1
|
|
if not isinstance(records, list):
|
|
sys.stderr.write('source JSON is not a list\n')
|
|
return 1
|
|
|
|
source = {}
|
|
for rec in records:
|
|
code = str(rec.get('cca2')) if rec.get('cca2') else None
|
|
name = str((rec.get('name') or {}).get('common')) if rec.get('name') else ''
|
|
if code:
|
|
source[code] = name
|
|
|
|
try:
|
|
with open(csv_path, encoding='utf-8') as fh:
|
|
rows = list(csv.reader(fh))
|
|
except OSError as exc:
|
|
sys.stderr.write('cannot read csv %s: %s\n' % (csv_path, exc))
|
|
return 1
|
|
if not rows:
|
|
sys.stderr.write('csv is empty\n')
|
|
return 1
|
|
header, data = rows[0], rows[1:]
|
|
problems = []
|
|
if len(data) != len(source):
|
|
problems.append('row count %d != source count %d'
|
|
% (len(data), len(source)))
|
|
|
|
csv_codes = {}
|
|
for row in data:
|
|
if len(row) != 14:
|
|
problems.append('row with %d fields: %r' % (len(row), row))
|
|
continue
|
|
csv_codes[row[2]] = row[0]
|
|
|
|
missing = sorted(set(source) - set(csv_codes))
|
|
extra = sorted(set(csv_codes) - set(source))
|
|
if missing:
|
|
problems.append('alpha-2 missing from csv: %r' % missing)
|
|
if extra:
|
|
problems.append('alpha-2 extra in csv: %r' % extra)
|
|
mismatches = [c for c in source if csv_codes.get(c) != source[c]]
|
|
if mismatches:
|
|
problems.append('name_common mismatches: %r' % mismatches)
|
|
|
|
if problems:
|
|
sys.stderr.write('VERIFY FAILED\n')
|
|
for problem in problems:
|
|
sys.stderr.write(' - %s\n' % problem)
|
|
return 1
|
|
print('verify: ok rows=%d codes=%d source=%d header=%r'
|
|
% (len(data), len(csv_codes), len(source), header))
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|