71 lines
2.6 KiB
Python
71 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract the UN member states table (Country | Date of admission) from the
|
|
Wikipedia 'List of member states of the United Nations' page.
|
|
|
|
Source (real web_extract result, saved from en.wikipedia.org):
|
|
/root/.hermes/profiles/ledger/cache/web/en.wikipedia.org-00a059c52c.md
|
|
|
|
Output (written to the artifact dir):
|
|
un_member_states.csv - Country, Date_of_admission, Year, Is_founding_member
|
|
un_member_states.json - same records as a JSON array
|
|
|
|
Only the 193 current member states are emitted; separator rows and the table
|
|
header are filtered out. Founding member == admitted in 1945 (the 51 original
|
|
members of the United Nations, per the page's "Current members" section).
|
|
"""
|
|
import csv
|
|
import json
|
|
import os
|
|
import re
|
|
from datetime import datetime
|
|
|
|
SRC = "/root/.hermes/profiles/ledger/cache/web/en.wikipedia.org-00a059c52c.md"
|
|
OUT = "/root/.hermes/wizard-outputs/ledger"
|
|
|
|
os.makedirs(OUT, exist_ok=True)
|
|
|
|
# Match rows like: || Afghanistan | 19 November 1946 | <notes> |
|
|
# country/date use non-greedy capture; notes capture is greedy so notes that
|
|
# contain ' | ' are kept whole.
|
|
ROW_RE = re.compile(r"^\| (.+?) \| (.+?) \| (.*)$", re.M)
|
|
|
|
records = []
|
|
for m in ROW_RE.finditer(open(SRC, encoding="utf-8").read()):
|
|
country = m.group(1).strip()
|
|
date = m.group(2).strip()
|
|
# Skip table header and Markdown separator rows.
|
|
if country.lower().startswith("member state") or date.lower().startswith("date"):
|
|
continue
|
|
if country in ("---", "") or date in ("---", ""):
|
|
continue
|
|
if not re.match(r"^\d{1,2} \w+ \d{4}$", date):
|
|
continue
|
|
d = datetime.strptime(date, "%d %B %Y")
|
|
records.append({
|
|
"country": country,
|
|
"date_of_admission": date,
|
|
"year": d.year,
|
|
"is_founding_member": (d.year == 1945),
|
|
})
|
|
|
|
records.sort(key=lambda r: datetime.strptime(r["date_of_admission"], "%d %B %Y"))
|
|
|
|
csv_path = os.path.join(OUT, "un_member_states.csv")
|
|
with open(csv_path, "w", encoding="utf-8", newline="") as fh:
|
|
writer = csv.DictWriter(
|
|
fh, fieldnames=["country", "date_of_admission", "year", "is_founding_member"]
|
|
)
|
|
writer.writeheader()
|
|
writer.writerows(records)
|
|
|
|
json_path = os.path.join(OUT, "un_member_states.json")
|
|
with open(json_path, "w", encoding="utf-8") as fh:
|
|
json.dump(records, fh, ensure_ascii=False, indent=2)
|
|
|
|
print("rows_emitted:", len(records))
|
|
print("columns:", list(records[0].keys()) if records else None)
|
|
print("founding_1945:", sum(1 for r in records if r["is_founding_member"]))
|
|
print("years:", sorted(set(r["year"] for r in records)))
|
|
print("csv:", csv_path)
|
|
print("json:", json_path)
|