201 lines
6.8 KiB
Python
201 lines
6.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Extract the countries-by-area table from Wikipedia into CSV + JSON + MD.
|
|
|
|
Source: https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_area
|
|
Method: ask the MediaWiki Action API for the article wikitext, locate the
|
|
wikitable with class "sticky-header col2left", and parse its data rows.
|
|
|
|
Output (same directory as this script):
|
|
country-area-<YYYY-MM-DD>.csv | .json | .md
|
|
"""
|
|
import csv
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
from html import unescape
|
|
|
|
WIKI_URL = ("https://en.wikipedia.org/w/api.php?action=parse&page="
|
|
"List_of_countries_and_dependencies_by_area&prop=wikitext&format=json")
|
|
USER_AGENT = "ledger-country-area-extract/1.0 (alexander; extraction lane)"
|
|
|
|
CSV_COLUMNS = [
|
|
"rank", "country", "total_area_km2", "land_area_km2",
|
|
"inland_water_km2", "pct_water", "notes",
|
|
]
|
|
|
|
TMPL_NUM = re.compile(r"^\{\{\s*km2\s+mi2\s*\|\s*([0-9.,]+)\s*\}\}$", re.S)
|
|
TMPL_FLAG = re.compile(r"^\{\{\s*flag\s*\|\s*(.*?)\s*\}\}$", re.S)
|
|
TMPL_NOFLAG = re.compile(r"^\{\{\s*noflag\s*\|\s*(.*?)\s*\}\}$", re.S)
|
|
TMPL_RANK = re.compile(r"^\{\{\s*(?:nts|hs|ntsh)\s*\|\s*([^}]*)\s*\}\}$", re.S)
|
|
LINK = re.compile(r"\[\[([^\]]*?)\]\]")
|
|
|
|
|
|
def clean_wikilinks(s):
|
|
"""Replace [[...]] with their display text (text after the last |)."""
|
|
def repl(m):
|
|
inner = m.group(1)
|
|
return inner.split("|", 1)[1] if "|" in inner else inner
|
|
return LINK.sub(repl, s)
|
|
|
|
|
|
def strip_wiki(s):
|
|
"""Remove wiki markup: templates, emphasis, wikilinks, HTML; unescape."""
|
|
s = re.sub(r"\{\{.*?\}\}", " ", s, flags=re.S)
|
|
s = re.sub(r"('{4,})(.*?)\1", r"\2", s, flags=re.S)
|
|
s = re.sub(r"('{2,})(.*?)\1", r"\2", s, flags=re.S)
|
|
s = re.sub(r"<ref[^>]*>.*?</ref>", " ", s, flags=re.S)
|
|
s = re.sub(r"<ref[^>]*>", " ", s)
|
|
s = re.sub(r"<[^>]+>", " ", s)
|
|
s = re.sub(r"\s+", " ", s)
|
|
return unescape(s).strip()
|
|
|
|
|
|
def param_after(s, prefix):
|
|
"""Return the content after `prefix`, up to the LAST '}}' in s."""
|
|
idx = s.find(prefix)
|
|
if idx == -1:
|
|
return s
|
|
rest = s[idx + len(prefix):]
|
|
if rest.endswith("}}"):
|
|
rest = rest[:-2]
|
|
return rest
|
|
|
|
|
|
def render_text(cell):
|
|
"""Convert a raw wiki cell into plain, human-readable text."""
|
|
cell = cell.strip()
|
|
# Strip surrounding bold/italic marks so templates inside are reachable.
|
|
cell = re.sub(r"^('{2,})(.*?)\1$", r"\2", cell, flags=re.S).strip()
|
|
cell = re.sub(r"[\s—–-]+$", "", cell).strip()
|
|
if cell.startswith("{{efn|"):
|
|
content = param_after(cell, "{{efn|")
|
|
return clean_wikilinks(strip_wiki(content))
|
|
m = TMPL_FLAG.match(cell)
|
|
if m:
|
|
return clean_wikilinks(strip_wiki(m.group(1)))
|
|
m = TMPL_NOFLAG.match(cell)
|
|
if m:
|
|
return clean_wikilinks(strip_wiki(m.group(1)))
|
|
if TMPL_NUM.match(cell):
|
|
return clean_wikilinks(strip_wiki(cell))
|
|
if TMPL_RANK.match(cell):
|
|
return clean_wikilinks(strip_wiki(cell))
|
|
return clean_wikilinks(strip_wiki(cell))
|
|
|
|
|
|
def extract_number(cell):
|
|
m = TMPL_NUM.match(cell.strip())
|
|
if m:
|
|
return float(m.group(1).replace(",", ""))
|
|
return None
|
|
|
|
|
|
def extract():
|
|
import urllib.request
|
|
req = urllib.request.Request(WIKI_URL, headers={"User-Agent": USER_AGENT})
|
|
with urllib.request.urlopen(req, timeout=60) as r:
|
|
d = json.load(r)
|
|
wikitext = d.get("parse", {}).get("wikitext", {}).get("*", "")
|
|
|
|
start = wikitext.find('{| class="wikitable')
|
|
if start == -1:
|
|
print("FAIL: no wikitable found", file=sys.stderr)
|
|
sys.exit(1)
|
|
end = wikitext.find("|}", start)
|
|
table = wikitext[start:end]
|
|
|
|
# Split into rows: "|-\n" marks a row break.
|
|
rows_raw = re.split(r"\n?\|-\s*\n", table)
|
|
data = []
|
|
rank = 0
|
|
for grp in rows_raw:
|
|
cells = re.split(r"\|\|\s*", grp)
|
|
if not cells:
|
|
continue
|
|
cells = [c.lstrip("|").strip() for c in cells]
|
|
if len(cells) < 6:
|
|
continue
|
|
country = render_text(cells[1])
|
|
if country in ("Earth", "Earth (total)"):
|
|
continue # grand-total row, not a country
|
|
rank += 1
|
|
total = extract_number(cells[2])
|
|
land = extract_number(cells[3])
|
|
water = extract_number(cells[4])
|
|
pct = render_text(cells[5])
|
|
notes = render_text(cells[6]) if len(cells) > 6 else ""
|
|
data.append({
|
|
"rank": rank,
|
|
"country": country,
|
|
"total_area_km2": total,
|
|
"land_area_km2": land,
|
|
"inland_water_km2": water,
|
|
"pct_water": pct,
|
|
"notes": notes,
|
|
})
|
|
return data
|
|
|
|
|
|
def stamp():
|
|
return datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
|
|
|
|
|
def main():
|
|
data = extract()
|
|
out = os.path.dirname(os.path.abspath(__file__))
|
|
st = stamp()
|
|
csv_path = os.path.join(out, "country-area-{}.csv".format(st))
|
|
json_path = os.path.join(out, "country-area-{}.json".format(st))
|
|
md_path = os.path.join(out, "country-area-{}.md".format(st))
|
|
|
|
with open(csv_path, "w", newline="", encoding="utf-8") as f:
|
|
w = csv.DictWriter(f, fieldnames=CSV_COLUMNS, lineterminator="\n")
|
|
w.writeheader()
|
|
w.writerows(data)
|
|
|
|
with open(json_path, "w", encoding="utf-8") as f:
|
|
json.dump({
|
|
"source": "https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_area",
|
|
"stamp": "{}T{}Z".format(st, datetime.now(timezone.utc).strftime("%H:%M:%SZ")),
|
|
"row_count": len(data),
|
|
"rows": data,
|
|
}, f, ensure_ascii=False, indent=2)
|
|
|
|
lines = [
|
|
"# Countries and dependencies by area",
|
|
"",
|
|
"Source: Wikipedia, \"List of countries and dependencies by area\"",
|
|
"Query: MediaWiki Action API -> parse -> wikitext -> first wikitable",
|
|
"Retrieved: {}T{}Z (UTC)".format(st, datetime.now(timezone.utc).strftime("%H:%M:%SZ")),
|
|
"Rows: {}".format(len(data)),
|
|
"",
|
|
"| Rank | Country | Total area (km2) | Land area (km2) | Inland water (km2) | % water |",
|
|
"|---:|---|---|---|---|---|",
|
|
]
|
|
for r in data:
|
|
lines.append(
|
|
"| {} | {} | {} | {} | {} | {} |".format(
|
|
r["rank"], r["country"], r["total_area_km2"], r["land_area_km2"],
|
|
r["inland_water_km2"], r["pct_water"]))
|
|
lines += [
|
|
"",
|
|
"## Evidence",
|
|
"",
|
|
"Each row is a verbatim cell of the Wikipedia table as of the snapshot time.",
|
|
"Numbers are the parsed km2 values of the `{{km2 mi2|...}}` templates; no value",
|
|
"was edited or invented. Ranks are positional (1 = largest country).",
|
|
"",
|
|
]
|
|
with open(md_path, "w", encoding="utf-8") as f:
|
|
f.write("\n".join(lines))
|
|
|
|
print("fetched {} data rows (stamp {}); wrote csv={}, json={}, md={}".format(
|
|
len(data), st, csv_path, json_path, md_path))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|