#!/usr/bin/env python3 """Extract the countries-by-area table from Wikipedia into CSV + JSON + MD. Source: https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_area Method: ask the MediaWiki Action API for the article wikitext, locate the wikitable with class "sticky-header col2left", and parse its data rows. Output (same directory as this script): country-area-.csv | .json | .md """ import csv import json import os import re import sys from datetime import datetime, timezone from html import unescape WIKI_URL = ("https://en.wikipedia.org/w/api.php?action=parse&page=" "List_of_countries_and_dependencies_by_area&prop=wikitext&format=json") USER_AGENT = "ledger-country-area-extract/1.0 (alexander; extraction lane)" CSV_COLUMNS = [ "rank", "country", "total_area_km2", "land_area_km2", "inland_water_km2", "pct_water", "notes", ] TMPL_NUM = re.compile(r"^\{\{\s*km2\s+mi2\s*\|\s*([0-9.,]+)\s*\}\}$", re.S) TMPL_FLAG = re.compile(r"^\{\{\s*flag\s*\|\s*(.*?)\s*\}\}$", re.S) TMPL_NOFLAG = re.compile(r"^\{\{\s*noflag\s*\|\s*(.*?)\s*\}\}$", re.S) TMPL_RANK = re.compile(r"^\{\{\s*(?:nts|hs|ntsh)\s*\|\s*([^}]*)\s*\}\}$", re.S) LINK = re.compile(r"\[\[([^\]]*?)\]\]") def clean_wikilinks(s): """Replace [[...]] with their display text (text after the last |).""" def repl(m): inner = m.group(1) return inner.split("|", 1)[1] if "|" in inner else inner return LINK.sub(repl, s) def strip_wiki(s): """Remove wiki markup: templates, emphasis, wikilinks, HTML; unescape.""" s = re.sub(r"\{\{.*?\}\}", " ", s, flags=re.S) s = re.sub(r"('{4,})(.*?)\1", r"\2", s, flags=re.S) s = re.sub(r"('{2,})(.*?)\1", r"\2", s, flags=re.S) s = re.sub(r"]*>.*?", " ", s, flags=re.S) s = re.sub(r"]*>", " ", s) s = re.sub(r"<[^>]+>", " ", s) s = re.sub(r"\s+", " ", s) return unescape(s).strip() def param_after(s, prefix): """Return the content after `prefix`, up to the LAST '}}' in s.""" idx = s.find(prefix) if idx == -1: return s rest = s[idx + len(prefix):] if rest.endswith("}}"): rest = rest[:-2] return rest def render_text(cell): """Convert a raw wiki cell into plain, human-readable text.""" cell = cell.strip() # Strip surrounding bold/italic marks so templates inside are reachable. cell = re.sub(r"^('{2,})(.*?)\1$", r"\2", cell, flags=re.S).strip() cell = re.sub(r"[\s—–-]+$", "", cell).strip() if cell.startswith("{{efn|"): content = param_after(cell, "{{efn|") return clean_wikilinks(strip_wiki(content)) m = TMPL_FLAG.match(cell) if m: return clean_wikilinks(strip_wiki(m.group(1))) m = TMPL_NOFLAG.match(cell) if m: return clean_wikilinks(strip_wiki(m.group(1))) if TMPL_NUM.match(cell): return clean_wikilinks(strip_wiki(cell)) if TMPL_RANK.match(cell): return clean_wikilinks(strip_wiki(cell)) return clean_wikilinks(strip_wiki(cell)) def extract_number(cell): m = TMPL_NUM.match(cell.strip()) if m: return float(m.group(1).replace(",", "")) return None def extract(): import urllib.request req = urllib.request.Request(WIKI_URL, headers={"User-Agent": USER_AGENT}) with urllib.request.urlopen(req, timeout=60) as r: d = json.load(r) wikitext = d.get("parse", {}).get("wikitext", {}).get("*", "") start = wikitext.find('{| class="wikitable') if start == -1: print("FAIL: no wikitable found", file=sys.stderr) sys.exit(1) end = wikitext.find("|}", start) table = wikitext[start:end] # Split into rows: "|-\n" marks a row break. rows_raw = re.split(r"\n?\|-\s*\n", table) data = [] rank = 0 for grp in rows_raw: cells = re.split(r"\|\|\s*", grp) if not cells: continue cells = [c.lstrip("|").strip() for c in cells] if len(cells) < 6: continue country = render_text(cells[1]) if country in ("Earth", "Earth (total)"): continue # grand-total row, not a country rank += 1 total = extract_number(cells[2]) land = extract_number(cells[3]) water = extract_number(cells[4]) pct = render_text(cells[5]) notes = render_text(cells[6]) if len(cells) > 6 else "" data.append({ "rank": rank, "country": country, "total_area_km2": total, "land_area_km2": land, "inland_water_km2": water, "pct_water": pct, "notes": notes, }) return data def stamp(): return datetime.now(timezone.utc).strftime("%Y-%m-%d") def main(): data = extract() out = os.path.dirname(os.path.abspath(__file__)) st = stamp() csv_path = os.path.join(out, "country-area-{}.csv".format(st)) json_path = os.path.join(out, "country-area-{}.json".format(st)) md_path = os.path.join(out, "country-area-{}.md".format(st)) with open(csv_path, "w", newline="", encoding="utf-8") as f: w = csv.DictWriter(f, fieldnames=CSV_COLUMNS, lineterminator="\n") w.writeheader() w.writerows(data) with open(json_path, "w", encoding="utf-8") as f: json.dump({ "source": "https://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_area", "stamp": "{}T{}Z".format(st, datetime.now(timezone.utc).strftime("%H:%M:%SZ")), "row_count": len(data), "rows": data, }, f, ensure_ascii=False, indent=2) lines = [ "# Countries and dependencies by area", "", "Source: Wikipedia, \"List of countries and dependencies by area\"", "Query: MediaWiki Action API -> parse -> wikitext -> first wikitable", "Retrieved: {}T{}Z (UTC)".format(st, datetime.now(timezone.utc).strftime("%H:%M:%SZ")), "Rows: {}".format(len(data)), "", "| Rank | Country | Total area (km2) | Land area (km2) | Inland water (km2) | % water |", "|---:|---|---|---|---|---|", ] for r in data: lines.append( "| {} | {} | {} | {} | {} | {} |".format( r["rank"], r["country"], r["total_area_km2"], r["land_area_km2"], r["inland_water_km2"], r["pct_water"])) lines += [ "", "## Evidence", "", "Each row is a verbatim cell of the Wikipedia table as of the snapshot time.", "Numbers are the parsed km2 values of the `{{km2 mi2|...}}` templates; no value", "was edited or invented. Ranks are positional (1 = largest country).", "", ] with open(md_path, "w", encoding="utf-8") as f: f.write("\n".join(lines)) print("fetched {} data rows (stamp {}); wrote csv={}, json={}, md={}".format( len(data), st, csv_path, json_path, md_path)) return 0 if __name__ == "__main__": sys.exit(main())