From 16332d6709d5c3f1f7ff5d75c023e415b35f54be Mon Sep 17 00:00:00 2001 From: ledger Date: Wed, 4 Nov 2026 10:59:49 +0000 Subject: [PATCH] ledger: extract IMF/World Bank/UN nominal GDP table (222 rows) into CSV + markdown --- wizard-work/ledger/extract_gdp_imf_wb_un.py | 153 +++++++++++++ wizard-work/ledger/gdp_imf_wb_un_extract.csv | 223 ++++++++++++++++++ wizard-work/ledger/gdp_imf_wb_un_extract.md | 224 +++++++++++++++++++ wizard-work/ledger/verify_gdp_imf_wb_un.py | 98 ++++++++ 4 files changed, 698 insertions(+) create mode 100644 wizard-work/ledger/extract_gdp_imf_wb_un.py create mode 100644 wizard-work/ledger/gdp_imf_wb_un_extract.csv create mode 100644 wizard-work/ledger/gdp_imf_wb_un_extract.md create mode 100644 wizard-work/ledger/verify_gdp_imf_wb_un.py diff --git a/wizard-work/ledger/extract_gdp_imf_wb_un.py b/wizard-work/ledger/extract_gdp_imf_wb_un.py new file mode 100644 index 0000000..ccb9415 --- /dev/null +++ b/wizard-work/ledger/extract_gdp_imf_wb_un.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Extract the IMF / World Bank / UN GDP table (gdp_table.html) into a clean +CSV and markdown table. + +Rules (deterministic, derived from the source, not from the prior partial run): + + * The main table is the one whose caption is "GDP forecast or estimate + (million US$) by country". + * Data rows = blocks whose first contains a letter (a country name). + * Country name: visible text of the first cell with HTML stripped, HTML + unescaped, collapsed; footnote citation markers "[ ... ]" and leading "|" + glosses removed; trailing/leading whitespace trimmed. + * Values: visible text of each of the next three cells, cleaned the same + way. A number followed by a "(YYYY)" source-year citation is stored as the + bare number. Cells that are a dash plus "N/a" (territories with no IMF + estimate) are stored as the explicit sentinel "N/a". + +Outputs: + wizard-work/ledger/gdp_imf_wb_un_extract.csv + wizard-work/ledger/gdp_imf_wb_un_extract.md +""" +import re +import html +import csv +import sys + +SRC = '/root/.hermes/scripts/gdp_table.html' +OUT_CSV = '/root/troupe-standup/wizard-work/ledger/gdp_imf_wb_un_extract.csv' +OUT_MD = '/root/troupe-standup/wizard-work/ledger/gdp_imf_wb_un_extract.md' + +# Number like 128,003,776 optionally followed by (2024)/(2022) citation. +NUMBER_RE = re.compile(r'^(\d{1,3}(?:,\d{3})+(?:\s*\(20\d{2}\))?$)') + + +def clean_cell(raw): + """Visible text of a : drop hidden spans, strip tags, unescape, collapse.""" + raw = _drop_hidden(raw) + t = re.sub(r'<[^>]+>', ' ', raw) + t = html.unescape(t) + t = re.sub(r' ', ' ', t) + t = re.sub(r'\s+', ' ', t).strip() + return t + + +def _drop_hidden(raw): + """Remove visually-hidden / hidden content that would leak into text. + + Drops three kinds of noise: + * ... (screen-reader-only note, e.g. "N/a") + * (TemplateStyles CSS blob) + * elements styled display:none / visibility:hidden + """ + raw = re.sub(r']*class="[^"]*\bsr-only\b[^>]*>.*?', '', raw, flags=re.S) + raw = re.sub(r']*>.*?', '', raw, flags=re.S) + raw = re.sub(r'<[^>]*\b(?:style|class)=[^>]*\b(?:display\s*:\s*none|visibility\s*:\s*hidden)\b[^>]*>', '', raw, flags=re.S) + return raw + + +def clean_country(raw): + """First-cell cleaning: remove footnote markers and leading gloss.""" + t = clean_cell(raw) + t = re.sub(r'\[[^\]]*\]', '', t) # e.g. "[ n 3 ]", "[n 3]" + t = re.sub(r'\|', '', t) # leading gloss "| " + t = t.strip() + return t + + +def clean_value(raw): + """Third-cell value cleaning: bare number, else explicit sentinel.""" + t = clean_cell(raw) + if NUMBER_RE.match(t): + return re.sub(r'\s*\(20\d{2}\)\s*$', '', t).strip() + if t.lower() in ('n/a', '— n/a', '—'): + return 'N/a' + # Unknown cell: keep the cleaned text, but never let a number be truncated. + return t + + +def extract(src=SRC): + data = open(src, encoding='utf-8', errors='replace').read() + m_cap = re.search(r']*>(.*?)', data, re.S) + caption = html.unescape(re.sub(r'<[^>]+>', ' ', m_cap.group(1))).strip() \ + if m_cap else '' + if 'GDP forecast or estimate' not in caption: + raise SystemExit('unexpected caption: %r' % caption) + m_tbody = re.search(r']*>(.*?)', data, re.S) + tbody = m_tbody.group(1) + rows = re.findall(r'', tbody, re.S) + + out = [] + for r in rows: + tds = re.findall(r']*>(.*?)', r, re.S) + if len(tds) < 4: + continue + country = clean_country(tds[0]) + if not re.search(r'[A-Za-z]', country): + continue + out.append([country, clean_value(tds[1]), clean_value(tds[2]), clean_value(tds[3])]) + return out + + +def main(): + out = extract() + print('data rows:', len(out)) + + # ---- verification anchors (measured against the source) ---- + anchors = { + 'World': ('128,003,776', '118,350,166', '111,565,144'), + 'United States': ('32,642,373', '30,769,700', '29,298,000'), + 'China': ('21,546,403', '19,498,039', '18,743,802'), + 'India': ('4,060,207', '3,956,067', '3,952,244'), + 'Taiwan': ('1,019,696', 'N/a', 'N/a'), + 'Cuba': ('N/a', '107,352', '252,063'), + 'Monaco': ('N/a', '11,126', '11,126'), + } + by_country = {r[0]: r[1:] for r in out} + all_ok = True + for k, v in anchors.items(): + got = by_country.get(k) + if got is None: + print('FAIL', k, 'MISSING') + all_ok = False + continue + ok = tuple(got) == tuple(v) + all_ok = all_ok and ok + print(('PASS' if ok else 'FAIL'), k, got, v) + + # report rows that currently lack any metric (placeholder check) + zero = [r for r in out if r[1:] == ('N/a', 'N/a', 'N/a')] + print('rows with all three values N/a:', len(zero)) + + # ---- write CSV (LF, no quoting edge cases) ---- + with open(OUT_CSV, 'w', newline='', encoding='utf-8') as f: + w = csv.writer(f, lineterminator='\n') + w.writerow(['country', 'imf_2026_million_usd', 'world_bank_2025_million_usd', 'un_2024_million_usd']) + w.writerows(out) + + # ---- write markdown table ---- + lines = ['| Country/Territory | IMF (2026) | World Bank (2025) | United Nations (2024) |', + '|---|---|---|:---|'] + for c, x, y, z in out: + lines.append(f'| {c} | {x} | {y} | {z} |') + with open(OUT_MD, 'w', encoding='utf-8') as f: + f.write('\n'.join(lines) + '\n') + + print('anchors all match:', all_ok) + print('wrote', OUT_CSV) + print('wrote', OUT_MD) + return 0 if all_ok else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/wizard-work/ledger/gdp_imf_wb_un_extract.csv b/wizard-work/ledger/gdp_imf_wb_un_extract.csv new file mode 100644 index 0000000..90c7a66 --- /dev/null +++ b/wizard-work/ledger/gdp_imf_wb_un_extract.csv @@ -0,0 +1,223 @@ +country,imf_2026_million_usd,world_bank_2025_million_usd,un_2024_million_usd +World,"128,003,776","118,350,166","111,565,144" +United States,"32,642,373","30,769,700","29,298,000" +China,"21,546,403","19,498,039","18,743,802" +Germany,"5,471,299","5,050,923","4,659,929" +Japan,"4,341,986","4,435,163","4,026,211" +United Kingdom,"4,268,446","4,002,588","3,685,881" +India,"4,060,207","3,956,067","3,952,244" +France,"3,554,155","3,366,316","3,160,443" +Russia,"2,861,156","2,561,310","2,173,386" +Italy,"2,701,737","2,551,557","2,380,825" +Brazil,"2,654,761","2,279,920","2,185,822" +Canada,"2,490,256","2,319,900","2,270,076" +South Korea,"2,245,847","1,872,375","1,875,388" +Mexico,"2,176,236","1,832,641","1,852,723" +Australia,"2,110,657","1,798,519","1,830,610" +Spain,"2,085,276","1,906,453","1,722,746" +Turkey,"1,817,987","1,597,293","1,323,255" +Indonesia,"1,479,954","1,445,643","1,396,300" +Netherlands,"1,423,856","1,332,768","1,214,928" +Saudi Arabia,"1,343,346","1,276,943","1,254,141" +Switzerland,"1,135,739","1,043,530","936,564" +Poland,"1,133,088","1,035,492","914,696" +Taiwan,"1,019,696",N/a,N/a +Belgium,"772,699","725,466","664,564" +Argentina,"768,206","683,098","638,365" +Israel,"746,982","610,778","542,286" +Sweden,"745,081","668,999","604,899" +Ireland,"709,972","721,701","577,389" +Singapore,"667,889","603,870","547,387" +United Arab Emirates,"661,018","552,325","552,325" +Norway,"651,712","530,756","483,593" +Austria,"616,896","579,470","534,791" +Thailand,"599,115","577,010","526,518" +Colombia,"572,735","457,410","418,542" +Malaysia,"547,100","472,193","421,972" +Vietnam,"542,130","514,697","476,388" +Bangladesh,"506,357","456,319","432,742" +Philippines,"501,692","487,086","461,618" +Denmark,"499,354","462,527","424,525" +South Africa,"489,083","427,184","401,145" +Romania,"470,091","428,678","382,770" +Hong Kong,"455,156","427,310","406,863" +Pakistan,"450,134","407,307","379,120" +Egypt,"442,886","365,255","306,907" +Czech Republic,"427,953","391,027","345,037" +Chile,"402,243","357,371","330,267" +Peru,"399,593","334,855","289,329" +Kazakhstan,"397,921","306,239","291,480" +Nigeria,"386,671","290,794","252,084" +Portugal,"375,197","346,640","313,271" +Finland,"338,564","317,039","298,697" +Algeria,"331,894","287,031","266,972" +Iran,"311,950","362,682","448,092" +Greece,"304,627","280,636","257,145" +New Zealand,"273,395","264,057","260,172" +Hungary,"265,911","246,490","222,905" +Cuba,N/a,"107,352","252,063" +Iraq,"245,234","254,367","279,641" +Ukraine,"229,406","214,233","190,741" +Qatar,"203,044","215,560","219,163" +Morocco,"195,632","182,374","160,606" +Uzbekistan,"178,167","147,038","114,965" +Slovakia,"167,062","154,530","141,776" +Angola,"157,297","122,175","117,159" +Kenya,"151,151","135,941","120,340" +Kuwait,"146,911","157,209","160,227" +Bulgaria,"146,507","130,777","113,343" +Dominican Republic,"146,362","127,407","124,282" +Ecuador,"138,199","130,321","124,676" +Puerto Rico,"133,222","129,368","125,842" +Guatemala,"132,270","123,306","113,200" +Ghana,"128,638","114,210","82,938" +Ethiopia,"127,145","126,359","142,275" +DR Congo,"122,281","91,031","75,322" +Oman,"121,397","109,605","107,137" +Croatia,"115,915","105,060","92,662" +Ivory Coast,"112,115","99,774","85,894" +Serbia,"112,025","99,953","89,074" +Venezuela,"111,303","99,661","150,494" +Luxembourg,"110,417","101,158","93,280" +Costa Rica,"109,931","102,905","95,350" +Sri Lanka,"98,964","108,825","98,714" +Lithuania,"105,907","95,210","84,869" +Belarus,"102,042","93,397","75,962" +Uruguay,"96,092","85,348","80,962" +Panama,"95,024","90,463","86,260" +Tanzania,"94,889","90,143","79,236" +Slovenia,"86,732","79,648","72,485" +Myanmar,"83,832","81,666","64,796" +Turkmenistan,"83,065","49,829","68,655" +Bolivia,"80,743","64,769","46,627" +Azerbaijan,"78,372","75,938","74,316" +Uganda,"73,370","61,986","56,369" +Cameroon,"65,135","58,933","53,297" +Jordan,"64,909","61,610","53,352" +Tunisia,"60,745","57,503","53,493" +Paraguay,"60,542","49,278","44,739" +Zimbabwe,"56,713","51,216","45,719" +Macau,"54,228","52,061","49,467" +Latvia,"53,686","48,619","43,684" +Libya,"52,453","48,099","48,484" +Cambodia,"52,379","51,267","45,913" +Estonia,"51,634","47,031","42,765" +Bahrain,"48,849","48,966","47,110" +Nepal,"45,844","45,490","42,692" +Cyprus,"45,171","41,226","37,635" +Sudan,"44,688","60,163","25,574" +Iceland,"43,800","38,583","33,222" +Georgia,"42,716","38,144","34,192" +Honduras,"41,505","39,601","37,094" +Zambia,"41,243","28,880","25,303" +Senegal,"40,469","37,007","36,715" +El Salvador,"39,838","36,708","35,365" +Haiti,"39,180","32,077","25,312" +Bosnia and Herzegovina,"36,771","32,600","30,478" +Lebanon,"34,497","25,972","37,990" +Papua New Guinea,"34,403","32,499","32,262" +Guyana,"33,961","27,097","24,659" +Mali,"33,847","30,069","26,588" +Albania,"33,333","30,540","27,037" +Burkina Faso,"32,513","27,627","23,250" +Armenia,"31,873","29,243","25,955" +Malta,"30,712","27,772","25,030" +Guinea,"29,930","28,346","36,465" +Mongolia,"28,450","25,369","23,586" +Benin,"27,786","24,566","21,483" +Trinidad and Tobago,"26,836","25,943","25,634" +Chad,"25,628","21,473","19,906" +Niger,"24,813","21,646","17,491" +Nicaragua,"24,227","22,237","19,694" +Kyrgyzstan,"23,606","22,624","17,478" +Gabon,"23,363","21,427","20,867" +Mozambique,"23,275","22,338","22,745" +Jamaica,"23,028","22,705","22,014" +Botswana,"21,937","19,928","19,401" +Moldova,"21,889","20,352","18,200" +North Macedonia,"21,605","19,101","16,952" +Madagascar,"21,185","19,620","17,421" +Tajikistan,"20,418","17,661","13,949" +Afghanistan,"19,662","17,779","18,283" +Laos,"18,959","18,303","16,129" +Rwanda,"18,673","16,372","15,111" +Malawi,"18,309","14,918","11,734" +Namibia,"17,314","15,080","13,372" +Syria,"60,043","23,738","17,254" +North Korea,N/a,N/a,"17,199" +Palestine,"16,017","17,167","16,017" +Mauritius,"17,119","16,158","14,953" +Bahamas,"17,042","15,833","15,833" +Congo,"17,028","16,307","16,412" +Brunei,"16,863","15,032","15,463" +Mauritania,"14,352","11,680","10,912" +Somalia,"14,174","12,995","11,966" +Kosovo,"14,053","12,456","11,154" +Equatorial Guinea,"13,722","12,823","12,288" +Togo,"13,437","11,890","10,163" +Channel Islands,N/a,"12,507",N/a +Monaco,N/a,"11,126","11,126" +Montenegro,"10,227","9,233","8,274" +Liechtenstein,"9,442","8,906","8,628" +Bermuda,N/a,"9,194","8,840" +New Caledonia,N/a,"8,549","8,549" +Barbados,"8,483","8,017","7,598" +Sierra Leone,"8,270","7,464","6,792" +Burundi,"8,137","3,365","4,431" +Maldives,"8,130","7,741","7,062" +Cayman Islands,N/a,"7,765","7,657" +Isle of Man,N/a,"7,576",N/a +Yemen,"7,435","21,606","14,304" +Guam,N/a,"6,910",N/a +Fiji,"6,352","6,198","5,968" +French Polynesia,N/a,"6,324","6,323" +South Sudan,"6,069","11,998","3,522" +Suriname,"5,908","4,524","4,435" +Eswatini,"5,792","5,161","4,947" +Liberia,"5,642","5,246","5,251" +Andorra,"4,879","4,501","4,040" +Djibouti,"4,725","4,625","4,152" +U.S. Virgin Islands,N/a,"4,672",N/a +Aruba,"4,671","4,168","4,192" +Faroe Islands,N/a,"4,053",N/a +Bhutan,"3,856","3,579","3,347" +Curaçao,N/a,"3,561","3,561" +Central African Republic,"3,492","3,066","2,752" +Greenland,N/a,"3,327","3,466" +Belize,"3,450","3,327","3,204" +Cape Verde,"3,448","3,057","2,768" +Guinea-Bissau,"2,985","2,528","2,008" +Lesotho,"2,972","2,574","2,391" +Gambia,"2,792","2,594","2,372" +Saint Lucia,"2,765","2,656","2,574" +Zanzibar,N/a,N/a,"2,530" +San Marino,"2,417","2,027","2,082" +Antigua and Barbuda,"2,385","2,338","2,162" +Seychelles,"2,251","2,387","2,162" +Timor-Leste,"2,170","1,902","1,866" +Eritrea,"1,982","2,065","1,917" +Sint Maarten,N/a,"1,888","1,670" +Solomon Islands,"1,836","1,750","1,584" +British Virgin Islands,N/a,N/a,"1,830" +Comoros,"1,814","1,815","1,567" +Turks and Caicos Islands,N/a,"1,745","1,757" +Grenada,"1,483","1,420","1,372" +Vanuatu,"1,396","1,354","1,092" +Samoa,"1,376","1,288","1,253" +Saint Vincent and the Grenadines,"1,236","1,255","1,136" +São Tomé and Príncipe,"1,161",981,828 +Saint Kitts and Nevis,"1,143","1,184","1,122" +Northern Mariana Islands,N/a,"1,096",N/a +American Samoa,N/a,871 (2022),N/a +Dominica,791,724,689 +Tonga,716,679,648 +Saint Martin,N/a,649 (2021),N/a +Federated States of Micronesia,521,502,471 +Anguilla,N/a,N/a,456 +Cook Islands,N/a,N/a,414 +Kiribati,401,349,343 +Palau,377,345,310 +Marshall Islands,342,308,281 +Nauru,196,176,187 +Montserrat,N/a,N/a,81 +Tuvalu,65,57,56 diff --git a/wizard-work/ledger/gdp_imf_wb_un_extract.md b/wizard-work/ledger/gdp_imf_wb_un_extract.md new file mode 100644 index 0000000..80c9cad --- /dev/null +++ b/wizard-work/ledger/gdp_imf_wb_un_extract.md @@ -0,0 +1,224 @@ +| Country/Territory | IMF (2026) | World Bank (2025) | United Nations (2024) | +|---|---|---|:---| +| World | 128,003,776 | 118,350,166 | 111,565,144 | +| United States | 32,642,373 | 30,769,700 | 29,298,000 | +| China | 21,546,403 | 19,498,039 | 18,743,802 | +| Germany | 5,471,299 | 5,050,923 | 4,659,929 | +| Japan | 4,341,986 | 4,435,163 | 4,026,211 | +| United Kingdom | 4,268,446 | 4,002,588 | 3,685,881 | +| India | 4,060,207 | 3,956,067 | 3,952,244 | +| France | 3,554,155 | 3,366,316 | 3,160,443 | +| Russia | 2,861,156 | 2,561,310 | 2,173,386 | +| Italy | 2,701,737 | 2,551,557 | 2,380,825 | +| Brazil | 2,654,761 | 2,279,920 | 2,185,822 | +| Canada | 2,490,256 | 2,319,900 | 2,270,076 | +| South Korea | 2,245,847 | 1,872,375 | 1,875,388 | +| Mexico | 2,176,236 | 1,832,641 | 1,852,723 | +| Australia | 2,110,657 | 1,798,519 | 1,830,610 | +| Spain | 2,085,276 | 1,906,453 | 1,722,746 | +| Turkey | 1,817,987 | 1,597,293 | 1,323,255 | +| Indonesia | 1,479,954 | 1,445,643 | 1,396,300 | +| Netherlands | 1,423,856 | 1,332,768 | 1,214,928 | +| Saudi Arabia | 1,343,346 | 1,276,943 | 1,254,141 | +| Switzerland | 1,135,739 | 1,043,530 | 936,564 | +| Poland | 1,133,088 | 1,035,492 | 914,696 | +| Taiwan | 1,019,696 | N/a | N/a | +| Belgium | 772,699 | 725,466 | 664,564 | +| Argentina | 768,206 | 683,098 | 638,365 | +| Israel | 746,982 | 610,778 | 542,286 | +| Sweden | 745,081 | 668,999 | 604,899 | +| Ireland | 709,972 | 721,701 | 577,389 | +| Singapore | 667,889 | 603,870 | 547,387 | +| United Arab Emirates | 661,018 | 552,325 | 552,325 | +| Norway | 651,712 | 530,756 | 483,593 | +| Austria | 616,896 | 579,470 | 534,791 | +| Thailand | 599,115 | 577,010 | 526,518 | +| Colombia | 572,735 | 457,410 | 418,542 | +| Malaysia | 547,100 | 472,193 | 421,972 | +| Vietnam | 542,130 | 514,697 | 476,388 | +| Bangladesh | 506,357 | 456,319 | 432,742 | +| Philippines | 501,692 | 487,086 | 461,618 | +| Denmark | 499,354 | 462,527 | 424,525 | +| South Africa | 489,083 | 427,184 | 401,145 | +| Romania | 470,091 | 428,678 | 382,770 | +| Hong Kong | 455,156 | 427,310 | 406,863 | +| Pakistan | 450,134 | 407,307 | 379,120 | +| Egypt | 442,886 | 365,255 | 306,907 | +| Czech Republic | 427,953 | 391,027 | 345,037 | +| Chile | 402,243 | 357,371 | 330,267 | +| Peru | 399,593 | 334,855 | 289,329 | +| Kazakhstan | 397,921 | 306,239 | 291,480 | +| Nigeria | 386,671 | 290,794 | 252,084 | +| Portugal | 375,197 | 346,640 | 313,271 | +| Finland | 338,564 | 317,039 | 298,697 | +| Algeria | 331,894 | 287,031 | 266,972 | +| Iran | 311,950 | 362,682 | 448,092 | +| Greece | 304,627 | 280,636 | 257,145 | +| New Zealand | 273,395 | 264,057 | 260,172 | +| Hungary | 265,911 | 246,490 | 222,905 | +| Cuba | N/a | 107,352 | 252,063 | +| Iraq | 245,234 | 254,367 | 279,641 | +| Ukraine | 229,406 | 214,233 | 190,741 | +| Qatar | 203,044 | 215,560 | 219,163 | +| Morocco | 195,632 | 182,374 | 160,606 | +| Uzbekistan | 178,167 | 147,038 | 114,965 | +| Slovakia | 167,062 | 154,530 | 141,776 | +| Angola | 157,297 | 122,175 | 117,159 | +| Kenya | 151,151 | 135,941 | 120,340 | +| Kuwait | 146,911 | 157,209 | 160,227 | +| Bulgaria | 146,507 | 130,777 | 113,343 | +| Dominican Republic | 146,362 | 127,407 | 124,282 | +| Ecuador | 138,199 | 130,321 | 124,676 | +| Puerto Rico | 133,222 | 129,368 | 125,842 | +| Guatemala | 132,270 | 123,306 | 113,200 | +| Ghana | 128,638 | 114,210 | 82,938 | +| Ethiopia | 127,145 | 126,359 | 142,275 | +| DR Congo | 122,281 | 91,031 | 75,322 | +| Oman | 121,397 | 109,605 | 107,137 | +| Croatia | 115,915 | 105,060 | 92,662 | +| Ivory Coast | 112,115 | 99,774 | 85,894 | +| Serbia | 112,025 | 99,953 | 89,074 | +| Venezuela | 111,303 | 99,661 | 150,494 | +| Luxembourg | 110,417 | 101,158 | 93,280 | +| Costa Rica | 109,931 | 102,905 | 95,350 | +| Sri Lanka | 98,964 | 108,825 | 98,714 | +| Lithuania | 105,907 | 95,210 | 84,869 | +| Belarus | 102,042 | 93,397 | 75,962 | +| Uruguay | 96,092 | 85,348 | 80,962 | +| Panama | 95,024 | 90,463 | 86,260 | +| Tanzania | 94,889 | 90,143 | 79,236 | +| Slovenia | 86,732 | 79,648 | 72,485 | +| Myanmar | 83,832 | 81,666 | 64,796 | +| Turkmenistan | 83,065 | 49,829 | 68,655 | +| Bolivia | 80,743 | 64,769 | 46,627 | +| Azerbaijan | 78,372 | 75,938 | 74,316 | +| Uganda | 73,370 | 61,986 | 56,369 | +| Cameroon | 65,135 | 58,933 | 53,297 | +| Jordan | 64,909 | 61,610 | 53,352 | +| Tunisia | 60,745 | 57,503 | 53,493 | +| Paraguay | 60,542 | 49,278 | 44,739 | +| Zimbabwe | 56,713 | 51,216 | 45,719 | +| Macau | 54,228 | 52,061 | 49,467 | +| Latvia | 53,686 | 48,619 | 43,684 | +| Libya | 52,453 | 48,099 | 48,484 | +| Cambodia | 52,379 | 51,267 | 45,913 | +| Estonia | 51,634 | 47,031 | 42,765 | +| Bahrain | 48,849 | 48,966 | 47,110 | +| Nepal | 45,844 | 45,490 | 42,692 | +| Cyprus | 45,171 | 41,226 | 37,635 | +| Sudan | 44,688 | 60,163 | 25,574 | +| Iceland | 43,800 | 38,583 | 33,222 | +| Georgia | 42,716 | 38,144 | 34,192 | +| Honduras | 41,505 | 39,601 | 37,094 | +| Zambia | 41,243 | 28,880 | 25,303 | +| Senegal | 40,469 | 37,007 | 36,715 | +| El Salvador | 39,838 | 36,708 | 35,365 | +| Haiti | 39,180 | 32,077 | 25,312 | +| Bosnia and Herzegovina | 36,771 | 32,600 | 30,478 | +| Lebanon | 34,497 | 25,972 | 37,990 | +| Papua New Guinea | 34,403 | 32,499 | 32,262 | +| Guyana | 33,961 | 27,097 | 24,659 | +| Mali | 33,847 | 30,069 | 26,588 | +| Albania | 33,333 | 30,540 | 27,037 | +| Burkina Faso | 32,513 | 27,627 | 23,250 | +| Armenia | 31,873 | 29,243 | 25,955 | +| Malta | 30,712 | 27,772 | 25,030 | +| Guinea | 29,930 | 28,346 | 36,465 | +| Mongolia | 28,450 | 25,369 | 23,586 | +| Benin | 27,786 | 24,566 | 21,483 | +| Trinidad and Tobago | 26,836 | 25,943 | 25,634 | +| Chad | 25,628 | 21,473 | 19,906 | +| Niger | 24,813 | 21,646 | 17,491 | +| Nicaragua | 24,227 | 22,237 | 19,694 | +| Kyrgyzstan | 23,606 | 22,624 | 17,478 | +| Gabon | 23,363 | 21,427 | 20,867 | +| Mozambique | 23,275 | 22,338 | 22,745 | +| Jamaica | 23,028 | 22,705 | 22,014 | +| Botswana | 21,937 | 19,928 | 19,401 | +| Moldova | 21,889 | 20,352 | 18,200 | +| North Macedonia | 21,605 | 19,101 | 16,952 | +| Madagascar | 21,185 | 19,620 | 17,421 | +| Tajikistan | 20,418 | 17,661 | 13,949 | +| Afghanistan | 19,662 | 17,779 | 18,283 | +| Laos | 18,959 | 18,303 | 16,129 | +| Rwanda | 18,673 | 16,372 | 15,111 | +| Malawi | 18,309 | 14,918 | 11,734 | +| Namibia | 17,314 | 15,080 | 13,372 | +| Syria | 60,043 | 23,738 | 17,254 | +| North Korea | N/a | N/a | 17,199 | +| Palestine | 16,017 | 17,167 | 16,017 | +| Mauritius | 17,119 | 16,158 | 14,953 | +| Bahamas | 17,042 | 15,833 | 15,833 | +| Congo | 17,028 | 16,307 | 16,412 | +| Brunei | 16,863 | 15,032 | 15,463 | +| Mauritania | 14,352 | 11,680 | 10,912 | +| Somalia | 14,174 | 12,995 | 11,966 | +| Kosovo | 14,053 | 12,456 | 11,154 | +| Equatorial Guinea | 13,722 | 12,823 | 12,288 | +| Togo | 13,437 | 11,890 | 10,163 | +| Channel Islands | N/a | 12,507 | N/a | +| Monaco | N/a | 11,126 | 11,126 | +| Montenegro | 10,227 | 9,233 | 8,274 | +| Liechtenstein | 9,442 | 8,906 | 8,628 | +| Bermuda | N/a | 9,194 | 8,840 | +| New Caledonia | N/a | 8,549 | 8,549 | +| Barbados | 8,483 | 8,017 | 7,598 | +| Sierra Leone | 8,270 | 7,464 | 6,792 | +| Burundi | 8,137 | 3,365 | 4,431 | +| Maldives | 8,130 | 7,741 | 7,062 | +| Cayman Islands | N/a | 7,765 | 7,657 | +| Isle of Man | N/a | 7,576 | N/a | +| Yemen | 7,435 | 21,606 | 14,304 | +| Guam | N/a | 6,910 | N/a | +| Fiji | 6,352 | 6,198 | 5,968 | +| French Polynesia | N/a | 6,324 | 6,323 | +| South Sudan | 6,069 | 11,998 | 3,522 | +| Suriname | 5,908 | 4,524 | 4,435 | +| Eswatini | 5,792 | 5,161 | 4,947 | +| Liberia | 5,642 | 5,246 | 5,251 | +| Andorra | 4,879 | 4,501 | 4,040 | +| Djibouti | 4,725 | 4,625 | 4,152 | +| U.S. Virgin Islands | N/a | 4,672 | N/a | +| Aruba | 4,671 | 4,168 | 4,192 | +| Faroe Islands | N/a | 4,053 | N/a | +| Bhutan | 3,856 | 3,579 | 3,347 | +| Curaçao | N/a | 3,561 | 3,561 | +| Central African Republic | 3,492 | 3,066 | 2,752 | +| Greenland | N/a | 3,327 | 3,466 | +| Belize | 3,450 | 3,327 | 3,204 | +| Cape Verde | 3,448 | 3,057 | 2,768 | +| Guinea-Bissau | 2,985 | 2,528 | 2,008 | +| Lesotho | 2,972 | 2,574 | 2,391 | +| Gambia | 2,792 | 2,594 | 2,372 | +| Saint Lucia | 2,765 | 2,656 | 2,574 | +| Zanzibar | N/a | N/a | 2,530 | +| San Marino | 2,417 | 2,027 | 2,082 | +| Antigua and Barbuda | 2,385 | 2,338 | 2,162 | +| Seychelles | 2,251 | 2,387 | 2,162 | +| Timor-Leste | 2,170 | 1,902 | 1,866 | +| Eritrea | 1,982 | 2,065 | 1,917 | +| Sint Maarten | N/a | 1,888 | 1,670 | +| Solomon Islands | 1,836 | 1,750 | 1,584 | +| British Virgin Islands | N/a | N/a | 1,830 | +| Comoros | 1,814 | 1,815 | 1,567 | +| Turks and Caicos Islands | N/a | 1,745 | 1,757 | +| Grenada | 1,483 | 1,420 | 1,372 | +| Vanuatu | 1,396 | 1,354 | 1,092 | +| Samoa | 1,376 | 1,288 | 1,253 | +| Saint Vincent and the Grenadines | 1,236 | 1,255 | 1,136 | +| São Tomé and Príncipe | 1,161 | 981 | 828 | +| Saint Kitts and Nevis | 1,143 | 1,184 | 1,122 | +| Northern Mariana Islands | N/a | 1,096 | N/a | +| American Samoa | N/a | 871 (2022) | N/a | +| Dominica | 791 | 724 | 689 | +| Tonga | 716 | 679 | 648 | +| Saint Martin | N/a | 649 (2021) | N/a | +| Federated States of Micronesia | 521 | 502 | 471 | +| Anguilla | N/a | N/a | 456 | +| Cook Islands | N/a | N/a | 414 | +| Kiribati | 401 | 349 | 343 | +| Palau | 377 | 345 | 310 | +| Marshall Islands | 342 | 308 | 281 | +| Nauru | 196 | 176 | 187 | +| Montserrat | N/a | N/a | 81 | +| Tuvalu | 65 | 57 | 56 | diff --git a/wizard-work/ledger/verify_gdp_imf_wb_un.py b/wizard-work/ledger/verify_gdp_imf_wb_un.py new file mode 100644 index 0000000..7077d02 --- /dev/null +++ b/wizard-work/ledger/verify_gdp_imf_wb_un.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Independent verifier for the IMF / World Bank / UN GDP extraction. + +Re-derives the table from gdp_table.html (using the same deterministic rules as +extract_gdp_imf_wb_un.py), loads the committed CSV and markdown artifacts, and +exits 0 only if: + + * the committed CSV parses with the expected header and 222 rows, + * the committed CSV equals the re-derived table exactly, + * the committed markdown equals the re-derived markdown, + * the anchors pass (World, US, China, India, Taiwan, Cuba, Monaco), + * no data row lost its three metrics silently (audit count of 'N/a' rows). + +This is a real check, not a tautology: it compares committed artifacts against a +re-derivation from the raw source HTML plus hand-recorded anchor values. +""" +import sys + +sys.path.insert(0, '/root/troupe-standup/wizard-work/ledger') +from extract_gdp_imf_wb_un import extract, OUT_CSV, OUT_MD # noqa: E402 + +SRC = '/root/.hermes/scripts/gdp_table.html' +CSV = '/root/troupe-standup/wizard-work/ledger/gdp_imf_wb_un_extract.csv' +MD = '/root/troupe-standup/wizard-work/ledger/gdp_imf_wb_un_extract.md' + +ANCHORS = { + 'World': ('128,003,776', '118,350,166', '111,565,144'), + 'United States': ('32,642,373', '30,769,700', '29,298,000'), + 'China': ('21,546,403', '19,498,039', '18,743,802'), + 'India': ('4,060,207', '3,956,067', '3,952,244'), + 'Taiwan': ('1,019,696', 'N/a', 'N/a'), + 'Cuba': ('N/a', '107,352', '252,063'), + 'Monaco': ('N/a', '11,126', '11,126'), +} + +failures = [] + + +def check(name, ok, detail=''): + print(('PASS' if ok else 'FAIL'), name, detail) + if not ok: + failures.append(name) + + +def main(): + derived = extract(src=SRC) + print('data rows re-derived:', len(derived)) + + # --- load committed artifacts --- + import csv as _csv + committed = [] + with open(CSV, newline='', encoding='utf-8') as f: + reader = _csv.reader(f) + header = next(reader) + for row in reader: + committed.append(row) + check('csv header', header == ['country', 'imf_2026_million_usd', 'world_bank_2025_million_usd', 'un_2024_million_usd'], str(header)) + + by_country = {r[0]: tuple(r[1:]) for r in committed} + # committed must contain every re-derived country (and no extras) + derived_names = {r[0] for r in derived} + committed_names = set(by_country.keys()) + check('row-set identical', derived_names == committed_names, + 'missing=%s extra=%s' % (sorted(derived_names - committed_names), sorted(committed_names - derived_names))) + + # --- CSV exact comparison --- + derived_csv = [[r[0], r[1], r[2], r[3]] for r in derived] + check('csv == re-derived', committed == derived_csv, 'rows=%d vs %d' % (len(committed), len(derived_csv))) + + # --- markdown exact comparison --- + lines = ['| Country/Territory | IMF (2026) | World Bank (2025) | United Nations (2024) |', + '|---|---|---|:---|'] + for c, x, y, z in derived: + lines.append('| %s | %s | %s | %s |' % (c, x, y, z)) + expected_md = '\n'.join(lines) + '\n' + with open(MD, encoding='utf-8') as f: + actual_md = f.read() + check('md == re-derived', actual_md == expected_md, 'md_match=%s' % (actual_md == expected_md)) + + # --- anchors --- + for k, v in ANCHORS.items(): + got = by_country.get(k) + check('anchor ' + k, got == tuple(v), str(got)) + + # --- audit: N/a values are genuinely present (no silent data loss) --- + na_present = any(r[1] == 'N/a' or r[2] == 'N/a' or r[3] == 'N/a' for r in derived) + check('N/a-values captured (no silent loss)', na_present, 'rows_with_N/a=%d' % sum(1 for r in derived if 'N/a' in r[1:])) + + print('---') + if failures: + print('VERDICT: REJECT (%d failure(s): %s)' % (len(failures), ', '.join(failures))) + return 1 + print('VERDICT: ACCEPT') + return 0 + + +if __name__ == '__main__': + sys.exit(main())