troupe-standup/wizard-work/ledger/gdp-us-states-2026-10-23/verify.py

#!/usr/bin/env python3
"""CHECK: re-fetches the live Wikipedia source for
'List of U.S. states and territories by GDP' and asserts the snapshot
(gdp-us-states-2026-10-23.{csv,json,md}) matches a fresh parse, plus
independent anchor values. Exit 0 = correct, nonzero = wrong.
"""
import csv
import hashlib
import json
import os
import re
import sys
import urllib.request

HERE = os.path.dirname(os.path.abspath(__file__))
SRC = "https://en.wikipedia.org/wiki/List_of_U.S._states_and_territories_by_GDP"
CSV_FN = "gdp-us-states-2026-10-23.csv"
JSON_FN = "gdp-us-states-2026-10-23.json"
MD_FN = "gdp-us-states-2026-10-23.md"
MAN_FN = "extraction_manifest.json"

failures = []


def fail(msg):
    failures.append(msg)
    print("FAIL:", msg)


def fetch(url):
    req = urllib.request.Request(url, headers={"User-Agent": "ledger-wizard/1.0 (check)"})
    with urllib.request.urlopen(req, timeout=60) as r:
        return r.read().decode("utf-8", errors="replace")


def norm(cells):
    """cells[0] carries an optional flagicon span; return [name, c1..c9]."""
    name = re.sub(r"<[^>]+>", " ", cells[0]).strip()
    name = re.sub(r"\s+", " ", name)
    name = re.sub(r"\[\d+\]", "", name).strip()
    return [name] + cells[1:]


def main():
    live_html = fetch(SRC)
    tables = re.findall(r"<table[^>]*>(.*?)</table>", live_html, re.S)
    target = None
    for t in tables:
        txt = re.sub(r"<[^>]+>", " ", t)
        txt = re.sub(r"\s+", " ", txt).strip()
        if "Nominal GDP at current prices" in txt and "State or federal district" in txt:
            target = t
            break
    assert target is not None, "source table not found"
    rows = re.findall(r"<tr[^>]*>(.*?)</tr>", target, re.S)
    live_cells = []
    for r in rows:
        cells = []
        for tag in ("th", "td"):
            for c in re.findall(r"<%s[^>]*>(.*?)</%s>" % (tag, tag), r, re.S):
                txt = re.sub(r"<[^>]+>", " ", c)
                txt = re.sub(r"\s+", " ", txt).strip()
                cells.append(txt)
        if any(cells):
            live_cells.append(norm(cells))
    # live_cells: [name, gdp2024, gdp2025, pct, chg, growth24, growth25, pop, pc24, pc25]
    by_name = {c[0]: c for c in live_cells}
    for anchor in ("California", "United States"):
        assert anchor in by_name, "anchor %r missing from live parse" % anchor

    # ---- read snapshot CSV ----
    csv_path = os.path.join(HERE, CSV_FN)
    if not os.path.exists(csv_path):
        fail("missing " + CSV_FN)
        sys.exit(1)
    snap = list(csv.DictReader(open(csv_path, encoding="utf-8")))
    if len(snap) != len(live_cells):
        fail("row count live=%d snapshot=%d" % (len(live_cells), len(snap)))

    # ---- compare live parse vs snapshot ----
    for i, lc in enumerate(live_cells):
        key, live_vals = lc[0], lc[1:]
        if i >= len(snap):
            break
        sv = snap[i]
        if sv["jurisdiction"] != key:
            fail("jurisdiction row %d live=%r snapshot=%r" % (i, key, sv["jurisdiction"]))
            continue
        for h, lv in zip(
            ["gdp_nominal_usd_2024", "gdp_nominal_usd_2025", "share_of_us_gdp_2025_pct",
             "gdp_change_2024_2025_millions_usd", "real_gdp_growth_2024_pct",
             "real_gdp_growth_2025_pct", "population_2025", "gdp_per_capita_usd_2024",
             "gdp_per_capita_usd_2025"], live_vals):
            if sv.get(h) != lv:
                fail("field drift row=%r col=%s live=%r snapshot=%r" % (key, h, lv, sv.get(h)))

    # ---- anchor checks (against the live parse values) ----
    def check(name, live_val, snapshot_val):
        if live_val != snapshot_val:
            fail("%s: live=%r snapshot=%r" % (name, live_val, snapshot_val))

    for r in snap:
        n = r["jurisdiction"]
        if n in by_name:
            a = by_name[n]
            check("C gdp2024" if n == "California" else "U gdp2024", a[1], r["gdp_nominal_usd_2024"])
            check("C gdp2025" if n == "California" else "U gdp2025", a[2], r["gdp_nominal_usd_2025"])
            check("C pct" if n == "California" else "U pct", a[3], r["share_of_us_gdp_2025_pct"])
            check("C chg" if n == "California" else "U chg", a[4], r["gdp_change_2024_2025_millions_usd"])
            check("C g24" if n == "California" else "U g24", a[5], r["real_gdp_growth_2024_pct"])
            check("C g25" if n == "California" else "U g25", a[6], r["real_gdp_growth_2025_pct"])
            check("C pop" if n == "California" else "U pop", a[7], r["population_2025"])
            check("C pc24" if n == "California" else "U pc24", a[8], r["gdp_per_capita_usd_2024"])
            check("C pc25" if n == "California" else "U pc25", a[9], r["gdp_per_capita_usd_2025"])

    # ---- JSON + manifest ----
    json_path = os.path.join(HERE, JSON_FN)
    if os.path.exists(json_path):
        d = json.load(open(json_path, encoding="utf-8"))
        if d.get("source") != SRC:
            fail("json source mismatch")
        if d.get("total_rows") != len(snap):
            fail("json total_rows=%r snapshot=%d" % (d.get("total_rows"), len(snap)))
        if len(d.get("rows", [])) != len(snap):
            fail("json rows count mismatch")

    man_path = os.path.join(HERE, MAN_FN)
    if os.path.exists(man_path):
        man = json.load(open(man_path, encoding="utf-8"))
        for fn, exp in man.get("files", {}).items():
            actual = hashlib.sha256(open(os.path.join(HERE, fn), "rb").read()).hexdigest()
            if actual != exp:
                fail("sha256 mismatch " + fn)

    if failures:
        print("CHECK FAILED (%d)" % len(failures))
        for f in failures:
            print("  -", f)
        sys.exit(1)
    print("PASS: snapshot matches live Wikipedia fetch (%d rows); anchors verified" % len(snap))


if __name__ == "__main__":
    main()