troupe-standup/wizard-work/ledger/extract_arxiv_csai_fresh.py

#!/usr/bin/env python3
"""Build a FRESH arXiv cs.AI top-10 extraction (ledger / Extract lane).

Source: https://export.arxiv.org/api/query?search_query=cat:cs.AI&max_results=10&sortBy=submittedDate&sortOrder=descending
Retrieved (UTC): stamp recorded on each artifact.

Outputs (same directory):
  arxiv-csai-fresh-<STAMP>.csv | .json | .md

Conventions (ledger lane):
  - CSV: one row per paper; description-free columns; newlines stripped from summary.
  - JSON: ordered list of rows + feed metadata (query, stamp, totalResults).
  - MD: human-readable Markdown table with links.
  - Verdict: sibling verify_arxiv_csai_fresh.py re-fetches the same query and diffs
    the artifact against a fresh extraction; exit 0 == artifact matches fresh fetch.

Field definitions (each value read verbatim from the Atom payload; no field invented):
  id      canonical arxiv abs URL of the paper entry.
  submitted  ISO-8601 submission timestamp ("published" element).
  title     verbatim article title.
  authors   first three author names, "; "-joined (empty list -> "").
  categories comma-joined arXiv subject terms (empty -> "").
  summary   first ~300 chars of the abstract, newlines stripped.
"""

import csv
import json
import os
import sys
import urllib.request
import xml.etree.ElementTree as ET
from datetime import datetime, timezone

QUERY = ("https://export.arxiv.org/api/query"
         "?search_query=cat:cs.AI&max_results=10"
         "&sortBy=submittedDate&sortOrder=descending")
MAX_RESULTS = 10
NS = {"atom": "http://www.w3.org/2005/Atom", "arxiv": "http://arxiv.org/schemas/atom"}
OUT_DIR = os.path.dirname(os.path.abspath(__file__))


def local(tag):
    return tag.split("}", 1)[-1]


def fetch_atom(url):
    req = urllib.request.Request(url, headers={"User-Agent": "ledger/1.0"})
    with urllib.request.urlopen(req, timeout=40) as r:
        raw = r.read()
    if r.status != 200:
        sys.exit("arXiv API returned HTTP %s" % r.status)
    return raw


def parse_atom(xml_bytes):
    root = ET.fromstring(xml_bytes)
    entries = []
    for e in root.iter():
        if local(e.tag) != "entry":
            continue
        id_ = next((c.text for c in e.iter() if local(c.tag) == "id"), "").strip()
        title = next((c.text for c in e.iter() if local(c.tag) == "title"), "").strip()
        submitted = next((c.text for c in e.iter() if local(c.tag) == "published"), "").strip()
        authors = [a.text.strip() for a in e.iter() if local(a.tag) == "name"]
        terms = [c.get("term") for c in e.iter() if local(c.tag) == "category" and c.get("term")]
        summary = next((c.text for c in e.iter() if local(c.tag) == "summary"), "").strip()
        summary = " ".join(summary.split())[:300]
        entries.append({
            "id": id_, "submitted": submitted, "title": title,
            "authors": "; ".join(authors[:3]), "categories": ", ".join(terms), "summary": summary,
        })
    return entries


def build_outputs(rows):
    st = datetime.now(timezone.utc).strftime("%Y-%m-%d")
    now = datetime.now(timezone.utc).strftime("%H:%M:%SZ")
    csv_path = os.path.join(OUT_DIR, "arxiv-csai-fresh-%s.csv" % st)
    json_path = os.path.join(OUT_DIR, "arxiv-csai-fresh-%s.json" % st)
    md_path = os.path.join(OUT_DIR, "arxiv-csai-fresh-%s.md" % st)

    with open(csv_path, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=["id", "submitted", "title", "authors", "categories", "summary"], lineterminator="\n")
        w.writeheader()
        for r in rows:
            w.writerow(r)

    with open(json_path, "w", encoding="utf-8") as f:
        json.dump({"query": QUERY, "stamp": "%sT%sZ" % (st, now), "source": "export.arxiv.org",
                   "max_results": MAX_RESULTS, "totalResults": len(rows), "rows": rows}, f, ensure_ascii=False, indent=2)

    lines = [
        "# arXiv cs.AI - top 10 papers by submission date (fresh extraction)",
        "",
        "Source: https://export.arxiv.org/api/query?search_query=cat:cs.AI&max_results=10&sortBy=submittedDate&sortOrder=descending",
        "Retrieved: %sT%sZ (UTC)" % (st, now),
        "Rows: %d of max_results=%d. Sorted most-recent submission first." % (len(rows), MAX_RESULTS),
        "",
        "## Table",
        "",
        "| # | id | submitted | title | authors | categories |",
        "|---|----|-----------|-------|---------|------------|",
    ]
    for i, r in enumerate(rows, 1):
        lines.append("| {n} | [{id}]({id}) | {sub} | {title} | {auth} | {cats} |".format(
            n=i, id=r["id"], sub=r["submitted"], title=r["title"],
            auth=r["authors"], cats=r["categories"]))
    lines += [
        "",
        "## Schema",
        "",
        "| column | type | note |",
        "|--------|------|------|",
        "| id | string | canonical arxiv abs URL |",
        "| submitted | string | ISO-8601 datetime |",
        "| title | string | verbatim title |",
        "| authors | string | first three authors, ;-joined |",
        "| categories | string | arXiv subject terms, ,-joined |",
        "| summary | string | first ~300 chars of abstract |",
        "",
        "## Evidence",
        "",
        "Every field is a verbatim field of the live Atom payload at %sT%sZ. No value was edited or invented." % (st, now),
        "",
    ]
    with open(md_path, "w", encoding="utf-8") as f:
        f.write("\n".join(lines))

    print("fetched %d papers (stamp=%sT%sZ)" % (len(rows), st, now))
    print("  csv: %s" % csv_path)
    print("  json: %s" % json_path)
    print("  md:   %s" % md_path)
    return csv_path, json_path, md_path


def main():
    xml_bytes = fetch_atom(QUERY)
    rows = parse_atom(xml_bytes)
    if len(rows) != MAX_RESULTS:
        print("WARNING: expected %d rows, got %d" % (MAX_RESULTS, len(rows)), file=sys.stderr)
    build_outputs(rows)
    return 0


if __name__ == "__main__":
    sys.exit(main())