84 lines
2.8 KiB
Python
84 lines
2.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Fresh extraction of the 20 most recent cs.LG arXiv papers into CSV/JSON/MD.
|
|
|
|
Source: public arXiv API Atom feed.
|
|
"""
|
|
import csv
|
|
import json
|
|
import re
|
|
import sys
|
|
import urllib.request
|
|
import xml.etree.ElementTree as ET
|
|
|
|
URL = ("https://export.arxiv.org/api/query"
|
|
"?search_query=cat:cs.LG&max_results=20&sortBy=submittedDate"
|
|
"&sortOrder=descending")
|
|
ATOM = "http://www.w3.org/2005/Atom"
|
|
HEADERS = {"User-Agent": "ledger-wizard/1.0"}
|
|
|
|
|
|
def fetch():
|
|
req = urllib.request.Request(URL, headers=HEADERS)
|
|
with urllib.request.urlopen(req, timeout=60) as r:
|
|
return r.read().decode("utf-8")
|
|
|
|
|
|
def parse(xml_text):
|
|
root = ET.fromstring(xml_text)
|
|
ns = {"a": ATOM}
|
|
entries = root.findall("a:entry", ns)
|
|
rows = []
|
|
for e in entries:
|
|
id_el = e.find("a:id", ns)
|
|
arxiv_id = id_el.text.rsplit("/", 1)[-1].split("v")[0] if id_el is not None else ""
|
|
title = _text(e.find("a:title", ns))
|
|
published = _text(e.find("a:published", ns))
|
|
summary = _text(e.find("a:summary", ns))
|
|
authors = [a.find("a:name", ns).text
|
|
for a in e.findall("a:author", ns)
|
|
if a.find("a:name", ns) is not None and a.find("a:name", ns).text]
|
|
rows.append({
|
|
"arxiv_id": arxiv_id,
|
|
"title": title,
|
|
"authors": "; ".join(authors),
|
|
"published": published.split("T")[0] if published else "",
|
|
"abstract": summary,
|
|
})
|
|
return rows
|
|
|
|
|
|
def _text(el):
|
|
return el.text.strip() if el is not None and el.text else ""
|
|
|
|
|
|
def main():
|
|
xml = fetch()
|
|
rows = parse(xml)
|
|
rows.sort(key=lambda r: r["published"], reverse=True)
|
|
# keep only the most recent 20 by published date
|
|
rows = rows[:20]
|
|
csv_path = "arxiv-lg-recent.csv"
|
|
with open(csv_path, "w", newline="", encoding="utf-8") as f:
|
|
w = csv.DictWriter(f, fieldnames=["arxiv_id", "title", "authors", "published", "abstract"], extrasaction="ignore")
|
|
w.writeheader()
|
|
w.writerows(rows)
|
|
json_path = "arxiv-lg-recent.json"
|
|
with open(json_path, "w", encoding="utf-8") as f:
|
|
json.dump(rows, f, ensure_ascii=False, indent=2)
|
|
# markdown table
|
|
md_path = "arxiv-lg-recent.md"
|
|
with open(md_path, "w", encoding="utf-8") as f:
|
|
f.write("| arxiv_id | title | authors | published | abstract |\n")
|
|
f.write("|---|---|---|---|---|\n")
|
|
for r in rows:
|
|
abstract = (r["abstract"] or "").replace("|", "\\|").replace("\r", " ").replace("\n", " ")
|
|
f.write(f"| {r['arxiv_id']} | {r['title']} | {r['authors']} | {r['published']} | {abstract} |\n")
|
|
print("rows:", len(rows))
|
|
print("csv:", csv_path, "json:", json_path, "md:", md_path)
|
|
for r in rows:
|
|
print(r["arxiv_id"], "|", r["published"], "|", r["title"][:60])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|