87 lines
2.7 KiB
Python
87 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Verification for the fresh cs.LG arXiv extraction.
|
|
|
|
Exits 0 only if every structural bar holds; otherwise prints the first failing
|
|
bar and exits 1. Run as: python3 verify_arxiv_csv.py
|
|
"""
|
|
import csv
|
|
import json
|
|
import re
|
|
import sys
|
|
|
|
import os
|
|
import sys
|
|
|
|
BASE = os.path.dirname(os.path.abspath(__file__))
|
|
CSV_PATH = os.path.join(BASE, "arxiv-lg-recent.csv")
|
|
JSON_PATH = os.path.join(BASE, "arxiv-lg-recent.json")
|
|
MD_PATH = os.path.join(BASE, "arxiv-lg-recent.md")
|
|
ARXIV_ID_RE = re.compile(r"^\d{4}\.\d{5}$")
|
|
DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
|
|
FIELDS = ["arxiv_id", "title", "authors", "published", "abstract"]
|
|
|
|
|
|
def read_csv():
|
|
with open(CSV_PATH, newline="", encoding="utf-8") as f:
|
|
return list(csv.DictReader(f))
|
|
|
|
|
|
def read_json():
|
|
with open(JSON_PATH, encoding="utf-8") as f:
|
|
return json.load(f)
|
|
|
|
|
|
def read_md():
|
|
with open(MD_PATH, encoding="utf-8") as f:
|
|
lines = f.read().splitlines()
|
|
# header + divider + separator rows
|
|
return [l for l in lines if l.startswith("|")]
|
|
|
|
|
|
def main():
|
|
errors = []
|
|
try:
|
|
rows = read_csv()
|
|
except Exception as e: # noqa: BLE001
|
|
print("FAIL: cannot read csv:", e)
|
|
return 1
|
|
if len(rows) != 20:
|
|
errors.append("csv row count %d != 20" % len(rows))
|
|
cols = None
|
|
for i, r in enumerate(rows):
|
|
if cols is None:
|
|
cols = list(r.keys())
|
|
if cols != FIELDS:
|
|
errors.append("csv header %s != %s" % (cols, FIELDS))
|
|
for f in FIELDS:
|
|
if f not in r:
|
|
errors.append("row %d missing column %s" % (i, f))
|
|
elif not r[f]:
|
|
errors.append("row %d empty %s" % (i, f))
|
|
if not ARXIV_ID_RE.match(r.get("arxiv_id", "")):
|
|
errors.append("row %d bad arxiv_id %r" % (i, r.get("arxiv_id")))
|
|
if not DATE_RE.match(r.get("published", "")):
|
|
errors.append("row %d bad published %r" % (i, r.get("published")))
|
|
try:
|
|
jr = read_json()
|
|
except Exception as e: # noqa: BLE001
|
|
print("FAIL: cannot read json:", e)
|
|
return 1
|
|
if len(jr) != 20:
|
|
errors.append("json row count %d != 20" % len(jr))
|
|
md = read_md()
|
|
data_rows = [l for l in md if l.startswith("| ") and "arxiv_id" not in l and not l.startswith("|---")]
|
|
if len(data_rows) != 20:
|
|
errors.append("md data rows %d != 20" % len(data_rows))
|
|
if errors:
|
|
print("FAIL: %d structural issue(s)" % len(errors))
|
|
for e in errors[:10]:
|
|
print(" -", e)
|
|
return 1
|
|
print("PASS: 20 rows, csv/json/md consistent, all arxiv_ids+published dates well-formed")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|