48 lines
1.9 KiB
Python
48 lines
1.9 KiB
Python
import xml.etree.ElementTree as ET, csv, re, os, urllib.request
|
|
|
|
NS = 'http://www.w3.org/2005/Atom'
|
|
def clean(s):
|
|
return re.sub(r'\s+', ' ', s or '').strip()
|
|
def field(e, tag):
|
|
el = e.find('a:' + tag, {'a': NS})
|
|
return clean(el.text) if el is not None else ''
|
|
|
|
url = 'https://export.arxiv.org/api/query?search_query=all:long+context&start=0&max_results=15'
|
|
xml_src = urllib.request.urlopen(url, timeout=30).read().decode('utf-8')
|
|
open('/root/.hermes/wizard-outputs/ledger/arxiv15.xml','w').write(xml_src)
|
|
|
|
root = ET.fromstring(xml_src)
|
|
entries = root.findall('a:entry', {'a': NS})
|
|
rows = []
|
|
for e in entries:
|
|
eid = clean(e.find('a:id', {'a': NS}).text)
|
|
aid = re.sub(r'^.*arxiv\.org\/abs\/', '', eid)
|
|
title = field(e, 'title')
|
|
authors = '; '.join(field(a, 'name') for a in e.findall('a:author', {'a': NS}))
|
|
published = field(e, 'published')
|
|
subject = '; '.join(clean(c.get('term')) for c in e.findall('a:category', {'a': NS}))
|
|
link = clean(e.find('a:link', {'a': NS}).get('href'))
|
|
rows.append([aid, title, authors, published, subject, link])
|
|
|
|
out = '/root/.hermes/wizard-outputs/ledger'
|
|
os.makedirs(out, exist_ok=True)
|
|
with open(out + '/arxiv_long_context.csv', 'w', newline='') as f:
|
|
w = csv.writer(f)
|
|
w.writerow(['arxiv_id','title','authors','published','subject','abs_url'])
|
|
w.writerows(rows)
|
|
|
|
# markdown table
|
|
def md_cell(s):
|
|
s = str(s).replace('|', '\\|')
|
|
return '|' + s + '|'
|
|
with open(out + '/arxiv_long_context.md', 'w') as f:
|
|
f.write('# arXiv: long-context (15 results)\n\nSource: ' + url + '\n\n')
|
|
f.write('| arxiv_id | title | authors | published | subject | abs_url |\n|---|---|---|---|---|---|\n')
|
|
for r in rows:
|
|
f.write(md_cell(r[0]) + md_cell(r[1]) + md_cell(r[2]) + md_cell(r[3]) + md_cell(r[4]) + md_cell(r[5]) + '\n')
|
|
|
|
print('HTTP_OK')
|
|
print('n_rows:', len(rows))
|
|
for r in rows:
|
|
print(r[0], '|', r[1][:60], '|', r[3])
|