import xml.etree.ElementTree as ET, csv, re, os, urllib.request

NS = 'http://www.w3.org/2005/Atom'
def clean(s):
    return re.sub(r'\s+', ' ', s or '').strip()
def field(e, tag):
    el = e.find('a:' + tag, {'a': NS})
    return clean(el.text) if el is not None else ''

url = 'https://export.arxiv.org/api/query?search_query=all:long+context&start=0&max_results=15'
xml_src = urllib.request.urlopen(url, timeout=30).read().decode('utf-8')
open('/root/.hermes/wizard-outputs/ledger/arxiv15.xml','w').write(xml_src)

root = ET.fromstring(xml_src)
entries = root.findall('a:entry', {'a': NS})
rows = []
for e in entries:
    eid = clean(e.find('a:id', {'a': NS}).text)
    aid = re.sub(r'^.*arxiv\.org\/abs\/', '', eid)
    title = field(e, 'title')
    authors = '; '.join(field(a, 'name') for a in e.findall('a:author', {'a': NS}))
    published = field(e, 'published')
    subject = '; '.join(clean(c.get('term')) for c in e.findall('a:category', {'a': NS}))
    link = clean(e.find('a:link', {'a': NS}).get('href'))
    rows.append([aid, title, authors, published, subject, link])

out = '/root/.hermes/wizard-outputs/ledger'
os.makedirs(out, exist_ok=True)
with open(out + '/arxiv_long_context.csv', 'w', newline='') as f:
    w = csv.writer(f)
    w.writerow(['arxiv_id','title','authors','published','subject','abs_url'])
    w.writerows(rows)

# markdown table
def md_cell(s):
    s = str(s).replace('|', '\\|')
    return '|' + s + '|'
with open(out + '/arxiv_long_context.md', 'w') as f:
    f.write('# arXiv: long-context (15 results)\n\nSource: ' + url + '\n\n')
    f.write('| arxiv_id | title | authors | published | subject | abs_url |\n|---|---|---|---|---|---|\n')
    for r in rows:
        f.write(md_cell(r[0]) + md_cell(r[1]) + md_cell(r[2]) + md_cell(r[3]) + md_cell(r[4]) + md_cell(r[5]) + '\n')

print('HTTP_OK')
print('n_rows:', len(rows))
for r in rows:
    print(r[0], '|', r[1][:60], '|', r[3])