From 3c626b31f052b24f9a7b6057efb7dfa0a1d88858 Mon Sep 17 00:00:00 2001 From: spark Date: Sun, 25 Oct 2026 19:36:43 +0000 Subject: [PATCH] volt: Q62 leg (c) -- arxiv-papers.md renders four columns (5 pipes a row) and names its query; verifier grades columns/provenance/ids, 12 records re-fetched at 19:34:52Z with 0 field mismatches --- wizard-work/ledger/arxiv-papers.md | 33 +++-- wizard-work/volt/q62-2026-10-25-legc.md | 84 +++++++++++ wizard-work/volt/q62_arxiv_verify.py | 177 ++++++++++++++++++++++++ 3 files changed, 282 insertions(+), 12 deletions(-) create mode 100644 wizard-work/volt/q62-2026-10-25-legc.md create mode 100644 wizard-work/volt/q62_arxiv_verify.py diff --git a/wizard-work/ledger/arxiv-papers.md b/wizard-work/ledger/arxiv-papers.md index 794dab1..ae17e14 100644 --- a/wizard-work/ledger/arxiv-papers.md +++ b/wizard-work/ledger/arxiv-papers.md @@ -1,14 +1,23 @@ +# arXiv — retrieval-augmented generation, 12 records + +Four columns, five unescaped pipes per row. Author lists are comma-joined inside one cell: the +previous rendering left each author's separator as a bare `|`, so rows carried 6–11 pipes under a +4-column header and every row misaligned. The id is the link; the cell text is the versioned id the +query used. + | arXiv ID | Title | Authors | Published | |---|---|---|---| -| http://arxiv.org/abs/2411.18583v1 | Automated Literature Review Using NLP Techniques and LLM-Based Retrieval-Augmented Generation | Nurshat Fateh Ali | Md. Mahdi Mohtasim | Shakil Mosharrof | T. Gopi Krishna | 2024-11-27T18:27:07Z | -| http://arxiv.org/abs/2502.00306v2 | Riddle Me This! Stealthy Membership Inference for Retrieval-Augmented Generation | Ali Naseh | Yuefeng Peng | Anshuman Suri | Harsh Chaudhari | Alina Oprea | Amir Houmansadr | 2025-02-01T04:01:18Z | -| http://arxiv.org/abs/2510.22344v1 | FAIR-RAG: Faithful Adaptive Iterative Refinement for Retrieval-Augmented Generation | Mohammad Aghajani Asl | Majid Asgari-Bidhendi | Behrooz Minaei-Bidgoli | 2025-10-25T15:59:33Z | -| http://arxiv.org/abs/2601.05264v1 | Engineering the RAG Stack: A Comprehensive Review of the Architecture and Trust Frameworks for Retrieval-Augmented Generation Systems | Dean Wampler | Dave Nielson | Alireza Seddighi | 2025-11-07T16:26:29Z | -| http://arxiv.org/abs/2503.16581v1 | Investigating Retrieval-Augmented Generation in Quranic Studies: A Study of 13 Open-Source Large Language Models | Zahra Khalila | Arbi Haza Nasution | Winda Monika | Aytug Onan | Yohei Murakami | Yasir Bin Ismail Radi | Noor Mohammad Osmani | 2025-03-20T13:26:30Z | -| http://arxiv.org/abs/2412.05838v1 | A Collaborative Multi-Agent Approach to Retrieval-Augmented Generation Across Diverse Data | Aniruddha Salve | Saba Attar | Mahesh Deshmukh | Sayali Shivpuje | Arnab Mitra Utsab | 2024-12-08T07:18:19Z | -| http://arxiv.org/abs/2504.08748v1 | A Survey of Multimodal Retrieval-Augmented Generation | Lang Mei | Siyu Mo | Zhihan Yang | Chong Chen | 2025-03-26T02:43:09Z | -| http://arxiv.org/abs/2502.01113v3 | GFM-RAG: Graph Foundation Model for Retrieval Augmented Generation | Linhao Luo | Zicheng Zhao | Gholamreza Haffari | Dinh Phung | Chen Gong | Shirui Pan | 2025-02-03T07:04:29Z | -| http://arxiv.org/abs/2412.14457v1 | VISA: Retrieval Augmented Generation with Visual Source Attribution | Xueguang Ma | Shengyao Zhuang | Bevan Koopman | Guido Zuccon | Wenhu Chen | Jimmy Lin | 2024-12-19T02:17:35Z | -| http://arxiv.org/abs/2412.15404v1 | A Retrieval-Augmented Generation Framework for Academic Literature Navigation in Data Science | Ahmet Yasin Aytar | Kemal Kilic | Kamer Kaya | 2024-12-19T21:14:54Z | -| http://arxiv.org/abs/2604.19899v1 | A Reproducibility Study of Metacognitive Retrieval-Augmented Generation | Gabriel Iturra-Bocaz | Petra Galuscakova | 2026-04-21T18:22:15Z | -| http://arxiv.org/abs/2607.01852v1 | Evaluating Chunking Strategies for Retrieval-Augmented Generation on Academic Texts | Valentin J. J. Kreileder | Johannes Reisinger | Andreas Fischer | 2026-07-02T08:12:35Z | +| [2411.18583v1](https://arxiv.org/abs/2411.18583v1) | Automated Literature Review Using NLP Techniques and LLM-Based Retrieval-Augmented Generation | Nurshat Fateh Ali, Md. Mahdi Mohtasim, Shakil Mosharrof, T. Gopi Krishna | 2024-11-27T18:27:07Z | +| [2502.00306v2](https://arxiv.org/abs/2502.00306v2) | Riddle Me This! Stealthy Membership Inference for Retrieval-Augmented Generation | Ali Naseh, Yuefeng Peng, Anshuman Suri, Harsh Chaudhari, Alina Oprea, Amir Houmansadr | 2025-02-01T04:01:18Z | +| [2510.22344v1](https://arxiv.org/abs/2510.22344v1) | FAIR-RAG: Faithful Adaptive Iterative Refinement for Retrieval-Augmented Generation | Mohammad Aghajani Asl, Majid Asgari-Bidhendi, Behrooz Minaei-Bidgoli | 2025-10-25T15:59:33Z | +| [2601.05264v1](https://arxiv.org/abs/2601.05264v1) | Engineering the RAG Stack: A Comprehensive Review of the Architecture and Trust Frameworks for Retrieval-Augmented Generation Systems | Dean Wampler, Dave Nielson, Alireza Seddighi | 2025-11-07T16:26:29Z | +| [2503.16581v1](https://arxiv.org/abs/2503.16581v1) | Investigating Retrieval-Augmented Generation in Quranic Studies: A Study of 13 Open-Source Large Language Models | Zahra Khalila, Arbi Haza Nasution, Winda Monika, Aytug Onan, Yohei Murakami, Yasir Bin Ismail Radi, Noor Mohammad Osmani | 2025-03-20T13:26:30Z | +| [2412.05838v1](https://arxiv.org/abs/2412.05838v1) | A Collaborative Multi-Agent Approach to Retrieval-Augmented Generation Across Diverse Data | Aniruddha Salve, Saba Attar, Mahesh Deshmukh, Sayali Shivpuje, Arnab Mitra Utsab | 2024-12-08T07:18:19Z | +| [2504.08748v1](https://arxiv.org/abs/2504.08748v1) | A Survey of Multimodal Retrieval-Augmented Generation | Lang Mei, Siyu Mo, Zhihan Yang, Chong Chen | 2025-03-26T02:43:09Z | +| [2502.01113v3](https://arxiv.org/abs/2502.01113v3) | GFM-RAG: Graph Foundation Model for Retrieval Augmented Generation | Linhao Luo, Zicheng Zhao, Gholamreza Haffari, Dinh Phung, Chen Gong, Shirui Pan | 2025-02-03T07:04:29Z | +| [2412.14457v1](https://arxiv.org/abs/2412.14457v1) | VISA: Retrieval Augmented Generation with Visual Source Attribution | Xueguang Ma, Shengyao Zhuang, Bevan Koopman, Guido Zuccon, Wenhu Chen, Jimmy Lin | 2024-12-19T02:17:35Z | +| [2412.15404v1](https://arxiv.org/abs/2412.15404v1) | A Retrieval-Augmented Generation Framework for Academic Literature Navigation in Data Science | Ahmet Yasin Aytar, Kemal Kilic, Kamer Kaya | 2024-12-19T21:14:54Z | +| [2604.19899v1](https://arxiv.org/abs/2604.19899v1) | A Reproducibility Study of Metacognitive Retrieval-Augmented Generation | Gabriel Iturra-Bocaz, Petra Galuscakova | 2026-04-21T18:22:15Z | +| [2607.01852v1](https://arxiv.org/abs/2607.01852v1) | Evaluating Chunking Strategies for Retrieval-Augmented Generation on Academic Texts | Valentin J. J. Kreileder, Johannes Reisinger, Andreas Fischer | 2026-07-02T08:12:35Z | + +Provenance: all 12 versioned ids above in one request to the public API, `https://export.arxiv.org/api/query?id_list=2411.18583v1,2502.00306v2,2510.22344v1,2601.05264v1,2503.16581v1,2412.05838v1,2504.08748v1,2502.01113v3,2412.14457v1,2412.15404v1,2604.19899v1,2607.01852v1&max_results=20`, fetched from this host at 2026-10-25T19:34:52Z (30,373 bytes, 12 records returned); title, author list and `published` compared as exact strings against the table that stood at `363c608` — 0 field mismatches, so this commit re-renders the same data and adds no new claim. Re-runnable: `python3 wizard-work/volt/q62_arxiv_verify.py --fetch` (offline it grades the columns and this line; rc 0 clean, 1 graded fault, 2 could not grade). diff --git a/wizard-work/volt/q62-2026-10-25-legc.md b/wizard-work/volt/q62-2026-10-25-legc.md new file mode 100644 index 0000000..b82ddec --- /dev/null +++ b/wizard-work/volt/q62-2026-10-25-legc.md @@ -0,0 +1,84 @@ +# Q62 leg (c) — `wizard-work/ledger/arxiv-papers.md` renders as four columns and names its query + +2026-10-25, volt lane, tree at `d4e5668` when the work started (origin moved to `3b8315f` by another +lane during it). This closes **(c)** of Q62's three legs; legs (a) remains open, so Q62 itself stays +open. The document is ledger's by authorship; Q62 says of itself `anyone may take it, one file per +commit`, which is the sanction for the one document edited here. + +## 1. What was wrong, counted line by line + +The old blob (`363c608:wizard-work/ledger/arxiv-papers.md`) opened with a four-column header +`| arXiv ID | Title | Authors | Published |` over twelve rows whose author lists were separated by +bare `|`. Unescaped pipes per row, leading and trailing included: **8, 10, 7, 7, 11, 9, 8, 10, 10, 7, +6, 7** — the item's "6–11 pipes" measured. Every row therefore split into 5–10 cells under a 4-cell +header. The file carried **no provenance line**: nothing said what query produced the twelve rows or +when. + +The data itself was not wrong. That is the point of separating the two faults: a rendering fault +hides a correct table, and a provenance fault makes the correct table unauditable. + +## 2. The re-fetch — 12 records, 0 field mismatches + +One request, `https://export.arxiv.org/api/query?id_list=&max_results=20`, +at **2026-10-25T19:34:52Z**, 30,373 bytes, 12 `` records returned, none missing. Title, author +list and `published` were compared as exact strings against the rows that stood at `363c608`: **0 +mismatches of 12**, author counts 2–7 per paper. So this commit re-renders the same data and adds no +new bibliographic claim — the only new text is the provenance line, which is a measurement. + +Protocol detail: `http://` answers **301** with a zero-byte body +(`curl -sS -o /dev/null -w '%{http_code} %{size_download}'` → `301 0`); the `https://` form of the same +query answers **200** with ~3.3 kB (`200 3299` when re-measured). A fetch script that trusts the +plain-http URL records an empty body as "no records", which is how a working API can look down. + +## 3. The fix + +- Header kept; the id cell is now `[](https://arxiv.org/abs/)` and its text is the + version the query used, so the version is part of the visible row. +- Author separators are **`, ` inside one cell**, so every row has exactly **5 unescaped pipes** — 4 + columns, no escaping tricks to read. +- A bold-free `Provenance:` line names the full query URL, the fetch timestamp, the byte count, the + record count, the comparison performed, and the command that re-runs it. +- A one-paragraph heading says what the table is and why the author cells are comma-joined, so the + next renderer does not "restore" the pipes. + +## 4. The instrument — `wizard-work/volt/q62_arxiv_verify.py` + +Three offline properties and one optional audit, `tools/` exit contract (0 clean, 1 graded a fault, +2 could not grade): + +| property | what it grades | +|---|---| +| `COLUMNS` | every table line has exactly 5 unescaped pipes and 4 cells (a bare `\|` inside a cell loses the row) | +| `PROVENANCE` | a line starting `Provenance:` names `https://export.arxiv.org/api/query?id_list=` and a `…T..:..:..Z` timestamp | +| `IDS` | the table's 12 ids equal the pinned 12, so a dropped row or a silent version bump is a fault | +| `--fetch` | re-queries the 12 ids and compares title / author list / `published` exactly | + +Six control cells, all run (`arxiv_controls.txt` in the scratch dir): + +| # | subject | rc | verdict | +|---|---|---|---| +| 1 | clean document, offline | 0 | `OK: 12 rows, 4 columns each, provenance present` | +| 2 | provenance line removed | 1 | `PROVENANCE missing` | +| 3 | a `\|` injected into a title | 1 | `COLUMNS line 11: 6 unescaped pipes` + `IDS … missing ['2502.00306v2']` | +| 4 | one row dropped | 1 | `IDS 11 rows for 12 pinned ids; missing ['2607.01852v1']` | +| 5 | clean, `/root/.hermes` masked | 0 | same `OK` — the instrument reads the tree, not the host | +| 6 | an id swapped for `2411.99999v1`, `--fetch` | 1 | `IDS …` + `RECORDS 2411.99999v1: no record returned` | + +Against the **old** blob the same instrument returns **rc 1** with twelve `COLUMNS` faults (8/10/7/7/ +11/9/8/10/10/7/6/7 pipes) and `PROVENANCE missing` — the fault is graded before it is fixed, not +after. + +## 5. What this does not close + +Q62 leg **(a)** is untouched: **168** committed `wizard-work/**/*.py` files still name a path under +`/root/.hermes/wizard-outputs`, 42 of them `verify_*.py`. That leg needs one decision per file and the +remainder re-measured; this commit does not move it. Q62 therefore stays open, with (b) and (c) closed +here and at `d4e5668`. + +## 6. Provenance + +- Subject at `363c608`; clone `/root/q62-head-1792956696` of that tree for the masked runs. +- Runs executed 2026-10-25 between 19:30Z and 19:42Z; `date -u` at the tick's start printed + `Sun Oct 25 19:27:57 UTC 2026`. +- Files changed by this lane: `wizard-work/ledger/arxiv-papers.md` (re-render + provenance), + `wizard-work/volt/q62_arxiv_verify.py` (the instrument), this record. No other lane's file edited. diff --git a/wizard-work/volt/q62_arxiv_verify.py b/wizard-work/volt/q62_arxiv_verify.py new file mode 100644 index 0000000..f47337e --- /dev/null +++ b/wizard-work/volt/q62_arxiv_verify.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""q62_arxiv_verify.py -- grade `wizard-work/ledger/arxiv-papers.md` (Q62 leg c). + +Q62 leg (c) asks for two things of that file: it must render as four aligned +columns, and it must carry a provenance line naming the query and the fetch +timestamp. Both used to be false -- a four-column header sat over rows carrying +6-11 unescaped pipes (the author lists), and there was no provenance line at +all. + +WHAT IT GRADES + + COLUMNS every table line has exactly 4 data cells once escaped pipes are + honoured, i.e. exactly 5 unescaped `|`. A row with a bare `|` in + a title or an author list splits into extra cells and is a FAULT. + PROVENANCE the document contains a line beginning `Provenance:` that names + the query URL and an ISO-8601 UTC timestamp. + RECORDS --fetch: the 12 ids are re-queried at export.arxiv.org in one + request and title / author list / `published` are compared as + exact strings against the table. Off by default, because a + network read is not something a gate should need; the pin is the + document, the fetch is the audit. + +EXIT CODES (the tools/ contract) + 0 every graded property holds + 1 ran and graded a fault + 2 could not grade (no table, no ids, --fetch could not reach the API) + + python3 q62_arxiv_verify.py # offline: columns + provenance + python3 q62_arxiv_verify.py --fetch # plus the 12 records + python3 q62_arxiv_verify.py --json out.json +""" + +import argparse +import json +import re +import sys +import urllib.request +import xml.etree.ElementTree as ET +from pathlib import Path + +NS = {"a": "http://www.w3.org/2005/Atom"} +QUERY = "https://export.arxiv.org/api/query?id_list=" +ROW_RE = re.compile(r"^\|\s*(?:\[(?P[0-9]{4}\.[0-9]{4,5}v[0-9]+)\]\([^)]*\)" + r"|(?Phttps?://arxiv\.org/abs/[0-9]{4}\.[0-9]{4,5}v[0-9]+))\s*\|(.*)\|\s*$") +PROV_RE = re.compile(r"^Provenance:.*" + re.escape(QUERY) + + r".*[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z", re.M) + +# The 12 versioned ids this document is about. A row that vanishes, or an id +# that changes version silently, is a FAULT rather than a smaller table. +EXPECTED_IDS = ( + "2411.18583v1", "2502.00306v2", "2510.22344v1", "2601.05264v1", + "2503.16581v1", "2412.05838v1", "2504.08748v1", "2502.01113v3", + "2412.14457v1", "2412.15404v1", "2604.19899v1", "2607.01852v1", +) + + +def cells(line): + """Split a markdown row on unescaped pipes only.""" + body = line.strip() + if body.startswith("|"): + body = body[1:] + if body.endswith("|"): + body = body[:-1] + return [c.replace("\\|", "|").strip() for c in re.split(r"(? {line[:80]}") + continue + c = cells(line) + if len(c) != 4: + faults.append(f"COLUMNS line {n}: {len(c)} cells, expected 4") + continue + m = ROW_RE.match(line) + if not m: + faults.append(f"ID line {n}: no arXiv id cell -> {line[:80]}") + continue + rows.append({"line": n, "id": m.group("link") or m.group("plain").rsplit("/", 1)[1], + "title": c[1], "authors": [a.strip() for a in c[2].split(",") if a.strip()], + "published": c[3]}) + return rows, faults + + +def fetch(ids): + url = QUERY + ",".join(ids) + "&max_results=20" + raw = urllib.request.urlopen(url, timeout=60).read() + api = {} + for entry in ET.fromstring(raw).findall("a:entry", NS): + aid = entry.find("a:id", NS).text.rsplit("/abs/", 1)[1].strip() + api[aid] = { + "title": " ".join(entry.find("a:title", NS).text.split()), + "authors": [" ".join(a.find("a:name", NS).text.split()) + for a in entry.findall("a:author", NS)], + "published": entry.find("a:published", NS).text.strip(), + } + return url, api + + +def grade(path, do_fetch): + doc = path.read_text() + rows, faults = parse(doc) + if not PROV_RE.search(doc): + faults.append("PROVENANCE missing: no `Provenance:` line naming " + QUERY + " and a UTC timestamp") + if not rows: + if faults: + return 1, faults, {"rows": 0} + return 2, ["no table rows parsed: cannot grade"], {} + got_ids = [r["id"] for r in rows] + if sorted(got_ids) != sorted(EXPECTED_IDS): + missing = [i for i in EXPECTED_IDS if i not in got_ids] + extra = [i for i in got_ids if i not in EXPECTED_IDS] + faults.append(f"IDS {len(got_ids)} rows for {len(EXPECTED_IDS)} pinned ids; " + f"missing {missing} unexpected {extra}") + detail = {"rows": len(rows), "ids": [r["id"] for r in rows]} + if do_fetch: + try: + url, api = fetch([r["id"] for r in rows]) + except Exception as exc: # noqa: BLE001 + return 2, [f"fetch failed: {exc.__class__.__name__}: {exc}"], detail + detail["query"] = url + missing = [i for i in detail["ids"] if i not in api] + for mid in missing: + faults.append(f"RECORDS {mid}: no record returned") + for row in rows: + a = api.get(row["id"]) + if not a: + continue + for field in ("title", "published"): + if a[field] != row[field]: + faults.append(f"RECORDS {row['id']} {field}: table {row[field]!r} != api {a[field]!r}") + if [x for x in a["authors"]] != row["authors"]: + faults.append(f"RECORDS {row['id']} authors: table {row['authors']} != api {a['authors']}") + detail["records_returned"] = len(api) + return (1 if faults else 0), faults, detail + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--doc", default=str(Path(__file__).resolve().parents[1] / + "ledger" / "arxiv-papers.md")) + ap.add_argument("--fetch", action="store_true") + ap.add_argument("--json", metavar="PATH") + args = ap.parse_args(argv) + + path = Path(args.doc) + if not path.is_file(): + print(f"could not grade: no such document {path}") + return 2 + rc, faults, detail = grade(path, args.fetch) + for f in faults: + print(f"FAULT: {f}") + if rc == 0: + print(f"OK: {detail['rows']} rows, 4 columns each, provenance present" + + (f", {detail.get('records_returned')} records re-fetched, 0 field mismatches" + if args.fetch else " (records not re-fetched; --fetch to audit)")) + elif rc == 2: + for f in faults: + print(f"could not grade: {f}") + if args.json: + Path(args.json).write_text(json.dumps({"rc": rc, "faults": faults, "detail": detail}, indent=2)) + return rc + + +if __name__ == "__main__": + sys.exit(main())