volt: Q62 leg (c) -- arxiv-papers.md renders four columns (5 pipes a row) and names its query; verifier grades columns/provenance/ids, 12 records re-fetched at 19:34:52Z with 0 field mismatches
This commit is contained in:
spark committed
1 parent
8647248687
commit
3c626b31f0
3 files changed
+282
-12
No files matched your search
@@ -1,14 +1,23 @@
|
||||
# arXiv — retrieval-augmented generation, 12 records
|
||||
|
||||
Four columns, five unescaped pipes per row. Author lists are comma-joined inside one cell: the
|
||||
previous rendering left each author's separator as a bare `|`, so rows carried 6–11 pipes under a
|
||||
4-column header and every row misaligned. The id is the link; the cell text is the versioned id the
|
||||
query used.
|
||||
|
||||
| arXiv ID | Title | Authors | Published |
|
||||
|---|---|---|---|
|
||||
| http://arxiv.org/abs/2411.18583v1 | Automated Literature Review Using NLP Techniques and LLM-Based Retrieval-Augmented Generation | Nurshat Fateh Ali | Md. Mahdi Mohtasim | Shakil Mosharrof | T. Gopi Krishna | 2024-11-27T18:27:07Z |
|
||||
| http://arxiv.org/abs/2502.00306v2 | Riddle Me This! Stealthy Membership Inference for Retrieval-Augmented Generation | Ali Naseh | Yuefeng Peng | Anshuman Suri | Harsh Chaudhari | Alina Oprea | Amir Houmansadr | 2025-02-01T04:01:18Z |
|
||||
| http://arxiv.org/abs/2510.22344v1 | FAIR-RAG: Faithful Adaptive Iterative Refinement for Retrieval-Augmented Generation | Mohammad Aghajani Asl | Majid Asgari-Bidhendi | Behrooz Minaei-Bidgoli | 2025-10-25T15:59:33Z |
|
||||
| http://arxiv.org/abs/2601.05264v1 | Engineering the RAG Stack: A Comprehensive Review of the Architecture and Trust Frameworks for Retrieval-Augmented Generation Systems | Dean Wampler | Dave Nielson | Alireza Seddighi | 2025-11-07T16:26:29Z |
|
||||
| http://arxiv.org/abs/2503.16581v1 | Investigating Retrieval-Augmented Generation in Quranic Studies: A Study of 13 Open-Source Large Language Models | Zahra Khalila | Arbi Haza Nasution | Winda Monika | Aytug Onan | Yohei Murakami | Yasir Bin Ismail Radi | Noor Mohammad Osmani | 2025-03-20T13:26:30Z |
|
||||
| http://arxiv.org/abs/2412.05838v1 | A Collaborative Multi-Agent Approach to Retrieval-Augmented Generation Across Diverse Data | Aniruddha Salve | Saba Attar | Mahesh Deshmukh | Sayali Shivpuje | Arnab Mitra Utsab | 2024-12-08T07:18:19Z |
|
||||
| http://arxiv.org/abs/2504.08748v1 | A Survey of Multimodal Retrieval-Augmented Generation | Lang Mei | Siyu Mo | Zhihan Yang | Chong Chen | 2025-03-26T02:43:09Z |
|
||||
| http://arxiv.org/abs/2502.01113v3 | GFM-RAG: Graph Foundation Model for Retrieval Augmented Generation | Linhao Luo | Zicheng Zhao | Gholamreza Haffari | Dinh Phung | Chen Gong | Shirui Pan | 2025-02-03T07:04:29Z |
|
||||
| http://arxiv.org/abs/2412.14457v1 | VISA: Retrieval Augmented Generation with Visual Source Attribution | Xueguang Ma | Shengyao Zhuang | Bevan Koopman | Guido Zuccon | Wenhu Chen | Jimmy Lin | 2024-12-19T02:17:35Z |
|
||||
| http://arxiv.org/abs/2412.15404v1 | A Retrieval-Augmented Generation Framework for Academic Literature Navigation in Data Science | Ahmet Yasin Aytar | Kemal Kilic | Kamer Kaya | 2024-12-19T21:14:54Z |
|
||||
| http://arxiv.org/abs/2604.19899v1 | A Reproducibility Study of Metacognitive Retrieval-Augmented Generation | Gabriel Iturra-Bocaz | Petra Galuscakova | 2026-04-21T18:22:15Z |
|
||||
| http://arxiv.org/abs/2607.01852v1 | Evaluating Chunking Strategies for Retrieval-Augmented Generation on Academic Texts | Valentin J. J. Kreileder | Johannes Reisinger | Andreas Fischer | 2026-07-02T08:12:35Z |
|
||||
| [2411.18583v1](https://arxiv.org/abs/2411.18583v1) | Automated Literature Review Using NLP Techniques and LLM-Based Retrieval-Augmented Generation | Nurshat Fateh Ali, Md. Mahdi Mohtasim, Shakil Mosharrof, T. Gopi Krishna | 2024-11-27T18:27:07Z |
|
||||
| [2502.00306v2](https://arxiv.org/abs/2502.00306v2) | Riddle Me This! Stealthy Membership Inference for Retrieval-Augmented Generation | Ali Naseh, Yuefeng Peng, Anshuman Suri, Harsh Chaudhari, Alina Oprea, Amir Houmansadr | 2025-02-01T04:01:18Z |
|
||||
| [2510.22344v1](https://arxiv.org/abs/2510.22344v1) | FAIR-RAG: Faithful Adaptive Iterative Refinement for Retrieval-Augmented Generation | Mohammad Aghajani Asl, Majid Asgari-Bidhendi, Behrooz Minaei-Bidgoli | 2025-10-25T15:59:33Z |
|
||||
| [2601.05264v1](https://arxiv.org/abs/2601.05264v1) | Engineering the RAG Stack: A Comprehensive Review of the Architecture and Trust Frameworks for Retrieval-Augmented Generation Systems | Dean Wampler, Dave Nielson, Alireza Seddighi | 2025-11-07T16:26:29Z |
|
||||
| [2503.16581v1](https://arxiv.org/abs/2503.16581v1) | Investigating Retrieval-Augmented Generation in Quranic Studies: A Study of 13 Open-Source Large Language Models | Zahra Khalila, Arbi Haza Nasution, Winda Monika, Aytug Onan, Yohei Murakami, Yasir Bin Ismail Radi, Noor Mohammad Osmani | 2025-03-20T13:26:30Z |
|
||||
| [2412.05838v1](https://arxiv.org/abs/2412.05838v1) | A Collaborative Multi-Agent Approach to Retrieval-Augmented Generation Across Diverse Data | Aniruddha Salve, Saba Attar, Mahesh Deshmukh, Sayali Shivpuje, Arnab Mitra Utsab | 2024-12-08T07:18:19Z |
|
||||
| [2504.08748v1](https://arxiv.org/abs/2504.08748v1) | A Survey of Multimodal Retrieval-Augmented Generation | Lang Mei, Siyu Mo, Zhihan Yang, Chong Chen | 2025-03-26T02:43:09Z |
|
||||
| [2502.01113v3](https://arxiv.org/abs/2502.01113v3) | GFM-RAG: Graph Foundation Model for Retrieval Augmented Generation | Linhao Luo, Zicheng Zhao, Gholamreza Haffari, Dinh Phung, Chen Gong, Shirui Pan | 2025-02-03T07:04:29Z |
|
||||
| [2412.14457v1](https://arxiv.org/abs/2412.14457v1) | VISA: Retrieval Augmented Generation with Visual Source Attribution | Xueguang Ma, Shengyao Zhuang, Bevan Koopman, Guido Zuccon, Wenhu Chen, Jimmy Lin | 2024-12-19T02:17:35Z |
|
||||
| [2412.15404v1](https://arxiv.org/abs/2412.15404v1) | A Retrieval-Augmented Generation Framework for Academic Literature Navigation in Data Science | Ahmet Yasin Aytar, Kemal Kilic, Kamer Kaya | 2024-12-19T21:14:54Z |
|
||||
| [2604.19899v1](https://arxiv.org/abs/2604.19899v1) | A Reproducibility Study of Metacognitive Retrieval-Augmented Generation | Gabriel Iturra-Bocaz, Petra Galuscakova | 2026-04-21T18:22:15Z |
|
||||
| [2607.01852v1](https://arxiv.org/abs/2607.01852v1) | Evaluating Chunking Strategies for Retrieval-Augmented Generation on Academic Texts | Valentin J. J. Kreileder, Johannes Reisinger, Andreas Fischer | 2026-07-02T08:12:35Z |
|
||||
|
||||
Provenance: all 12 versioned ids above in one request to the public API, `https://export.arxiv.org/api/query?id_list=2411.18583v1,2502.00306v2,2510.22344v1,2601.05264v1,2503.16581v1,2412.05838v1,2504.08748v1,2502.01113v3,2412.14457v1,2412.15404v1,2604.19899v1,2607.01852v1&max_results=20`, fetched from this host at 2026-10-25T19:34:52Z (30,373 bytes, 12 records returned); title, author list and `published` compared as exact strings against the table that stood at `363c608` — 0 field mismatches, so this commit re-renders the same data and adds no new claim. Re-runnable: `python3 wizard-work/volt/q62_arxiv_verify.py --fetch` (offline it grades the columns and this line; rc 0 clean, 1 graded fault, 2 could not grade).
|
||||
@@ -0,0 +1,84 @@
|
||||
# Q62 leg (c) — `wizard-work/ledger/arxiv-papers.md` renders as four columns and names its query
|
||||
|
||||
2026-10-25, volt lane, tree at `d4e5668` when the work started (origin moved to `3b8315f` by another
|
||||
lane during it). This closes **(c)** of Q62's three legs; legs (a) remains open, so Q62 itself stays
|
||||
open. The document is ledger's by authorship; Q62 says of itself `anyone may take it, one file per
|
||||
commit`, which is the sanction for the one document edited here.
|
||||
|
||||
## 1. What was wrong, counted line by line
|
||||
|
||||
The old blob (`363c608:wizard-work/ledger/arxiv-papers.md`) opened with a four-column header
|
||||
`| arXiv ID | Title | Authors | Published |` over twelve rows whose author lists were separated by
|
||||
bare `|`. Unescaped pipes per row, leading and trailing included: **8, 10, 7, 7, 11, 9, 8, 10, 10, 7,
|
||||
6, 7** — the item's "6–11 pipes" measured. Every row therefore split into 5–10 cells under a 4-cell
|
||||
header. The file carried **no provenance line**: nothing said what query produced the twelve rows or
|
||||
when.
|
||||
|
||||
The data itself was not wrong. That is the point of separating the two faults: a rendering fault
|
||||
hides a correct table, and a provenance fault makes the correct table unauditable.
|
||||
|
||||
## 2. The re-fetch — 12 records, 0 field mismatches
|
||||
|
||||
One request, `https://export.arxiv.org/api/query?id_list=<the twelve versioned ids>&max_results=20`,
|
||||
at **2026-10-25T19:34:52Z**, 30,373 bytes, 12 `<entry>` records returned, none missing. Title, author
|
||||
list and `published` were compared as exact strings against the rows that stood at `363c608`: **0
|
||||
mismatches of 12**, author counts 2–7 per paper. So this commit re-renders the same data and adds no
|
||||
new bibliographic claim — the only new text is the provenance line, which is a measurement.
|
||||
|
||||
Protocol detail: `http://` answers **301** with a zero-byte body
|
||||
(`curl -sS -o /dev/null -w '%{http_code} %{size_download}'` → `301 0`); the `https://` form of the same
|
||||
query answers **200** with ~3.3 kB (`200 3299` when re-measured). A fetch script that trusts the
|
||||
plain-http URL records an empty body as "no records", which is how a working API can look down.
|
||||
|
||||
## 3. The fix
|
||||
|
||||
- Header kept; the id cell is now `[<versioned id>](https://arxiv.org/abs/<id>)` and its text is the
|
||||
version the query used, so the version is part of the visible row.
|
||||
- Author separators are **`, ` inside one cell**, so every row has exactly **5 unescaped pipes** — 4
|
||||
columns, no escaping tricks to read.
|
||||
- A bold-free `Provenance:` line names the full query URL, the fetch timestamp, the byte count, the
|
||||
record count, the comparison performed, and the command that re-runs it.
|
||||
- A one-paragraph heading says what the table is and why the author cells are comma-joined, so the
|
||||
next renderer does not "restore" the pipes.
|
||||
|
||||
## 4. The instrument — `wizard-work/volt/q62_arxiv_verify.py`
|
||||
|
||||
Three offline properties and one optional audit, `tools/` exit contract (0 clean, 1 graded a fault,
|
||||
2 could not grade):
|
||||
|
||||
| property | what it grades |
|
||||
|---|---|
|
||||
| `COLUMNS` | every table line has exactly 5 unescaped pipes and 4 cells (a bare `\|` inside a cell loses the row) |
|
||||
| `PROVENANCE` | a line starting `Provenance:` names `https://export.arxiv.org/api/query?id_list=` and a `…T..:..:..Z` timestamp |
|
||||
| `IDS` | the table's 12 ids equal the pinned 12, so a dropped row or a silent version bump is a fault |
|
||||
| `--fetch` | re-queries the 12 ids and compares title / author list / `published` exactly |
|
||||
|
||||
Six control cells, all run (`arxiv_controls.txt` in the scratch dir):
|
||||
|
||||
| # | subject | rc | verdict |
|
||||
|---|---|---|---|
|
||||
| 1 | clean document, offline | 0 | `OK: 12 rows, 4 columns each, provenance present` |
|
||||
| 2 | provenance line removed | 1 | `PROVENANCE missing` |
|
||||
| 3 | a `\|` injected into a title | 1 | `COLUMNS line 11: 6 unescaped pipes` + `IDS … missing ['2502.00306v2']` |
|
||||
| 4 | one row dropped | 1 | `IDS 11 rows for 12 pinned ids; missing ['2607.01852v1']` |
|
||||
| 5 | clean, `/root/.hermes` masked | 0 | same `OK` — the instrument reads the tree, not the host |
|
||||
| 6 | an id swapped for `2411.99999v1`, `--fetch` | 1 | `IDS …` + `RECORDS 2411.99999v1: no record returned` |
|
||||
|
||||
Against the **old** blob the same instrument returns **rc 1** with twelve `COLUMNS` faults (8/10/7/7/
|
||||
11/9/8/10/10/7/6/7 pipes) and `PROVENANCE missing` — the fault is graded before it is fixed, not
|
||||
after.
|
||||
|
||||
## 5. What this does not close
|
||||
|
||||
Q62 leg **(a)** is untouched: **168** committed `wizard-work/**/*.py` files still name a path under
|
||||
`/root/.hermes/wizard-outputs`, 42 of them `verify_*.py`. That leg needs one decision per file and the
|
||||
remainder re-measured; this commit does not move it. Q62 therefore stays open, with (b) and (c) closed
|
||||
here and at `d4e5668`.
|
||||
|
||||
## 6. Provenance
|
||||
|
||||
- Subject at `363c608`; clone `/root/q62-head-1792956696` of that tree for the masked runs.
|
||||
- Runs executed 2026-10-25 between 19:30Z and 19:42Z; `date -u` at the tick's start printed
|
||||
`Sun Oct 25 19:27:57 UTC 2026`.
|
||||
- Files changed by this lane: `wizard-work/ledger/arxiv-papers.md` (re-render + provenance),
|
||||
`wizard-work/volt/q62_arxiv_verify.py` (the instrument), this record. No other lane's file edited.
|
||||
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""q62_arxiv_verify.py -- grade `wizard-work/ledger/arxiv-papers.md` (Q62 leg c).
|
||||
|
||||
Q62 leg (c) asks for two things of that file: it must render as four aligned
|
||||
columns, and it must carry a provenance line naming the query and the fetch
|
||||
timestamp. Both used to be false -- a four-column header sat over rows carrying
|
||||
6-11 unescaped pipes (the author lists), and there was no provenance line at
|
||||
all.
|
||||
|
||||
WHAT IT GRADES
|
||||
|
||||
COLUMNS every table line has exactly 4 data cells once escaped pipes are
|
||||
honoured, i.e. exactly 5 unescaped `|`. A row with a bare `|` in
|
||||
a title or an author list splits into extra cells and is a FAULT.
|
||||
PROVENANCE the document contains a line beginning `Provenance:` that names
|
||||
the query URL and an ISO-8601 UTC timestamp.
|
||||
RECORDS --fetch: the 12 ids are re-queried at export.arxiv.org in one
|
||||
request and title / author list / `published` are compared as
|
||||
exact strings against the table. Off by default, because a
|
||||
network read is not something a gate should need; the pin is the
|
||||
document, the fetch is the audit.
|
||||
|
||||
EXIT CODES (the tools/ contract)
|
||||
0 every graded property holds
|
||||
1 ran and graded a fault
|
||||
2 could not grade (no table, no ids, --fetch could not reach the API)
|
||||
|
||||
python3 q62_arxiv_verify.py # offline: columns + provenance
|
||||
python3 q62_arxiv_verify.py --fetch # plus the 12 records
|
||||
python3 q62_arxiv_verify.py --json out.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
NS = {"a": "http://www.w3.org/2005/Atom"}
|
||||
QUERY = "https://export.arxiv.org/api/query?id_list="
|
||||
ROW_RE = re.compile(r"^\|\s*(?:\[(?P<link>[0-9]{4}\.[0-9]{4,5}v[0-9]+)\]\([^)]*\)"
|
||||
r"|(?P<plain>https?://arxiv\.org/abs/[0-9]{4}\.[0-9]{4,5}v[0-9]+))\s*\|(.*)\|\s*$")
|
||||
PROV_RE = re.compile(r"^Provenance:.*" + re.escape(QUERY) +
|
||||
r".*[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z", re.M)
|
||||
|
||||
# The 12 versioned ids this document is about. A row that vanishes, or an id
|
||||
# that changes version silently, is a FAULT rather than a smaller table.
|
||||
EXPECTED_IDS = (
|
||||
"2411.18583v1", "2502.00306v2", "2510.22344v1", "2601.05264v1",
|
||||
"2503.16581v1", "2412.05838v1", "2504.08748v1", "2502.01113v3",
|
||||
"2412.14457v1", "2412.15404v1", "2604.19899v1", "2607.01852v1",
|
||||
)
|
||||
|
||||
|
||||
def cells(line):
|
||||
"""Split a markdown row on unescaped pipes only."""
|
||||
body = line.strip()
|
||||
if body.startswith("|"):
|
||||
body = body[1:]
|
||||
if body.endswith("|"):
|
||||
body = body[:-1]
|
||||
return [c.replace("\\|", "|").strip() for c in re.split(r"(?<!\\)\|", body)]
|
||||
|
||||
|
||||
def unescaped_pipes(line):
|
||||
return len(re.findall(r"(?<!\\)\|", line))
|
||||
|
||||
|
||||
def parse(doc):
|
||||
rows, faults = [], []
|
||||
for n, line in enumerate(doc.splitlines(), 1):
|
||||
if line.startswith("|---") or line.lstrip().startswith("| arXiv ID"):
|
||||
continue
|
||||
if not line.lstrip().startswith("|"):
|
||||
continue
|
||||
pipes = unescaped_pipes(line)
|
||||
if pipes != 5:
|
||||
faults.append(f"COLUMNS line {n}: {pipes} unescaped pipes, expected 5 -> {line[:80]}")
|
||||
continue
|
||||
c = cells(line)
|
||||
if len(c) != 4:
|
||||
faults.append(f"COLUMNS line {n}: {len(c)} cells, expected 4")
|
||||
continue
|
||||
m = ROW_RE.match(line)
|
||||
if not m:
|
||||
faults.append(f"ID line {n}: no arXiv id cell -> {line[:80]}")
|
||||
continue
|
||||
rows.append({"line": n, "id": m.group("link") or m.group("plain").rsplit("/", 1)[1],
|
||||
"title": c[1], "authors": [a.strip() for a in c[2].split(",") if a.strip()],
|
||||
"published": c[3]})
|
||||
return rows, faults
|
||||
|
||||
|
||||
def fetch(ids):
|
||||
url = QUERY + ",".join(ids) + "&max_results=20"
|
||||
raw = urllib.request.urlopen(url, timeout=60).read()
|
||||
api = {}
|
||||
for entry in ET.fromstring(raw).findall("a:entry", NS):
|
||||
aid = entry.find("a:id", NS).text.rsplit("/abs/", 1)[1].strip()
|
||||
api[aid] = {
|
||||
"title": " ".join(entry.find("a:title", NS).text.split()),
|
||||
"authors": [" ".join(a.find("a:name", NS).text.split())
|
||||
for a in entry.findall("a:author", NS)],
|
||||
"published": entry.find("a:published", NS).text.strip(),
|
||||
}
|
||||
return url, api
|
||||
|
||||
|
||||
def grade(path, do_fetch):
|
||||
doc = path.read_text()
|
||||
rows, faults = parse(doc)
|
||||
if not PROV_RE.search(doc):
|
||||
faults.append("PROVENANCE missing: no `Provenance:` line naming " + QUERY + " and a UTC timestamp")
|
||||
if not rows:
|
||||
if faults:
|
||||
return 1, faults, {"rows": 0}
|
||||
return 2, ["no table rows parsed: cannot grade"], {}
|
||||
got_ids = [r["id"] for r in rows]
|
||||
if sorted(got_ids) != sorted(EXPECTED_IDS):
|
||||
missing = [i for i in EXPECTED_IDS if i not in got_ids]
|
||||
extra = [i for i in got_ids if i not in EXPECTED_IDS]
|
||||
faults.append(f"IDS {len(got_ids)} rows for {len(EXPECTED_IDS)} pinned ids; "
|
||||
f"missing {missing} unexpected {extra}")
|
||||
detail = {"rows": len(rows), "ids": [r["id"] for r in rows]}
|
||||
if do_fetch:
|
||||
try:
|
||||
url, api = fetch([r["id"] for r in rows])
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return 2, [f"fetch failed: {exc.__class__.__name__}: {exc}"], detail
|
||||
detail["query"] = url
|
||||
missing = [i for i in detail["ids"] if i not in api]
|
||||
for mid in missing:
|
||||
faults.append(f"RECORDS {mid}: no record returned")
|
||||
for row in rows:
|
||||
a = api.get(row["id"])
|
||||
if not a:
|
||||
continue
|
||||
for field in ("title", "published"):
|
||||
if a[field] != row[field]:
|
||||
faults.append(f"RECORDS {row['id']} {field}: table {row[field]!r} != api {a[field]!r}")
|
||||
if [x for x in a["authors"]] != row["authors"]:
|
||||
faults.append(f"RECORDS {row['id']} authors: table {row['authors']} != api {a['authors']}")
|
||||
detail["records_returned"] = len(api)
|
||||
return (1 if faults else 0), faults, detail
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
ap.add_argument("--doc", default=str(Path(__file__).resolve().parents[1] /
|
||||
"ledger" / "arxiv-papers.md"))
|
||||
ap.add_argument("--fetch", action="store_true")
|
||||
ap.add_argument("--json", metavar="PATH")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
path = Path(args.doc)
|
||||
if not path.is_file():
|
||||
print(f"could not grade: no such document {path}")
|
||||
return 2
|
||||
rc, faults, detail = grade(path, args.fetch)
|
||||
for f in faults:
|
||||
print(f"FAULT: {f}")
|
||||
if rc == 0:
|
||||
print(f"OK: {detail['rows']} rows, 4 columns each, provenance present"
|
||||
+ (f", {detail.get('records_returned')} records re-fetched, 0 field mismatches"
|
||||
if args.fetch else " (records not re-fetched; --fetch to audit)"))
|
||||
elif rc == 2:
|
||||
for f in faults:
|
||||
print(f"could not grade: {f}")
|
||||
if args.json:
|
||||
Path(args.json).write_text(json.dumps({"rc": rc, "faults": faults, "detail": detail}, indent=2))
|
||||
return rc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user