466 lines
24 KiB
Python
466 lines
24 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
build_dossier.py -- build the 5-paper agentic-memory grounded dossier.
|
||
|
||
Every quote in the dossier is EXTRACTED from the pinned corpus bytes
|
||
(anchor + span), never hand-typed, so the dossier cannot drift from its
|
||
sources. verify_claims.py re-checks each quote against the same pinned
|
||
bytes and runs a mutation battery proving the check is not vacuous.
|
||
|
||
Run: python3 build_dossier.py
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parent
|
||
CORPUS = ROOT / "corpus"
|
||
|
||
PAPERS = {
|
||
"2610.29671": "Uni-Mem: Learning a Long-Horizon Agentic Memory Manager with Unified Online Policy Optimization",
|
||
"2610.29823": "MemoryTax: The Bitter Lesson of Agent Memory",
|
||
"2610.29869": "Compaction Laundering: Provenance Destruction in Context-Compacting Coding Agents",
|
||
"2610.30485": "Why Bother Me Again? Benchmarking Personal Agent Memory for Interaction Preferences",
|
||
"2610.30755": "Is Complex Agentic Memory Always Needed? Naive RAG Is Stronger Than We Thought",
|
||
}
|
||
|
||
# id, paper, theme, claim, value (headline number, must be inside the quote),
|
||
# anchor (verbatim start marker in the paper text), span (chars to take), note.
|
||
SPEC = [
|
||
# ---------------- 2610.30755 -- Raw-RAG / naive RAG ----------------
|
||
dict(id="R1", paper="2610.30755", theme="T1",
|
||
claim="Raw-RAG ranks first among compared memory systems in six of seven benchmark-model settings.",
|
||
value="six of seven",
|
||
anchor="Raw-RAG ranks first among the compared memory systems in six",
|
||
span=210,
|
||
note="The headline result: the naive baseline wins most settings."),
|
||
dict(id="R2", paper="2610.30755", theme="T1",
|
||
claim="Raw-RAG beats the strongest compared agentic memory system by 0.13–5.53 points.",
|
||
value="0.13–5.53 points",
|
||
anchor="outperforming the strongest compared agentic memory system by 0.13",
|
||
span=130,
|
||
note="Margin is small on LoCoMo/LongMemEval, large on MemoryAgentBench."),
|
||
dict(id="R3", paper="2610.30755", theme="T4",
|
||
claim="Mem0's UPDATE+DELETE account for only 4.39% of 33,902 LoCoMo write decisions.",
|
||
value="4.39%",
|
||
anchor="together account for only 4.39% of the 33,902 write decisions on LoCoMo",
|
||
span=140,
|
||
note="Mutation is rare in practice -- most proposed writes are duplicates or malformed."),
|
||
dict(id="R4", paper="2610.30755", theme="T4",
|
||
claim="Replacing UPDATE/DELETE with append-only ADD/NONE raises LoCoMo accuracy 67.36% -> 69.42%.",
|
||
value="67.36% to 69.42%",
|
||
anchor="Replacing UPDATE with append-only ADD and DELETE with NONE raises",
|
||
span=170,
|
||
note="Explicit mutation gives no accuracy gain; the simpler store is better."),
|
||
dict(id="R5", paper="2610.30755", theme="T4",
|
||
claim="Removing reranking lowers MemoryAgentBench Total by 3.92 points.",
|
||
value="3.92 points",
|
||
anchor="Removing reranking lowers Total by 3.92 points",
|
||
span=190,
|
||
note="Reranking is the one component worth keeping."),
|
||
dict(id="R6", paper="2610.30755", theme="T4",
|
||
claim="Returning only one item lowers Total by 11.73 points.",
|
||
value="11.73 points",
|
||
anchor="while returning only one item lowers it by 11.73 points",
|
||
span=130,
|
||
note="Retrieval depth matters far more than chunk size."),
|
||
dict(id="R7", paper="2610.30755", theme="T4",
|
||
claim="Top-120 retrieval loses 14.57 points.",
|
||
value="14.57 points",
|
||
anchor="Top-120 loses 14.57 points",
|
||
span=130,
|
||
note="Driven by long-range understanding and conflict resolution."),
|
||
dict(id="R8", paper="2610.30755", theme="T1",
|
||
claim="No memory-management variant improves on Raw-RAG (48.32%).",
|
||
value="48.32%",
|
||
anchor="None improves Raw-RAG (48.32%)",
|
||
span=150,
|
||
note="RQ2 answer: added management does not help."),
|
||
dict(id="R9", paper="2610.30755", theme="T1",
|
||
claim="Atomic-fact extraction collapses test-time learning from 47.68% to 15.37%.",
|
||
value="47.68% to 15.37%",
|
||
anchor="Atomic-fact extraction reduces test-time learning from 47.68% to 15.37%",
|
||
span=130,
|
||
note="The most damaging of the three management operations."),
|
||
dict(id="R10", paper="2610.30755", theme="T2",
|
||
claim="Full-context beats Raw-RAG by 19.98 points on DeepSeek-V4-Flash MEME.",
|
||
value="19.98 points",
|
||
anchor="Full-context leads Raw-RAG by 19.98 points",
|
||
span=130,
|
||
note="The one setting where the naive baseline is not first."),
|
||
# ---------------- 2610.30485 -- MemPABench ----------------
|
||
dict(id="P1", paper="2610.30485", theme="T5",
|
||
claim="MemPABench has four simulated users holding 14 preferences.",
|
||
value="14 preferences",
|
||
anchor="Four grounded simulated users hold 14 preferences",
|
||
span=140,
|
||
note="Preferences span work and personal contexts."),
|
||
dict(id="P2", paper="2610.30485", theme="T5",
|
||
claim="Feedback spans 99 sessions over four months.",
|
||
value="99 sessions over four months",
|
||
anchor="Their feedback spans 99 sessions over four months",
|
||
span=130,
|
||
note="Five event-driven preference changes per user."),
|
||
dict(id="P3", paper="2610.30485", theme="T5",
|
||
claim="No Memory reaches 49.6% PSA against 45.2% random.",
|
||
value="49.6%",
|
||
anchor="No Memory, where the PA selects by its defaults, reaches 49.6%",
|
||
span=190,
|
||
note="Defaults alone nearly match chance-plus-margin."),
|
||
dict(id="P4", paper="2610.30485", theme="T5",
|
||
claim="File memory raises PSA to 59.2%, +9.6 points over No Memory.",
|
||
value="59.2%",
|
||
anchor="It raises PSA to 59.2%",
|
||
span=170,
|
||
note="The only condition with a clear gain; CI [+3.9, +15.6]."),
|
||
dict(id="P5", paper="2610.30485", theme="T5",
|
||
claim="Mem0, MemOS and Simple RAG stay at No Memory level (PSA 50.0/50.0/49.9).",
|
||
value="50.0%, 50.0%, and 49.9%",
|
||
anchor="with PSA of 50.0%, 50.0%, and 49.9%",
|
||
span=130,
|
||
note="Retrieval-based memory does not beat having none."),
|
||
dict(id="P6", paper="2610.30485", theme="T5",
|
||
claim="File memory PET is 25.4%, more than twice No Memory's 11.2%.",
|
||
value="25.4%",
|
||
anchor="Its PET is 25.4%",
|
||
span=170,
|
||
note="But its CS does not improve (14.4% vs 21.8%): context is not tracked."),
|
||
dict(id="P7", paper="2610.30485", theme="T5",
|
||
claim="Oracle lifts correct selection 33%->96% but BRS>=4 only 24%->59%.",
|
||
value="33% to 96%",
|
||
anchor="Oracle raises correct selections of the five settings",
|
||
span=210,
|
||
note="Knowing the preference does not override the model's interaction habits."),
|
||
dict(id="P8", paper="2610.30485", theme="T5",
|
||
claim="Information Flow is the hardest dimension, PSA 34–44%.",
|
||
value="34–44%",
|
||
anchor="Information Flow is the hardest under every condition",
|
||
span=160,
|
||
note="Expression Style (57-69%) and Disclosure (55-58%) are easier."),
|
||
dict(id="P9", paper="2610.30485", theme="T5",
|
||
claim="Anonymizing users drops shortcut accuracy 53.2% -> 46.4%, near the 45.2% chance.",
|
||
value="53.2%",
|
||
anchor="reaches 53.2% accuracy from the source identity but 46.4%",
|
||
span=140,
|
||
note="Confirms the benchmark measures memory, not name-guessing."),
|
||
# ---------------- 2610.29823 -- MemoryTax ----------------
|
||
dict(id="M1", paper="2610.29823", theme="T2",
|
||
claim="Study scope: one LoCoMo conversation, 419 turns, 152 questions.",
|
||
value="419 turns, 152 questions",
|
||
anchor="On one LoCoMo conversation (419 turns, 152 questions)",
|
||
span=160,
|
||
note="Deliberately narrow: one conversation, one run per cell."),
|
||
dict(id="M2", paper="2610.29823", theme="T2",
|
||
claim="With GPT-4o-mini, Mem0 scores 78% vs 74% for full context and BM25.",
|
||
value="78%",
|
||
anchor="With GPT-4o-mini, Mem0 scores highest",
|
||
span=160,
|
||
note="Agent memory is a patch for a weak model."),
|
||
dict(id="M3", paper="2610.29823", theme="T2",
|
||
claim="With GPT-6 Luna, full context scores 92%, BM25 82%, Mem0 80%, Zep 51%.",
|
||
value="92%",
|
||
anchor="With GPT-6 Luna, full context scores 92%",
|
||
span=200,
|
||
note="The patch stops helping as the model gets stronger."),
|
||
dict(id="M4", paper="2610.29823", theme="T2",
|
||
claim="With Claude Opus 5.5, agents score highest: Pi 99%, Codex 98%, Claude Code 97%.",
|
||
value="Pi 99%, Codex 98% and Claude Code 97%",
|
||
anchor="With Claude Opus 5.5, Mem0 and full context both score 95%",
|
||
span=170,
|
||
note="Searching the raw record beats reading all of it at the top end."),
|
||
dict(id="M5", paper="2610.29823", theme="T2",
|
||
claim="Pi scores 99% for $6.95/rollout vs full context 95% for $17.84.",
|
||
value="$6.95",
|
||
anchor="Pi scores 99% for $6.95 a rollout",
|
||
span=160,
|
||
note="Less than half the cost for a higher score."),
|
||
dict(id="M6", paper="2610.29823", theme="T4",
|
||
claim="On the 88 questions without photo evidence, Pi scores 100%.",
|
||
value="88 questions without photo evidence",
|
||
anchor="On the 88 questions without photo evidence",
|
||
span=160,
|
||
note="Pi matches full context once captions are removed from the comparison."),
|
||
dict(id="M7", paper="2610.29823", theme="T2",
|
||
claim="Pi reads ~10,000 input tokens/question (7,900 cached) vs 28,900 for full context.",
|
||
value="10,000 input tokens",
|
||
anchor="Pi reads about 10,000 input tokens a question",
|
||
span=150,
|
||
note="The efficiency mechanism behind the cost lead."),
|
||
dict(id="M8", paper="2610.29823", theme="T3",
|
||
claim="Writer ceiling: on Zep's GPT-4o-mini graph readers score 43/43/45%; on Opus 5.5's, 66/71/87%.",
|
||
value="43%, 43% and 45%",
|
||
anchor="GPT-4o-mini graph, GPT-4o-mini, GPT-6 Luna and Claude Opus 5.5 score 43%, 43% and 45%",
|
||
span=210,
|
||
note="A stronger reader cannot recover what a weaker writer left out."),
|
||
dict(id="M9", paper="2610.29823", theme="T3",
|
||
claim="Pi's one miss is a LoCoMo answer-key error, not a memory failure.",
|
||
value="151 questions",
|
||
anchor="Its one miss is LoCoMo",
|
||
span=590,
|
||
note="The model is right and the benchmark key is wrong."),
|
||
# ---------------- 2610.29869 -- Compaction Laundering ----------------
|
||
dict(id="C1", paper="2610.29869", theme="T3",
|
||
claim="Forged approval launders into memory 87–100% of the time across five flattening harnesses.",
|
||
value="87–100%",
|
||
anchor="the forged approval launders into the",
|
||
span=210,
|
||
note="Up to a ~908,000-token context window."),
|
||
dict(id="C2", paper="2610.29869", theme="T3",
|
||
claim="The identical attack succeeds 0% of the time across four role-typed harnesses.",
|
||
value="0% of the time",
|
||
anchor="succeeds 0% of the time",
|
||
span=130,
|
||
note="Role-typing is a built-in control."),
|
||
dict(id="C3", paper="2610.29869", theme="T3",
|
||
claim="Live in the real pi TUI the attack runs at ~50% against a 0% paired clean arm.",
|
||
value="~50%",
|
||
anchor="live in the real pi TUI at ~50%",
|
||
span=130,
|
||
note="Runtime evidence, not a simulation."),
|
||
dict(id="C4", paper="2610.29869", theme="T3",
|
||
claim="A code-level provenance check catches 77% of laundering, demoting 2% of legitimate content.",
|
||
value="77%",
|
||
anchor="we measure a code-level provenance check that catches 77% of",
|
||
span=160,
|
||
note="The only defense that works; prompt-level defenses cannot restore provenance."),
|
||
dict(id="C5", paper="2610.29869", theme="T3",
|
||
claim="In-prompt anti-injection rules stop 13–25%; published classifier defenses report 67% at best.",
|
||
value="13–25%",
|
||
anchor="stops 13",
|
||
span=140,
|
||
note="Baselines are far weaker than the code-level check."),
|
||
dict(id="C6", paper="2610.29869", theme="T3",
|
||
claim="Laundering still succeeds 100% of the time at every context size.",
|
||
value="100%",
|
||
anchor="The laundering still succeeds 100% of the",
|
||
span=160,
|
||
note="Size does not attenuate the attack."),
|
||
dict(id="C7", paper="2610.29869", theme="T3",
|
||
claim="Executed rate settles to ~20–40% in a full window.",
|
||
value="~20–40%",
|
||
anchor="settles to ~20",
|
||
span=130,
|
||
note="Real agents compact at this size, so this is the realistic case."),
|
||
# ---------------- 2610.29671 -- Uni-Mem ----------------
|
||
dict(id="U1", paper="2610.29671", theme="T1",
|
||
claim="Uni-Mem reaches an overall LoCoMo JJ of 88.80%.",
|
||
value="88.80%",
|
||
anchor="Uni-Mem achieves the best performance across all four question types",
|
||
span=220,
|
||
note="Best across all four question types."),
|
||
dict(id="U2", paper="2610.29671", theme="T1",
|
||
claim="Uni-Mem surpasses Memory-R2 (OSS) by 10.93 points and Mem-T by 12.31.",
|
||
value="10.93 percentage points",
|
||
anchor="It surpasses the strongest baseline, Memory-R2 (OSS), by 10.93",
|
||
span=210,
|
||
note="Gains of 12.93 and 9.29 on multi-hop and temporal."),
|
||
dict(id="U3", paper="2610.29671", theme="T1",
|
||
claim="Uni-Mem still reaches 80.99% without SFT, +3.12% over the strongest baseline.",
|
||
value="80.99%",
|
||
anchor="Uni-Mem still achieves 80.99% without SFT",
|
||
span=140,
|
||
note="RL alone is enough to beat the baselines."),
|
||
dict(id="U4", paper="2610.29671", theme="T1",
|
||
claim="ECA improves overall JJ by 1.29 and 3.03 points without and with SFT.",
|
||
value="1.29 and 3.03 percentage points",
|
||
anchor="improves overall JJ by 1.29 and 3.03 percentage points",
|
||
span=130,
|
||
note="Event-wise credit assignment localizes sparse feedback."),
|
||
dict(id="U5", paper="2610.29671", theme="T1",
|
||
claim="Stage-Decoupled GRPO improves overall JJ by 2.48 points without SFT.",
|
||
value="2.48 percentage points",
|
||
anchor="improves overall JJ by 2.48 percentage points without SFT",
|
||
span=140,
|
||
note="Temporal accuracy rises 10.55 points in this setting."),
|
||
dict(id="U6", paper="2610.29671", theme="T1",
|
||
claim="Trained construction alone improves overall JJ by 8.18 points.",
|
||
value="8.18 percentage points",
|
||
anchor="improves the overall JJ score by 8.18 percentage points",
|
||
span=130,
|
||
note="Each capability contributes even paired with its untrained counterpart."),
|
||
dict(id="U7", paper="2610.29671", theme="T1",
|
||
claim="Extending update blocks 1:1 -> 3:2 reduces overall JJ from 88.80% to 83.75%.",
|
||
value="83.75%",
|
||
anchor="the overall JJ score from 88.80",
|
||
span=120,
|
||
note="Frequent alternation matters as much as balanced allocation."),
|
||
dict(id="U8", paper="2610.29671", theme="T2",
|
||
claim="SFT followed by RL improves the 4B manager's JJ from 76.58% to 88.80%.",
|
||
value="76.58% to 88.80%",
|
||
anchor="SFT followed by RL improves the 4B manager",
|
||
span=150,
|
||
note="A small trained manager beats Qwen3.5-27B (87.05%) and 397B-A17B (85.86%)."),
|
||
dict(id="U9", paper="2610.29671", theme="T2",
|
||
claim="Uni-Mem outperforms all off-the-shelf managers including Qwen3.5-27B (87.05%).",
|
||
value="87.05%",
|
||
anchor="Uni-Mem also outperforms all off-the-shelf managers",
|
||
span=210,
|
||
note="Scale is not the binding constraint; joint training is."),
|
||
]
|
||
|
||
THEMES = {
|
||
"T1": "Is designed memory needed at all?",
|
||
"T2": "Cost, ceiling, and scale",
|
||
"T3": "Provenance: what compaction destroys and what writers omit",
|
||
"T4": "What raw evidence must be preserved",
|
||
"T5": "Interaction preferences are a different problem",
|
||
}
|
||
|
||
|
||
def sha256(p: Path) -> str:
|
||
return hashlib.sha256(p.read_bytes()).hexdigest()
|
||
|
||
|
||
def norm(s: str) -> str:
|
||
return " ".join(s.split())
|
||
|
||
|
||
def main() -> int:
|
||
texts, pins, sizes = {}, {}, {}
|
||
for pid in PAPERS:
|
||
p = CORPUS / f"{pid}.txt"
|
||
texts[pid] = p.read_text(encoding="utf-8")
|
||
pins[pid] = sha256(p)
|
||
sizes[pid] = p.stat().st_size
|
||
|
||
claims, seen = [], set()
|
||
for s in SPEC:
|
||
t = texts[s["paper"]]
|
||
idx = t.find(s["anchor"])
|
||
if idx < 0:
|
||
raise SystemExit(f"ANCHOR NOT FOUND: {s['id']} :: {s['anchor']!r}")
|
||
quote = t[idx: idx + s["span"]]
|
||
if s["value"] not in quote:
|
||
raise SystemExit(f"VALUE NOT IN QUOTE: {s['id']} :: {s['value']!r}")
|
||
if s["id"] in seen:
|
||
raise SystemExit(f"DUPLICATE ID: {s['id']}")
|
||
seen.add(s["id"])
|
||
claims.append(dict(
|
||
id=s["id"], paper=s["paper"], theme=s["theme"],
|
||
claim=s["claim"], value=s["value"], note=s["note"],
|
||
anchor=s["anchor"], quote=quote,
|
||
corpus_sha256=pins[s["paper"]],
|
||
))
|
||
|
||
by_paper = {pid: sum(1 for c in claims if c["paper"] == pid) for pid in PAPERS}
|
||
|
||
# ---- claims.json ----
|
||
(ROOT / "claims.json").write_text(json.dumps(dict(
|
||
built="2026-10-25",
|
||
query='all:"agentic memory"',
|
||
feed_updated="2026-10-25T00:40:42Z",
|
||
feed_total_results=583,
|
||
papers=[dict(arxiv_id=pid, title=PAPERS[pid], bytes=sizes[pid],
|
||
chars=len(texts[pid]), sha256=pins[pid],
|
||
claims=by_paper[pid]) for pid in PAPERS],
|
||
themes=THEMES,
|
||
claims=claims,
|
||
), indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||
|
||
# ---- DOSSIER.md ----
|
||
L = []
|
||
L.append("# Agentic Memory: A Five-Paper Grounded Dossier\n")
|
||
L.append("Archive lane -- sphinx. Built 2026-10-25.\n")
|
||
L.append('Scope: the five most recent arXiv papers matching `all:"agentic memory"`, '
|
||
"taken from a single feed snapshot (updated 2026-10-25T00:40:42Z; 583 total results).\n")
|
||
L.append("Every number below is a verbatim quote from the pinned corpus text. "
|
||
"Nothing is paraphrased into a figure. `verify_claims.py` re-derives every "
|
||
"quote from the same pinned bytes and fails the build if any drifts.\n")
|
||
|
||
L.append("## Provenance (pinned)\n")
|
||
L.append("| paper | title | bytes | chars | sha256 (first 16) | claims |")
|
||
L.append("|---|---|---|---|---|---|")
|
||
for pid in PAPERS:
|
||
L.append(f"| {pid} | {PAPERS[pid]} | {sizes[pid]} | {len(texts[pid])} | "
|
||
f"`{pins[pid][:16]}` | {by_paper[pid]} |")
|
||
L.append("")
|
||
|
||
L.append("## How this dossier is verified\n")
|
||
L.append("```")
|
||
L.append("python3 verify_claims.py # or: bash CHECK.txt")
|
||
L.append("```")
|
||
L.append("")
|
||
L.append("The verifier checks, for each of the %d claims:" % len(claims))
|
||
L.append("1. the corpus file's sha256 equals the pin recorded at build time;")
|
||
L.append("2. the claim's `quote` is a literal substring of that corpus file;")
|
||
L.append("3. the claim's headline `value` appears inside that quote;")
|
||
L.append("4. the headline `value` appears in this dossier.")
|
||
L.append("")
|
||
L.append("It then runs a mutation battery: corrupting one quote at a time in a scratch")
|
||
L.append("copy of `claims.json` must make the verification fail. If any mutation passes")
|
||
L.append("undetected, the build fails -- the check is falsified, not assumed.\n")
|
||
|
||
for tid, tname in THEMES.items():
|
||
L.append(f"## {tid}: {tname}\n")
|
||
for c in [c for c in claims if c["theme"] == tid]:
|
||
L.append(f"### {c['id']} -- {c['paper']}\n")
|
||
L.append(f"**{c['claim']}**\n")
|
||
L.append(f"> {norm(c['quote'])}\n")
|
||
L.append(f"*Why it matters:* {c['note']}\n")
|
||
|
||
L.append("## Cross-paper synthesis\n")
|
||
L.append("Four of the five papers converge on one answer: **the elaborate part of agent "
|
||
"memory is not where the value is.**\n")
|
||
L.append("- **Raw-RAG wins most settings** (2610.30755) -- naive dense retrieval over raw "
|
||
"history ranks first in six of seven benchmark-model settings, and *no* added "
|
||
"memory-management operation improves on it. The one setting where it loses is "
|
||
"the one where full context is affordable.\n")
|
||
L.append("- **Mutation is nearly free to drop** (2610.30755) -- UPDATE+DELETE are 4.39% of "
|
||
"LoCoMo writes and 2.46% of LongMemEval writes; replacing them with append-only "
|
||
"ADD/NONE *raises* accuracy 67.36% -> 69.42%.\n")
|
||
L.append("- **Agent memory is a patch for a weak model** (2610.29823) -- Mem0 helps "
|
||
"GPT-4o-mini (78% vs 74%) but not GPT-6 Luna or Claude Opus 5.5, and at the top "
|
||
"end Pi scores 99% for $6.95 against full context's 95% for $17.84.\n")
|
||
L.append("- **A stronger reader does not recover what a weaker writer left out** "
|
||
"(2610.29823) -- on Zep's GPT-4o-mini graph the three readers score 43/43/45%; "
|
||
"on the graph Claude Opus 5.5 wrote they score 66/71/87%. Memory has a ceiling "
|
||
"set by its writer.\n")
|
||
L.append("- **What must be preserved is raw evidence, not extracted facts** (2610.30755, "
|
||
"2610.29823) -- verbatim chunks beat lossy artifacts; on the 88 questions with no "
|
||
"photo evidence, Pi's lead over full context vanishes (100% vs 99%).\n")
|
||
L.append("- **Compaction destroys provenance deterministically** (2610.29869) -- a forged "
|
||
"approval launders into memory 87-100% of the time in flattening harnesses and "
|
||
"0% in role-typed ones; no prompt-level defense restores it, only a code-level "
|
||
"check (77% catch, 2% collateral).\n")
|
||
L.append("- **Interaction preferences are a harder problem than factual recall** "
|
||
"(2610.30485) -- retrieval-based memory does not beat No Memory (PSA 50.0/50.0/49.9 "
|
||
"vs 49.6%), and even an Oracle that hands the agent the right preference lifts "
|
||
"delivered behavior only 24% -> 59%.\n")
|
||
L.append("- **The counter-example is training the memory manager itself** (2610.29671) -- "
|
||
"Uni-Mem jointly learns construction and retrieval and reaches 88.80% JJ, beating "
|
||
"Memory-R2 (OSS) by 10.93 points and off-the-shelf Qwen3.5-27B (87.05%). The gain "
|
||
"comes from the training scheme (Stage-Decoupled GRPO + ECA), not from a bigger model.\n")
|
||
|
||
L.append("## Where the five disagree\n")
|
||
L.append("2610.30755 and 2610.29823 both conclude that raw-history search is competitive, "
|
||
"but they disagree on *why the alternative fails*: 2610.30755 blames the rarity of "
|
||
"useful mutation (4.39% of writes), while 2610.29823 blames the writer's ceiling. "
|
||
"These are compatible -- a weak writer produces few useful mutations *and* a "
|
||
"lossy store -- but neither paper measures the other's mechanism, so the dossier "
|
||
"records the disagreement rather than resolving it.\n")
|
||
L.append("2610.30485 is the outlier on the headline question: it studies *interaction* "
|
||
"preferences, not factual recall, and finds memory nearly useless there. That is a "
|
||
"different task family, not a refutation of the other four.\n")
|
||
|
||
L.append("## Claim ledger\n")
|
||
L.append(f"{len(claims)} claims across {len(PAPERS)} papers "
|
||
f"({', '.join(f'{pid}:{by_paper[pid]}' for pid in PAPERS)}).\n")
|
||
L.append("| id | paper | theme | value |")
|
||
L.append("|---|---|---|---|")
|
||
for c in claims:
|
||
L.append(f"| {c['id']} | {c['paper']} | {c['theme']} | {c['value']} |")
|
||
L.append("")
|
||
|
||
(ROOT / "DOSSIER.md").write_text("\n".join(L) + "\n", encoding="utf-8")
|
||
|
||
print(f"claims: {len(claims)} papers: {len(PAPERS)}")
|
||
for pid in PAPERS:
|
||
print(f" {pid} {pins[pid][:16]} {by_paper[pid]:>2} claims")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|