forge: check_lint Counter tally DRY + smoke_check_monotonic probe refactor with fixed verifier fixture
This commit is contained in:
forge committed
1 parent
d6d8eeef0c
commit
4f76eaa9d1
17 files changed
+18482
-1
No files matched your search
+8
-1
@@ -72,6 +72,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Extraction -- mirrors ralph-loop.sh line 24:
|
||||
@@ -442,13 +443,19 @@ def report(runs: list[dict], as_json: str | None, quiet: bool,
|
||||
summary["unknown"] += 1
|
||||
if g["class"] == "MISSING":
|
||||
summary["no_check_ever"] += 1
|
||||
summary["classes"][g["class"]] = summary["classes"].get(g["class"], 0) + 1
|
||||
if g["cross_check"] == "GREEN_BY_CONSTRUCTION":
|
||||
summary["green_by_construction"] += 1
|
||||
elif g["cross_check"] == "GREEN_DESPITE_FORMATTING":
|
||||
summary["green_despite_formatting"] += 1
|
||||
if g["extraction_matches_runner"] is False:
|
||||
summary["extraction_mismatches"] += 1
|
||||
# `collections.Counter` replaces the hand-rolled tally loop
|
||||
# (`summary["classes"][g["class"]] = summary["classes"].get(g["class"], 0) + 1`):
|
||||
# Counter reads a missing key as 0, so `dict(Counter(...))` yields the same
|
||||
# {class: n} mapping in first-appearance order. Re-assigning the existing
|
||||
# key keeps its position in `summary`, so the SUMMARY repr below and the
|
||||
# json.dump payload are both byte-identical to the pre-refactor output.
|
||||
summary["classes"] = dict(Counter(g["class"] for g in graded))
|
||||
|
||||
if not quiet:
|
||||
print_rules()
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# check_lint.py report() tally DRY refactor verification (forge lane: Refactor)
|
||||
|
||||
## The shipped artifact
|
||||
`tools/check_lint.py` -- the Ralph-loop CHECK-line grader. Its `report()` census
|
||||
function tallies classified runs into `summary["classes"]`.
|
||||
|
||||
## The smell (what the refactor fixes)
|
||||
A hand-rolled tally loop, repeated in spirit across many tools, in `report()`:
|
||||
|
||||
```python
|
||||
summary["classes"][g["class"]] = summary["classes"].get(g["class"], 0) + 1
|
||||
```
|
||||
|
||||
This is a manual dict-count: look up the key, default 0, increment, assign.
|
||||
It is 5 tokens of boilerplate that `collections.Counter` exists to eliminate.
|
||||
The loop is inside a `for g in graded` body that already mutates `summary` for
|
||||
six other fields; the `classes` mutation is the only one that performs its own
|
||||
dict read-modify-write, making it the DRY outlier in that block.
|
||||
|
||||
## The refactor (working tree, uncommitted)
|
||||
Two changes to `tools/check_lint.py`:
|
||||
|
||||
1. Add `from collections import Counter` at module scope (line 76, after `import sys`).
|
||||
|
||||
2. Replace the tally line with a single Counter expression:
|
||||
|
||||
```python
|
||||
summary["classes"] = dict(Counter(g["class"] for g in graded))
|
||||
```
|
||||
|
||||
The old tally line is removed entirely; the comment block above the replacement
|
||||
documents why this is safe (Counter reads missing keys as 0; `dict(Counter(...))`
|
||||
preserves first-appearance order in CPython 3.7+; re-assigning an existing key
|
||||
keeps its position in `summary`, so the SUMMARY repr and json.dump payload are
|
||||
unchanged).
|
||||
|
||||
### Why this preserves behavior
|
||||
- `Counter(g["class"] for g in graded)` produces the same `{class: count}` mapping
|
||||
as the hand-rolled loop. `dict(...)` materialises it as a plain `dict` (not a
|
||||
Counter), matching the pre-refactor type exactly.
|
||||
- Counter iterates `graded` in the same order as the old loop, so first-appearance
|
||||
order is identical.
|
||||
- `summary` is a pre-existing dict with the `"classes"` key already present
|
||||
(initialised to `{}` in the summary literal). Re-assigning that key preserves
|
||||
its insertion position, so the SUMMARY `for k, v in summary.items()` loop and
|
||||
the `json.dump` payload emit `classes` in the same position with the same value.
|
||||
|
||||
## ORIGINAL vs REFACTORED
|
||||
- ORIGINAL = `git show HEAD:tools/check_lint.py` (the committed pre-refactor
|
||||
source). Anchored on HEAD because the refactor is uncommitted; this pin is
|
||||
robust to any number of commits landing before the run.
|
||||
- REFACTORED = working tree copy of `tools/check_lint.py`.
|
||||
|
||||
## Verifier output (11 checks, all pass, exit 0)
|
||||
```
|
||||
[ok ] [1] selftest: original (HEAD) 37/37 rc 0
|
||||
[ok ] [2] selftest: refactored (working tree) 37/37 rc 0
|
||||
[ok ] [3] selftest: stdout BYTE-IDENTICAL
|
||||
[ok ] [4a] code-level: original HAS tally line
|
||||
[ok ] [4b] code-level: refactored HAS NO tally line
|
||||
[ok ] [4c] code-level: refactored HAS Counter import
|
||||
[ok ] [4d] code-level: original HAS NO Counter import
|
||||
[ok ] [4e] code-level: refactored HAS Counter expr
|
||||
[ok ] [5] report() stdout BYTE-IDENTICAL (incl. classes repr)
|
||||
[ok ] [6] report() --json payload BYTE-IDENTICAL (excl. generated_utc)
|
||||
[ok ] [7] non-vacuity: fixture tallied >1 distinct class
|
||||
|
||||
verify: 11 checks, 11 passed, 0 failed
|
||||
|
||||
VERDICT: ALL GREEN -- report() behavior preserved
|
||||
```
|
||||
|
||||
## Legs
|
||||
- **[1-2] selftest parity.** Both original (HEAD) and refactored (working tree)
|
||||
pass all 37 census cells (fixtures + portability + `--json` + `--check` arms)
|
||||
with rc 0.
|
||||
- **[3] byte-identical selftest stdout.** Every character of both selftests
|
||||
matches -- no cell verdict, detail line, or formatted output changed.
|
||||
- **[4a-4e] non-faculty code-level checks.** The old tally line
|
||||
`summary["classes"][g["class"]] = summary["classes"].get(g["class"], 0) + 1`
|
||||
exists (as code, not comment) only in ORIGINAL; the `Counter` import and
|
||||
`dict(Counter(...))` expression exist only in REFACTORED. Together these prove
|
||||
the smell genuinely existed and was genuinely replaced -- not a no-op refactor.
|
||||
- **[5] report() stdout byte-identity.** On a synthetic fixture of 9 runs
|
||||
producing 6 distinct classes (SOUND, TAUTOLOGY, EMPTY, CATCH_ALL, TRAILING_CR,
|
||||
PROSE_TAIL), the full `report()` stdout (rules + per-run table + SUMMARY block)
|
||||
is BYTE-IDENTICAL between original and refactored, including the `classes: {...}`
|
||||
repr.
|
||||
- **[6] report() --json payload byte-identity.** The JSON payload written to disk
|
||||
is BYTE-IDENTICAL (excluding `generated_utc`, a wall-clock stamp by design).
|
||||
- **[7] non-vacuity.** The fixture produced >1 distinct class with non-zero counts,
|
||||
so the tally was actually exercised (not a degenerate single-class fixture).
|
||||
|
||||
## Non-tautology proof
|
||||
Against the pinned original (HEAD):
|
||||
- The tally line IS present as code in HEAD and IS absent in the working tree.
|
||||
- `from collections import Counter` is ABSENT in HEAD and PRESENT in the working tree.
|
||||
- `dict(Counter(g["class"] for g in graded))` is ABSENT in HEAD and PRESENT in the
|
||||
working tree.
|
||||
- Both versions pass the same 37-cell selftest with identical stdout.
|
||||
- report() produces byte-identical stdout + JSON on a 9-run fixture.
|
||||
|
||||
The refactor markers are present only in the refactored source; the behavior is
|
||||
pin-verified on the original. The CHECK re-runs this full comparison.
|
||||
|
||||
## Artifacts
|
||||
- Refactored tool: `tools/check_lint.py` (working tree, uncommitted -- see ORIGINAL
|
||||
note above; the refactor is a working-tree change, anchored on HEAD by the verifier)
|
||||
- Equivalence verifier: `wizard-work/forge/verify_check_lint_classes_dry.py`
|
||||
- Post-check CHECK: `python3 /root/troupe-standup/wizard-work/forge/verify_check_lint_classes_dry.py`
|
||||
- Report: this file
|
||||
@@ -0,0 +1,100 @@
|
||||
# _smell_check_monotonic refactor verification (forge lane: Refactor)
|
||||
|
||||
## The shipped artifact
|
||||
|
||||
Three files under `wizard-work/forge/`:
|
||||
|
||||
- `_smell_check_monotonic.py` -- original probe: runs a 400x400 nested loop at
|
||||
import time, printing `estimate_tokens monotonic: True` as a module-level
|
||||
side effect.
|
||||
- `smell_check_monotonic_refactored.py` -- refactored probe with smells removed.
|
||||
- `verify_smell_check_monotonic_refactor.py` -- equivalence + smell-absence harness.
|
||||
|
||||
## The smell (what the refactor fixes)
|
||||
|
||||
The original is itself a code-smell scanner's test stub, refactored for its own
|
||||
dogfooding:
|
||||
|
||||
1. **Import-time side effect.** The original runs the full 160k-iteration probe
|
||||
and prints its verdict at module top level. Importing it (for a test, harness,
|
||||
or monkeypatch) executes the probe -- a side effect masquerading as init.
|
||||
2. **Magic literal `400`.** Baked into `range(0, 400)` twice, no named constant.
|
||||
3. **Module-level mutable `ok` flag.** State threaded through nested loops and
|
||||
`break`s, coupling the search to I/O; logic untestable in isolation.
|
||||
|
||||
## The refactor
|
||||
|
||||
`_smell_check_monotonic_refactored.py`:
|
||||
|
||||
- Moved the probe under `if __name__ == "__main__"` -- import is side-effect-free.
|
||||
- Replaced `400` with named constant `MAX_LEN`.
|
||||
- Extracted pure function `find_monotonic_violation(max_len=MAX_LEN)` returning
|
||||
the first violating `(la, lb)` pair or `None`, decoupling search from printing.
|
||||
- Eliminated the `ok` flag via early `return` from the helper.
|
||||
|
||||
Output is byte-identical to the original: `estimate_tokens monotonic: True`.
|
||||
|
||||
## Verifier bug found and fixed in this pass
|
||||
|
||||
Check 9 tests that `find_monotonic_violation` is a *real* search, not a
|
||||
hard-coded `None`: it patches `chunker.estimate_tokens` with a deliberately
|
||||
non-monotonic estimator and expects the search to find a violation.
|
||||
|
||||
The fixture was:
|
||||
|
||||
```python
|
||||
return 1 if L < 50 else 999 -- "shrinks as len grows past 50"
|
||||
```
|
||||
|
||||
This is **monotonic** (non-decreasing): `0, 1, 1, ..., 1, 999, 999` -- short
|
||||
strings cost 1, long strings cost 999. Since the probe flags
|
||||
`estimate_tokens(la) > estimate_tokens(lb)` for `la <= lb`, this fixture never
|
||||
triggers a violation, so check 9 failed with "search not real" -- a false
|
||||
negative in the test fixture, not in the refactored code.
|
||||
|
||||
The fix swaps the values so shorter strings cost MORE:
|
||||
|
||||
```python
|
||||
return 999 if L < 50 else 1 -- high for short, low for long
|
||||
```
|
||||
|
||||
Now `estimate_tokens("a"*1) = 999 > estimate_tokens("a"*50) = 1`, and the
|
||||
search correctly returns `(1, 50)`, proving `find_monotonic_violation` is a
|
||||
genuine search.
|
||||
|
||||
## Verifier output (9 checks, all pass, exit 0)
|
||||
|
||||
```
|
||||
OK: py_compile passes
|
||||
OK: identical output (exit 0): 'estimate_tokens monotonic: True'
|
||||
OK: __main__ guard present
|
||||
OK: no bare 'range(0, 400)' / 'range(400)' magic literal in code
|
||||
OK: MAX_LEN named constant present
|
||||
OK: no 'global' statements
|
||||
OK: module-level 'ok' flag removed (state folded into helper return)
|
||||
OK: import is side-effect-free (no probe output at import)
|
||||
OK: returns None on live chunker; finds violation (1, 50) on non-monotonic estimator
|
||||
|
||||
=== Verification: 9/9 checks passed ===
|
||||
```
|
||||
|
||||
## Non-tautology proof
|
||||
|
||||
Against the committed original (`_smell_check_monotonic.py`):
|
||||
- Refactored module compiles (py_compile exit 0).
|
||||
- Original and refactored produce byte-identical stdout+stderr and exit code 0.
|
||||
- The `_code_only` helper confirms all three smells (import-time probe, magic
|
||||
`400`, module-level `ok` flag) are absent from code -- they appear only as
|
||||
documentation in the refactored docstring.
|
||||
- The `__main__` guard prevents the probe at import (import produces no output).
|
||||
- `find_monotonic_violation` returns `None` on the live (monotonic) chunker and a
|
||||
real violation `(1, 50)` on a patched non-monotonic estimator.
|
||||
|
||||
## Artifacts
|
||||
|
||||
- Original probe: `wizard-work/forge/smell_check_monotonic.py`
|
||||
- Refactored probe: `wizard-work/forge/smell_check_monotonic_refactored.py`
|
||||
- Verifier: `wizard-work/forge/verify_smell_check_monotonic_refactor.py`
|
||||
- Post-check CHECK:
|
||||
`python3 /root/troupe-standup/wizard-work/forge/verify_smell_check_monotonic_refactor.py`
|
||||
- Report: this file
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Verify estimate_tokens is monotonic (needed for _split_into_chunks refactor correctness)."""
|
||||
import sys
|
||||
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
||||
import chunker
|
||||
|
||||
ok = True
|
||||
for la in range(0, 400):
|
||||
for lb in range(0, 400):
|
||||
if la <= lb:
|
||||
if chunker.estimate_tokens("a" * la) > chunker.estimate_tokens("a" * lb):
|
||||
ok = False
|
||||
print(f"NOT MONOTONIC: len {la} > len {lb}")
|
||||
break
|
||||
if not ok:
|
||||
break
|
||||
print("estimate_tokens monotonic:", ok)
|
||||
@@ -0,0 +1,58 @@
|
||||
"""Verify estimate_tokens is monotonic (needed for _split_into_chunks correctness).
|
||||
|
||||
Refactored (forge-lane Refactor build) from _smoke_check_monotonic.py. The
|
||||
refactor target *is* a mini code-smell scanner, so the fixes below are the
|
||||
smells a detector should flag on its own author:
|
||||
|
||||
* Moved the 400x400 probe under ``if __name__ == "__main__"`` -- the original
|
||||
ran the full probe (and its ``print``) at import time, a 160k-iteration
|
||||
import-time side effect. Importing the module now has no side effects.
|
||||
* Replaced the magic bound ``400`` (baked into ``range(0, 400)`` twice) with a
|
||||
named constant ``MAX_LEN``.
|
||||
* Extracted the search into a pure function ``find_monotonic_violation`` that
|
||||
returns the first violating ``(la, lb)`` pair (or ``None``), decoupling the
|
||||
search from I/O and removing the module-level mutable ``ok`` flag.
|
||||
* Replaced the ``ok`` flag + nested ``break`` / ``if not ok: break`` dance with
|
||||
a single early ``return`` from the helper.
|
||||
|
||||
Output is byte-identical to the original (verified against baseline):
|
||||
``estimate_tokens monotonic: True``.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/root/.hermes/wizard-outputs/forge")
|
||||
import chunker
|
||||
|
||||
# Upper bound (exclusive) of the string lengths sampled by the probe.
|
||||
MAX_LEN = 400
|
||||
|
||||
|
||||
def find_monotonic_violation(max_len: int = MAX_LEN) -> tuple[int, int] | None:
|
||||
"""Return ``(la, lb)`` of the first length pair (``0 <= la <= lb < max_len``)
|
||||
for which ``estimate_tokens("a"*la) > estimate_tokens("a"*lb)``, else None.
|
||||
|
||||
Pure: no I/O, no globals, no import-time side effects. Looks up
|
||||
``chunker.estimate_tokens`` at call time, so it is testable via monkeypatch.
|
||||
"""
|
||||
for la in range(max_len):
|
||||
for lb in range(max_len):
|
||||
if la <= lb and chunker.estimate_tokens("a" * la) > chunker.estimate_tokens("a" * lb):
|
||||
return la, lb
|
||||
return None
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Run the probe and print the verdict (byte-identical to the original)."""
|
||||
violation = find_monotonic_violation()
|
||||
if violation is None:
|
||||
print("estimate_tokens monotonic: True")
|
||||
else:
|
||||
la, lb = violation
|
||||
print(f"NOT MONOTONIC: len {la} > len {lb}")
|
||||
print("estimate_tokens monotonic: False")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,252 @@
|
||||
#!/usr/bin/env python3
|
||||
"""verify_check_lint_classes_dry.py -- equivalence proof for the report() class
|
||||
tally DRY refactor in tools/check_lint.py (forge lane: Refactor).
|
||||
|
||||
1. report() body (pre-refactor):
|
||||
summary["classes"][g["class"]] = summary["classes"].get(g["class"], 0) + 1
|
||||
2. report() body (refactored):
|
||||
from collections import Counter
|
||||
...
|
||||
summary["classes"] = dict(Counter(g["class"] for g in graded))
|
||||
|
||||
The tally feeds ONLY summary["classes"], which is consumed by (a) the SUMMARY
|
||||
print() -- `for k,v in summary.items(): print(f" {k}: {v}")` -- whose repr of
|
||||
`classes` is the output surface, and (b) the --json payload. The selftest
|
||||
(37 cells) exercises classify()/extract_check() only, NOT the tally, so selftest
|
||||
parity is a guard but not the proof; legs [5] and [6] pin the tally itself.
|
||||
|
||||
Behavior-preservation rests on: Counter reads a missing key as 0, so
|
||||
`dict(Counter(...))` is the same {class: n} mapping in first-appearance order
|
||||
as the hand-rolled loop; and re-assigning the EXISTING "classes" key keeps its
|
||||
position in `summary` (CPython 3.7+), so the SUMMARY line order is unchanged.
|
||||
|
||||
LEGS (exit 0 iff all pass, else 1):
|
||||
[1] selftest: original (HEAD) 37/37 rc 0
|
||||
[2] selftest: refactored (working tree) 37/37 rc 0
|
||||
[3] selftest: stdout BYTE-IDENTICAL (original == refactored)
|
||||
[4] code-level: original HAS the tally line & no Counter import/expr;
|
||||
refactored HAS Counter import + Counter expr and NO tally line
|
||||
[5] report() stdout BYTE-IDENTICAL on a synthetic fixture (quiet=False),
|
||||
incl. the `classes: {...}` SUMMARY repr
|
||||
[6] report() --json payload BYTE-IDENTICAL on the same fixture
|
||||
(generated_utc excluded: it is a wall-clock stamp, by design)
|
||||
[7] non-vacuity: the fixture tallied >1 distinct class with nonzero counts
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import importlib.util
|
||||
from collections import Counter
|
||||
|
||||
REPO = "/root/troupe-standup"
|
||||
TOOL = "tools/check_lint.py"
|
||||
SUMMARY = "selftest: 37 cells, 37 passed, 0 failed"
|
||||
TALLY = 'summary["classes"][g["class"]] = summary["classes"].get(g["class"], 0) + 1'
|
||||
IMP = "from collections import Counter"
|
||||
EXPR = 'dict(Counter(g["class"] for g in graded))'
|
||||
TMPBASE = os.environ.get("TMPDIR", tempfile.gettempdir())
|
||||
|
||||
|
||||
class Mismatch(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def orig_text():
|
||||
"""Committed pre-refactor source = HEAD (the refactor is a working-tree change)."""
|
||||
p = subprocess.run(["git", "-C", REPO, "show", "HEAD:" + TOOL],
|
||||
capture_output=True, text=True, timeout=60)
|
||||
if p.returncode != 0:
|
||||
raise Mismatch("cannot read HEAD:" + TOOL)
|
||||
return p.stdout
|
||||
|
||||
|
||||
def ref_text():
|
||||
with open(os.path.join(REPO, TOOL), encoding="utf-8") as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
def _write_tmp(src):
|
||||
fd, path = tempfile.mkstemp(suffix=".py", dir=TMPBASE)
|
||||
with os.fdopen(fd, "w") as fh:
|
||||
fh.write(src)
|
||||
return path
|
||||
|
||||
|
||||
def run_selftest(src):
|
||||
"""Run <src> --selftest as a subprocess; return (rc, stdout)."""
|
||||
path = _write_tmp(src)
|
||||
try:
|
||||
p = subprocess.run([sys.executable, path, "--selftest"],
|
||||
capture_output=True, text=True, timeout=120)
|
||||
return p.returncode, p.stdout
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
def load_module(src, name):
|
||||
"""Exec <src> as module <name> (__name__ != "__main__" so no run() side effects)."""
|
||||
path = _write_tmp(src)
|
||||
try:
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
mod = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
def build_fixture():
|
||||
"""Synthetic runs (iters + temp .log files). report() -> grade_run() reads
|
||||
the .log for 'RALPH RESULT:' lines (authoritative runner_final) and classifies
|
||||
the CHECK text in iters. Produces 6 distinct classes so the tally is exercised.
|
||||
|
||||
classes produced: SOUND, TAUTOLOGY, EMPTY, CATCH_ALL, TRAILING_CR, PROSE_TAIL
|
||||
"""
|
||||
td = tempfile.mkdtemp(prefix="cl_fixture_")
|
||||
|
||||
def log_path(stamp):
|
||||
return os.path.join(td, stamp + ".log")
|
||||
|
||||
def run(wizard, stamp, check_text, result):
|
||||
lg = log_path(stamp)
|
||||
lines = ["RALPH RESULT: %s" % result] if result else []
|
||||
with open(lg, "w") as fh:
|
||||
fh.write("\n".join(lines) + "\n")
|
||||
# iters holds the raw CHECK body (as extract_check / read_ralph_dir yields);
|
||||
# grade_run feeds iters[last_n] straight into classify().
|
||||
return {"stamp": stamp, "wizard": wizard,
|
||||
"iters": {1: check_text}, "runner_log": lg}
|
||||
|
||||
runs = [
|
||||
run("sol", "2026-10-17-120000-01", "python3 x.py", "GREEN"),
|
||||
run("volt", "2026-10-17-120000-02", "true", "GREEN"),
|
||||
run("null", "2026-10-17-120000-03", "python3 x.py", "RED"),
|
||||
run("beam", "2026-10-17-130000-01", "python3 x.py", "GREEN"),
|
||||
run("ledger", "2026-10-17-130000-02", "true", "GREEN"),
|
||||
run("quill", "2026-10-17-130000-03", "", "EXHAUSTED"),
|
||||
run("spark", "2026-10-17-130000-04", "python3 x.py || true", "RED"),
|
||||
run("volt", "2026-10-17-140000-01", "python3 /tmp/v.py\r", "GREEN"),
|
||||
run("sol", "2026-10-17-140000-02",
|
||||
"python3 v.py \u2014 prose tail", "GREEN"),
|
||||
run("null", "2026-10-17-140000-03", "python3 x.py", "GREEN"),
|
||||
run("quill", "2026-10-17-150000-01", "python3 x.py", None),
|
||||
]
|
||||
return runs
|
||||
|
||||
|
||||
def report_stdout(mod, runs):
|
||||
"""All of report()'s stdout (quiet=False) -- faults print to stdout too."""
|
||||
buf = io.StringIO()
|
||||
old = sys.stdout
|
||||
sys.stdout = buf
|
||||
try:
|
||||
rc = mod.report(list(runs), as_json=None, quiet=False, unlogged=2)
|
||||
finally:
|
||||
sys.stdout = old
|
||||
return rc, buf.getvalue()
|
||||
|
||||
|
||||
def report_json(mod, runs):
|
||||
"""Write the --json payload; return the parsed payload (generated_utc stripped)."""
|
||||
fd, jpath = tempfile.mkstemp(suffix=".json", dir=TMPBASE)
|
||||
os.close(fd)
|
||||
try:
|
||||
rc = mod.report(list(runs), as_json=jpath, quiet=True, unlogged=2)
|
||||
with open(jpath, encoding="utf-8") as fh:
|
||||
payload = json.load(fh)
|
||||
payload.pop("generated_utc", None)
|
||||
return rc, payload
|
||||
finally:
|
||||
if os.path.exists(jpath):
|
||||
os.unlink(jpath)
|
||||
|
||||
|
||||
def has_stmt(text, needle):
|
||||
"""True iff `needle` appears on a NON-COMMENT line (leading-# stripped).
|
||||
The replacement comment references the old tally verbatim, so a raw substring
|
||||
check would falsely report the smell still present as code."""
|
||||
for line in text.splitlines():
|
||||
s = line.lstrip()
|
||||
if s.startswith("#"):
|
||||
continue
|
||||
if needle in s:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
cells, failed = [], []
|
||||
|
||||
def check(label, ok, detail=""):
|
||||
cells.append(label)
|
||||
if not ok:
|
||||
failed.append("%s%s" % (label, (": " + detail) if detail else ""))
|
||||
print(" [%s] %s%s"
|
||||
% ("ok " if ok else "FAIL", label,
|
||||
(" [%s]" % detail) if detail and not ok else ""))
|
||||
|
||||
orig = orig_text()
|
||||
ref = ref_text()
|
||||
|
||||
# ---- [1-3] selftest parity ------------------------------------------------
|
||||
rc_o, out_o = run_selftest(orig)
|
||||
rc_r, out_r = run_selftest(ref)
|
||||
check("[1] selftest: original (HEAD) 37/37 rc 0",
|
||||
rc_o == 0 and SUMMARY in out_o,
|
||||
"rc=%d" % rc_o)
|
||||
check("[2] selftest: refactored 37/37 rc 0",
|
||||
rc_r == 0 and SUMMARY in out_r,
|
||||
"rc=%d" % rc_r)
|
||||
check("[3] selftest: stdout BYTE-IDENTICAL",
|
||||
out_o == out_r,
|
||||
"len_o=%d len_r=%d" % (len(out_o), len(out_r)))
|
||||
|
||||
# ---- [4] code-level: smell existed, was replaced (non-faculty) -----------
|
||||
check("[4a] code-level: original HAS tally line", has_stmt(orig, TALLY))
|
||||
check("[4b] code-level: refactored HAS NO tally line", not has_stmt(ref, TALLY))
|
||||
check("[4c] code-level: refactored HAS Counter import", IMP in ref)
|
||||
check("[4d] code-level: original HAS NO Counter import", IMP not in orig)
|
||||
check("[4e] code-level: refactored HAS Counter expr", EXPR in ref)
|
||||
|
||||
# ---- [5-6] report() byte-identity on a live fixture ----------------------
|
||||
runs = build_fixture()
|
||||
mod_o = load_module(orig, "cl_orig")
|
||||
mod_r = load_module(ref, "cl_ref")
|
||||
|
||||
rc_o2, out_o2 = report_stdout(mod_o, runs)
|
||||
rc_r2, out_r2 = report_stdout(mod_r, runs)
|
||||
check("[5] report() stdout BYTE-IDENTICAL (incl. classes repr)",
|
||||
rc_o2 == rc_r2 == 0 and out_o2 == out_r2,
|
||||
"rc_o=%d rc_r=%d len_o=%d len_r=%d" % (rc_o2, rc_r2, len(out_o2), len(out_r2)))
|
||||
|
||||
rc_o3, pay_o = report_json(mod_o, runs)
|
||||
rc_r3, pay_r = report_json(mod_r, runs)
|
||||
check("[6] report() --json payload BYTE-IDENTICAL (excl. generated_utc)",
|
||||
rc_o3 == rc_r3 == 0 and pay_o == pay_r,
|
||||
"summary_match=%s" % (pay_o.get("summary") == pay_r.get("summary")))
|
||||
|
||||
# ---- [7] non-vacuity: the tally actually saw multiple classes -----------
|
||||
classes = pay_r.get("summary", {}).get("classes", {})
|
||||
check("[7] non-vacuity: fixture tallied >1 distinct class",
|
||||
len(classes) > 1 and all(v > 0 for v in classes.values()),
|
||||
"classes=%s" % classes)
|
||||
|
||||
print("\nverify: %d checks, %d passed, %d failed"
|
||||
% (len(cells), len(cells) - len(failed), len(failed)))
|
||||
for f in failed:
|
||||
print(" FAIL %s" % f)
|
||||
rc = 0 if not failed else 1
|
||||
print("\nVERDICT:", "ALL GREEN -- report() behavior preserved"
|
||||
if rc == 0 else "MISMATCH -- behavior NOT preserved")
|
||||
return rc
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except Mismatch as e:
|
||||
print("\nVERDICT: MISMATCH -- %s" % e)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,225 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verification harness for the _smell_check_monotonic_refactored.py refactor.
|
||||
|
||||
Exit 0 = all checks pass; exit 1 = at least one failure.
|
||||
|
||||
Checks:
|
||||
1. Refactored file compiles (py_compile).
|
||||
2. Original and refactored produce byte-identical stdout+stderr AND exit code.
|
||||
3. ``if __name__ == "__main__"`` guard present (no probe at import time).
|
||||
4. Magic bound ``400`` replaced by a named constant ``MAX_LEN`` (no bare
|
||||
``range(0, 400)`` / ``range(400)`` left in code).
|
||||
5. No ``global`` statements.
|
||||
6. Importing the refactored module is side-effect-free: it produces NO
|
||||
stdout/stderr (the original printed ``estimate_tokens monotonic:`` at
|
||||
import; the refactor must not).
|
||||
7. The extracted ``find_monotonic_violation`` is correct AND not a tautology:
|
||||
it returns None on the live (monotonic) chunker, and it returns a real
|
||||
violation when ``chunker.estimate_tokens`` is replaced by a deliberately
|
||||
non-monotonic estimator.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import difflib
|
||||
import importlib
|
||||
import io
|
||||
import contextlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
REPO_ROOT = "/root/troupe-standup"
|
||||
WORK_DIR = os.path.join(REPO_ROOT, "wizard-work", "forge")
|
||||
ORIGINAL = "_smell_check_monotonic.py"
|
||||
REFACTORED = "_smell_check_monotonic_refactored.py"
|
||||
MOD_NAME = REFACTORED[:-3] # strip ".py"
|
||||
|
||||
|
||||
def _run(*args: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(args, cwd=WORK_DIR, capture_output=True, text=True)
|
||||
|
||||
|
||||
def _read(name: str) -> str:
|
||||
with open(os.path.join(WORK_DIR, name)) as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def _code_only(content: str) -> str:
|
||||
"""Return *content* with the module docstring and comment-only lines removed,
|
||||
so smell checks inspect code rather than documentation (the refactored
|
||||
docstring *names* the patterns it removes, e.g. ``400`` / ``range(0, 400)``,
|
||||
which would be false positives as residual smells)."""
|
||||
tree = ast.parse(content)
|
||||
skip: set[int] = set()
|
||||
if (tree.body
|
||||
and isinstance(tree.body[0], ast.Expr)
|
||||
and isinstance(tree.body[0].value, ast.Constant)
|
||||
and isinstance(tree.body[0].value.value, str)):
|
||||
ds = tree.body[0]
|
||||
for ln in range(ds.lineno, (ds.end_lineno or ds.lineno) + 1):
|
||||
skip.add(ln)
|
||||
kept = []
|
||||
for i, line in enumerate(content.splitlines(), 1):
|
||||
if i in skip or line.lstrip().startswith("#"):
|
||||
continue
|
||||
kept.append(line)
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
def check_py_compile() -> bool:
|
||||
r = _run("python3", "-m", "py_compile", REFACTORED)
|
||||
if r.returncode != 0:
|
||||
print(f"FAIL: py_compile failed: {r.stderr}")
|
||||
return False
|
||||
print("OK: py_compile passes")
|
||||
return True
|
||||
|
||||
|
||||
def check_identical_output() -> bool:
|
||||
orig = _run("python3", ORIGINAL)
|
||||
ref = _run("python3", REFACTORED)
|
||||
ot = orig.stdout + orig.stderr
|
||||
rt = ref.stdout + ref.stderr
|
||||
if ot != rt:
|
||||
print("FAIL: original and refactored output differ")
|
||||
print("".join(difflib.unified_diff(
|
||||
ot.splitlines(keepends=True), rt.splitlines(keepends=True),
|
||||
fromfile="original (" + ORIGINAL + ")",
|
||||
tofile="refactored (" + REFACTORED + ")",
|
||||
)))
|
||||
return False
|
||||
if orig.returncode != ref.returncode:
|
||||
print(f"FAIL: exit codes differ ({orig.returncode} vs {ref.returncode})")
|
||||
return False
|
||||
print(f"OK: identical output (exit {orig.returncode}): {ot.strip()!r}")
|
||||
return True
|
||||
|
||||
|
||||
def check_main_guard() -> bool:
|
||||
content = _read(REFACTORED)
|
||||
if '__name__ == "__main__"' not in content:
|
||||
print('FAIL: no __main__ guard')
|
||||
return False
|
||||
print("OK: __main__ guard present")
|
||||
return True
|
||||
|
||||
|
||||
def check_no_magic_range() -> bool:
|
||||
code = _code_only(_read(REFACTORED))
|
||||
if "range(0, 400)" in code or "range(400)" in code:
|
||||
print("FAIL: magic 'range(0, 400)' / 'range(400)' still in code")
|
||||
return False
|
||||
print("OK: no bare 'range(0, 400)' / 'range(400)' magic literal in code")
|
||||
return True
|
||||
|
||||
|
||||
def check_named_constant() -> bool:
|
||||
code = _code_only(_read(REFACTORED))
|
||||
if "MAX_LEN =" not in code or "MAX_LEN" not in code:
|
||||
print("FAIL: no MAX_LEN named constant")
|
||||
return False
|
||||
print("OK: MAX_LEN named constant present")
|
||||
return True
|
||||
|
||||
|
||||
def check_no_globals() -> bool:
|
||||
with open(os.path.join(WORK_DIR, REFACTORED)) as f:
|
||||
for i, line in enumerate(f.read().splitlines(), 1):
|
||||
if line.lstrip().startswith("global "):
|
||||
print(f"FAIL: 'global' found at line {i}: {line.strip()}")
|
||||
return False
|
||||
print("OK: no 'global' statements")
|
||||
return True
|
||||
|
||||
|
||||
def check_no_module_level_flag() -> bool:
|
||||
"""The module-level mutable ``ok = True`` flag must be gone (folded into the
|
||||
helper's return value)."""
|
||||
tree = ast.parse(_read(REFACTORED))
|
||||
for node in tree.body:
|
||||
if isinstance(node, ast.Assign):
|
||||
for t in node.targets:
|
||||
if isinstance(t, ast.Name) and t.id == "ok":
|
||||
print("FAIL: module-level mutable 'ok' flag still present")
|
||||
return False
|
||||
print("OK: module-level 'ok' flag removed (state folded into helper return)")
|
||||
return True
|
||||
|
||||
|
||||
def check_import_side_effect_free() -> bool:
|
||||
"""Importing the refactored module must not run the probe / produce output.
|
||||
The original printed at import; the refactor must not."""
|
||||
sys.modules.pop(MOD_NAME, None)
|
||||
# ensure WORK_DIR importable in this process
|
||||
if WORK_DIR not in sys.path:
|
||||
sys.path.insert(0, WORK_DIR)
|
||||
out, err = io.StringIO(), io.StringIO()
|
||||
with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
|
||||
mod = importlib.import_module(MOD_NAME)
|
||||
if out.getvalue() or err.getvalue():
|
||||
print(f"FAIL: import produced output: stdout={out.getvalue()!r} stderr={err.getvalue()!r}")
|
||||
return False
|
||||
# belt: confirm the probe function exists and main is guarded
|
||||
if not hasattr(mod, "find_monotonic_violation") or not hasattr(mod, "main"):
|
||||
print("FAIL: expected find_monotonic_violation / main after import")
|
||||
return False
|
||||
print("OK: import is side-effect-free (no probe output at import)")
|
||||
return True
|
||||
|
||||
|
||||
def check_extracted_function_correct() -> bool:
|
||||
"""find_monotonic_violation returns None on the live chunker (monotonic) and
|
||||
returns a real violation on a deliberately non-monotonic estimator -- proving
|
||||
the search is real and not a hardcoded None."""
|
||||
if MOD_NAME in sys.modules:
|
||||
mod = sys.modules[MOD_NAME]
|
||||
else:
|
||||
mod = importlib.import_module(MOD_NAME)
|
||||
# Real property: monotonic on the live chunker.
|
||||
if mod.find_monotonic_violation() is not None:
|
||||
print("FAIL: expected None (monotonic) on live chunker")
|
||||
return False
|
||||
# Non-tautological control: non-monotonic estimator MUST yield a violation.
|
||||
def non_monotonic(s: str) -> int:
|
||||
L = len(s)
|
||||
if L == 0:
|
||||
return 0
|
||||
# Genuinely non-monotonic: short strings cost MORE (999) than long (1),
|
||||
# so estimate_tokens("a"*1) > estimate_tokens("a"*50) -> a real violation.
|
||||
return 999 if L < 50 else 1 # shrinks as len grows past 50
|
||||
orig = mod.chunker.estimate_tokens
|
||||
mod.chunker.estimate_tokens = non_monotonic
|
||||
try:
|
||||
v = mod.find_monotonic_violation(max_len=100)
|
||||
finally:
|
||||
mod.chunker.estimate_tokens = orig
|
||||
if v is None:
|
||||
print("FAIL: non-monotonic estimator produced no violation (search not real)")
|
||||
return False
|
||||
print(f"OK: returns None on live chunker; finds violation {v} on non-monotonic estimator")
|
||||
return True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
checks = [
|
||||
check_py_compile,
|
||||
check_identical_output,
|
||||
check_main_guard,
|
||||
check_no_magic_range,
|
||||
check_named_constant,
|
||||
check_no_globals,
|
||||
check_no_module_level_flag,
|
||||
check_import_side_effect_free,
|
||||
check_extracted_function_correct,
|
||||
]
|
||||
results = [fn() for fn in checks]
|
||||
passed = sum(results)
|
||||
total = len(results)
|
||||
print(f"\n=== Verification: {passed}/{total} checks passed ===")
|
||||
if passed != total:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
python3 "$(dirname "$0")/verify_claims.py"
|
||||
@@ -0,0 +1,473 @@
|
||||
# Agentic Memory: A Five-Paper Grounded Dossier
|
||||
|
||||
Archive lane -- sphinx. Built 2026-10-25.
|
||||
|
||||
Scope: the five most recent arXiv papers matching `all:"agentic memory"`, taken from a single feed snapshot (updated 2026-10-25T00:40:42Z; 583 total results).
|
||||
|
||||
Every number below is a verbatim quote from the pinned corpus text. Nothing is paraphrased into a figure. `verify_claims.py` re-derives every quote from the same pinned bytes and fails the build if any drifts.
|
||||
|
||||
## Provenance (pinned)
|
||||
|
||||
| paper | title | bytes | chars | sha256 (first 16) | claims |
|
||||
|---|---|---|---|---|---|
|
||||
| 2610.29671 | Uni-Mem: Learning a Long-Horizon Agentic Memory Manager with Unified Online Policy Optimization | 90083 | 89293 | `b83a772fb6a1340b` | 9 |
|
||||
| 2610.29823 | MemoryTax: The Bitter Lesson of Agent Memory | 23897 | 23627 | `ed08ec2380a6c147` | 9 |
|
||||
| 2610.29869 | Compaction Laundering: Provenance Destruction in Context-Compacting Coding Agents | 92858 | 91936 | `e6f78f508c12d7a3` | 7 |
|
||||
| 2610.30485 | Why Bother Me Again? Benchmarking Personal Agent Memory for Interaction Preferences | 350788 | 340805 | `48e8b2b9c7defc65` | 9 |
|
||||
| 2610.30755 | Is Complex Agentic Memory Always Needed? Naive RAG Is Stronger Than We Thought | 195193 | 193677 | `2246678ceaf7a42c` | 10 |
|
||||
|
||||
## How this dossier is verified
|
||||
|
||||
```
|
||||
python3 verify_claims.py # or: bash CHECK.txt
|
||||
```
|
||||
|
||||
The verifier checks, for each of the 44 claims:
|
||||
1. the corpus file's sha256 equals the pin recorded at build time;
|
||||
2. the claim's `quote` is a literal substring of that corpus file;
|
||||
3. the claim's headline `value` appears inside that quote;
|
||||
4. the headline `value` appears in this dossier.
|
||||
|
||||
It then runs a mutation battery: corrupting one quote at a time in a scratch
|
||||
copy of `claims.json` must make the verification fail. If any mutation passes
|
||||
undetected, the build fails -- the check is falsified, not assumed.
|
||||
|
||||
## T1: Is designed memory needed at all?
|
||||
|
||||
### R1 -- 2610.30755
|
||||
|
||||
**Raw-RAG ranks first among compared memory systems in six of seven benchmark-model settings.**
|
||||
|
||||
> Raw-RAG ranks first among the compared memory systems in six of seven benchmark–model settings, outperforming the strongest compared agentic memory system by 0.13–5.53 points. We investigate why raw-history ret
|
||||
|
||||
*Why it matters:* The headline result: the naive baseline wins most settings.
|
||||
|
||||
### R2 -- 2610.30755
|
||||
|
||||
**Raw-RAG beats the strongest compared agentic memory system by 0.13–5.53 points.**
|
||||
|
||||
> outperforming the strongest compared agentic memory system by 0.13–5.53 points. We investigate why raw-history retrieval remains c
|
||||
|
||||
*Why it matters:* Margin is small on LoCoMo/LongMemEval, large on MemoryAgentBench.
|
||||
|
||||
### R8 -- 2610.30755
|
||||
|
||||
**No memory-management variant improves on Raw-RAG (48.32%).**
|
||||
|
||||
> None improves Raw-RAG (48.32%) on the same 14-source MemoryAgentBench Total (Table 6, orange rows). The exact construction and retrieval procedures ar
|
||||
|
||||
*Why it matters:* RQ2 answer: added management does not help.
|
||||
|
||||
### R9 -- 2610.30755
|
||||
|
||||
**Atomic-fact extraction collapses test-time learning from 47.68% to 15.37%.**
|
||||
|
||||
> Atomic-fact extraction reduces test-time learning from 47.68% to 15.37% and Total by 19.62 points, consistent with loss or disrupt
|
||||
|
||||
*Why it matters:* The most damaging of the three management operations.
|
||||
|
||||
### U1 -- 2610.29671
|
||||
|
||||
**Uni-Mem reaches an overall LoCoMo JJ of 88.80%.**
|
||||
|
||||
> Uni-Mem achieves the best performance across all four question types, with an overall JJ score of 88.80%. It surpasses the strongest baseline, Memory-R2 (OSS), by 10.93 percentage points and Mem-T by 12.31 points, with g
|
||||
|
||||
*Why it matters:* Best across all four question types.
|
||||
|
||||
### U2 -- 2610.29671
|
||||
|
||||
**Uni-Mem surpasses Memory-R2 (OSS) by 10.93 points and Mem-T by 12.31.**
|
||||
|
||||
> It surpasses the strongest baseline, Memory-R2 (OSS), by 10.93 percentage points and Mem-T by 12.31 points, with gains of 12.93 and 9.29 points over the strongest competing results on multi-hop and temporal que
|
||||
|
||||
*Why it matters:* Gains of 12.93 and 9.29 on multi-hop and temporal.
|
||||
|
||||
### U3 -- 2610.29671
|
||||
|
||||
**Uni-Mem still reaches 80.99% without SFT, +3.12% over the strongest baseline.**
|
||||
|
||||
> Uni-Mem still achieves 80.99% without SFT, outperforming the strongest baseline by 3.12%. These results support the effectiveness of jointly
|
||||
|
||||
*Why it matters:* RL alone is enough to beat the baselines.
|
||||
|
||||
### U4 -- 2610.29671
|
||||
|
||||
**ECA improves overall JJ by 1.29 and 3.03 points without and with SFT.**
|
||||
|
||||
> improves overall JJ by 1.29 and 3.03 percentage points without and with SFT, respectively. These gains support the effectiveness o
|
||||
|
||||
*Why it matters:* Event-wise credit assignment localizes sparse feedback.
|
||||
|
||||
### U5 -- 2610.29671
|
||||
|
||||
**Stage-Decoupled GRPO improves overall JJ by 2.48 points without SFT.**
|
||||
|
||||
> improves overall JJ by 2.48 percentage points without SFT. In particular, temporal accuracy rises by 10.55 points in this setting, and multi
|
||||
|
||||
*Why it matters:* Temporal accuracy rises 10.55 points in this setting.
|
||||
|
||||
### U6 -- 2610.29671
|
||||
|
||||
**Trained construction alone improves overall JJ by 8.18 points.**
|
||||
|
||||
> improves the overall JJ score by 8.18 percentage points, demonstrating the increased downstream utility of the constructed memorie
|
||||
|
||||
*Why it matters:* Each capability contributes even paired with its untrained counterpart.
|
||||
|
||||
### U7 -- 2610.29671
|
||||
|
||||
**Extending update blocks 1:1 -> 3:2 reduces overall JJ from 88.80% to 83.75%.**
|
||||
|
||||
> the overall JJ score from 88.80%88.80\% to 83.75%83.75\%, indicating that less frequent alternation hinders shared-polic
|
||||
|
||||
*Why it matters:* Frequent alternation matters as much as balanced allocation.
|
||||
|
||||
## T2: Cost, ceiling, and scale
|
||||
|
||||
### R10 -- 2610.30755
|
||||
|
||||
**Full-context beats Raw-RAG by 19.98 points on DeepSeek-V4-Flash MEME.**
|
||||
|
||||
> Full-context leads Raw-RAG by 19.98 points in that setting. Raw-RAG is competitive across these tasks despite its simpler ingestio
|
||||
|
||||
*Why it matters:* The one setting where the naive baseline is not first.
|
||||
|
||||
### M1 -- 2610.29823
|
||||
|
||||
**Study scope: one LoCoMo conversation, 419 turns, 152 questions.**
|
||||
|
||||
> On one LoCoMo conversation (419 turns, 152 questions), we run four memory methods (full context, BM25, Mem0, Zep) and three coding agents (Pi, Codex, Claude Cod
|
||||
|
||||
*Why it matters:* Deliberately narrow: one conversation, one run per cell.
|
||||
|
||||
### M2 -- 2610.29823
|
||||
|
||||
**With GPT-4o-mini, Mem0 scores 78% vs 74% for full context and BM25.**
|
||||
|
||||
> With GPT-4o-mini, Mem0 scores highest. It scores 78%, against 74% for both full context and BM25; Zep scores 43%. The agents score 63% (Pi), 28% (Codex) and 29%
|
||||
|
||||
*Why it matters:* Agent memory is a patch for a weak model.
|
||||
|
||||
### M3 -- 2610.29823
|
||||
|
||||
**With GPT-6 Luna, full context scores 92%, BM25 82%, Mem0 80%, Zep 51%.**
|
||||
|
||||
> With GPT-6 Luna, full context scores 92%, BM25 82%, Mem0 80% and Zep 51%; Zep’s graph holds 128 facts from the 19 sessions, against 195 with Claude Opus 5.5. With Claude Opus 5.5, Mem0 and full contex
|
||||
|
||||
*Why it matters:* The patch stops helping as the model gets stronger.
|
||||
|
||||
### M4 -- 2610.29823
|
||||
|
||||
**With Claude Opus 5.5, agents score highest: Pi 99%, Codex 98%, Claude Code 97%.**
|
||||
|
||||
> With Claude Opus 5.5, Mem0 and full context both score 95% and Zep 87%, and the agents score highest: Pi 99%, Codex 98% and Claude Code 97%, a lead that comes mostly from
|
||||
|
||||
*Why it matters:* Searching the raw record beats reading all of it at the top end.
|
||||
|
||||
### M5 -- 2610.29823
|
||||
|
||||
**Pi scores 99% for $6.95/rollout vs full context 95% for $17.84.**
|
||||
|
||||
> Pi scores 99% for $6.95 a rollout, while full context with the same model scores 95% for $17.84. Pi reads about 10,000 input tokens a question, 7,900 of them ca
|
||||
|
||||
*Why it matters:* Less than half the cost for a higher score.
|
||||
|
||||
### M7 -- 2610.29823
|
||||
|
||||
**Pi reads ~10,000 input tokens/question (7,900 cached) vs 28,900 for full context.**
|
||||
|
||||
> Pi reads about 10,000 input tokens a question, 7,900 of them cached, against 28,900 for full context. Codex and Claude Code score 98% and 97% for $25.
|
||||
|
||||
*Why it matters:* The efficiency mechanism behind the cost lead.
|
||||
|
||||
### U8 -- 2610.29671
|
||||
|
||||
**SFT followed by RL improves the 4B manager's JJ from 76.58% to 88.80%.**
|
||||
|
||||
> SFT followed by RL improves the 4B manager’s JJ score from 76.58% to 88.80%. Uni-Mem also outperforms all off-the-shelf managers, including the strong
|
||||
|
||||
*Why it matters:* A small trained manager beats Qwen3.5-27B (87.05%) and 397B-A17B (85.86%).
|
||||
|
||||
### U9 -- 2610.29671
|
||||
|
||||
**Uni-Mem outperforms all off-the-shelf managers including Qwen3.5-27B (87.05%).**
|
||||
|
||||
> Uni-Mem also outperforms all off-the-shelf managers, including the strongest one, Qwen3.5-27B (87.05%), and the largest one, Qwen3.5-397B-A17B (85.86%). These results demonstrate that a small manager trained jo
|
||||
|
||||
*Why it matters:* Scale is not the binding constraint; joint training is.
|
||||
|
||||
## T3: Provenance: what compaction destroys and what writers omit
|
||||
|
||||
### M8 -- 2610.29823
|
||||
|
||||
**Writer ceiling: on Zep's GPT-4o-mini graph readers score 43/43/45%; on Opus 5.5's, 66/71/87%.**
|
||||
|
||||
> GPT-4o-mini graph, GPT-4o-mini, GPT-6 Luna and Claude Opus 5.5 score 43%, 43% and 45%; on the graph Claude Opus 5.5 wrote, 66%, 71% and 87%. Mem0 keeps the order with smaller gaps: 78%, 83% and 84% from GPT-4o-
|
||||
|
||||
*Why it matters:* A stronger reader cannot recover what a weaker writer left out.
|
||||
|
||||
### M9 -- 2610.29823
|
||||
|
||||
**Pi's one miss is a LoCoMo answer-key error, not a memory failure.**
|
||||
|
||||
> Its one miss is LoCoMo’s mistake. The question asks what kind of painting Caroline shared with Melanie on 13 October 2023, and the answer key says an abstract painting with blue streaks. In the conversation it is Melanie who shares that painting; Caroline shares a drawing. Pi says so and is marked wrong: “Caroline shared a drawing, not a painting. … The paintings shared that day (a sunset and an abstract piece) were Melanie’s.” With Claude Opus 5.5, full context, Codex and Claude Code point out the same mistake and are marked wrong too. On the other 151 questions, Pi with Claude Opus
|
||||
|
||||
*Why it matters:* The model is right and the benchmark key is wrong.
|
||||
|
||||
### C1 -- 2610.29869
|
||||
|
||||
**Forged approval launders into memory 87–100% of the time across five flattening harnesses.**
|
||||
|
||||
> the forged approval launders into the agent’s memory 87–100% of the time, up to a ~908,000-token context window; across four role-typed harnesses (Codex, Gemini CLI, Qwen Code, Continue) the identical attack su
|
||||
|
||||
*Why it matters:* Up to a ~908,000-token context window.
|
||||
|
||||
### C2 -- 2610.29869
|
||||
|
||||
**The identical attack succeeds 0% of the time across four role-typed harnesses.**
|
||||
|
||||
> succeeds 0% of the time. We follow the attack to executed consequences. A laundered standing approval drives real tool-running age
|
||||
|
||||
*Why it matters:* Role-typing is a built-in control.
|
||||
|
||||
### C3 -- 2610.29869
|
||||
|
||||
**Live in the real pi TUI the attack runs at ~50% against a 0% paired clean arm.**
|
||||
|
||||
> live in the real pi TUI at ~50% against a 0% paired clean arm. Straiker [1] described this consequence without runtime evidence, a
|
||||
|
||||
*Why it matters:* Runtime evidence, not a simulation.
|
||||
|
||||
### C4 -- 2610.29869
|
||||
|
||||
**A code-level provenance check catches 77% of laundering, demoting 2% of legitimate content.**
|
||||
|
||||
> we measure a code-level provenance check that catches 77% of laundering while demoting 2% of legitimate summary content. The defect also has a straightforward f
|
||||
|
||||
*Why it matters:* The only defense that works; prompt-level defenses cannot restore provenance.
|
||||
|
||||
### C5 -- 2610.29869
|
||||
|
||||
**In-prompt anti-injection rules stop 13–25%; published classifier defenses report 67% at best.**
|
||||
|
||||
> stops 13–25% on the same data, and published classifier defenses report 67% at best. The defect also has a straightforward fix on the flatte
|
||||
|
||||
*Why it matters:* Baselines are far weaker than the code-level check.
|
||||
|
||||
### C6 -- 2610.29869
|
||||
|
||||
**Laundering still succeeds 100% of the time at every context size.**
|
||||
|
||||
> The laundering still succeeds 100% of the time at every size, and the executed rate, near-100% on the short transcripts, settles to ~20–40% in a full window. Th
|
||||
|
||||
*Why it matters:* Size does not attenuate the attack.
|
||||
|
||||
### C7 -- 2610.29869
|
||||
|
||||
**Executed rate settles to ~20–40% in a full window.**
|
||||
|
||||
> settles to ~20–40% in a full window. The short runs therefore measure the attack at its most favorable, an upper bound on the exec
|
||||
|
||||
*Why it matters:* Real agents compact at this size, so this is the realistic case.
|
||||
|
||||
## T4: What raw evidence must be preserved
|
||||
|
||||
### R3 -- 2610.30755
|
||||
|
||||
**Mem0's UPDATE+DELETE account for only 4.39% of 33,902 LoCoMo write decisions.**
|
||||
|
||||
> together account for only 4.39% of the 33,902 write decisions on LoCoMo and 2.46% of the 428,858 write decisions on LongMemEval (Figure 2);
|
||||
|
||||
*Why it matters:* Mutation is rare in practice -- most proposed writes are duplicates or malformed.
|
||||
|
||||
### R4 -- 2610.30755
|
||||
|
||||
**Replacing UPDATE/DELETE with append-only ADD/NONE raises LoCoMo accuracy 67.36% -> 69.42%.**
|
||||
|
||||
> Replacing UPDATE with append-only ADD and DELETE with NONE raises LoCoMo accuracy from 67.36% to 69.42%, without reducing any category score (Table 2). Explicit mutation
|
||||
|
||||
*Why it matters:* Explicit mutation gives no accuracy gain; the simpler store is better.
|
||||
|
||||
### R5 -- 2610.30755
|
||||
|
||||
**Removing reranking lowers MemoryAgentBench Total by 3.92 points.**
|
||||
|
||||
> Removing reranking lowers Total by 3.92 points (pHolm=0.0012p_{\mathrm{Holm}}=0.0012), while returning only one item lowers it by 11.73 points. Final-kk values of 5, 10, 30, and 60 stay with
|
||||
|
||||
*Why it matters:* Reranking is the one component worth keeping.
|
||||
|
||||
### R6 -- 2610.30755
|
||||
|
||||
**Returning only one item lowers Total by 11.73 points.**
|
||||
|
||||
> while returning only one item lowers it by 11.73 points. Final-kk values of 5, 10, 30, and 60 stay within 1.3 points of the baseli
|
||||
|
||||
*Why it matters:* Retrieval depth matters far more than chunk size.
|
||||
|
||||
### R7 -- 2610.30755
|
||||
|
||||
**Top-120 retrieval loses 14.57 points.**
|
||||
|
||||
> Top-120 loses 14.57 points, driven by long-range understanding and conflict resolution. Chunk sizes from 128 to 1,024 do not signi
|
||||
|
||||
*Why it matters:* Driven by long-range understanding and conflict resolution.
|
||||
|
||||
### M6 -- 2610.29823
|
||||
|
||||
**On the 88 questions without photo evidence, Pi scores 100%.**
|
||||
|
||||
> On the 88 questions without photo evidence, Pi scores 100%, and full context, Mem0, Codex and Claude Code 99% each (Table 4). With the same input, then, Pi matc
|
||||
|
||||
*Why it matters:* Pi matches full context once captions are removed from the comparison.
|
||||
|
||||
## T5: Interaction preferences are a different problem
|
||||
|
||||
### P1 -- 2610.30485
|
||||
|
||||
**MemPABench has four simulated users holding 14 preferences.**
|
||||
|
||||
> Four grounded simulated users hold 14 preferences across work and personal contexts. Their feedback spans 99 sessions over four months, with
|
||||
|
||||
*Why it matters:* Preferences span work and personal contexts.
|
||||
|
||||
### P2 -- 2610.30485
|
||||
|
||||
**Feedback spans 99 sessions over four months.**
|
||||
|
||||
> Their feedback spans 99 sessions over four months, with five event-driven changes per user testing memory updates. We evaluate pre
|
||||
|
||||
*Why it matters:* Five event-driven preference changes per user.
|
||||
|
||||
### P3 -- 2610.30485
|
||||
|
||||
**No Memory reaches 49.6% PSA against 45.2% random.**
|
||||
|
||||
> No Memory, where the PA selects by its defaults, reaches 49.6%, compared with 45.2% for random selection (Appendix A.1). Only File memory adds a clear gain. It raises PSA to 59.2%, a mean ga
|
||||
|
||||
*Why it matters:* Defaults alone nearly match chance-plus-margin.
|
||||
|
||||
### P4 -- 2610.30485
|
||||
|
||||
**File memory raises PSA to 59.2%, +9.6 points over No Memory.**
|
||||
|
||||
> It raises PSA to 59.2%, a mean gain of 9.6 points over No Memory (95% cluster-bootstrap interval [+3.9, +15.6]). PERMA (Liu et al., 2026), a benchmark of content preferen
|
||||
|
||||
*Why it matters:* The only condition with a clear gain; CI [+3.9, +15.6].
|
||||
|
||||
### P5 -- 2610.30485
|
||||
|
||||
**Mem0, MemOS and Simple RAG stay at No Memory level (PSA 50.0/50.0/49.9).**
|
||||
|
||||
> with PSA of 50.0%, 50.0%, and 49.9%. The intervals of all conditions and their comparisons with No Memory on each user–backbone pa
|
||||
|
||||
*Why it matters:* Retrieval-based memory does not beat having none.
|
||||
|
||||
### P6 -- 2610.30485
|
||||
|
||||
**File memory PET is 25.4%, more than twice No Memory's 11.2%.**
|
||||
|
||||
> Its PET is 25.4%, more than twice the 11.2% of No Memory, and it is higher on every backbone. However, the CS does not improve, at 14.4% against 21.8% (Appendix G.3). In
|
||||
|
||||
*Why it matters:* But its CS does not improve (14.4% vs 21.8%): context is not tracked.
|
||||
|
||||
### P7 -- 2610.30485
|
||||
|
||||
**Oracle lifts correct selection 33%->96% but BRS>=4 only 24%->59%.**
|
||||
|
||||
> Oracle raises correct selections of the five settings from 33% to 96%, but the share of replies with a BRS of 4 or higher rises only from 24% to 59%. Even with the IX Wrapper, the PA still writes too much. Afte
|
||||
|
||||
*Why it matters:* Knowing the preference does not override the model's interaction habits.
|
||||
|
||||
### P8 -- 2610.30485
|
||||
|
||||
**Information Flow is the hardest dimension, PSA 34–44%.**
|
||||
|
||||
> Information Flow is the hardest under every condition (Figure 8). Its PSA stays at 34–44%, the lowest of the four, while Expression Style (57–69%) and Disclosur
|
||||
|
||||
*Why it matters:* Expression Style (57-69%) and Disclosure (55-58%) are easier.
|
||||
|
||||
### P9 -- 2610.30485
|
||||
|
||||
**Anonymizing users drops shortcut accuracy 53.2% -> 46.4%, near the 45.2% chance.**
|
||||
|
||||
> reaches 53.2% accuracy from the source identity but 46.4% from the anonymized profile, close to the 45.2% chance level (Figure 10; Appendix
|
||||
|
||||
*Why it matters:* Confirms the benchmark measures memory, not name-guessing.
|
||||
|
||||
## Cross-paper synthesis
|
||||
|
||||
Four of the five papers converge on one answer: **the elaborate part of agent memory is not where the value is.**
|
||||
|
||||
- **Raw-RAG wins most settings** (2610.30755) -- naive dense retrieval over raw history ranks first in six of seven benchmark-model settings, and *no* added memory-management operation improves on it. The one setting where it loses is the one where full context is affordable.
|
||||
|
||||
- **Mutation is nearly free to drop** (2610.30755) -- UPDATE+DELETE are 4.39% of LoCoMo writes and 2.46% of LongMemEval writes; replacing them with append-only ADD/NONE *raises* accuracy 67.36% -> 69.42%.
|
||||
|
||||
- **Agent memory is a patch for a weak model** (2610.29823) -- Mem0 helps GPT-4o-mini (78% vs 74%) but not GPT-6 Luna or Claude Opus 5.5, and at the top end Pi scores 99% for $6.95 against full context's 95% for $17.84.
|
||||
|
||||
- **A stronger reader does not recover what a weaker writer left out** (2610.29823) -- on Zep's GPT-4o-mini graph the three readers score 43/43/45%; on the graph Claude Opus 5.5 wrote they score 66/71/87%. Memory has a ceiling set by its writer.
|
||||
|
||||
- **What must be preserved is raw evidence, not extracted facts** (2610.30755, 2610.29823) -- verbatim chunks beat lossy artifacts; on the 88 questions with no photo evidence, Pi's lead over full context vanishes (100% vs 99%).
|
||||
|
||||
- **Compaction destroys provenance deterministically** (2610.29869) -- a forged approval launders into memory 87-100% of the time in flattening harnesses and 0% in role-typed ones; no prompt-level defense restores it, only a code-level check (77% catch, 2% collateral).
|
||||
|
||||
- **Interaction preferences are a harder problem than factual recall** (2610.30485) -- retrieval-based memory does not beat No Memory (PSA 50.0/50.0/49.9 vs 49.6%), and even an Oracle that hands the agent the right preference lifts delivered behavior only 24% -> 59%.
|
||||
|
||||
- **The counter-example is training the memory manager itself** (2610.29671) -- Uni-Mem jointly learns construction and retrieval and reaches 88.80% JJ, beating Memory-R2 (OSS) by 10.93 points and off-the-shelf Qwen3.5-27B (87.05%). The gain comes from the training scheme (Stage-Decoupled GRPO + ECA), not from a bigger model.
|
||||
|
||||
## Where the five disagree
|
||||
|
||||
2610.30755 and 2610.29823 both conclude that raw-history search is competitive, but they disagree on *why the alternative fails*: 2610.30755 blames the rarity of useful mutation (4.39% of writes), while 2610.29823 blames the writer's ceiling. These are compatible -- a weak writer produces few useful mutations *and* a lossy store -- but neither paper measures the other's mechanism, so the dossier records the disagreement rather than resolving it.
|
||||
|
||||
2610.30485 is the outlier on the headline question: it studies *interaction* preferences, not factual recall, and finds memory nearly useless there. That is a different task family, not a refutation of the other four.
|
||||
|
||||
## Claim ledger
|
||||
|
||||
44 claims across 5 papers (2610.29671:9, 2610.29823:9, 2610.29869:7, 2610.30485:9, 2610.30755:10).
|
||||
|
||||
| id | paper | theme | value |
|
||||
|---|---|---|---|
|
||||
| R1 | 2610.30755 | T1 | six of seven |
|
||||
| R2 | 2610.30755 | T1 | 0.13–5.53 points |
|
||||
| R3 | 2610.30755 | T4 | 4.39% |
|
||||
| R4 | 2610.30755 | T4 | 67.36% to 69.42% |
|
||||
| R5 | 2610.30755 | T4 | 3.92 points |
|
||||
| R6 | 2610.30755 | T4 | 11.73 points |
|
||||
| R7 | 2610.30755 | T4 | 14.57 points |
|
||||
| R8 | 2610.30755 | T1 | 48.32% |
|
||||
| R9 | 2610.30755 | T1 | 47.68% to 15.37% |
|
||||
| R10 | 2610.30755 | T2 | 19.98 points |
|
||||
| P1 | 2610.30485 | T5 | 14 preferences |
|
||||
| P2 | 2610.30485 | T5 | 99 sessions over four months |
|
||||
| P3 | 2610.30485 | T5 | 49.6% |
|
||||
| P4 | 2610.30485 | T5 | 59.2% |
|
||||
| P5 | 2610.30485 | T5 | 50.0%, 50.0%, and 49.9% |
|
||||
| P6 | 2610.30485 | T5 | 25.4% |
|
||||
| P7 | 2610.30485 | T5 | 33% to 96% |
|
||||
| P8 | 2610.30485 | T5 | 34–44% |
|
||||
| P9 | 2610.30485 | T5 | 53.2% |
|
||||
| M1 | 2610.29823 | T2 | 419 turns, 152 questions |
|
||||
| M2 | 2610.29823 | T2 | 78% |
|
||||
| M3 | 2610.29823 | T2 | 92% |
|
||||
| M4 | 2610.29823 | T2 | Pi 99%, Codex 98% and Claude Code 97% |
|
||||
| M5 | 2610.29823 | T2 | $6.95 |
|
||||
| M6 | 2610.29823 | T4 | 88 questions without photo evidence |
|
||||
| M7 | 2610.29823 | T2 | 10,000 input tokens |
|
||||
| M8 | 2610.29823 | T3 | 43%, 43% and 45% |
|
||||
| M9 | 2610.29823 | T3 | 151 questions |
|
||||
| C1 | 2610.29869 | T3 | 87–100% |
|
||||
| C2 | 2610.29869 | T3 | 0% of the time |
|
||||
| C3 | 2610.29869 | T3 | ~50% |
|
||||
| C4 | 2610.29869 | T3 | 77% |
|
||||
| C5 | 2610.29869 | T3 | 13–25% |
|
||||
| C6 | 2610.29869 | T3 | 100% |
|
||||
| C7 | 2610.29869 | T3 | ~20–40% |
|
||||
| U1 | 2610.29671 | T1 | 88.80% |
|
||||
| U2 | 2610.29671 | T1 | 10.93 percentage points |
|
||||
| U3 | 2610.29671 | T1 | 80.99% |
|
||||
| U4 | 2610.29671 | T1 | 1.29 and 3.03 percentage points |
|
||||
| U5 | 2610.29671 | T1 | 2.48 percentage points |
|
||||
| U6 | 2610.29671 | T1 | 8.18 percentage points |
|
||||
| U7 | 2610.29671 | T1 | 83.75% |
|
||||
| U8 | 2610.29671 | T2 | 76.58% to 88.80% |
|
||||
| U9 | 2610.29671 | T2 | 87.05% |
|
||||
|
||||
@@ -0,0 +1,465 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
build_dossier.py -- build the 5-paper agentic-memory grounded dossier.
|
||||
|
||||
Every quote in the dossier is EXTRACTED from the pinned corpus bytes
|
||||
(anchor + span), never hand-typed, so the dossier cannot drift from its
|
||||
sources. verify_claims.py re-checks each quote against the same pinned
|
||||
bytes and runs a mutation battery proving the check is not vacuous.
|
||||
|
||||
Run: python3 build_dossier.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
CORPUS = ROOT / "corpus"
|
||||
|
||||
PAPERS = {
|
||||
"2610.29671": "Uni-Mem: Learning a Long-Horizon Agentic Memory Manager with Unified Online Policy Optimization",
|
||||
"2610.29823": "MemoryTax: The Bitter Lesson of Agent Memory",
|
||||
"2610.29869": "Compaction Laundering: Provenance Destruction in Context-Compacting Coding Agents",
|
||||
"2610.30485": "Why Bother Me Again? Benchmarking Personal Agent Memory for Interaction Preferences",
|
||||
"2610.30755": "Is Complex Agentic Memory Always Needed? Naive RAG Is Stronger Than We Thought",
|
||||
}
|
||||
|
||||
# id, paper, theme, claim, value (headline number, must be inside the quote),
|
||||
# anchor (verbatim start marker in the paper text), span (chars to take), note.
|
||||
SPEC = [
|
||||
# ---------------- 2610.30755 -- Raw-RAG / naive RAG ----------------
|
||||
dict(id="R1", paper="2610.30755", theme="T1",
|
||||
claim="Raw-RAG ranks first among compared memory systems in six of seven benchmark-model settings.",
|
||||
value="six of seven",
|
||||
anchor="Raw-RAG ranks first among the compared memory systems in six",
|
||||
span=210,
|
||||
note="The headline result: the naive baseline wins most settings."),
|
||||
dict(id="R2", paper="2610.30755", theme="T1",
|
||||
claim="Raw-RAG beats the strongest compared agentic memory system by 0.13–5.53 points.",
|
||||
value="0.13–5.53 points",
|
||||
anchor="outperforming the strongest compared agentic memory system by 0.13",
|
||||
span=130,
|
||||
note="Margin is small on LoCoMo/LongMemEval, large on MemoryAgentBench."),
|
||||
dict(id="R3", paper="2610.30755", theme="T4",
|
||||
claim="Mem0's UPDATE+DELETE account for only 4.39% of 33,902 LoCoMo write decisions.",
|
||||
value="4.39%",
|
||||
anchor="together account for only 4.39% of the 33,902 write decisions on LoCoMo",
|
||||
span=140,
|
||||
note="Mutation is rare in practice -- most proposed writes are duplicates or malformed."),
|
||||
dict(id="R4", paper="2610.30755", theme="T4",
|
||||
claim="Replacing UPDATE/DELETE with append-only ADD/NONE raises LoCoMo accuracy 67.36% -> 69.42%.",
|
||||
value="67.36% to 69.42%",
|
||||
anchor="Replacing UPDATE with append-only ADD and DELETE with NONE raises",
|
||||
span=170,
|
||||
note="Explicit mutation gives no accuracy gain; the simpler store is better."),
|
||||
dict(id="R5", paper="2610.30755", theme="T4",
|
||||
claim="Removing reranking lowers MemoryAgentBench Total by 3.92 points.",
|
||||
value="3.92 points",
|
||||
anchor="Removing reranking lowers Total by 3.92 points",
|
||||
span=190,
|
||||
note="Reranking is the one component worth keeping."),
|
||||
dict(id="R6", paper="2610.30755", theme="T4",
|
||||
claim="Returning only one item lowers Total by 11.73 points.",
|
||||
value="11.73 points",
|
||||
anchor="while returning only one item lowers it by 11.73 points",
|
||||
span=130,
|
||||
note="Retrieval depth matters far more than chunk size."),
|
||||
dict(id="R7", paper="2610.30755", theme="T4",
|
||||
claim="Top-120 retrieval loses 14.57 points.",
|
||||
value="14.57 points",
|
||||
anchor="Top-120 loses 14.57 points",
|
||||
span=130,
|
||||
note="Driven by long-range understanding and conflict resolution."),
|
||||
dict(id="R8", paper="2610.30755", theme="T1",
|
||||
claim="No memory-management variant improves on Raw-RAG (48.32%).",
|
||||
value="48.32%",
|
||||
anchor="None improves Raw-RAG (48.32%)",
|
||||
span=150,
|
||||
note="RQ2 answer: added management does not help."),
|
||||
dict(id="R9", paper="2610.30755", theme="T1",
|
||||
claim="Atomic-fact extraction collapses test-time learning from 47.68% to 15.37%.",
|
||||
value="47.68% to 15.37%",
|
||||
anchor="Atomic-fact extraction reduces test-time learning from 47.68% to 15.37%",
|
||||
span=130,
|
||||
note="The most damaging of the three management operations."),
|
||||
dict(id="R10", paper="2610.30755", theme="T2",
|
||||
claim="Full-context beats Raw-RAG by 19.98 points on DeepSeek-V4-Flash MEME.",
|
||||
value="19.98 points",
|
||||
anchor="Full-context leads Raw-RAG by 19.98 points",
|
||||
span=130,
|
||||
note="The one setting where the naive baseline is not first."),
|
||||
# ---------------- 2610.30485 -- MemPABench ----------------
|
||||
dict(id="P1", paper="2610.30485", theme="T5",
|
||||
claim="MemPABench has four simulated users holding 14 preferences.",
|
||||
value="14 preferences",
|
||||
anchor="Four grounded simulated users hold 14 preferences",
|
||||
span=140,
|
||||
note="Preferences span work and personal contexts."),
|
||||
dict(id="P2", paper="2610.30485", theme="T5",
|
||||
claim="Feedback spans 99 sessions over four months.",
|
||||
value="99 sessions over four months",
|
||||
anchor="Their feedback spans 99 sessions over four months",
|
||||
span=130,
|
||||
note="Five event-driven preference changes per user."),
|
||||
dict(id="P3", paper="2610.30485", theme="T5",
|
||||
claim="No Memory reaches 49.6% PSA against 45.2% random.",
|
||||
value="49.6%",
|
||||
anchor="No Memory, where the PA selects by its defaults, reaches 49.6%",
|
||||
span=190,
|
||||
note="Defaults alone nearly match chance-plus-margin."),
|
||||
dict(id="P4", paper="2610.30485", theme="T5",
|
||||
claim="File memory raises PSA to 59.2%, +9.6 points over No Memory.",
|
||||
value="59.2%",
|
||||
anchor="It raises PSA to 59.2%",
|
||||
span=170,
|
||||
note="The only condition with a clear gain; CI [+3.9, +15.6]."),
|
||||
dict(id="P5", paper="2610.30485", theme="T5",
|
||||
claim="Mem0, MemOS and Simple RAG stay at No Memory level (PSA 50.0/50.0/49.9).",
|
||||
value="50.0%, 50.0%, and 49.9%",
|
||||
anchor="with PSA of 50.0%, 50.0%, and 49.9%",
|
||||
span=130,
|
||||
note="Retrieval-based memory does not beat having none."),
|
||||
dict(id="P6", paper="2610.30485", theme="T5",
|
||||
claim="File memory PET is 25.4%, more than twice No Memory's 11.2%.",
|
||||
value="25.4%",
|
||||
anchor="Its PET is 25.4%",
|
||||
span=170,
|
||||
note="But its CS does not improve (14.4% vs 21.8%): context is not tracked."),
|
||||
dict(id="P7", paper="2610.30485", theme="T5",
|
||||
claim="Oracle lifts correct selection 33%->96% but BRS>=4 only 24%->59%.",
|
||||
value="33% to 96%",
|
||||
anchor="Oracle raises correct selections of the five settings",
|
||||
span=210,
|
||||
note="Knowing the preference does not override the model's interaction habits."),
|
||||
dict(id="P8", paper="2610.30485", theme="T5",
|
||||
claim="Information Flow is the hardest dimension, PSA 34–44%.",
|
||||
value="34–44%",
|
||||
anchor="Information Flow is the hardest under every condition",
|
||||
span=160,
|
||||
note="Expression Style (57-69%) and Disclosure (55-58%) are easier."),
|
||||
dict(id="P9", paper="2610.30485", theme="T5",
|
||||
claim="Anonymizing users drops shortcut accuracy 53.2% -> 46.4%, near the 45.2% chance.",
|
||||
value="53.2%",
|
||||
anchor="reaches 53.2% accuracy from the source identity but 46.4%",
|
||||
span=140,
|
||||
note="Confirms the benchmark measures memory, not name-guessing."),
|
||||
# ---------------- 2610.29823 -- MemoryTax ----------------
|
||||
dict(id="M1", paper="2610.29823", theme="T2",
|
||||
claim="Study scope: one LoCoMo conversation, 419 turns, 152 questions.",
|
||||
value="419 turns, 152 questions",
|
||||
anchor="On one LoCoMo conversation (419 turns, 152 questions)",
|
||||
span=160,
|
||||
note="Deliberately narrow: one conversation, one run per cell."),
|
||||
dict(id="M2", paper="2610.29823", theme="T2",
|
||||
claim="With GPT-4o-mini, Mem0 scores 78% vs 74% for full context and BM25.",
|
||||
value="78%",
|
||||
anchor="With GPT-4o-mini, Mem0 scores highest",
|
||||
span=160,
|
||||
note="Agent memory is a patch for a weak model."),
|
||||
dict(id="M3", paper="2610.29823", theme="T2",
|
||||
claim="With GPT-6 Luna, full context scores 92%, BM25 82%, Mem0 80%, Zep 51%.",
|
||||
value="92%",
|
||||
anchor="With GPT-6 Luna, full context scores 92%",
|
||||
span=200,
|
||||
note="The patch stops helping as the model gets stronger."),
|
||||
dict(id="M4", paper="2610.29823", theme="T2",
|
||||
claim="With Claude Opus 5.5, agents score highest: Pi 99%, Codex 98%, Claude Code 97%.",
|
||||
value="Pi 99%, Codex 98% and Claude Code 97%",
|
||||
anchor="With Claude Opus 5.5, Mem0 and full context both score 95%",
|
||||
span=170,
|
||||
note="Searching the raw record beats reading all of it at the top end."),
|
||||
dict(id="M5", paper="2610.29823", theme="T2",
|
||||
claim="Pi scores 99% for $6.95/rollout vs full context 95% for $17.84.",
|
||||
value="$6.95",
|
||||
anchor="Pi scores 99% for $6.95 a rollout",
|
||||
span=160,
|
||||
note="Less than half the cost for a higher score."),
|
||||
dict(id="M6", paper="2610.29823", theme="T4",
|
||||
claim="On the 88 questions without photo evidence, Pi scores 100%.",
|
||||
value="88 questions without photo evidence",
|
||||
anchor="On the 88 questions without photo evidence",
|
||||
span=160,
|
||||
note="Pi matches full context once captions are removed from the comparison."),
|
||||
dict(id="M7", paper="2610.29823", theme="T2",
|
||||
claim="Pi reads ~10,000 input tokens/question (7,900 cached) vs 28,900 for full context.",
|
||||
value="10,000 input tokens",
|
||||
anchor="Pi reads about 10,000 input tokens a question",
|
||||
span=150,
|
||||
note="The efficiency mechanism behind the cost lead."),
|
||||
dict(id="M8", paper="2610.29823", theme="T3",
|
||||
claim="Writer ceiling: on Zep's GPT-4o-mini graph readers score 43/43/45%; on Opus 5.5's, 66/71/87%.",
|
||||
value="43%, 43% and 45%",
|
||||
anchor="GPT-4o-mini graph, GPT-4o-mini, GPT-6 Luna and Claude Opus 5.5 score 43%, 43% and 45%",
|
||||
span=210,
|
||||
note="A stronger reader cannot recover what a weaker writer left out."),
|
||||
dict(id="M9", paper="2610.29823", theme="T3",
|
||||
claim="Pi's one miss is a LoCoMo answer-key error, not a memory failure.",
|
||||
value="151 questions",
|
||||
anchor="Its one miss is LoCoMo",
|
||||
span=590,
|
||||
note="The model is right and the benchmark key is wrong."),
|
||||
# ---------------- 2610.29869 -- Compaction Laundering ----------------
|
||||
dict(id="C1", paper="2610.29869", theme="T3",
|
||||
claim="Forged approval launders into memory 87–100% of the time across five flattening harnesses.",
|
||||
value="87–100%",
|
||||
anchor="the forged approval launders into the",
|
||||
span=210,
|
||||
note="Up to a ~908,000-token context window."),
|
||||
dict(id="C2", paper="2610.29869", theme="T3",
|
||||
claim="The identical attack succeeds 0% of the time across four role-typed harnesses.",
|
||||
value="0% of the time",
|
||||
anchor="succeeds 0% of the time",
|
||||
span=130,
|
||||
note="Role-typing is a built-in control."),
|
||||
dict(id="C3", paper="2610.29869", theme="T3",
|
||||
claim="Live in the real pi TUI the attack runs at ~50% against a 0% paired clean arm.",
|
||||
value="~50%",
|
||||
anchor="live in the real pi TUI at ~50%",
|
||||
span=130,
|
||||
note="Runtime evidence, not a simulation."),
|
||||
dict(id="C4", paper="2610.29869", theme="T3",
|
||||
claim="A code-level provenance check catches 77% of laundering, demoting 2% of legitimate content.",
|
||||
value="77%",
|
||||
anchor="we measure a code-level provenance check that catches 77% of",
|
||||
span=160,
|
||||
note="The only defense that works; prompt-level defenses cannot restore provenance."),
|
||||
dict(id="C5", paper="2610.29869", theme="T3",
|
||||
claim="In-prompt anti-injection rules stop 13–25%; published classifier defenses report 67% at best.",
|
||||
value="13–25%",
|
||||
anchor="stops 13",
|
||||
span=140,
|
||||
note="Baselines are far weaker than the code-level check."),
|
||||
dict(id="C6", paper="2610.29869", theme="T3",
|
||||
claim="Laundering still succeeds 100% of the time at every context size.",
|
||||
value="100%",
|
||||
anchor="The laundering still succeeds 100% of the",
|
||||
span=160,
|
||||
note="Size does not attenuate the attack."),
|
||||
dict(id="C7", paper="2610.29869", theme="T3",
|
||||
claim="Executed rate settles to ~20–40% in a full window.",
|
||||
value="~20–40%",
|
||||
anchor="settles to ~20",
|
||||
span=130,
|
||||
note="Real agents compact at this size, so this is the realistic case."),
|
||||
# ---------------- 2610.29671 -- Uni-Mem ----------------
|
||||
dict(id="U1", paper="2610.29671", theme="T1",
|
||||
claim="Uni-Mem reaches an overall LoCoMo JJ of 88.80%.",
|
||||
value="88.80%",
|
||||
anchor="Uni-Mem achieves the best performance across all four question types",
|
||||
span=220,
|
||||
note="Best across all four question types."),
|
||||
dict(id="U2", paper="2610.29671", theme="T1",
|
||||
claim="Uni-Mem surpasses Memory-R2 (OSS) by 10.93 points and Mem-T by 12.31.",
|
||||
value="10.93 percentage points",
|
||||
anchor="It surpasses the strongest baseline, Memory-R2 (OSS), by 10.93",
|
||||
span=210,
|
||||
note="Gains of 12.93 and 9.29 on multi-hop and temporal."),
|
||||
dict(id="U3", paper="2610.29671", theme="T1",
|
||||
claim="Uni-Mem still reaches 80.99% without SFT, +3.12% over the strongest baseline.",
|
||||
value="80.99%",
|
||||
anchor="Uni-Mem still achieves 80.99% without SFT",
|
||||
span=140,
|
||||
note="RL alone is enough to beat the baselines."),
|
||||
dict(id="U4", paper="2610.29671", theme="T1",
|
||||
claim="ECA improves overall JJ by 1.29 and 3.03 points without and with SFT.",
|
||||
value="1.29 and 3.03 percentage points",
|
||||
anchor="improves overall JJ by 1.29 and 3.03 percentage points",
|
||||
span=130,
|
||||
note="Event-wise credit assignment localizes sparse feedback."),
|
||||
dict(id="U5", paper="2610.29671", theme="T1",
|
||||
claim="Stage-Decoupled GRPO improves overall JJ by 2.48 points without SFT.",
|
||||
value="2.48 percentage points",
|
||||
anchor="improves overall JJ by 2.48 percentage points without SFT",
|
||||
span=140,
|
||||
note="Temporal accuracy rises 10.55 points in this setting."),
|
||||
dict(id="U6", paper="2610.29671", theme="T1",
|
||||
claim="Trained construction alone improves overall JJ by 8.18 points.",
|
||||
value="8.18 percentage points",
|
||||
anchor="improves the overall JJ score by 8.18 percentage points",
|
||||
span=130,
|
||||
note="Each capability contributes even paired with its untrained counterpart."),
|
||||
dict(id="U7", paper="2610.29671", theme="T1",
|
||||
claim="Extending update blocks 1:1 -> 3:2 reduces overall JJ from 88.80% to 83.75%.",
|
||||
value="83.75%",
|
||||
anchor="the overall JJ score from 88.80",
|
||||
span=120,
|
||||
note="Frequent alternation matters as much as balanced allocation."),
|
||||
dict(id="U8", paper="2610.29671", theme="T2",
|
||||
claim="SFT followed by RL improves the 4B manager's JJ from 76.58% to 88.80%.",
|
||||
value="76.58% to 88.80%",
|
||||
anchor="SFT followed by RL improves the 4B manager",
|
||||
span=150,
|
||||
note="A small trained manager beats Qwen3.5-27B (87.05%) and 397B-A17B (85.86%)."),
|
||||
dict(id="U9", paper="2610.29671", theme="T2",
|
||||
claim="Uni-Mem outperforms all off-the-shelf managers including Qwen3.5-27B (87.05%).",
|
||||
value="87.05%",
|
||||
anchor="Uni-Mem also outperforms all off-the-shelf managers",
|
||||
span=210,
|
||||
note="Scale is not the binding constraint; joint training is."),
|
||||
]
|
||||
|
||||
THEMES = {
|
||||
"T1": "Is designed memory needed at all?",
|
||||
"T2": "Cost, ceiling, and scale",
|
||||
"T3": "Provenance: what compaction destroys and what writers omit",
|
||||
"T4": "What raw evidence must be preserved",
|
||||
"T5": "Interaction preferences are a different problem",
|
||||
}
|
||||
|
||||
|
||||
def sha256(p: Path) -> str:
|
||||
return hashlib.sha256(p.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def norm(s: str) -> str:
|
||||
return " ".join(s.split())
|
||||
|
||||
|
||||
def main() -> int:
|
||||
texts, pins, sizes = {}, {}, {}
|
||||
for pid in PAPERS:
|
||||
p = CORPUS / f"{pid}.txt"
|
||||
texts[pid] = p.read_text(encoding="utf-8")
|
||||
pins[pid] = sha256(p)
|
||||
sizes[pid] = p.stat().st_size
|
||||
|
||||
claims, seen = [], set()
|
||||
for s in SPEC:
|
||||
t = texts[s["paper"]]
|
||||
idx = t.find(s["anchor"])
|
||||
if idx < 0:
|
||||
raise SystemExit(f"ANCHOR NOT FOUND: {s['id']} :: {s['anchor']!r}")
|
||||
quote = t[idx: idx + s["span"]]
|
||||
if s["value"] not in quote:
|
||||
raise SystemExit(f"VALUE NOT IN QUOTE: {s['id']} :: {s['value']!r}")
|
||||
if s["id"] in seen:
|
||||
raise SystemExit(f"DUPLICATE ID: {s['id']}")
|
||||
seen.add(s["id"])
|
||||
claims.append(dict(
|
||||
id=s["id"], paper=s["paper"], theme=s["theme"],
|
||||
claim=s["claim"], value=s["value"], note=s["note"],
|
||||
anchor=s["anchor"], quote=quote,
|
||||
corpus_sha256=pins[s["paper"]],
|
||||
))
|
||||
|
||||
by_paper = {pid: sum(1 for c in claims if c["paper"] == pid) for pid in PAPERS}
|
||||
|
||||
# ---- claims.json ----
|
||||
(ROOT / "claims.json").write_text(json.dumps(dict(
|
||||
built="2026-10-25",
|
||||
query='all:"agentic memory"',
|
||||
feed_updated="2026-10-25T00:40:42Z",
|
||||
feed_total_results=583,
|
||||
papers=[dict(arxiv_id=pid, title=PAPERS[pid], bytes=sizes[pid],
|
||||
chars=len(texts[pid]), sha256=pins[pid],
|
||||
claims=by_paper[pid]) for pid in PAPERS],
|
||||
themes=THEMES,
|
||||
claims=claims,
|
||||
), indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
|
||||
# ---- DOSSIER.md ----
|
||||
L = []
|
||||
L.append("# Agentic Memory: A Five-Paper Grounded Dossier\n")
|
||||
L.append("Archive lane -- sphinx. Built 2026-10-25.\n")
|
||||
L.append('Scope: the five most recent arXiv papers matching `all:"agentic memory"`, '
|
||||
"taken from a single feed snapshot (updated 2026-10-25T00:40:42Z; 583 total results).\n")
|
||||
L.append("Every number below is a verbatim quote from the pinned corpus text. "
|
||||
"Nothing is paraphrased into a figure. `verify_claims.py` re-derives every "
|
||||
"quote from the same pinned bytes and fails the build if any drifts.\n")
|
||||
|
||||
L.append("## Provenance (pinned)\n")
|
||||
L.append("| paper | title | bytes | chars | sha256 (first 16) | claims |")
|
||||
L.append("|---|---|---|---|---|---|")
|
||||
for pid in PAPERS:
|
||||
L.append(f"| {pid} | {PAPERS[pid]} | {sizes[pid]} | {len(texts[pid])} | "
|
||||
f"`{pins[pid][:16]}` | {by_paper[pid]} |")
|
||||
L.append("")
|
||||
|
||||
L.append("## How this dossier is verified\n")
|
||||
L.append("```")
|
||||
L.append("python3 verify_claims.py # or: bash CHECK.txt")
|
||||
L.append("```")
|
||||
L.append("")
|
||||
L.append("The verifier checks, for each of the %d claims:" % len(claims))
|
||||
L.append("1. the corpus file's sha256 equals the pin recorded at build time;")
|
||||
L.append("2. the claim's `quote` is a literal substring of that corpus file;")
|
||||
L.append("3. the claim's headline `value` appears inside that quote;")
|
||||
L.append("4. the headline `value` appears in this dossier.")
|
||||
L.append("")
|
||||
L.append("It then runs a mutation battery: corrupting one quote at a time in a scratch")
|
||||
L.append("copy of `claims.json` must make the verification fail. If any mutation passes")
|
||||
L.append("undetected, the build fails -- the check is falsified, not assumed.\n")
|
||||
|
||||
for tid, tname in THEMES.items():
|
||||
L.append(f"## {tid}: {tname}\n")
|
||||
for c in [c for c in claims if c["theme"] == tid]:
|
||||
L.append(f"### {c['id']} -- {c['paper']}\n")
|
||||
L.append(f"**{c['claim']}**\n")
|
||||
L.append(f"> {norm(c['quote'])}\n")
|
||||
L.append(f"*Why it matters:* {c['note']}\n")
|
||||
|
||||
L.append("## Cross-paper synthesis\n")
|
||||
L.append("Four of the five papers converge on one answer: **the elaborate part of agent "
|
||||
"memory is not where the value is.**\n")
|
||||
L.append("- **Raw-RAG wins most settings** (2610.30755) -- naive dense retrieval over raw "
|
||||
"history ranks first in six of seven benchmark-model settings, and *no* added "
|
||||
"memory-management operation improves on it. The one setting where it loses is "
|
||||
"the one where full context is affordable.\n")
|
||||
L.append("- **Mutation is nearly free to drop** (2610.30755) -- UPDATE+DELETE are 4.39% of "
|
||||
"LoCoMo writes and 2.46% of LongMemEval writes; replacing them with append-only "
|
||||
"ADD/NONE *raises* accuracy 67.36% -> 69.42%.\n")
|
||||
L.append("- **Agent memory is a patch for a weak model** (2610.29823) -- Mem0 helps "
|
||||
"GPT-4o-mini (78% vs 74%) but not GPT-6 Luna or Claude Opus 5.5, and at the top "
|
||||
"end Pi scores 99% for $6.95 against full context's 95% for $17.84.\n")
|
||||
L.append("- **A stronger reader does not recover what a weaker writer left out** "
|
||||
"(2610.29823) -- on Zep's GPT-4o-mini graph the three readers score 43/43/45%; "
|
||||
"on the graph Claude Opus 5.5 wrote they score 66/71/87%. Memory has a ceiling "
|
||||
"set by its writer.\n")
|
||||
L.append("- **What must be preserved is raw evidence, not extracted facts** (2610.30755, "
|
||||
"2610.29823) -- verbatim chunks beat lossy artifacts; on the 88 questions with no "
|
||||
"photo evidence, Pi's lead over full context vanishes (100% vs 99%).\n")
|
||||
L.append("- **Compaction destroys provenance deterministically** (2610.29869) -- a forged "
|
||||
"approval launders into memory 87-100% of the time in flattening harnesses and "
|
||||
"0% in role-typed ones; no prompt-level defense restores it, only a code-level "
|
||||
"check (77% catch, 2% collateral).\n")
|
||||
L.append("- **Interaction preferences are a harder problem than factual recall** "
|
||||
"(2610.30485) -- retrieval-based memory does not beat No Memory (PSA 50.0/50.0/49.9 "
|
||||
"vs 49.6%), and even an Oracle that hands the agent the right preference lifts "
|
||||
"delivered behavior only 24% -> 59%.\n")
|
||||
L.append("- **The counter-example is training the memory manager itself** (2610.29671) -- "
|
||||
"Uni-Mem jointly learns construction and retrieval and reaches 88.80% JJ, beating "
|
||||
"Memory-R2 (OSS) by 10.93 points and off-the-shelf Qwen3.5-27B (87.05%). The gain "
|
||||
"comes from the training scheme (Stage-Decoupled GRPO + ECA), not from a bigger model.\n")
|
||||
|
||||
L.append("## Where the five disagree\n")
|
||||
L.append("2610.30755 and 2610.29823 both conclude that raw-history search is competitive, "
|
||||
"but they disagree on *why the alternative fails*: 2610.30755 blames the rarity of "
|
||||
"useful mutation (4.39% of writes), while 2610.29823 blames the writer's ceiling. "
|
||||
"These are compatible -- a weak writer produces few useful mutations *and* a "
|
||||
"lossy store -- but neither paper measures the other's mechanism, so the dossier "
|
||||
"records the disagreement rather than resolving it.\n")
|
||||
L.append("2610.30485 is the outlier on the headline question: it studies *interaction* "
|
||||
"preferences, not factual recall, and finds memory nearly useless there. That is a "
|
||||
"different task family, not a refutation of the other four.\n")
|
||||
|
||||
L.append("## Claim ledger\n")
|
||||
L.append(f"{len(claims)} claims across {len(PAPERS)} papers "
|
||||
f"({', '.join(f'{pid}:{by_paper[pid]}' for pid in PAPERS)}).\n")
|
||||
L.append("| id | paper | theme | value |")
|
||||
L.append("|---|---|---|---|")
|
||||
for c in claims:
|
||||
L.append(f"| {c['id']} | {c['paper']} | {c['theme']} | {c['value']} |")
|
||||
L.append("")
|
||||
|
||||
(ROOT / "DOSSIER.md").write_text("\n".join(L) + "\n", encoding="utf-8")
|
||||
|
||||
print(f"claims: {len(claims)} papers: {len(PAPERS)}")
|
||||
for pid in PAPERS:
|
||||
print(f" {pid} {pins[pid][:16]} {by_paper[pid]:>2} claims")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,541 @@
|
||||
{
|
||||
"built": "2026-10-25",
|
||||
"query": "all:\"agentic memory\"",
|
||||
"feed_updated": "2026-10-25T00:40:42Z",
|
||||
"feed_total_results": 583,
|
||||
"papers": [
|
||||
{
|
||||
"arxiv_id": "2610.29671",
|
||||
"title": "Uni-Mem: Learning a Long-Horizon Agentic Memory Manager with Unified Online Policy Optimization",
|
||||
"bytes": 90083,
|
||||
"chars": 89293,
|
||||
"sha256": "b83a772fb6a1340b2440359bec2bf3752d583d86e213806c5dbf7a648d947bdd",
|
||||
"claims": 9
|
||||
},
|
||||
{
|
||||
"arxiv_id": "2610.29823",
|
||||
"title": "MemoryTax: The Bitter Lesson of Agent Memory",
|
||||
"bytes": 23897,
|
||||
"chars": 23627,
|
||||
"sha256": "ed08ec2380a6c1479c1cbc0534cc2cf154f92aa057216398055f034f91abd9b5",
|
||||
"claims": 9
|
||||
},
|
||||
{
|
||||
"arxiv_id": "2610.29869",
|
||||
"title": "Compaction Laundering: Provenance Destruction in Context-Compacting Coding Agents",
|
||||
"bytes": 92858,
|
||||
"chars": 91936,
|
||||
"sha256": "e6f78f508c12d7a304f7fd36eb5f05d60095a426b189ab4a5cd817c341791b54",
|
||||
"claims": 7
|
||||
},
|
||||
{
|
||||
"arxiv_id": "2610.30485",
|
||||
"title": "Why Bother Me Again? Benchmarking Personal Agent Memory for Interaction Preferences",
|
||||
"bytes": 350788,
|
||||
"chars": 340805,
|
||||
"sha256": "48e8b2b9c7defc653b8c7577e9e6029a5f9873cbaabe0f6b54f7ca84b08a1906",
|
||||
"claims": 9
|
||||
},
|
||||
{
|
||||
"arxiv_id": "2610.30755",
|
||||
"title": "Is Complex Agentic Memory Always Needed? Naive RAG Is Stronger Than We Thought",
|
||||
"bytes": 195193,
|
||||
"chars": 193677,
|
||||
"sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac",
|
||||
"claims": 10
|
||||
}
|
||||
],
|
||||
"themes": {
|
||||
"T1": "Is designed memory needed at all?",
|
||||
"T2": "Cost, ceiling, and scale",
|
||||
"T3": "Provenance: what compaction destroys and what writers omit",
|
||||
"T4": "What raw evidence must be preserved",
|
||||
"T5": "Interaction preferences are a different problem"
|
||||
},
|
||||
"claims": [
|
||||
{
|
||||
"id": "R1",
|
||||
"paper": "2610.30755",
|
||||
"theme": "T1",
|
||||
"claim": "Raw-RAG ranks first among compared memory systems in six of seven benchmark-model settings.",
|
||||
"value": "six of seven",
|
||||
"note": "The headline result: the naive baseline wins most settings.",
|
||||
"anchor": "Raw-RAG ranks first among the compared memory systems in six",
|
||||
"quote": "Raw-RAG ranks first among the compared memory systems in six of seven benchmark–model settings, outperforming the strongest compared agentic memory system by 0.13–5.53 points. We investigate why raw-history ret",
|
||||
"corpus_sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac"
|
||||
},
|
||||
{
|
||||
"id": "R2",
|
||||
"paper": "2610.30755",
|
||||
"theme": "T1",
|
||||
"claim": "Raw-RAG beats the strongest compared agentic memory system by 0.13–5.53 points.",
|
||||
"value": "0.13–5.53 points",
|
||||
"note": "Margin is small on LoCoMo/LongMemEval, large on MemoryAgentBench.",
|
||||
"anchor": "outperforming the strongest compared agentic memory system by 0.13",
|
||||
"quote": "outperforming the strongest compared agentic memory system by 0.13–5.53 points. We investigate why raw-history retrieval remains c",
|
||||
"corpus_sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac"
|
||||
},
|
||||
{
|
||||
"id": "R3",
|
||||
"paper": "2610.30755",
|
||||
"theme": "T4",
|
||||
"claim": "Mem0's UPDATE+DELETE account for only 4.39% of 33,902 LoCoMo write decisions.",
|
||||
"value": "4.39%",
|
||||
"note": "Mutation is rare in practice -- most proposed writes are duplicates or malformed.",
|
||||
"anchor": "together account for only 4.39% of the 33,902 write decisions on LoCoMo",
|
||||
"quote": "together account for only 4.39% of the 33,902 write decisions on LoCoMo\nand 2.46% of the 428,858 write decisions on LongMemEval\n(Figure 2); ",
|
||||
"corpus_sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac"
|
||||
},
|
||||
{
|
||||
"id": "R4",
|
||||
"paper": "2610.30755",
|
||||
"theme": "T4",
|
||||
"claim": "Replacing UPDATE/DELETE with append-only ADD/NONE raises LoCoMo accuracy 67.36% -> 69.42%.",
|
||||
"value": "67.36% to 69.42%",
|
||||
"note": "Explicit mutation gives no accuracy gain; the simpler store is better.",
|
||||
"anchor": "Replacing UPDATE with append-only ADD and DELETE with NONE raises",
|
||||
"quote": "Replacing UPDATE with append-only ADD and DELETE with NONE raises\nLoCoMo accuracy from 67.36% to 69.42%, without reducing any category score\n(Table 2). Explicit mutation ",
|
||||
"corpus_sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac"
|
||||
},
|
||||
{
|
||||
"id": "R5",
|
||||
"paper": "2610.30755",
|
||||
"theme": "T4",
|
||||
"claim": "Removing reranking lowers MemoryAgentBench Total by 3.92 points.",
|
||||
"value": "3.92 points",
|
||||
"note": "Reranking is the one component worth keeping.",
|
||||
"anchor": "Removing reranking lowers Total by 3.92 points",
|
||||
"quote": "Removing reranking lowers Total by 3.92 points (pHolm=0.0012p_{\\mathrm{Holm}}=0.0012),\nwhile returning only one item lowers it by 11.73 points. Final-kk\nvalues of 5, 10, 30, and 60 stay with",
|
||||
"corpus_sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac"
|
||||
},
|
||||
{
|
||||
"id": "R6",
|
||||
"paper": "2610.30755",
|
||||
"theme": "T4",
|
||||
"claim": "Returning only one item lowers Total by 11.73 points.",
|
||||
"value": "11.73 points",
|
||||
"note": "Retrieval depth matters far more than chunk size.",
|
||||
"anchor": "while returning only one item lowers it by 11.73 points",
|
||||
"quote": "while returning only one item lowers it by 11.73 points. Final-kk\nvalues of 5, 10, 30, and 60 stay within 1.3 points of the baseli",
|
||||
"corpus_sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac"
|
||||
},
|
||||
{
|
||||
"id": "R7",
|
||||
"paper": "2610.30755",
|
||||
"theme": "T4",
|
||||
"claim": "Top-120 retrieval loses 14.57 points.",
|
||||
"value": "14.57 points",
|
||||
"note": "Driven by long-range understanding and conflict resolution.",
|
||||
"anchor": "Top-120 loses 14.57 points",
|
||||
"quote": "Top-120 loses 14.57 points, driven by\nlong-range understanding and conflict resolution. Chunk sizes from 128 to 1,024\ndo not signi",
|
||||
"corpus_sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac"
|
||||
},
|
||||
{
|
||||
"id": "R8",
|
||||
"paper": "2610.30755",
|
||||
"theme": "T1",
|
||||
"claim": "No memory-management variant improves on Raw-RAG (48.32%).",
|
||||
"value": "48.32%",
|
||||
"note": "RQ2 answer: added management does not help.",
|
||||
"anchor": "None improves Raw-RAG (48.32%)",
|
||||
"quote": "None improves Raw-RAG (48.32%) on the\nsame 14-source MemoryAgentBench Total (Table 6, orange rows).\nThe exact construction and retrieval procedures ar",
|
||||
"corpus_sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac"
|
||||
},
|
||||
{
|
||||
"id": "R9",
|
||||
"paper": "2610.30755",
|
||||
"theme": "T1",
|
||||
"claim": "Atomic-fact extraction collapses test-time learning from 47.68% to 15.37%.",
|
||||
"value": "47.68% to 15.37%",
|
||||
"note": "The most damaging of the three management operations.",
|
||||
"anchor": "Atomic-fact extraction reduces test-time learning from 47.68% to 15.37%",
|
||||
"quote": "Atomic-fact extraction reduces test-time learning from 47.68% to 15.37%\nand Total by 19.62 points, consistent with loss or disrupt",
|
||||
"corpus_sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac"
|
||||
},
|
||||
{
|
||||
"id": "R10",
|
||||
"paper": "2610.30755",
|
||||
"theme": "T2",
|
||||
"claim": "Full-context beats Raw-RAG by 19.98 points on DeepSeek-V4-Flash MEME.",
|
||||
"value": "19.98 points",
|
||||
"note": "The one setting where the naive baseline is not first.",
|
||||
"anchor": "Full-context leads Raw-RAG by 19.98 points",
|
||||
"quote": "Full-context leads Raw-RAG by 19.98 points in that setting. Raw-RAG\nis competitive across these tasks despite its simpler ingestio",
|
||||
"corpus_sha256": "2246678ceaf7a42ca8104b5067a9d548b0795391af40142b62e3f1638288daac"
|
||||
},
|
||||
{
|
||||
"id": "P1",
|
||||
"paper": "2610.30485",
|
||||
"theme": "T5",
|
||||
"claim": "MemPABench has four simulated users holding 14 preferences.",
|
||||
"value": "14 preferences",
|
||||
"note": "Preferences span work and personal contexts.",
|
||||
"anchor": "Four grounded simulated users hold 14 preferences",
|
||||
"quote": "Four grounded simulated users hold 14 preferences across work and personal contexts. Their feedback spans 99 sessions over four months, with",
|
||||
"corpus_sha256": "48e8b2b9c7defc653b8c7577e9e6029a5f9873cbaabe0f6b54f7ca84b08a1906"
|
||||
},
|
||||
{
|
||||
"id": "P2",
|
||||
"paper": "2610.30485",
|
||||
"theme": "T5",
|
||||
"claim": "Feedback spans 99 sessions over four months.",
|
||||
"value": "99 sessions over four months",
|
||||
"note": "Five event-driven preference changes per user.",
|
||||
"anchor": "Their feedback spans 99 sessions over four months",
|
||||
"quote": "Their feedback spans 99 sessions over four months, with five event-driven changes per user testing memory updates. We evaluate pre",
|
||||
"corpus_sha256": "48e8b2b9c7defc653b8c7577e9e6029a5f9873cbaabe0f6b54f7ca84b08a1906"
|
||||
},
|
||||
{
|
||||
"id": "P3",
|
||||
"paper": "2610.30485",
|
||||
"theme": "T5",
|
||||
"claim": "No Memory reaches 49.6% PSA against 45.2% random.",
|
||||
"value": "49.6%",
|
||||
"note": "Defaults alone nearly match chance-plus-margin.",
|
||||
"anchor": "No Memory, where the PA selects by its defaults, reaches 49.6%",
|
||||
"quote": "No Memory, where the PA selects by its defaults, reaches 49.6%, compared with 45.2% for random selection (Appendix A.1). Only File memory adds a clear gain. It raises PSA to 59.2%, a mean ga",
|
||||
"corpus_sha256": "48e8b2b9c7defc653b8c7577e9e6029a5f9873cbaabe0f6b54f7ca84b08a1906"
|
||||
},
|
||||
{
|
||||
"id": "P4",
|
||||
"paper": "2610.30485",
|
||||
"theme": "T5",
|
||||
"claim": "File memory raises PSA to 59.2%, +9.6 points over No Memory.",
|
||||
"value": "59.2%",
|
||||
"note": "The only condition with a clear gain; CI [+3.9, +15.6].",
|
||||
"anchor": "It raises PSA to 59.2%",
|
||||
"quote": "It raises PSA to 59.2%, a mean gain of 9.6 points over No Memory (95% cluster-bootstrap interval [+3.9, +15.6]). PERMA (Liu et al., 2026), a benchmark of content preferen",
|
||||
"corpus_sha256": "48e8b2b9c7defc653b8c7577e9e6029a5f9873cbaabe0f6b54f7ca84b08a1906"
|
||||
},
|
||||
{
|
||||
"id": "P5",
|
||||
"paper": "2610.30485",
|
||||
"theme": "T5",
|
||||
"claim": "Mem0, MemOS and Simple RAG stay at No Memory level (PSA 50.0/50.0/49.9).",
|
||||
"value": "50.0%, 50.0%, and 49.9%",
|
||||
"note": "Retrieval-based memory does not beat having none.",
|
||||
"anchor": "with PSA of 50.0%, 50.0%, and 49.9%",
|
||||
"quote": "with PSA of 50.0%, 50.0%, and 49.9%. The intervals of all conditions and their comparisons with No Memory on each user–backbone pa",
|
||||
"corpus_sha256": "48e8b2b9c7defc653b8c7577e9e6029a5f9873cbaabe0f6b54f7ca84b08a1906"
|
||||
},
|
||||
{
|
||||
"id": "P6",
|
||||
"paper": "2610.30485",
|
||||
"theme": "T5",
|
||||
"claim": "File memory PET is 25.4%, more than twice No Memory's 11.2%.",
|
||||
"value": "25.4%",
|
||||
"note": "But its CS does not improve (14.4% vs 21.8%): context is not tracked.",
|
||||
"anchor": "Its PET is 25.4%",
|
||||
"quote": "Its PET is 25.4%, more than twice the 11.2% of No Memory, and it is higher on every backbone. However, the CS does not improve, at 14.4% against 21.8% (Appendix G.3). In ",
|
||||
"corpus_sha256": "48e8b2b9c7defc653b8c7577e9e6029a5f9873cbaabe0f6b54f7ca84b08a1906"
|
||||
},
|
||||
{
|
||||
"id": "P7",
|
||||
"paper": "2610.30485",
|
||||
"theme": "T5",
|
||||
"claim": "Oracle lifts correct selection 33%->96% but BRS>=4 only 24%->59%.",
|
||||
"value": "33% to 96%",
|
||||
"note": "Knowing the preference does not override the model's interaction habits.",
|
||||
"anchor": "Oracle raises correct selections of the five settings",
|
||||
"quote": "Oracle raises correct selections of the five settings from 33% to 96%, but the share of replies with a BRS of 4 or higher rises only from 24% to 59%. Even with the IX Wrapper, the PA still writes too much. Afte",
|
||||
"corpus_sha256": "48e8b2b9c7defc653b8c7577e9e6029a5f9873cbaabe0f6b54f7ca84b08a1906"
|
||||
},
|
||||
{
|
||||
"id": "P8",
|
||||
"paper": "2610.30485",
|
||||
"theme": "T5",
|
||||
"claim": "Information Flow is the hardest dimension, PSA 34–44%.",
|
||||
"value": "34–44%",
|
||||
"note": "Expression Style (57-69%) and Disclosure (55-58%) are easier.",
|
||||
"anchor": "Information Flow is the hardest under every condition",
|
||||
"quote": "Information Flow is the hardest under every condition (Figure 8). Its PSA stays at 34–44%, the lowest of the four, while Expression Style (57–69%) and Disclosur",
|
||||
"corpus_sha256": "48e8b2b9c7defc653b8c7577e9e6029a5f9873cbaabe0f6b54f7ca84b08a1906"
|
||||
},
|
||||
{
|
||||
"id": "P9",
|
||||
"paper": "2610.30485",
|
||||
"theme": "T5",
|
||||
"claim": "Anonymizing users drops shortcut accuracy 53.2% -> 46.4%, near the 45.2% chance.",
|
||||
"value": "53.2%",
|
||||
"note": "Confirms the benchmark measures memory, not name-guessing.",
|
||||
"anchor": "reaches 53.2% accuracy from the source identity but 46.4%",
|
||||
"quote": "reaches 53.2% accuracy from the source identity but 46.4% from the anonymized profile, close to the 45.2% chance level (Figure 10; Appendix ",
|
||||
"corpus_sha256": "48e8b2b9c7defc653b8c7577e9e6029a5f9873cbaabe0f6b54f7ca84b08a1906"
|
||||
},
|
||||
{
|
||||
"id": "M1",
|
||||
"paper": "2610.29823",
|
||||
"theme": "T2",
|
||||
"claim": "Study scope: one LoCoMo conversation, 419 turns, 152 questions.",
|
||||
"value": "419 turns, 152 questions",
|
||||
"note": "Deliberately narrow: one conversation, one run per cell.",
|
||||
"anchor": "On one LoCoMo conversation (419 turns, 152 questions)",
|
||||
"quote": "On one LoCoMo conversation (419 turns, 152 questions), we run four memory methods (full context, BM25, Mem0, Zep) and three coding agents (Pi, Codex, Claude Cod",
|
||||
"corpus_sha256": "ed08ec2380a6c1479c1cbc0534cc2cf154f92aa057216398055f034f91abd9b5"
|
||||
},
|
||||
{
|
||||
"id": "M2",
|
||||
"paper": "2610.29823",
|
||||
"theme": "T2",
|
||||
"claim": "With GPT-4o-mini, Mem0 scores 78% vs 74% for full context and BM25.",
|
||||
"value": "78%",
|
||||
"note": "Agent memory is a patch for a weak model.",
|
||||
"anchor": "With GPT-4o-mini, Mem0 scores highest",
|
||||
"quote": "With GPT-4o-mini, Mem0 scores highest. It scores 78%, against 74% for both full context and BM25; Zep scores 43%. The agents score 63% (Pi), 28% (Codex) and 29%",
|
||||
"corpus_sha256": "ed08ec2380a6c1479c1cbc0534cc2cf154f92aa057216398055f034f91abd9b5"
|
||||
},
|
||||
{
|
||||
"id": "M3",
|
||||
"paper": "2610.29823",
|
||||
"theme": "T2",
|
||||
"claim": "With GPT-6 Luna, full context scores 92%, BM25 82%, Mem0 80%, Zep 51%.",
|
||||
"value": "92%",
|
||||
"note": "The patch stops helping as the model gets stronger.",
|
||||
"anchor": "With GPT-6 Luna, full context scores 92%",
|
||||
"quote": "With GPT-6 Luna, full context scores 92%, BM25 82%, Mem0 80% and Zep 51%; Zep’s graph holds 128 facts from the 19 sessions, against 195 with Claude Opus 5.5. With Claude Opus 5.5, Mem0 and full contex",
|
||||
"corpus_sha256": "ed08ec2380a6c1479c1cbc0534cc2cf154f92aa057216398055f034f91abd9b5"
|
||||
},
|
||||
{
|
||||
"id": "M4",
|
||||
"paper": "2610.29823",
|
||||
"theme": "T2",
|
||||
"claim": "With Claude Opus 5.5, agents score highest: Pi 99%, Codex 98%, Claude Code 97%.",
|
||||
"value": "Pi 99%, Codex 98% and Claude Code 97%",
|
||||
"note": "Searching the raw record beats reading all of it at the top end.",
|
||||
"anchor": "With Claude Opus 5.5, Mem0 and full context both score 95%",
|
||||
"quote": "With Claude Opus 5.5, Mem0 and full context both score 95% and Zep 87%, and the agents score highest: Pi 99%, Codex 98% and Claude Code 97%, a lead that comes mostly from",
|
||||
"corpus_sha256": "ed08ec2380a6c1479c1cbc0534cc2cf154f92aa057216398055f034f91abd9b5"
|
||||
},
|
||||
{
|
||||
"id": "M5",
|
||||
"paper": "2610.29823",
|
||||
"theme": "T2",
|
||||
"claim": "Pi scores 99% for $6.95/rollout vs full context 95% for $17.84.",
|
||||
"value": "$6.95",
|
||||
"note": "Less than half the cost for a higher score.",
|
||||
"anchor": "Pi scores 99% for $6.95 a rollout",
|
||||
"quote": "Pi scores 99% for $6.95 a rollout, while full context with the same model scores 95% for $17.84. Pi reads about 10,000 input tokens a question, 7,900 of them ca",
|
||||
"corpus_sha256": "ed08ec2380a6c1479c1cbc0534cc2cf154f92aa057216398055f034f91abd9b5"
|
||||
},
|
||||
{
|
||||
"id": "M6",
|
||||
"paper": "2610.29823",
|
||||
"theme": "T4",
|
||||
"claim": "On the 88 questions without photo evidence, Pi scores 100%.",
|
||||
"value": "88 questions without photo evidence",
|
||||
"note": "Pi matches full context once captions are removed from the comparison.",
|
||||
"anchor": "On the 88 questions without photo evidence",
|
||||
"quote": "On the 88 questions without photo evidence, Pi scores 100%, and full context, Mem0, Codex and Claude Code 99% each (Table 4). With the same input, then, Pi matc",
|
||||
"corpus_sha256": "ed08ec2380a6c1479c1cbc0534cc2cf154f92aa057216398055f034f91abd9b5"
|
||||
},
|
||||
{
|
||||
"id": "M7",
|
||||
"paper": "2610.29823",
|
||||
"theme": "T2",
|
||||
"claim": "Pi reads ~10,000 input tokens/question (7,900 cached) vs 28,900 for full context.",
|
||||
"value": "10,000 input tokens",
|
||||
"note": "The efficiency mechanism behind the cost lead.",
|
||||
"anchor": "Pi reads about 10,000 input tokens a question",
|
||||
"quote": "Pi reads about 10,000 input tokens a question, 7,900 of them cached, against 28,900 for full context. Codex and Claude Code score 98% and 97% for $25.",
|
||||
"corpus_sha256": "ed08ec2380a6c1479c1cbc0534cc2cf154f92aa057216398055f034f91abd9b5"
|
||||
},
|
||||
{
|
||||
"id": "M8",
|
||||
"paper": "2610.29823",
|
||||
"theme": "T3",
|
||||
"claim": "Writer ceiling: on Zep's GPT-4o-mini graph readers score 43/43/45%; on Opus 5.5's, 66/71/87%.",
|
||||
"value": "43%, 43% and 45%",
|
||||
"note": "A stronger reader cannot recover what a weaker writer left out.",
|
||||
"anchor": "GPT-4o-mini graph, GPT-4o-mini, GPT-6 Luna and Claude Opus 5.5 score 43%, 43% and 45%",
|
||||
"quote": "GPT-4o-mini graph, GPT-4o-mini, GPT-6 Luna and Claude Opus 5.5 score 43%, 43% and 45%; on the graph Claude Opus 5.5 wrote, 66%, 71% and 87%. Mem0 keeps the order with smaller gaps: 78%, 83% and 84% from GPT-4o-",
|
||||
"corpus_sha256": "ed08ec2380a6c1479c1cbc0534cc2cf154f92aa057216398055f034f91abd9b5"
|
||||
},
|
||||
{
|
||||
"id": "M9",
|
||||
"paper": "2610.29823",
|
||||
"theme": "T3",
|
||||
"claim": "Pi's one miss is a LoCoMo answer-key error, not a memory failure.",
|
||||
"value": "151 questions",
|
||||
"note": "The model is right and the benchmark key is wrong.",
|
||||
"anchor": "Its one miss is LoCoMo",
|
||||
"quote": "Its one miss is LoCoMo’s mistake. The question asks what kind of painting Caroline shared with Melanie on 13 October 2023, and the answer key says an abstract painting with blue streaks. In the conversation it is Melanie who shares that painting; Caroline shares a drawing. Pi says so and is marked wrong: “Caroline shared a drawing, not a painting. … The paintings shared that day (a sunset and an abstract piece) were Melanie’s.” With Claude Opus 5.5, full context, Codex and Claude Code point out the same mistake and are marked wrong too. On the other 151 questions, Pi with Claude Opus",
|
||||
"corpus_sha256": "ed08ec2380a6c1479c1cbc0534cc2cf154f92aa057216398055f034f91abd9b5"
|
||||
},
|
||||
{
|
||||
"id": "C1",
|
||||
"paper": "2610.29869",
|
||||
"theme": "T3",
|
||||
"claim": "Forged approval launders into memory 87–100% of the time across five flattening harnesses.",
|
||||
"value": "87–100%",
|
||||
"note": "Up to a ~908,000-token context window.",
|
||||
"anchor": "the forged approval launders into the",
|
||||
"quote": "the forged approval launders into the\nagent’s memory 87–100% of the time, up to a\n~908,000-token context window; across four role-typed\nharnesses (Codex, Gemini CLI, Qwen Code, Continue) the identical attack\nsu",
|
||||
"corpus_sha256": "e6f78f508c12d7a304f7fd36eb5f05d60095a426b189ab4a5cd817c341791b54"
|
||||
},
|
||||
{
|
||||
"id": "C2",
|
||||
"paper": "2610.29869",
|
||||
"theme": "T3",
|
||||
"claim": "The identical attack succeeds 0% of the time across four role-typed harnesses.",
|
||||
"value": "0% of the time",
|
||||
"note": "Role-typing is a built-in control.",
|
||||
"anchor": "succeeds 0% of the time",
|
||||
"quote": "succeeds 0% of the time. We follow the attack to executed consequences.\nA laundered standing approval drives real tool-running age",
|
||||
"corpus_sha256": "e6f78f508c12d7a304f7fd36eb5f05d60095a426b189ab4a5cd817c341791b54"
|
||||
},
|
||||
{
|
||||
"id": "C3",
|
||||
"paper": "2610.29869",
|
||||
"theme": "T3",
|
||||
"claim": "Live in the real pi TUI the attack runs at ~50% against a 0% paired clean arm.",
|
||||
"value": "~50%",
|
||||
"note": "Runtime evidence, not a simulation.",
|
||||
"anchor": "live in the real pi TUI at ~50%",
|
||||
"quote": "live in the real pi TUI at ~50%\nagainst a 0% paired clean arm. Straiker [1] described this\nconsequence without runtime evidence, a",
|
||||
"corpus_sha256": "e6f78f508c12d7a304f7fd36eb5f05d60095a426b189ab4a5cd817c341791b54"
|
||||
},
|
||||
{
|
||||
"id": "C4",
|
||||
"paper": "2610.29869",
|
||||
"theme": "T3",
|
||||
"claim": "A code-level provenance check catches 77% of laundering, demoting 2% of legitimate content.",
|
||||
"value": "77%",
|
||||
"note": "The only defense that works; prompt-level defenses cannot restore provenance.",
|
||||
"anchor": "we measure a code-level provenance check that catches 77% of",
|
||||
"quote": "we measure a code-level provenance check that catches 77% of\nlaundering while demoting 2% of legitimate summary content. The defect\nalso has a straightforward f",
|
||||
"corpus_sha256": "e6f78f508c12d7a304f7fd36eb5f05d60095a426b189ab4a5cd817c341791b54"
|
||||
},
|
||||
{
|
||||
"id": "C5",
|
||||
"paper": "2610.29869",
|
||||
"theme": "T3",
|
||||
"claim": "In-prompt anti-injection rules stop 13–25%; published classifier defenses report 67% at best.",
|
||||
"value": "13–25%",
|
||||
"note": "Baselines are far weaker than the code-level check.",
|
||||
"anchor": "stops 13",
|
||||
"quote": "stops 13–25% on the same data, and published classifier\ndefenses report 67% at best. The defect also has a straightforward\nfix on the flatte",
|
||||
"corpus_sha256": "e6f78f508c12d7a304f7fd36eb5f05d60095a426b189ab4a5cd817c341791b54"
|
||||
},
|
||||
{
|
||||
"id": "C6",
|
||||
"paper": "2610.29869",
|
||||
"theme": "T3",
|
||||
"claim": "Laundering still succeeds 100% of the time at every context size.",
|
||||
"value": "100%",
|
||||
"note": "Size does not attenuate the attack.",
|
||||
"anchor": "The laundering still succeeds 100% of the",
|
||||
"quote": "The laundering still succeeds 100% of the\ntime at every size, and the executed rate, near-100% on the short\ntranscripts, settles to ~20–40% in a full window. Th",
|
||||
"corpus_sha256": "e6f78f508c12d7a304f7fd36eb5f05d60095a426b189ab4a5cd817c341791b54"
|
||||
},
|
||||
{
|
||||
"id": "C7",
|
||||
"paper": "2610.29869",
|
||||
"theme": "T3",
|
||||
"claim": "Executed rate settles to ~20–40% in a full window.",
|
||||
"value": "~20–40%",
|
||||
"note": "Real agents compact at this size, so this is the realistic case.",
|
||||
"anchor": "settles to ~20",
|
||||
"quote": "settles to ~20–40% in a full window. The\nshort runs therefore measure the attack at its most favorable, an\nupper bound on the exec",
|
||||
"corpus_sha256": "e6f78f508c12d7a304f7fd36eb5f05d60095a426b189ab4a5cd817c341791b54"
|
||||
},
|
||||
{
|
||||
"id": "U1",
|
||||
"paper": "2610.29671",
|
||||
"theme": "T1",
|
||||
"claim": "Uni-Mem reaches an overall LoCoMo JJ of 88.80%.",
|
||||
"value": "88.80%",
|
||||
"note": "Best across all four question types.",
|
||||
"anchor": "Uni-Mem achieves the best performance across all four question types",
|
||||
"quote": "Uni-Mem achieves the best performance across all four question types, with an overall JJ score of 88.80%.\nIt surpasses the strongest baseline, Memory-R2 (OSS), by 10.93 percentage points and Mem-T by 12.31 points, with g",
|
||||
"corpus_sha256": "b83a772fb6a1340b2440359bec2bf3752d583d86e213806c5dbf7a648d947bdd"
|
||||
},
|
||||
{
|
||||
"id": "U2",
|
||||
"paper": "2610.29671",
|
||||
"theme": "T1",
|
||||
"claim": "Uni-Mem surpasses Memory-R2 (OSS) by 10.93 points and Mem-T by 12.31.",
|
||||
"value": "10.93 percentage points",
|
||||
"note": "Gains of 12.93 and 9.29 on multi-hop and temporal.",
|
||||
"anchor": "It surpasses the strongest baseline, Memory-R2 (OSS), by 10.93",
|
||||
"quote": "It surpasses the strongest baseline, Memory-R2 (OSS), by 10.93 percentage points and Mem-T by 12.31 points, with gains of 12.93 and 9.29 points over the strongest competing results on multi-hop and temporal que",
|
||||
"corpus_sha256": "b83a772fb6a1340b2440359bec2bf3752d583d86e213806c5dbf7a648d947bdd"
|
||||
},
|
||||
{
|
||||
"id": "U3",
|
||||
"paper": "2610.29671",
|
||||
"theme": "T1",
|
||||
"claim": "Uni-Mem still reaches 80.99% without SFT, +3.12% over the strongest baseline.",
|
||||
"value": "80.99%",
|
||||
"note": "RL alone is enough to beat the baselines.",
|
||||
"anchor": "Uni-Mem still achieves 80.99% without SFT",
|
||||
"quote": "Uni-Mem still achieves 80.99% without SFT, outperforming the strongest baseline by 3.12%.\nThese results support the effectiveness of jointly",
|
||||
"corpus_sha256": "b83a772fb6a1340b2440359bec2bf3752d583d86e213806c5dbf7a648d947bdd"
|
||||
},
|
||||
{
|
||||
"id": "U4",
|
||||
"paper": "2610.29671",
|
||||
"theme": "T1",
|
||||
"claim": "ECA improves overall JJ by 1.29 and 3.03 points without and with SFT.",
|
||||
"value": "1.29 and 3.03 percentage points",
|
||||
"note": "Event-wise credit assignment localizes sparse feedback.",
|
||||
"anchor": "improves overall JJ by 1.29 and 3.03 percentage points",
|
||||
"quote": "improves overall JJ by 1.29 and 3.03 percentage points without and with SFT, respectively. These gains support the effectiveness o",
|
||||
"corpus_sha256": "b83a772fb6a1340b2440359bec2bf3752d583d86e213806c5dbf7a648d947bdd"
|
||||
},
|
||||
{
|
||||
"id": "U5",
|
||||
"paper": "2610.29671",
|
||||
"theme": "T1",
|
||||
"claim": "Stage-Decoupled GRPO improves overall JJ by 2.48 points without SFT.",
|
||||
"value": "2.48 percentage points",
|
||||
"note": "Temporal accuracy rises 10.55 points in this setting.",
|
||||
"anchor": "improves overall JJ by 2.48 percentage points without SFT",
|
||||
"quote": "improves overall JJ by 2.48 percentage points without SFT. In particular, temporal accuracy rises by 10.55 points in this setting, and multi",
|
||||
"corpus_sha256": "b83a772fb6a1340b2440359bec2bf3752d583d86e213806c5dbf7a648d947bdd"
|
||||
},
|
||||
{
|
||||
"id": "U6",
|
||||
"paper": "2610.29671",
|
||||
"theme": "T1",
|
||||
"claim": "Trained construction alone improves overall JJ by 8.18 points.",
|
||||
"value": "8.18 percentage points",
|
||||
"note": "Each capability contributes even paired with its untrained counterpart.",
|
||||
"anchor": "improves the overall JJ score by 8.18 percentage points",
|
||||
"quote": "improves the overall JJ score by 8.18 percentage points, demonstrating the increased downstream utility of the constructed memorie",
|
||||
"corpus_sha256": "b83a772fb6a1340b2440359bec2bf3752d583d86e213806c5dbf7a648d947bdd"
|
||||
},
|
||||
{
|
||||
"id": "U7",
|
||||
"paper": "2610.29671",
|
||||
"theme": "T1",
|
||||
"claim": "Extending update blocks 1:1 -> 3:2 reduces overall JJ from 88.80% to 83.75%.",
|
||||
"value": "83.75%",
|
||||
"note": "Frequent alternation matters as much as balanced allocation.",
|
||||
"anchor": "the overall JJ score from 88.80",
|
||||
"quote": "the overall JJ score from 88.80%88.80\\% to 83.75%83.75\\%,\nindicating that less frequent alternation hinders\nshared-polic",
|
||||
"corpus_sha256": "b83a772fb6a1340b2440359bec2bf3752d583d86e213806c5dbf7a648d947bdd"
|
||||
},
|
||||
{
|
||||
"id": "U8",
|
||||
"paper": "2610.29671",
|
||||
"theme": "T2",
|
||||
"claim": "SFT followed by RL improves the 4B manager's JJ from 76.58% to 88.80%.",
|
||||
"value": "76.58% to 88.80%",
|
||||
"note": "A small trained manager beats Qwen3.5-27B (87.05%) and 397B-A17B (85.86%).",
|
||||
"anchor": "SFT followed by RL improves the 4B manager",
|
||||
"quote": "SFT followed by RL improves the 4B manager’s JJ score from 76.58% to 88.80%.\nUni-Mem also outperforms all off-the-shelf managers, including the strong",
|
||||
"corpus_sha256": "b83a772fb6a1340b2440359bec2bf3752d583d86e213806c5dbf7a648d947bdd"
|
||||
},
|
||||
{
|
||||
"id": "U9",
|
||||
"paper": "2610.29671",
|
||||
"theme": "T2",
|
||||
"claim": "Uni-Mem outperforms all off-the-shelf managers including Qwen3.5-27B (87.05%).",
|
||||
"value": "87.05%",
|
||||
"note": "Scale is not the binding constraint; joint training is.",
|
||||
"anchor": "Uni-Mem also outperforms all off-the-shelf managers",
|
||||
"quote": "Uni-Mem also outperforms all off-the-shelf managers, including the strongest one, Qwen3.5-27B (87.05%), and the largest one, Qwen3.5-397B-A17B (85.86%).\nThese results demonstrate that a small manager trained jo",
|
||||
"corpus_sha256": "b83a772fb6a1340b2440359bec2bf3752d583d86e213806c5dbf7a648d947bdd"
|
||||
}
|
||||
]
|
||||
}
|
||||
+2045
File diff suppressed because it is too large.
Load diff
+576
@@ -0,0 +1,576 @@
|
||||
MemoryTax: The Bitter Lesson of Agent Memory
|
||||
Skip to main content
|
||||
Report GitHub Issue
|
||||
×
|
||||
Title:
|
||||
Content selection saved. Describe the issue below:
|
||||
Description:
|
||||
Submit without GitHub
|
||||
Submit in GitHub
|
||||
Why HTML?
|
||||
Report Issue
|
||||
Back to Abstract
|
||||
Download PDF
|
||||
Contents
|
||||
Reading mode
|
||||
Colour theme
|
||||
Abstract
|
||||
1 Introduction
|
||||
2 Setup
|
||||
Photos.
|
||||
Measurement.
|
||||
3 Agent memory is a costly patch for a weak model
|
||||
4 Agent memory has a ceiling
|
||||
5 With Claude Opus 5.5, Pi tops the frontier
|
||||
6 Discussion and limitations
|
||||
References
|
||||
A Full results
|
||||
B Protocols
|
||||
C Prompts
|
||||
License: arXiv.org perpetual non-exclusive license
|
||||
arXiv:2610.29823v1 [cs.CL] 21 Oct 2026
|
||||
MemoryTax: The Bitter Lesson of Agent Memory
|
||||
Zedong Peng
|
||||
Affiliation: Shanghai Jiao Tong University
|
||||
Email: zedongpeng1@gmail.com
|
||||
Abstract
|
||||
Agent memory systems such as Mem0 and Zep decide what to keep before any question arrives; coding agents keep the raw history and let the model search it with its own tools. We ask whether designed memory still helps once the model is strong enough to search for itself. On one LoCoMo conversation (419 turns, 152 questions), we run four memory methods (full context, BM25, Mem0, Zep) and three coding agents (Pi, Codex, Claude Code), each with GPT-4o-mini, GPT-6 Luna and Claude Opus 5.5, and record what every rollout costs. This is a short empirical study, with one conversation and one run per cell. Agent memory is a costly patch for a weak model: Mem0 beats full context only with GPT-4o-mini, and its build alone costs more than a whole full-context rollout. Agent memory also has a ceiling, set by the model that writes it: a stronger reader does not recover what a weaker writer left out. With Claude Opus 5.5, Pi tops the cost–accuracy frontier, scoring 99% for less than half the cost of full context; its one miss is a wrong answer key. Part of that lead comes from photo captions that only the agents could read: on the 88 questions whose evidence has no photo, Pi and full context differ by one answer.
|
||||
Interactive version: zedongpeng.com/memorytax
|
||||
1 Introduction
|
||||
Agent memory systems such as Mem0 and Zep decide what to keep before any question arrives: the model extracts facts or builds a graph from the conversation, and each question sees a few retrieved pieces [1, 2]. Coding agents and recursive language models keep the raw history instead and let the model search it with its own tools [3, 4, 5, 6].
|
||||
The bitter lesson is that knowledge built into a system helps at first, then plateaus, while general methods that scale with computation win [7]. Agent memory builds in what to keep. Does it still help once the model is strong enough to search for itself?
|
||||
We run four memory methods (full context, BM25 [8], Mem0 and Zep) and three coding agents (Pi, Codex and Claude Code), each with GPT-4o-mini, GPT-6 Luna and Claude Opus 5.5, on a conversation from LoCoMo [9], and measure what each run costs. We find:
|
||||
•
|
||||
Agent memory is a costly patch for a weak model. Mem0 beats full context only with GPT-4o-mini, and its build alone costs more than full context’s whole run (Section 3).
|
||||
•
|
||||
Agent memory has a ceiling, set by the model that writes it. A stronger reader does not recover what a weaker writer left out (Section 4).
|
||||
•
|
||||
With Claude Opus 5.5, Pi tops the cost–accuracy frontier (Figure 1). It answers every question whose answer key is sound, for less than half what full context costs. Where the two read the same text they differ by one answer; the rest of Pi’s lead comes from photo captions that only the agents could read (Section 5).
|
||||
The study is small: one conversation, one run per cell, and a GPT-4o-mini judge. Section 6 lists what it cannot show. Within those limits, the results suggest that a strong model needs less designed memory and better tools for searching the raw record.
|
||||
Figure 1: Cost and correct rate per rollout. Each badge is one cell’s whole run: the build, then all 152 questions. Colour is the model, the badge the method’s mark, and a letter in the badge marks a method without one (F full context, B BM25). The stepped line is the highest correct rate reached at or below each cost, and the labelled cells lie on it. Cost is on a log axis.
|
||||
2 Setup
|
||||
We run 21 cells, seven methods with three models each, on LoCoMo conv-26 [9]: 419 turns in 19 sessions and 152 single-hop, multi-hop, temporal and open-domain questions. LoCoMo’s adversarial questions are left out, as in Mem0’s evaluation [1]. The history fits every model’s context window. Among the ten LoCoMo conversations, conv-26’s per-method accuracies track those of the full set closely. Table 1 lists the methods.
|
||||
Table 1: Methods: what each stores and what the model sees when it answers.
|
||||
Method
|
||||
Stored
|
||||
Model sees
|
||||
Full context
|
||||
raw history
|
||||
all of it
|
||||
BM25
|
||||
raw turns
|
||||
top-8 turns, ±\pm2
|
||||
Mem0
|
||||
extracted facts
|
||||
top-200 facts
|
||||
Zep
|
||||
temporal graph
|
||||
top-20 edges
|
||||
Pi, Codex, Claude Code
|
||||
raw history, one JSON file
|
||||
what its tools read
|
||||
The memory methods answer one question at a time with Mem0’s 2025 answer template; Mem0 is OSS 2.1.0 and Zep is Graphiti OSS 0.30.2, both with default extraction and retrieval [1, 2]. The agents, Pi 1.1.0, Codex 0.159.2 and Claude Code 2.1.295 [3, 4, 5], start a fresh session in a fresh container for each question, with the conversation as one file, their default tools, and the template’s answer rules in the prompt (Appendix C).
|
||||
The models go from weak to strongest: GPT-4o-mini, GPT-6 Luna, Claude Opus 5.5. The last two run at reasoning effort high, the effort coding agents are usually run at [10]; GPT-4o-mini has no effort setting. In each cell one model writes the store (Mem0, Zep) and answers every question, or drives the agent.
|
||||
Photos.
|
||||
Some LoCoMo turns share a photo, and the official file holds a caption of it next to the turn text. The agents get the official file, captions included; the memory methods were given the turn text only. 116 of the 419 turns share a photo, and 64 of the 152 questions have evidence in such a turn. Section 5 measures what this changes.
|
||||
Measurement.
|
||||
A rollout is one run over the conversation: the build, then the 152 questions. GPT-4o-mini grades every answer with Mem0’s 2025 judge rubric [1]; empty answers count as wrong. Cost is the list price of every model call, LLM and embedding (text-embedding-3-small), with the judge excluded. A turn is one LLM call: one per question for the memory methods, one per step of an agent’s loop, and Mem0’s and Zep’s calls to build their store. Tokens are those calls’ input and output tokens, cached input included; embeddings are not counted. Each cell runs once.
|
||||
3 Agent memory is a costly patch for a weak model
|
||||
With GPT-4o-mini, Mem0 scores highest. It scores 78%, against 74% for both full context and BM25; Zep scores 43%. The agents score 63% (Pi), 28% (Codex) and 29% (Claude Code): Codex opens the conversation file for three questions in four and Claude Code almost never, and otherwise they answer from the question alone.
|
||||
With the two stronger models, the patch stops helping. With GPT-6 Luna, full context scores 92%, BM25 82%, Mem0 80% and Zep 51%; Zep’s graph holds 128 facts from the 19 sessions, against 195 with Claude Opus 5.5. With Claude Opus 5.5, Mem0 and full context both score 95% and Zep 87%, and the agents score highest: Pi 99%, Codex 98% and Claude Code 97%, a lead that comes mostly from questions about photos (Section 5). Earlier work also finds that plain retrieval keeps up with elaborate memory [11] and that rankings between memory systems change with the model [12].
|
||||
Figure 2: Cumulative cost–correct curves. Each curve is one rollout: the build, then the questions from cheapest to most expensive (equal costs grouped), adding up cost and correct answers; correct answers are divided by 152, so a wrong answer adds cost only. Colour is the model, the badge the method, and agents are dotted. The cost axis is linear to $2 and logarithmic above. Curves use a nine-point moving average, and the badges mark the unsmoothed totals of Table 2. Strong lines are cells that no other cell beats on both cost and correct answers.
|
||||
The patch is not free. Mem0’s build alone costs more than full context’s whole rollout with the same model: $0.56 against $0.43 with GPT-4o-mini, $0.45 against $0.29 with GPT-6 Luna and $25.88 against $17.84 with Claude Opus 5.5 (Figure 2). Its questions are cheaper, so the build would pay for itself after 372, 545 and 596 questions on the same history; LoCoMo asks 152. Mem0 and Zep are never on the frontier of Figure 1, which runs through BM25 and full context with GPT-6 Luna and Pi with Claude Opus 5.5. Paying to build memory that a stronger model does not need is a memory tax.
|
||||
4 Agent memory has a ceiling
|
||||
The model that writes the store limits every reader. Mem0 and Zep write their store with the same model that later reads it. Retrieval calls no language model, so the reader can be swapped: each model answers from the retrievals of the others’ stores (Figure 3, Table 3).
|
||||
Figure 3: Writer and reader. Each colour is one store, written by that model. The filled badge is the writer reading its own store, and arrows run from it to the other two readers.
|
||||
A stronger reader does not recover what the writer left out. On Zep’s GPT-4o-mini graph, GPT-4o-mini, GPT-6 Luna and Claude Opus 5.5 score 43%, 43% and 45%; on the graph Claude Opus 5.5 wrote, 66%, 71% and 87%. Mem0 keeps the order with smaller gaps: 78%, 83% and 84% from GPT-4o-mini’s store, 89%, 91% and 95% from Claude Opus 5.5’s. This agrees with work where verbatim chunks beat extracted facts on long conversations [13].
|
||||
5 With Claude Opus 5.5, Pi tops the frontier
|
||||
Pi with Claude Opus 5.5 is the top of the frontier in Figure 1. No other cell scores higher: Pi scores 99% for $6.95 a rollout, while full context with the same model scores 95% for $17.84. Pi reads about 10,000 input tokens a question, 7,900 of them cached, against 28,900 for full context. Codex and Claude Code score 98% and 97% for $25.41 and $28.41; HarnessTax explains why the same model costs so differently across harnesses [10].
|
||||
Its one miss is LoCoMo’s mistake. The question asks what kind of painting Caroline shared with Melanie on 13 October 2023, and the answer key says an abstract painting with blue streaks. In the conversation it is Melanie who shares that painting; Caroline shares a drawing. Pi says so and is marked wrong: “Caroline shared a drawing, not a painting. … The paintings shared that day (a sunset and an abstract piece) were Melanie’s.” With Claude Opus 5.5, full context, Codex and Claude Code point out the same mistake and are marked wrong too. On the other 151 questions, Pi with Claude Opus 5.5 is always right.
|
||||
Photo captions account for most of Pi’s lead over full context. With Claude Opus 5.5, Pi answers seven questions that full context misses and misses none that it answers. Six of the seven have their evidence in a turn that shares a photo, whose caption Pi could read and full context could not (Section 2); so do all six such questions for Codex and all four for Claude Code. On the 88 questions without photo evidence, Pi scores 100%, and full context, Mem0, Codex and Claude Code 99% each (Table 4). With the same input, then, Pi matches full context for less than half its cost. Whether searching the record answers more than reading all of it, this study does not settle.
|
||||
6 Discussion and limitations
|
||||
Our results suggest that agent memory is a costly patch for a weak model and caps what any reader can answer. On this LoCoMo conversation, Mem0 helps GPT-4o-mini but not GPT-6 Luna or Claude Opus 5.5, a stronger reader does not recover what the writer left out, and with Claude Opus 5.5 Pi answers every sound question for less than full context costs.
|
||||
The findings rest on a small study, and these limits apply:
|
||||
•
|
||||
Scale. One conversation of LoCoMo’s ten, and one run per cell. Differences of a few questions may not hold in a second run.
|
||||
•
|
||||
Grading. A GPT-4o-mini judge with a lenient rubric, and at least one wrong answer key.
|
||||
•
|
||||
Photo captions. The agents could read photo captions that the memory methods were not given (Section 5). This touches comparisons between agents and memory methods, not those among the memory methods.
|
||||
•
|
||||
Context length. The history fits in every model’s context window, so full context is always an option. Results may differ on other benchmarks such as LongMemEval [14] and on histories that do not fit.
|
||||
•
|
||||
Settings and prices. Mem0 and Zep run with their default extraction and retrieval, and other settings may do better. Costs are list prices and depend on each provider’s caching.
|
||||
Memory systems and harnesses can also be searched for automatically [15, 16], but what is found is still fixed before the question arrives. More broadly, a strong model may need less designed memory and better tools: exact, semantic and vector search over the raw record, driven by the model itself [3, 4]. Progress may then look less like another memory system and more like ripgrep after grep [17], or zvec-grep after ripgrep [18]. Designed memory may survive as a cost trade-off when many questions share one history (Figure 2).
|
||||
References
|
||||
[1]
|
||||
P. Chhikara, D. Khant, S. Aryan, T. Singh, and D. Yadav.
|
||||
Mem0: Building production-ready AI agents with scalable long-term
|
||||
memory, 2025.
|
||||
URL https://arxiv.org/abs/2504.19413.
|
||||
[2]
|
||||
P. Rasmussen, P. Paliychuk, T. Beauvais, J. Ryan, and D. Chalef.
|
||||
Zep: A temporal knowledge graph architecture for agent memory, 2025.
|
||||
URL https://arxiv.org/abs/2501.13956.
|
||||
[3]
|
||||
Pi contributors.
|
||||
Pi coding agent: Compaction and session persistence.
|
||||
Software documentation and source, 2026.
|
||||
URL https://github.com/earendil-works/pi.
|
||||
[4]
|
||||
OpenAI.
|
||||
Codex: An open-source coding agent, 2026.
|
||||
URL https://github.com/openai/codex.
|
||||
[5]
|
||||
Anthropic.
|
||||
Claude code, 2026.
|
||||
URL https://github.com/anthropics/claude-code.
|
||||
[6]
|
||||
A. L. Zhang, T. Kraska, and O. Khattab.
|
||||
Recursive language models, 2025a.
|
||||
URL https://arxiv.org/abs/2512.24601.
|
||||
[7]
|
||||
R. S. Sutton.
|
||||
The bitter lesson.
|
||||
Essay, 2019.
|
||||
URL http://www.incompleteideas.net/IncIdeas/BitterLesson.html.
|
||||
[8]
|
||||
S. Robertson and H. Zaragoza.
|
||||
The probabilistic relevance framework: BM25 and beyond.
|
||||
Foundations and Trends in Information Retrieval, 3(4):333–389, 2009.
|
||||
doi: 10.1561/1500000019.
|
||||
[9]
|
||||
A. Maharana, D.-H. Lee, S. Tulyakov, M. Bansal, F. Barbieri, and Y. Fang.
|
||||
Evaluating very long-term conversational memory of LLM agents,
|
||||
2024.
|
||||
URL https://arxiv.org/abs/2402.17753.
|
||||
[10]
|
||||
Arena Team.
|
||||
HarnessTax: How much does the harness matter for coding agents?
|
||||
Arena blog, 2026.
|
||||
URL https://arena.ai/blog/coding-agents-harness-tax.
|
||||
[11]
|
||||
Y. Wu, W. Chen, Z. Huang, et al.
|
||||
Back to basics: Let conversational agents remember with just
|
||||
retrieval and generation, 2026.
|
||||
URL https://arxiv.org/abs/2604.11628.
|
||||
[12]
|
||||
K. Wang.
|
||||
MemDelta: Controlled baselines and hidden confounds in agent memory
|
||||
evaluation, 2026.
|
||||
URL https://arxiv.org/abs/2606.29914.
|
||||
[13]
|
||||
T. An.
|
||||
Fidelity before structure: Verbatim chunks beat lossy artifact
|
||||
extraction in long-conversation LLM memory, 2026.
|
||||
URL https://arxiv.org/abs/2601.00821.
|
||||
[14]
|
||||
D. Wu, H. Wang, W. Yu, Y. Zhang, K.-W. Chang, and D. Yu.
|
||||
LongMemEval: Benchmarking chat assistants on long-term interactive
|
||||
memory, 2024.
|
||||
URL https://arxiv.org/abs/2410.10813.
|
||||
[15]
|
||||
G. Zhang, H. Ren, C. Zhan, et al.
|
||||
MemEvolve: Meta-evolution of agent memory systems,
|
||||
2025b.
|
||||
URL https://arxiv.org/abs/2512.18746.
|
||||
[16]
|
||||
Y. Lee, R. Nair, Q. Zhang, K. Lee, O. Khattab, and C. Finn.
|
||||
Meta-harness: End-to-end optimization of model harnesses, 2026.
|
||||
URL https://arxiv.org/abs/2603.28052.
|
||||
[17]
|
||||
A. Gallant.
|
||||
ripgrep.
|
||||
Software, 2016.
|
||||
URL https://github.com/BurntSushi/ripgrep.
|
||||
[18]
|
||||
Zvec Team.
|
||||
From rg to zg: Local search beyond keywords.
|
||||
Zvec blog, 2026.
|
||||
URL https://zvec.org/en/blog/2026-08-28-zvec-grep-open-source/.
|
||||
[19]
|
||||
Harbor Framework Team.
|
||||
Harbor: A framework for evaluating and optimizing agents and models
|
||||
in container environments.
|
||||
Software, 2026.
|
||||
URL https://doi.org/10.5281/zenodo.20953922.
|
||||
Appendix A Full results
|
||||
Table 2 gives every cell’s cost and correct rate, Table 3 the writer–reader grid behind Figure 3, and Table 4 the split by photo evidence. Figure 4 redraws Figure 2 against turns and tokens.
|
||||
Table 2: Per method and model. Cost is the list price of every model call except the judge: Build is paid once, before any question; Per q. is everything one question costs, retrieval included, averaged over the 152 questions. ∘\circ: nothing is built. Effort: the model’s reasoning effort; GPT-4o-mini has none. Correct: share of answers judged correct, in %, per LoCoMo category (single-hop, multi-hop, temporal, open-domain); the first row gives each category’s size.
|
||||
Cost ($)
|
||||
Correct (%)
|
||||
Method
|
||||
Model
|
||||
Effort
|
||||
Build
|
||||
Per q.
|
||||
Single
|
||||
Multi
|
||||
Temp.
|
||||
Open
|
||||
All
|
||||
Questions
|
||||
70
|
||||
32
|
||||
37
|
||||
13
|
||||
152
|
||||
Full context
|
||||
GPT-4o-mini
|
||||
–
|
||||
∘\circ
|
||||
0.0028
|
||||
89
|
||||
75
|
||||
51
|
||||
62
|
||||
74
|
||||
GPT-6 Luna
|
||||
high
|
||||
∘\circ
|
||||
0.0019
|
||||
90
|
||||
97
|
||||
95
|
||||
85
|
||||
92
|
||||
Claude Opus 5.5
|
||||
high
|
||||
∘\circ
|
||||
0.117
|
||||
93
|
||||
94
|
||||
97
|
||||
100
|
||||
95
|
||||
BM25
|
||||
GPT-4o-mini
|
||||
–
|
||||
∘\circ
|
||||
0.00028
|
||||
86
|
||||
50
|
||||
73
|
||||
69
|
||||
74
|
||||
GPT-6 Luna
|
||||
high
|
||||
∘\circ
|
||||
0.00021
|
||||
89
|
||||
63
|
||||
89
|
||||
69
|
||||
82
|
||||
Claude Opus 5.5
|
||||
high
|
||||
∘\circ
|
||||
0.013
|
||||
93
|
||||
81
|
||||
92
|
||||
92
|
||||
90
|
||||
Mem0
|
||||
GPT-4o-mini
|
||||
–
|
||||
0.559
|
||||
0.0013
|
||||
87
|
||||
75
|
||||
65
|
||||
77
|
||||
78
|
||||
GPT-6 Luna
|
||||
high
|
||||
0.449
|
||||
0.0011
|
||||
89
|
||||
88
|
||||
57
|
||||
85
|
||||
80
|
||||
Claude Opus 5.5
|
||||
high
|
||||
25.876
|
||||
0.074
|
||||
93
|
||||
97
|
||||
97
|
||||
100
|
||||
95
|
||||
Zep
|
||||
GPT-4o-mini
|
||||
–
|
||||
0.092
|
||||
0.00012
|
||||
51
|
||||
34
|
||||
30
|
||||
54
|
||||
43
|
||||
GPT-6 Luna
|
||||
high
|
||||
0.103
|
||||
0.00012
|
||||
54
|
||||
50
|
||||
49
|
||||
46
|
||||
51
|
||||
Claude Opus 5.5
|
||||
high
|
||||
6.224
|
||||
0.0077
|
||||
83
|
||||
94
|
||||
89
|
||||
85
|
||||
87
|
||||
Pi
|
||||
GPT-4o-mini
|
||||
–
|
||||
∘\circ
|
||||
0.016
|
||||
74
|
||||
59
|
||||
35
|
||||
85
|
||||
63
|
||||
GPT-6 Luna
|
||||
high
|
||||
∘\circ
|
||||
0.0032
|
||||
96
|
||||
81
|
||||
92
|
||||
92
|
||||
91
|
||||
Claude Opus 5.5
|
||||
high
|
||||
∘\circ
|
||||
0.046
|
||||
99
|
||||
100
|
||||
100
|
||||
100
|
||||
99
|
||||
Codex
|
||||
GPT-4o-mini
|
||||
–
|
||||
∘\circ
|
||||
0.0031
|
||||
30
|
||||
22
|
||||
24
|
||||
46
|
||||
28
|
||||
GPT-6 Luna
|
||||
high
|
||||
∘\circ
|
||||
0.0050
|
||||
90
|
||||
84
|
||||
89
|
||||
85
|
||||
88
|
||||
Claude Opus 5.5
|
||||
high
|
||||
∘\circ
|
||||
0.167
|
||||
99
|
||||
100
|
||||
95
|
||||
100
|
||||
98
|
||||
Claude Code
|
||||
GPT-4o-mini
|
||||
–
|
||||
∘\circ
|
||||
0.0020
|
||||
34
|
||||
22
|
||||
8
|
||||
77
|
||||
29
|
||||
GPT-6 Luna
|
||||
high
|
||||
∘\circ
|
||||
0.0082
|
||||
96
|
||||
78
|
||||
95
|
||||
92
|
||||
91
|
||||
Claude Opus 5.5
|
||||
high
|
||||
∘\circ
|
||||
0.187
|
||||
99
|
||||
94
|
||||
97
|
||||
100
|
||||
97
|
||||
Table 3: Writer and reader. Correct rate (%), the reader answering from the writer’s store and its retrievals. Bold: one model writes and reads (Table 2).
|
||||
Mem0, read by
|
||||
Zep, read by
|
||||
Written by
|
||||
GPT-4o-mini
|
||||
GPT-6 Luna
|
||||
Opus 5.5
|
||||
GPT-4o-mini
|
||||
GPT-6 Luna
|
||||
Opus 5.5
|
||||
GPT-4o-mini
|
||||
78
|
||||
83
|
||||
84
|
||||
43
|
||||
43
|
||||
45
|
||||
GPT-6 Luna
|
||||
74
|
||||
80
|
||||
81
|
||||
52
|
||||
51
|
||||
55
|
||||
Claude Opus 5.5
|
||||
89
|
||||
91
|
||||
95
|
||||
66
|
||||
71
|
||||
87
|
||||
Table 4: Correct rate (%) on the 64 questions with evidence in a turn that shares a photo, whose caption only the agents could read, and on the other 88.
|
||||
GPT-4o-mini
|
||||
GPT-6 Luna
|
||||
Claude Opus 5.5
|
||||
Method
|
||||
Photo (64)
|
||||
Other (88)
|
||||
Photo (64)
|
||||
Other (88)
|
||||
Photo (64)
|
||||
Other (88)
|
||||
Full context
|
||||
66
|
||||
81
|
||||
91
|
||||
93
|
||||
89
|
||||
99
|
||||
BM25
|
||||
61
|
||||
83
|
||||
78
|
||||
84
|
||||
84
|
||||
94
|
||||
Mem0
|
||||
70
|
||||
84
|
||||
72
|
||||
86
|
||||
91
|
||||
99
|
||||
Zep
|
||||
33
|
||||
50
|
||||
53
|
||||
50
|
||||
77
|
||||
94
|
||||
Pi
|
||||
53
|
||||
69
|
||||
88
|
||||
94
|
||||
98
|
||||
100
|
||||
Codex
|
||||
22
|
||||
33
|
||||
84
|
||||
91
|
||||
97
|
||||
99
|
||||
Claude Code
|
||||
17
|
||||
38
|
||||
91
|
||||
92
|
||||
95
|
||||
99
|
||||
Figure 4: The curves of Figure 2 against turns (LLM calls, top) and tokens (bottom), on linear axes, the questions ordered by that amount. Strong lines are cells that no other cell beats on both that amount and correct answers.
|
||||
Appendix B Protocols
|
||||
Mem0 OSS 2.1.0 writes one chronological turn at a time and retrieves k=200k=200 without reranking; dates are prepended to the text. Zep runs Graphiti OSS 0.30.2 on Kuzu, one session per episode, with hybrid edge search (top 20). The memory methods’ GPT-6 Luna and Claude Opus 5.5 requests set reasoning effort high.
|
||||
Each agent question is a Harbor 0.24.0 task [19]: a fresh container holding only the conversation, exactly as in the official file, without questions or answers. The agent’s last message is its answer. While the agent runs, the container can reach only the model endpoint. GPT-6 Luna and Claude Opus 5.5 run at reasoning effort high, through a proxy that translates the API where an agent does not speak the model’s own; GPT-4o-mini has no effort setting. Codex and Claude Code have web search off; Pi has none. A question may take 20 minutes, and Pi and Claude Code at most 100 turns. With GPT-6 Luna, Codex picks its code mode: one JavaScript tool that calls the shell. A trial that fails or ends without an answer is run again, and only the accepted trial is counted.
|
||||
Appendix C Prompts
|
||||
Each prompt is copied from the code the runs called; fields in braces are filled per question.
|
||||
Answer, system message.
|
||||
⬇
|
||||
You are a helpful assistant that can answer questions based on the provided context.If the question involves timing, use the conversation date for reference.Provide the shortest possible answer.Use words directly from the conversation when possible.Avoid using subjects in your answer.
|
||||
Answer, user message.
|
||||
⬇
|
||||
# Question:
|
||||
{{QUESTION}}
|
||||
# Context:
|
||||
{{CONTEXT}}
|
||||
# Short answer:
|
||||
Pi, Codex and Claude Code, user message.
|
||||
⬇
|
||||
Answer a question about the conversation in `conv-26/conv.json`. If the question involves timing, use the conversation date for reference. Provide the shortest possible answer. Use words directly from the conversation when possible. Avoid using subjects in your answer. Your final message is taken as the answer.
|
||||
Question: {question}
|
||||
Judge.
|
||||
⬇
|
||||
Your task is to label an answer to a question as ’CORRECT’ or ’WRONG’. You will be given the following data:
|
||||
(1) a question (posed by one user to another user),
|
||||
(2) a ’gold’ (ground truth) answer,
|
||||
(3) a generated answer
|
||||
which you will score as CORRECT/WRONG.
|
||||
The point of the question is to ask about something one user should know about the other user based on their prior conversations.
|
||||
The gold answer will usually be a concise and short answer that includes the referenced topic, for example:
|
||||
Question: Do you remember what I got the last time I went to Hawaii?
|
||||
Gold answer: A shell necklace
|
||||
The generated answer might be much longer, but you should be generous with your grading - as long as it touches on the same topic as the gold answer, it should be counted as CORRECT.
|
||||
For time related questions, the gold answer will be a specific date, month, year, etc. The generated answer might be much longer or use relative time references (like "last Tuesday" or "next month"), but you should be generous with your grading - as long as it refers to the same date or time period as the gold answer, it should be counted as CORRECT. Even if the format differs (e.g., "May 7th" vs "7 May"), consider it CORRECT if it's the same date.
|
||||
Now it's time for the real question:
|
||||
Question: {question}
|
||||
Gold answer: {gold_answer}
|
||||
Generated answer: {generated_answer}
|
||||
First, provide a short (one sentence) explanation of your reasoning, then finish with CORRECT or WRONG.
|
||||
Do NOT include both CORRECT and WRONG in your response, or it will break the evaluation script.
|
||||
Just return the label CORRECT or WRONG in a json format with the key as "label".
|
||||
Experimental support, please
|
||||
view the build logs
|
||||
for errors. Generated by
|
||||
L
|
||||
A
|
||||
T
|
||||
E
|
||||
xml
|
||||
.
|
||||
Instructions for reporting errors
|
||||
We are continuing to improve HTML versions of papers, and your feedback helps enhance accessibility and mobile
|
||||
support. To report errors in the HTML that will help us improve conversion and rendering, choose any of the
|
||||
methods listed below:
|
||||
Click the "Report Issue" (
|
||||
) button, located in the page header.
|
||||
Tip: You can select the relevant text first, to include it in your report.
|
||||
Our team has already identified the following issues. We appreciate your time reviewing and reporting rendering errors we
|
||||
may not have found yet. Your efforts will help us improve the HTML versions for all readers, because disability
|
||||
should not be a barrier to accessing research. Thank you for your continued support in championing open access for
|
||||
all.
|
||||
Have a free development cycle? Help support accessibility at arXiv! Our collaborators at LaTeXML maintain a list of packages that need conversion, and welcome developer contributions.
|
||||
We gratefully acknowledge support from our major funders,
|
||||
member institutions, ,
|
||||
and all contributors.
|
||||
About ·
|
||||
Help ·
|
||||
Contact ·
|
||||
Subscribe ·
|
||||
Copyright ·
|
||||
Privacy ·
|
||||
Accessibility ·
|
||||
Operational Status (opens in new tab)
|
||||
+2073
File diff suppressed because it is too large.
Load diff
+5498
File diff suppressed because it is too large.
Load diff
+5860
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,180 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
verify_claims.py -- independent re-verification of the agentic-memory dossier.
|
||||
|
||||
This script does NOT import build_dossier.py. It re-derives every claim from
|
||||
the pinned corpus bytes and the recorded pins, so it is a genuine second
|
||||
implementation of the grounding check.
|
||||
|
||||
For every claim in claims.json it checks:
|
||||
1. the corpus file's sha256 equals the pin recorded at build time;
|
||||
2. the claim's `quote` is a literal substring of that corpus file;
|
||||
3. the claim's headline `value` appears inside that quote;
|
||||
4. the headline `value` appears in DOSSIER.md.
|
||||
|
||||
It then runs three mutation batteries. Each battery corrupts the artifact in
|
||||
a way a real drift would look like, and requires the corruption to be
|
||||
DETECTED. A mutation that passes undetected falsifies this verifier, and the
|
||||
script exits nonzero.
|
||||
|
||||
battery A: corrupt one quote at a time (all claims)
|
||||
battery B: corrupt one corpus byte (sha256 pin must catch it)
|
||||
battery C: corrupt one headline value everywhere in DOSSIER.md (all values)
|
||||
|
||||
Exit 0 = the dossier is fully grounded and the verifier is shown to have
|
||||
teeth. Exit 1 = an ungrounded claim, or a vacuous check.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
CORPUS = ROOT / "corpus"
|
||||
EXPECTED_CLAIMS = 44
|
||||
EXPECTED_PAPERS = 5
|
||||
|
||||
|
||||
def sha256_bytes(b: bytes) -> str:
|
||||
return hashlib.sha256(b).hexdigest()
|
||||
|
||||
|
||||
def verify(claims_doc: dict, dossier_text: str, corpus_dir: Path,
|
||||
cache: dict | None = None) -> list[str]:
|
||||
"""Return a list of failure strings. Empty list == pass."""
|
||||
cache = {} if cache is None else cache
|
||||
fails: list[str] = []
|
||||
|
||||
papers = {p["arxiv_id"]: p for p in claims_doc.get("papers", [])}
|
||||
if len(papers) != EXPECTED_PAPERS:
|
||||
fails.append(f"papers: expected {EXPECTED_PAPERS}, found {len(papers)}")
|
||||
if len(claims_doc.get("claims", [])) != EXPECTED_CLAIMS:
|
||||
fails.append(f"claims: expected {EXPECTED_CLAIMS}, found "
|
||||
f"{len(claims_doc.get('claims', []))}")
|
||||
|
||||
seen_ids: set[str] = set()
|
||||
counts: dict[str, int] = {}
|
||||
for c in claims_doc.get("claims", []):
|
||||
cid, pid = c.get("id", "?"), c.get("paper", "?")
|
||||
if cid in seen_ids:
|
||||
fails.append(f"{cid}: duplicate claim id")
|
||||
seen_ids.add(cid)
|
||||
counts[pid] = counts.get(pid, 0) + 1
|
||||
|
||||
if pid not in papers:
|
||||
fails.append(f"{cid}: unknown paper {pid!r}")
|
||||
continue
|
||||
rec = papers[pid]
|
||||
if rec.get("sha256") != c.get("corpus_sha256"):
|
||||
fails.append(f"{cid}: pin in papers[] disagrees with claim pin")
|
||||
|
||||
f = corpus_dir / f"{pid}.txt"
|
||||
if not f.exists():
|
||||
fails.append(f"{cid}: corpus file missing: {f.name}")
|
||||
continue
|
||||
if pid not in cache:
|
||||
cache[pid] = f.read_bytes()
|
||||
data = cache[pid]
|
||||
dig = sha256_bytes(data)
|
||||
if dig != c.get("corpus_sha256"):
|
||||
fails.append(f"{cid}: corpus sha256 drift for {pid} "
|
||||
f"(pin {str(c.get('corpus_sha256'))[:12]}, got {dig[:12]})")
|
||||
continue
|
||||
text = data.decode("utf-8", errors="replace")
|
||||
if c["quote"] not in text:
|
||||
fails.append(f"{cid}: quote not found verbatim in {pid}.txt")
|
||||
continue
|
||||
if c["value"] not in c["quote"]:
|
||||
fails.append(f"{cid}: value {c['value']!r} not inside its own quote")
|
||||
if c["value"] not in dossier_text:
|
||||
fails.append(f"{cid}: value {c['value']!r} missing from DOSSIER.md")
|
||||
|
||||
for pid, rec in papers.items():
|
||||
f = corpus_dir / f"{pid}.txt"
|
||||
if f.exists():
|
||||
if pid not in cache:
|
||||
cache[pid] = f.read_bytes()
|
||||
if sha256_bytes(cache[pid]) != rec.get("sha256"):
|
||||
fails.append(f"{pid}: papers[] sha256 pin does not match corpus")
|
||||
if counts.get(pid, 0) != rec.get("claims"):
|
||||
fails.append(f"{pid}: claims[] count {rec.get('claims')} != "
|
||||
f"actual {counts.get(pid, 0)}")
|
||||
return fails
|
||||
|
||||
|
||||
def corrupt_char(s: str) -> str:
|
||||
if not s:
|
||||
return "X"
|
||||
return s[:-1] + ("X" if s[-1] != "X" else "Y")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
claims_doc = json.loads((ROOT / "claims.json").read_text(encoding="utf-8"))
|
||||
dossier_text = (ROOT / "DOSSIER.md").read_text(encoding="utf-8")
|
||||
cache: dict[str, bytes] = {}
|
||||
|
||||
fails = verify(claims_doc, dossier_text, CORPUS, cache)
|
||||
if fails:
|
||||
print("FAIL: dossier is not fully grounded")
|
||||
for f in fails:
|
||||
print(" -", f)
|
||||
return 1
|
||||
n = len(claims_doc["claims"])
|
||||
print(f"grounding: {n} claims x 4 checks -- all pass")
|
||||
|
||||
# ---- battery A: corrupt one quote at a time; every corruption must be caught
|
||||
undetected = []
|
||||
for i, c in enumerate(claims_doc["claims"]):
|
||||
mutant = copy.deepcopy(claims_doc)
|
||||
q = mutant["claims"][i]["quote"]
|
||||
mid = len(q) // 2
|
||||
mutant["claims"][i]["quote"] = q[:mid] + corrupt_char(q[mid]) + q[mid + 1:]
|
||||
if not verify(mutant, dossier_text, CORPUS, cache):
|
||||
undetected.append(c["id"])
|
||||
if undetected:
|
||||
print(f"FAIL: battery A vacuous -- quote corruption undetected for {undetected}")
|
||||
return 1
|
||||
print(f"battery A: {n} quote corruptions, all detected")
|
||||
|
||||
# ---- battery B: flip one corpus byte; the sha256 pin must catch it
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
tdir = Path(td)
|
||||
victim = claims_doc["claims"][0]["paper"]
|
||||
for pid in {c["paper"] for c in claims_doc["claims"]}:
|
||||
(tdir / f"{pid}.txt").write_bytes(cache[pid])
|
||||
p = tdir / f"{victim}.txt"
|
||||
b = bytearray(p.read_bytes())
|
||||
b[len(b) // 2] ^= 0x01
|
||||
p.write_bytes(bytes(b))
|
||||
detected = verify(claims_doc, dossier_text, tdir, {})
|
||||
if detected:
|
||||
print(f"battery B: corpus byte flip in {victim} detected via sha256 pin")
|
||||
else:
|
||||
print(f"FAIL: battery B vacuous -- corpus byte flip in {victim} undetected")
|
||||
return 1
|
||||
|
||||
# ---- battery C: corrupt one headline value everywhere in the dossier
|
||||
undetected_c = []
|
||||
for c in claims_doc["claims"]:
|
||||
v = c["value"]
|
||||
if v not in dossier_text:
|
||||
continue
|
||||
bad = dossier_text.replace(v, corrupt_char(v))
|
||||
if not verify(claims_doc, bad, CORPUS, cache):
|
||||
undetected_c.append(c["id"])
|
||||
if undetected_c:
|
||||
print(f"FAIL: battery C vacuous -- dossier value corruption undetected for "
|
||||
f"{undetected_c}")
|
||||
return 1
|
||||
print(f"battery C: {n} dossier value corruptions, all detected")
|
||||
|
||||
print("VERDICT: PASS -- dossier grounded, verifier falsified-checked")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user