fix: eliminate 3x SQLite message duplication in gateway sessions (#860)
Three separate code paths all wrote to the same SQLite state.db with no deduplication, inflating session transcripts by 3-4x: 1. _log_msg_to_db() — wrote each message individually after append 2. _flush_messages_to_session_db() — re-wrote ALL new messages at every _persist_session() call (~18 exit points), with no tracking of what was already written 3. gateway append_to_transcript() — wrote everything a third time after the agent returned Since load_transcript() prefers SQLite over JSONL, the inflated data was loaded on every session resume, causing proportional token waste. Fix: - Remove _log_msg_to_db() and all 16 call sites (redundant with flush) - Add _last_flushed_db_idx tracking in _flush_messages_to_session_db() so repeated _persist_session() calls only write truly new messages - Reset flush cursor on compression (new session ID) - Add skip_db parameter to SessionStore.append_to_transcript() so the gateway skips SQLite writes when the agent already persisted them - Gateway now passes skip_db=True for agent-managed messages, still writes to JSONL as backup Verified: a 12-message CLI session with tool calls produces exactly 12 SQLite rows with zero duplicates (previously would be 36-48). Tests: 9 new tests covering flush deduplication, skip_db behavior, compression reset, and initialization. Full suite passes (2869 tests).
This commit is contained in:
@@ -1322,6 +1322,11 @@ class GatewayRunner:
|
||||
{"role": "assistant", "content": response, "timestamp": ts}
|
||||
)
|
||||
else:
|
||||
# The agent already persisted these messages to SQLite via
|
||||
# _flush_messages_to_session_db(), so skip the DB write here
|
||||
# to prevent the duplicate-write bug (#860). We still write
|
||||
# to JSONL for backward compatibility and as a backup.
|
||||
agent_persisted = self._session_db is not None
|
||||
for msg in new_messages:
|
||||
# Skip system messages (they're rebuilt each run)
|
||||
if msg.get("role") == "system":
|
||||
@@ -1329,7 +1334,8 @@ class GatewayRunner:
|
||||
# Add timestamp to each message for debugging
|
||||
entry = {**msg, "timestamp": ts}
|
||||
self.session_store.append_to_transcript(
|
||||
session_entry.session_id, entry
|
||||
session_entry.session_id, entry,
|
||||
skip_db=agent_persisted,
|
||||
)
|
||||
|
||||
# Update session
|
||||
|
||||
@@ -677,10 +677,17 @@ class SessionStore:
|
||||
"""Get the path to a session's legacy transcript file."""
|
||||
return self.sessions_dir / f"{session_id}.jsonl"
|
||||
|
||||
def append_to_transcript(self, session_id: str, message: Dict[str, Any]) -> None:
|
||||
"""Append a message to a session's transcript (SQLite + legacy JSONL)."""
|
||||
# Write to SQLite
|
||||
if self._db:
|
||||
def append_to_transcript(self, session_id: str, message: Dict[str, Any], skip_db: bool = False) -> None:
|
||||
"""Append a message to a session's transcript (SQLite + legacy JSONL).
|
||||
|
||||
Args:
|
||||
skip_db: When True, only write to JSONL and skip the SQLite write.
|
||||
Used when the agent already persisted messages to SQLite
|
||||
via its own _flush_messages_to_session_db(), preventing
|
||||
the duplicate-write bug (#860).
|
||||
"""
|
||||
# Write to SQLite (unless the agent already handled it)
|
||||
if self._db and not skip_db:
|
||||
try:
|
||||
self._db.append_message(
|
||||
session_id=session_id,
|
||||
|
||||
Reference in New Issue
Block a user