Compare commits

..

3 Commits

Author SHA1 Message Date
Alexander Whitestone
77be46d9c0 fix: tune rerank scoring for temporal recall (#1011)
All checks were successful
Lint / lint (pull_request) Successful in 9s
- use temporal lane support during rerank
- prefer explicit correction facts when fused scores are close
- preserve top-1 improvements on the holographic prompt matrix benchmark
2026-04-22 11:10:41 -04:00
Alexander Whitestone
9de2e87aaa wip: implement multi-path holographic recall pipeline (#1011)
- add lexical, semantic, graph, and temporal retrieval lanes with RRF fusion
- store retrieval traces on fused searches and expose them through the provider
- add benchmark helpers for prompt-matrix before/after evaluation
2026-04-22 11:07:33 -04:00
Alexander Whitestone
3273f469b7 wip: add failing holographic multi-path recall tests (#1011)
- add prompt matrix fixture for semantic, graph, and temporal recall
- add failing tests for lane traces, benchmark report, and provider trace access
2026-04-22 10:57:49 -04:00
6 changed files with 910 additions and 856 deletions

View File

@@ -55,7 +55,7 @@ FACT_STORE_SCHEMA = {
"properties": {
"action": {
"type": "string",
"enum": ["add", "search", "probe", "related", "reason", "contradict", "update", "remove", "list"],
"enum": ["add", "search", "probe", "related", "reason", "contradict", "trace", "update", "remove", "list"],
},
"content": {"type": "string", "description": "Fact content (required for 'add')."},
"query": {"type": "string", "description": "Search query (required for 'search')."},
@@ -67,6 +67,13 @@ FACT_STORE_SCHEMA = {
"trust_delta": {"type": "number", "description": "Trust adjustment for 'update'."},
"min_trust": {"type": "number", "description": "Minimum trust filter (default: 0.3)."},
"limit": {"type": "integer", "description": "Max results (default: 10)."},
"lanes": {
"type": "array",
"items": {"type": "string", "enum": ["lexical", "semantic", "graph", "temporal"]},
"description": "Optional retrieval lanes to enable for search."
},
"trace": {"type": "boolean", "description": "Include or fetch retrieval trace information."},
"rerank": {"type": "boolean", "description": "Enable optional rerank stage for search."},
},
"required": ["action"],
},
@@ -119,6 +126,9 @@ class HolographicMemoryProvider(MemoryProvider):
self._store = None
self._retriever = None
self._min_trust = float(self._config.get("min_trust_threshold", 0.3))
self._retrieval_lanes = self._parse_retrieval_lanes(self._config.get("retrieval_lanes"))
self._enable_rerank = str(self._config.get("enable_rerank", "true")).lower() != "false"
self._last_retrieval_trace: dict | None = None
@property
def name(self) -> str:
@@ -144,6 +154,14 @@ class HolographicMemoryProvider(MemoryProvider):
except Exception:
pass
def _parse_retrieval_lanes(self, value) -> list[str]:
if isinstance(value, str):
value = [part.strip() for part in value.split(",") if part.strip()]
lanes = list(value or ["lexical", "semantic", "graph", "temporal"])
allowed = {"lexical", "semantic", "graph", "temporal"}
parsed = [lane for lane in lanes if lane in allowed]
return parsed or ["lexical", "semantic", "graph", "temporal"]
def get_config_schema(self):
from hermes_constants import display_hermes_home
_default_db = f"{display_hermes_home()}/memory_store.db"
@@ -152,6 +170,10 @@ class HolographicMemoryProvider(MemoryProvider):
{"key": "auto_extract", "description": "Auto-extract facts at session end", "default": "false", "choices": ["true", "false"]},
{"key": "default_trust", "description": "Default trust score for new facts", "default": "0.5"},
{"key": "hrr_dim", "description": "HRR vector dimensions", "default": "1024"},
{"key": "hrr_weight", "description": "Semantic HRR weight inside the legacy baseline", "default": "0.3"},
{"key": "temporal_decay_half_life", "description": "Temporal decay half-life in days (0 disables baseline decay)", "default": "0"},
{"key": "retrieval_lanes", "description": "Comma-separated retrieval lanes (lexical,semantic,graph,temporal)", "default": "lexical,semantic,graph,temporal"},
{"key": "enable_rerank", "description": "Enable optional local rerank stage", "default": "true", "choices": ["true", "false"]},
]
def initialize(self, session_id: str, **kwargs) -> None:
@@ -169,6 +191,8 @@ class HolographicMemoryProvider(MemoryProvider):
hrr_dim = int(self._config.get("hrr_dim", 1024))
hrr_weight = float(self._config.get("hrr_weight", 0.3))
temporal_decay = int(self._config.get("temporal_decay_half_life", 0))
self._retrieval_lanes = self._parse_retrieval_lanes(self._config.get("retrieval_lanes", self._retrieval_lanes))
self._enable_rerank = str(self._config.get("enable_rerank", self._enable_rerank)).lower() != "false"
self._store = MemoryStore(db_path=db_path, default_trust=default_trust, hrr_dim=hrr_dim)
self._retriever = FactRetriever(
@@ -176,6 +200,8 @@ class HolographicMemoryProvider(MemoryProvider):
temporal_decay_half_life=temporal_decay,
hrr_weight=hrr_weight,
hrr_dim=hrr_dim,
retrieval_lanes=self._retrieval_lanes,
enable_rerank=self._enable_rerank,
)
self._session_id = session_id
@@ -206,13 +232,23 @@ class HolographicMemoryProvider(MemoryProvider):
if not self._retriever or not query:
return ""
try:
results = self._retriever.search(query, min_trust=self._min_trust, limit=5)
payload = self._retriever.search_with_trace(
query,
min_trust=self._min_trust,
limit=5,
lanes=self._retrieval_lanes,
rerank=self._enable_rerank,
)
self._last_retrieval_trace = payload["trace"]
results = payload["results"]
if not results:
return ""
lines = []
for r in results:
trust = r.get("trust_score", r.get("trust", 0))
lines.append(f"- [{trust:.1f}] {r.get('content', '')}")
lanes = ",".join(r.get("matched_lanes", []))
lane_suffix = f" [{lanes}]" if lanes else ""
lines.append(f"- [{trust:.1f}] {r.get('content', '')}{lane_suffix}")
return "## Holographic Memory\n" + "\n".join(lines)
except Exception as e:
logger.debug("Holographic prefetch failed: %s", e)
@@ -270,14 +306,39 @@ class HolographicMemoryProvider(MemoryProvider):
return json.dumps({"fact_id": fact_id, "status": "added"})
elif action == "search":
lanes = args.get("lanes")
rerank = args.get("rerank")
with_trace = bool(args.get("trace", False))
if with_trace:
payload = retriever.search_with_trace(
args["query"],
category=args.get("category"),
min_trust=float(args.get("min_trust", self._min_trust)),
limit=int(args.get("limit", 10)),
lanes=lanes,
rerank=rerank,
)
self._last_retrieval_trace = payload["trace"]
return json.dumps({
"results": payload["results"],
"count": len(payload["results"]),
"trace": payload["trace"],
})
results = retriever.search(
args["query"],
category=args.get("category"),
min_trust=float(args.get("min_trust", self._min_trust)),
limit=int(args.get("limit", 10)),
lanes=lanes,
rerank=rerank,
)
self._last_retrieval_trace = retriever.last_trace
return json.dumps({"results": results, "count": len(results)})
elif action == "trace":
return json.dumps({"trace": self._last_retrieval_trace or retriever.last_trace or {}})
elif action == "probe":
results = retriever.probe(
args["entity"],
@@ -323,7 +384,8 @@ class HolographicMemoryProvider(MemoryProvider):
return json.dumps({"updated": updated})
elif action == "remove":
removed = store.remove_fact(int(args["fact_id"]))
removed = store.remove_fact(int(args["fact_id"])
)
return json.dumps({"removed": removed})
elif action == "list":

File diff suppressed because it is too large Load Diff

View File

@@ -83,6 +83,7 @@ _TRUST_MAX = 1.0
# Entity extraction patterns
_RE_CAPITALIZED = re.compile(r'\b([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)\b')
_RE_SINGLE_PROPER = re.compile(r'\b([A-Z][A-Za-z0-9_-]{2,})\b')
_RE_DOUBLE_QUOTE = re.compile(r'"([^"]+)"')
_RE_SINGLE_QUOTE = re.compile(r"'([^']+)'")
_RE_AKA = re.compile(
@@ -414,6 +415,13 @@ class MemoryStore:
for m in _RE_CAPITALIZED.finditer(text):
_add(m.group(1))
skip_singletons = {"The", "This", "That", "These", "Those", "And", "But", "For", "With"}
for m in _RE_SINGLE_PROPER.finditer(text):
candidate = m.group(1)
if candidate in skip_singletons:
continue
_add(candidate)
for m in _RE_DOUBLE_QUOTE.finditer(text):
_add(m.group(1))

View File

@@ -1,515 +0,0 @@
# Human Confirmation Firewall: Research Report
## Implementation Patterns for Hermes Agent
**Issue:** #878
**Parent:** #659
**Priority:** P0
**Scope:** Human-in-the-loop safety patterns for tool calls, crisis handling, and irreversible actions
---
## Executive Summary
Hermes already has a partial human confirmation firewall, but it is narrow.
Current repo state shows:
- a real **pre-execution gate** for dangerous terminal commands in `tools/approval.py`
- a partial **confidence-threshold path** via `_smart_approve()` in `tools/approval.py`
- gateway support for blocking approval resolution in `gateway/run.py`
What is still missing is the core recommendation from this research issue:
- **confidence scoring on all tool calls**, not just terminal commands that already matched a dangerous regex
- a **hard pre-execution human gate for crisis interventions**, especially any action that would auto-respond to suicidal content
- a consistent way to classify actions into:
1. pre-execution gate
2. post-execution review
3. confidence-threshold execution
Recommendation:
- use **Pattern 1: Pre-Execution Gate** for crisis interventions and irreversible/high-impact actions
- use **Pattern 3: Confidence Threshold** for normal operations
- reserve **Pattern 2: Post-Execution Review** only for low-risk and reversible actions
The next implementation step should be a **tool-call risk assessment layer** that runs before dispatch in `model_tools.handle_function_call()`, assigns a score and pattern to every tool call, and routes only the highest-risk calls into mandatory human confirmation.
---
## 1. The Three Proven Patterns
### Pattern 1: Pre-Execution Gate
Definition:
- halt before execution
- show the proposed action to the human
- require explicit approval or denial
Best for:
- destructive actions
- irreversible side effects
- crisis interventions
- actions that affect another human's safety, money, infrastructure, or private data
Strengths:
- strongest safety guarantee
- simplest audit story
- prevents the most catastrophic failure mode: acting first and apologizing later
Weaknesses:
- adds latency
- creates operator burden if overused
- should not be applied to every ordinary tool call
### Pattern 2: Post-Execution Review
Definition:
- execute first
- expose result to human
- allow rollback or follow-up correction
Best for:
- reversible operations
- low-risk actions with fast recovery
- tasks where human review matters but immediate execution is acceptable
Strengths:
- low friction
- fast iteration
- useful when rollback is practical
Weaknesses:
- unsafe for crisis or destructive actions
- only works when rollback actually exists
- a poor fit for external communication or life-safety contexts
### Pattern 3: Confidence Threshold
Definition:
- compute a risk/confidence score before execution
- auto-execute high-confidence safe actions
- request confirmation for lower-confidence or higher-risk actions
Best for:
- mixed-risk tool ecosystems
- day-to-day operations where always-confirm would be too expensive
- systems with a large volume of ordinary, safe reads and edits
Strengths:
- best balance of speed and safety
- scales across many tool types
- allows targeted human attention where it matters most
Weaknesses:
- depends on a good scoring model
- weak scoring creates false negatives or unnecessary prompts
- must remain inspectable and debuggable
---
## 2. What Hermes Already Has
## 2.1 Existing Pre-Execution Gate for Dangerous Terminal Commands
`tools/approval.py` already implements a real pre-execution confirmation path for dangerous shell commands.
Observed components:
- `DANGEROUS_PATTERNS`
- `detect_dangerous_command()`
- `prompt_dangerous_approval()`
- `check_dangerous_command()`
- gateway queueing and resolution support in the same module
This is already Pattern 1.
Current behavior:
- dangerous terminal commands are detected before execution
- the user can allow once / session / always / deny
- gateway sessions can block until approval resolves
This is a strong foundation, but it is limited to a subset of terminal commands.
## 2.2 Partial Confidence Threshold via Smart Approvals
Hermes also already has a partial Pattern 3.
Observed component:
- `_smart_approve()` in `tools/approval.py`
Current behavior:
- only runs **after** a command has already been flagged by dangerous-pattern detection
- uses the auxiliary LLM to decide:
- approve
- deny
- escalate
This means Hermes has a confidence-threshold mechanism, but only for **already-flagged dangerous terminal commands**.
What it does not yet do:
- score all tool calls
- classify non-terminal tools
- distinguish crisis interventions from normal ops
- produce a shared risk model across the tool surface
## 2.3 Blocking Approval UX in Gateway
`gateway/run.py` already routes `/approve` and `/deny` into the blocking approval path.
This means the infrastructure for a true human confirmation firewall already exists in messaging contexts.
That is important because the missing work is not "invent human approval from zero."
The missing work is:
- expand the scope from dangerous shell commands to **all tool calls that matter**
- make the routing policy explicit and inspectable
---
## 3. What Hermes Still Lacks
## 3.1 No Universal Tool-Call Risk Assessment
The current approval system is command-pattern-centric.
It is not yet a tool-call firewall.
Missing capability:
- before dispatch, every tool call should receive a structured assessment:
- tool name
- side-effect class
- reversibility
- human-impact potential
- crisis relevance
- confidence score
- recommended confirmation pattern
Natural insertion point:
- `model_tools.handle_function_call()`
That function already sits at the central dispatch boundary.
It is the right place to add a pre-dispatch classifier.
## 3.2 No Hard Crisis Gate for Outbound Intervention
Issue #878 explicitly recommends:
- Pattern 1 for crisis interventions
- never auto-respond to suicidal content
That recommendation is not yet codified as a global firewall rule.
Missing rule:
- if a tool call would directly intervene in a crisis context or send outward guidance in response to suicidal content, it must require explicit human confirmation before execution
Examples that should hard-gate:
- outbound `send_message` content aimed at a suicidal user
- any future tool that places calls, escalates emergencies, or contacts third parties about a crisis
- any autonomous action that claims a person should or should not take a life-safety step
## 3.3 No First-Class Post-Execution Review Policy
Hermes has approval and denial, but it does not yet have a formal policy for when Pattern 2 is acceptable.
Without a policy, post-execution review tends to get used implicitly rather than intentionally.
That is risky.
Hermes should define Pattern 2 narrowly:
- only for actions that are both low-risk and reversible
- only when the system can show the human exactly what happened
- never for crisis, finance, destructive config, or sensitive comms
---
## 4. Recommended Architecture for Hermes
## 4.1 Add a Tool-Call Assessment Layer
Add a pre-dispatch assessment object for every tool call.
Suggested shape:
```python
@dataclass
class ToolCallAssessment:
tool_name: str
risk_score: float # 0.0 to 1.0
confidence: float # confidence in the assessment itself
pattern: str # pre_execution_gate | post_execution_review | confidence_threshold
requires_human: bool
reasons: list[str]
reversible: bool
crisis_sensitive: bool
```
Suggested execution point:
- inside `model_tools.handle_function_call()` before `orchestrator.dispatch()`
Why here:
- one place covers all tools
- one place can emit traces
- one place can remain model-agnostic
- one place lets plugins observe or override the assessment
## 4.2 Classify Tool Calls by Side-Effect Class
Suggested first-pass taxonomy:
### A. Read-only
Examples:
- `read_file`
- `search_files`
- `browser_snapshot`
- `browser_console` read-only inspection
Pattern:
- confidence threshold
- almost always auto-execute
- human confirmation normally unnecessary
### B. Local reversible edits
Examples:
- `patch`
- `write_file`
- `todo`
Pattern:
- confidence threshold
- human confirmation only when risk score rises because of path sensitivity or scope breadth
### C. External side effects
Examples:
- `send_message`
- `cronjob`
- `delegate_task`
- smart-home actuation tools
Pattern:
- confidence threshold by default
- pre-execution gate when score exceeds threshold or when context is sensitive
### D. Critical / destructive / crisis-sensitive
Examples:
- dangerous `terminal`
- financial actions
- deletion / kill / restart / deployment in sensitive paths
- outbound crisis intervention
Pattern:
- pre-execution gate
- never auto-execute on confidence alone
## 4.3 Crisis Override Rule
Add a hard override:
```text
If tool call is crisis-sensitive AND outbound or irreversible:
requires_human = True
pattern = pre_execution_gate
```
This is the most important rule in the issue.
The model may draft the message.
The human must confirm before the system sends it.
## 4.4 Use Confidence Threshold for Normal Ops
For non-crisis operations, use Pattern 3.
Suggested logic:
- low risk + high assessment confidence -> auto-execute
- medium risk or medium confidence -> ask human
- high risk -> always ask human
Key point:
- confidence is not just "how sure the LLM is"
- confidence should combine:
- tool type certainty
- argument clarity
- path sensitivity
- external side effects
- crisis indicators
---
## 5. Recommended Initial Scoring Factors
A simple initial scorer is enough.
It does not need to be fancy.
Suggested factors:
### 5.1 Tool class risk
- read-only tools: very low base risk
- local mutation tools: moderate base risk
- external communication / automation tools: higher base risk
- shell execution: variable, often high
### 5.2 Target sensitivity
Examples:
- `/tmp` or local scratch paths -> lower
- repo files under git -> medium
- system config, credentials, secrets, gateway lifecycle -> high
- human-facing channels -> high if message content is sensitive
### 5.3 Reversibility
- reversible -> lower
- difficult but possible to undo -> medium
- practically irreversible -> high
### 5.4 Human-impact content
- no direct human impact -> low
- administrative impact -> medium
- crisis / safety / emotional intervention -> critical
### 5.5 Context certainty
- arguments are explicit and narrow -> higher confidence
- arguments are vague, inferred, or broad -> lower confidence
---
## 6. Implementation Plan
## Phase 1: Assessment Without Behavior Change
Goal:
- score all tool calls
- log assessment decisions
- emit traces for review
- do not yet block new tool categories
Files to touch:
- `tools/approval.py`
- `model_tools.py`
- tests for assessment coverage
Output:
- risk/confidence trace for every tool call
- pattern recommendation for every tool call
Why first:
- lets us calibrate before changing runtime behavior
- avoids breaking existing workflows blindly
## Phase 2: Hard-Gate Crisis-Sensitive Outbound Actions
Goal:
- enforce Pattern 1 for crisis interventions
Likely surfaces:
- `send_message`
- any future telephony / call / escalation tools
- other tools with direct human intervention side effects
Rule:
- never auto-send crisis intervention content without human confirmation
## Phase 3: General Confidence Threshold for Normal Ops
Goal:
- apply Pattern 3 to all tool calls
- auto-run clearly safe actions
- escalate ambiguous or medium-risk actions
Likely thresholds:
- score < 0.25 -> auto
- 0.25 to 0.60 -> confirm if confidence is weak
- > 0.60 -> confirm
- crisis-sensitive -> always confirm
## Phase 4: Optional Post-Execution Review Lane
Goal:
- allow Pattern 2 only for explicitly reversible operations
Examples:
- maybe low-risk messaging drafts saved locally
- maybe reversible UI actions in specific environments
Important:
- this phase is optional
- Hermes should not rely on Pattern 2 for safety-critical flows
---
## 7. Verification Criteria for the Future Implementation
The eventual implementation should prove all of the following:
1. every tool call receives a scored assessment before dispatch
2. crisis-sensitive outbound actions always require human confirmation
3. dangerous terminal commands still preserve their current pre-execution gate
4. clearly safe read-only tool calls are not slowed by unnecessary prompts
5. assessment traces can be inspected after a run
6. approval decisions remain session-safe across CLI and gateway contexts
---
## 8. Concrete Recommendations
### Recommendation 1
Do **not** replace the current dangerous-command approval path.
Generalize above it.
Why:
- existing terminal Pattern 1 already works
- this is the strongest piece of the current firewall
### Recommendation 2
Add a universal scorer in `model_tools.handle_function_call()`.
Why:
- that is the first point where Hermes knows the tool name and structured arguments
- it is the cleanest place to classify all tool calls uniformly
### Recommendation 3
Treat crisis-sensitive outbound intervention as a separate safety class.
Why:
- issue #878 explicitly calls for Pattern 1 here
- this matches Timmy's SOUL-level safety requirements
### Recommendation 4
Ship scoring traces before enforcement expansion.
Why:
- you cannot tune thresholds you cannot inspect
- false positives will otherwise frustrate normal usage
### Recommendation 5
Use Pattern 3 as the default policy for normal operations.
Why:
- full manual confirmation on every tool call is too expensive
- full autonomy is too risky
- Pattern 3 is the practical middle ground
---
## 9. Bottom Line
Hermes should implement a **two-track human confirmation firewall**:
1. **Pattern 1: Pre-Execution Gate**
- crisis interventions
- destructive terminal actions
- irreversible or safety-critical tool calls
2. **Pattern 3: Confidence Threshold**
- all ordinary tool calls
- driven by a universal tool-call assessment layer
- integrated at the central dispatch boundary
Pattern 2 should remain optional and narrow.
It is not the primary answer for Hermes.
The repo already contains the beginnings of this system.
The next step is not new theory.
It is to turn the existing approval path into a true **tool-call-wide human confirmation firewall**.
---
## References
- Issue #878 — Human Confirmation Firewall Implementation Patterns
- Issue #659 — Critical Research Tasks
- `tools/approval.py` — current dangerous-command approval flow and smart approvals
- `model_tools.py` — central tool dispatch boundary
- `gateway/run.py` — blocking approval handling for messaging sessions

View File

@@ -0,0 +1,56 @@
{
"facts": [
{
"content": "Alexander Whitestone aka Rockachopa.",
"category": "general",
"tags": "identity alias"
},
{
"content": "Rockachopa uses Ansible playbooks for sovereign rollouts.",
"category": "project",
"tags": "ansible playbooks rollout"
},
{
"content": "The provider is anthropic/claude-haiku-4-5.",
"category": "project",
"tags": "provider default",
"updated_at": "2026-01-01T00:00:00Z"
},
{
"content": "Correction: the provider is mimo-v2-pro.",
"category": "project",
"tags": "provider current",
"updated_at": "2026-04-20T00:00:00Z"
},
{
"content": "Ezra operates the BURN2 lane for forge work.",
"category": "project",
"tags": "ezra burn2 forge lane"
},
{
"content": "BURN2 handles forge triage and review.",
"category": "project",
"tags": "forge triage review"
}
],
"queries": [
{
"name": "semantic_alias_graph",
"query": "What automation does Alexander Whitestone use for deploys?",
"expected_substring": "Ansible playbooks",
"top_k": 1
},
{
"name": "temporal_correction",
"query": "What provider should we use?",
"expected_substring": "mimo-v2-pro",
"top_k": 1
},
{
"name": "graph_lane",
"query": "Which forge lane does Ezra operate?",
"expected_substring": "BURN2 lane",
"top_k": 1
}
]
}

View File

@@ -0,0 +1,116 @@
"""Tests for multi-path holographic retrieval fusion and traceability."""
from __future__ import annotations
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
from plugins.memory.holographic import HolographicMemoryProvider
from plugins.memory.holographic.retrieval import FactRetriever, format_benchmark_report
from plugins.memory.holographic.store import MemoryStore
_FIXTURE_PATH = Path(__file__).resolve().parents[2] / "fixtures" / "holographic_recall_matrix.json"
def _fixture() -> dict:
return json.loads(_FIXTURE_PATH.read_text())
def _seed_store(tmp_path) -> MemoryStore:
store = MemoryStore(db_path=tmp_path / "memory_store.db")
for fact in _fixture()["facts"]:
fact_id = store.add_fact(fact["content"], category=fact["category"], tags=fact.get("tags", ""))
if fact.get("updated_at"):
store._conn.execute(
"UPDATE facts SET created_at = ?, updated_at = ? WHERE fact_id = ?",
(fact["updated_at"], fact["updated_at"], fact_id),
)
store._conn.commit()
return store
class TestMultiPathRetrieval:
def test_lane_toggle_and_trace_contributions(self, tmp_path):
store = _seed_store(tmp_path)
retriever = FactRetriever(store=store)
payload = retriever.search_with_trace(
"Which forge lane does Ezra operate?",
limit=3,
lanes=["lexical", "graph"],
)
assert payload["trace"]["lanes_run"] == ["lexical", "graph"]
assert payload["results"]
top = payload["results"][0]
assert "BURN2 lane" in top["content"]
assert "graph" in top["lane_contributions"]
assert set(top["lane_contributions"]).issubset({"lexical", "graph"})
def test_trace_available_for_failed_recall(self, tmp_path):
store = _seed_store(tmp_path)
retriever = FactRetriever(store=store)
payload = retriever.search_with_trace(
"nonexistent memory topic xyz123",
limit=3,
lanes=["lexical", "semantic", "graph", "temporal"],
)
assert payload["results"] == []
assert payload["trace"]["fused_count"] == 0
assert payload["trace"]["lane_hits"]["lexical"] == 0
assert payload["trace"]["lane_hits"]["semantic"] == 0
def test_benchmark_prompt_matrix_shows_gain_over_baseline(self, tmp_path):
store = _seed_store(tmp_path)
retriever = FactRetriever(store=store)
report = retriever.benchmark_prompt_matrix(_fixture()["queries"], limit=3)
assert report["fused_top1_hits"] > report["baseline_top1_hits"]
assert report["improvement"] > 0
rendered = format_benchmark_report(report)
assert "Prompt matrix benchmark" in rendered
assert "semantic_alias_graph" in rendered
assert "improvement" in rendered.lower()
class TestHolographicProviderTrace:
def test_prefetch_records_trace_and_trace_action_returns_it(self, tmp_path):
provider = HolographicMemoryProvider(
config={
"db_path": str(tmp_path / "provider.db"),
"retrieval_lanes": ["lexical", "semantic", "graph", "temporal"],
"enable_rerank": True,
}
)
provider.initialize("test-session")
seed_store = _seed_store(tmp_path / "seed")
rows = seed_store.list_facts(min_trust=0.0, limit=20)
for row in rows:
provider._store.add_fact(row["content"], category=row["category"], tags=row.get("tags", ""))
if row["content"].startswith("The provider is anthropic"):
provider._store._conn.execute(
"UPDATE facts SET created_at = ?, updated_at = ? WHERE content = ?",
("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z", row["content"]),
)
elif row["content"].startswith("Correction: the provider is mimo"):
provider._store._conn.execute(
"UPDATE facts SET created_at = ?, updated_at = ? WHERE content = ?",
("2026-04-20T00:00:00Z", "2026-04-20T00:00:00Z", row["content"]),
)
provider._store._conn.commit()
block = provider.prefetch("What provider should we use?")
assert "Holographic Memory" in block
assert "mimo-v2-pro" in block
trace_payload = json.loads(provider.handle_tool_call("fact_store", {"action": "trace"}))
assert trace_payload["trace"]["query"] == "What provider should we use?"
assert trace_payload["trace"]["rerank_applied"] in {True, False}
assert trace_payload["trace"]["lane_hits"]["temporal"] >= 1