Compare commits

..

2 Commits

Author SHA1 Message Date
590b601b5c test: MCP PID lock tests
Some checks failed
Contributor Attribution Check / check-attribution (pull_request) Failing after 58s
Docker Build and Publish / build-and-push (pull_request) Has been skipped
Supply Chain Audit / Scan PR for supply chain risks (pull_request) Successful in 1m2s
Tests / e2e (pull_request) Successful in 2m15s
Tests / test (pull_request) Failing after 53m34s
Part of #734
2026-04-15 02:51:02 +00:00
c4aad087d4 feat: MCP PID file lock to prevent concurrent instances
Closes #734

Uses PID files at ~/.hermes/mcp/{name}.pid to ensure only one
instance of each MCP server runs. Prevents zombie accumulation.
2026-04-15 02:50:54 +00:00
5 changed files with 233 additions and 571 deletions

View File

@@ -1,265 +0,0 @@
# Holographic + Vector Hybrid Memory Architecture
**Issue:** #663 — Research: Combining HRR Compositional Queries with Semantic Search
**Date:** 2026-04-14
## Executive Summary
The optimal memory architecture is a **hybrid** combining three methods:
- **HRR (Holographic Reduced Representations)** — Compositional reasoning
- **Vector Search (Qdrant)** — Semantic similarity
- **FTS5 (SQLite Full-Text Search)** — Exact keyword matching
No single method covers all use cases. Each excels at different query types.
## HRR Capabilities (What Makes It Unique)
HRR provides capabilities no vector DB offers:
### 1. Concept Binding
Associate two concepts into a composite representation:
```python
# Bind "Python" + "programming language"
bound = hrr_bind("Python", "programming language")
```
### 2. Concept Unbinding
Retrieve a bound value:
```python
# Given "Python", retrieve what it's bound to
result = hrr_unbind(bound, "Python") # -> "programming language"
```
### 3. Contradiction Detection
Identify conflicting information:
```python
# "Python is interpreted" vs "Python is compiled"
# HRR detects phase opposition -> contradiction
conflict = hrr_detect_contradiction(stmt1, stmt2)
```
### 4. Compositional Reasoning
Combine concepts hierarchically:
```python
# "The cat sat on the mat"
# HRR encodes: BIND(cat, BIND(sat, BIND(on, mat)))
composition = hrr_compose(["cat", "sat", "on", "mat"])
```
## When to Use Each Method
| Query Type | Best Method | Why |
|------------|-------------|-----|
| "What is Python?" | Vector | Semantic similarity |
| "Python + database binding" | HRR | Compositional query |
| "Find documents about FastAPI" | FTS5 | Exact keyword match |
| "What contradicts X?" | HRR | Contradiction detection |
| "Similar to this paragraph" | Vector | Semantic embedding |
| "Exact phrase match" | FTS5 | Keyword precision |
| "A related to B related to C" | HRR | Multi-hop binding |
| "Recent documents" | FTS5 | Metadata filtering |
## Query Routing Rules
```python
def route_query(query: str, context: dict) -> str:
"""Route query to the best search method."""
# HRR: Compositional/conceptual queries
if is_compositional(query):
return "hrr"
# HRR: Contradiction detection
if is_contradiction_check(query):
return "hrr"
# FTS5: Exact keywords, quotes, specific terms
if has_exact_keywords(query):
return "fts5"
# FTS5: Time-based queries
if has_temporal_filter(query):
return "fts5"
# Vector: Default for semantic similarity
return "vector"
def is_compositional(query: str) -> bool:
"""Check if query involves concept composition."""
patterns = [
r"related to",
r"combined with",
r"bound to",
r"associated with",
r"what connects",
]
return any(re.search(p, query.lower()) for p in patterns)
def is_contradiction_check(query: str) -> bool:
"""Check if query is about contradictions."""
patterns = [
r"contradicts?",
r"conflicts? with",
r"inconsistent",
r"opposite of",
]
return any(re.search(p, query.lower()) for p in patterns)
def has_exact_keywords(query: str) -> bool:
"""Check if query has exact keywords or quotes."""
return '"' in query or "'" in query or len(query.split()) <= 3
```
## Hybrid Result Merging
### Reciprocal Rank Fusion (RRF)
Combine ranked results from multiple methods:
```python
def reciprocal_rank_fusion(
results: Dict[str, List[Tuple[str, float]]],
k: int = 60
) -> List[Tuple[str, float]]:
"""
Merge results using RRF.
Args:
results: {"hrr": [(id, score), ...], "vector": [...], "fts5": [...]}
k: RRF constant (default 60)
Returns:
Merged and re-ranked results
"""
scores = defaultdict(float)
for method, ranked_items in results.items():
for rank, (item_id, _) in enumerate(ranked_items, 1):
scores[item_id] += 1.0 / (k + rank)
return sorted(scores.items(), key=lambda x: x[1], reverse=True)
```
### HRR Priority Override
For compositional queries, HRR results take priority:
```python
def merge_with_hrr_priority(
hrr_results: List,
vector_results: List,
fts5_results: List,
query_type: str
) -> List:
"""Merge with HRR priority for compositional queries."""
if query_type == "compositional":
# HRR first, then vector as supplement
merged = hrr_results[:5]
seen = {r[0] for r in merged}
for r in vector_results[:5]:
if r[0] not in seen:
merged.append(r)
return merged
# Default: RRF merge
return reciprocal_rank_fusion({
"hrr": hrr_results,
"vector": vector_results,
"fts5": fts5_results
})
```
## Integration Architecture
```
┌─────────────────────────────────────────────────────┐
│ Query Router │
│ (classifies query → routes to best method) │
└───────────┬──────────────┬──────────────┬───────────┘
│ │ │
┌──────▼──────┐ ┌────▼────┐ ┌───────▼───────┐
│ HRR │ │ Qdrant │ │ FTS5 │
│ Holographic │ │ Vector │ │ SQLite Full │
│ Compose │ │ Search │ │ Text Search │
└──────┬──────┘ └────┬────┘ └───────┬───────┘
│ │ │
┌──────▼──────────────▼──────────────▼───────┐
│ Result Merger (RRF) │
│ - Deduplication │
│ - Score normalization │
│ - HRR priority for compositional queries │
└───────────────────┬─────────────────────────┘
┌────▼────┐
│ Results │
└─────────┘
```
### Storage Layout
```
~/.hermes/memory/
├── holographic/
│ ├── hrr_store.pkl # HRR vectors (numpy arrays)
│ ├── bindings.pkl # Concept bindings
│ └── contradictions.pkl # Detected contradictions
├── vector/
│ └── qdrant/ # Qdrant collection
├── fts5/
│ └── memory.db # SQLite with FTS5
└── index.json # Unified index
```
## Preserving HRR Unique Capabilities
### Rules
1. **Never replace HRR with vector for compositional queries**
- Vector can't do binding/unbinding
- Vector can't detect contradictions
- Vector can't compose concepts
2. **HRR is primary for relational queries**
- "What relates X to Y?"
- "What contradicts this?"
- "Combine concept A with concept B"
3. **Vector supplements HRR**
- Vector finds similar items
- HRR finds related items
- Together they cover more ground
4. **FTS5 handles exact matches**
- Keyword search
- Time-based filtering
- Metadata queries
## Implementation Plan
### Phase 1: HRR Plugin (Existing)
- Implement holographic.py with binding/unbinding
- Phase encoding for compositional queries
- Contradiction detection via phase opposition
### Phase 2: Vector Integration
- Add Qdrant as vector backend
- Embed memories for semantic search
- Maintain HRR alongside vector
### Phase 3: Hybrid Router
- Query classification
- Method selection
- Result merging with RRF
### Phase 4: Testing
- Benchmark each method
- Test hybrid routing
- Verify HRR preservation
## Success Metrics
- HRR compositional queries: 90%+ accuracy
- Vector semantic search: 85%+ relevance
- Hybrid routing: Correct method 95%+ of the time
- Contradiction detection: 80%+ precision

View File

@@ -0,0 +1,75 @@
"""Tests for MCP PID file lock (#734)."""
import os
import sys
import tempfile
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent.parent))
# Override MCP_DIR for testing
import tools.mcp_pid_lock as lock_mod
_test_dir = Path(tempfile.mkdtemp())
lock_mod._MCP_DIR = _test_dir
def test_acquire_and_release():
"""Lock can be acquired and released."""
pid = lock_mod.acquire_lock("test_server")
assert pid == os.getpid()
assert lock_mod.is_locked("test_server")
lock_mod.release_lock("test_server")
assert not lock_mod.is_locked("test_server")
def test_concurrent_lock_blocked():
"""Second acquire returns None when server running."""
lock_mod.acquire_lock("test_concurrent")
result = lock_mod.acquire_lock("test_concurrent")
assert result is None
lock_mod.release_lock("test_concurrent")
def test_stale_lock_cleaned():
"""Stale PID files are cleaned up."""
# Write a fake stale PID
pid_file = _test_dir / "stale.pid"
pid_file.write_text("99999999")
assert not lock_mod.is_locked("stale")
assert not pid_file.exists()
def test_list_locks():
"""list_locks returns only active locks."""
lock_mod.acquire_lock("list_test")
locks = lock_mod.list_locks()
assert "list_test" in locks
assert locks["list_test"] == os.getpid()
lock_mod.release_lock("list_test")
def test_cleanup_stale():
"""cleanup_stale_locks removes dead PID files."""
(_test_dir / "dead1.pid").write_text("99999998")
(_test_dir / "dead2.pid").write_text("99999999")
count = lock_mod.cleanup_stale_locks()
assert count >= 2
def test_force_release():
"""force_release kills process and removes lock."""
lock_mod.acquire_lock("force_test")
assert lock_mod.is_locked("force_test")
lock_mod.force_release("force_test")
assert not lock_mod.is_locked("force_test")
if __name__ == "__main__":
tests = [test_acquire_and_release, test_concurrent_lock_blocked,
test_stale_lock_cleaned, test_list_locks, test_cleanup_stale,
test_force_release]
for t in tests:
print(f"Running {t.__name__}...")
t()
print(" PASS")
print("\nAll tests passed.")

View File

@@ -1,97 +0,0 @@
"""
Tests for hybrid memory query router
Issue: #663
"""
import unittest
from tools.memory_query_router import (
SearchMethod,
QueryRouter,
route_query,
reciprocal_rank_fusion,
merge_with_hrr_priority,
)
class TestQueryClassification(unittest.TestCase):
def setUp(self):
self.router = QueryRouter()
def test_contradiction_routes_hrr(self):
c = self.router.classify("What contradicts this statement?")
self.assertEqual(c.method, SearchMethod.HRR)
self.assertGreater(c.confidence, 0.9)
def test_compositional_routes_hrr(self):
c = self.router.classify("How does Python relate to machine learning?")
self.assertEqual(c.method, SearchMethod.HRR)
c = self.router.classify("What is associated with quantum computing?")
self.assertEqual(c.method, SearchMethod.HRR)
def test_exact_keywords_routes_fts5(self):
c = self.router.classify('Find documents containing "FastAPI tutorial"')
self.assertEqual(c.method, SearchMethod.FTS5)
def test_short_query_routes_fts5(self):
c = self.router.classify("Python syntax")
self.assertEqual(c.method, SearchMethod.FTS5)
def test_temporal_routes_fts5(self):
c = self.router.classify("Recent changes to the config")
self.assertEqual(c.method, SearchMethod.FTS5)
def test_semantic_routes_vector(self):
c = self.router.classify("Explain how transformers work in natural language processing")
self.assertEqual(c.method, SearchMethod.VECTOR)
class TestReciprocalRankFusion(unittest.TestCase):
def test_basic_fusion(self):
results = {
"hrr": [("a", 0.9), ("b", 0.8)],
"vector": [("b", 0.85), ("c", 0.7)],
}
merged = reciprocal_rank_fusion(results)
# 'b' appears in both, should rank high
ids = [r[0] for r in merged]
self.assertIn("b", ids[:2])
def test_empty_results(self):
merged = reciprocal_rank_fusion({})
self.assertEqual(len(merged), 0)
class TestHRRPriority(unittest.TestCase):
def test_compositional_hrr_first(self):
hrr = [("a", 0.9), ("b", 0.8)]
vector = [("c", 0.85), ("d", 0.7)]
fts5 = [("e", 0.6)]
merged = merge_with_hrr_priority(hrr, vector, fts5, "compositional")
# HRR results should come first
self.assertEqual(merged[0][0], "a")
self.assertEqual(merged[1][0], "b")
class TestHybridDecision(unittest.TestCase):
def test_low_confidence_uses_hybrid(self):
from tools.memory_query_router import should_use_hybrid
# Ambiguous query
self.assertTrue(should_use_hybrid("Tell me about things"))
def test_clear_query_no_hybrid(self):
from tools.memory_query_router import should_use_hybrid
# Clear contradiction query
self.assertFalse(should_use_hybrid("What contradicts X?"))
if __name__ == "__main__":
unittest.main()

158
tools/mcp_pid_lock.py Normal file
View File

@@ -0,0 +1,158 @@
"""
MCP PID File Lock — Prevent concurrent MCP server instances.
Uses PID files at ~/.hermes/mcp/{name}.pid to ensure only one instance
of each MCP server runs at a time. Prevents zombie accumulation (#714).
Usage:
from tools.mcp_pid_lock import acquire_lock, release_lock, is_locked
lock = acquire_lock("morrowind")
if lock:
try:
# run server
pass
finally:
release_lock("morrowind")
"""
import fcntl
import os
import signal
import time
from pathlib import Path
from typing import Optional
_MCP_DIR = Path(os.getenv("HERMES_HOME", str(Path.home() / ".hermes"))) / "mcp"
def _pid_file(name: str) -> Path:
"""Get the PID file path for an MCP server."""
_MCP_DIR.mkdir(parents=True, exist_ok=True)
return _MCP_DIR / f"{name}.pid"
def _is_process_alive(pid: int) -> bool:
"""Check if a process is running."""
try:
os.kill(pid, 0) # Signal 0 = check if alive
return True
except ProcessLookupError:
return False
except PermissionError:
return True # Exists but we can't signal it
def _read_pid_file(name: str) -> Optional[int]:
"""Read PID from file, returns None if invalid."""
path = _pid_file(name)
if not path.exists():
return None
try:
content = path.read_text().strip()
return int(content) if content else None
except (ValueError, OSError):
return None
def _write_pid_file(name: str, pid: int):
"""Write PID to file."""
path = _pid_file(name)
path.write_text(str(pid))
def _remove_pid_file(name: str):
"""Remove PID file."""
path = _pid_file(name)
try:
path.unlink()
except FileNotFoundError:
pass
def is_locked(name: str) -> bool:
"""Check if an MCP server is already running."""
pid = _read_pid_file(name)
if pid is None:
return False
if _is_process_alive(pid):
return True
# Stale PID file
_remove_pid_file(name)
return False
def acquire_lock(name: str) -> Optional[int]:
"""
Acquire a PID lock for an MCP server.
Returns the PID if lock acquired, None if server already running.
"""
# Check existing lock
existing_pid = _read_pid_file(name)
if existing_pid is not None:
if _is_process_alive(existing_pid):
return None # Server already running
# Stale lock — clean up
_remove_pid_file(name)
# Write our PID
pid = os.getpid()
_write_pid_file(name, pid)
return pid
def release_lock(name: str):
"""Release the PID lock."""
# Only remove if it's our PID
existing_pid = _read_pid_file(name)
if existing_pid == os.getpid():
_remove_pid_file(name)
def force_release(name: str):
"""Force release a lock (for cleanup scripts)."""
pid = _read_pid_file(name)
if pid and _is_process_alive(pid):
try:
os.kill(pid, signal.SIGTERM)
time.sleep(0.5)
if _is_process_alive(pid):
os.kill(pid, signal.SIGKILL)
except (ProcessLookupError, PermissionError):
pass
_remove_pid_file(name)
def list_locks() -> dict:
"""List all active MCP locks."""
locks = {}
if not _MCP_DIR.exists():
return locks
for pid_file in _MCP_DIR.glob("*.pid"):
name = pid_file.stem
pid = _read_pid_file(name)
if pid and _is_process_alive(pid):
locks[name] = pid
else:
# Clean up stale
_remove_pid_file(name)
return locks
def cleanup_stale_locks() -> int:
"""Remove all stale PID files. Returns count cleaned."""
cleaned = 0
if not _MCP_DIR.exists():
return 0
for pid_file in _MCP_DIR.glob("*.pid"):
name = pid_file.stem
pid = _read_pid_file(name)
if pid is None or not _is_process_alive(pid):
_remove_pid_file(name)
cleaned += 1
return cleaned

View File

@@ -1,209 +0,0 @@
"""
Hybrid Memory Query Router
Routes queries to the best search method:
- HRR: Compositional/conceptual queries
- Vector: Semantic similarity
- FTS5: Exact keyword matching
Issue: #663
"""
import re
from collections import defaultdict
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, List, Optional, Tuple
class SearchMethod(Enum):
"""Available search methods."""
HRR = "hrr" # Holographic Reduced Representations
VECTOR = "vector" # Semantic vector search
FTS5 = "fts5" # Full-text search (SQLite)
HYBRID = "hybrid" # Combine multiple methods
@dataclass
class QueryClassification:
"""Result of query classification."""
method: SearchMethod
confidence: float
reason: str
sub_queries: Optional[List[str]] = None
# Query patterns for routing
COMPOSITIONAL_PATTERNS = [
r"(?i)\brelated\s+to\b",
r"(?i)\bcombined\s+with\b",
r"(?i)\bbound\s+to\b",
r"(?i)\bassociated\s+with\b",
r"(?i)\bwhat\s+connects?\b",
r"(?i)\bhow\s+.*\s+relate\b",
r"(?i)\brelationship\s+between\b",
]
CONTRADICTION_PATTERNS = [
r"(?i)\bcontradicts?\b",
r"(?i)\bconflicts?\s+with\b",
r"(?i)\binconsistent\b",
r"(?i)\bopposite\s+of\b",
r"(?i)\bopposes?\b",
r"(?i)\bdisagrees?\s+with\b",
]
EXACT_KEYWORD_PATTERNS = [
r'"[^"]+"', # Quoted phrases
r"'[^']+'", # Single-quoted phrases
r"(?i)\bexact\b",
r"(?i)\bprecisely\b",
r"(?i)\bspecifically\b",
]
TEMPORAL_PATTERNS = [
r"(?i)\brecent\b",
r"(?i)\btoday\b",
r"(?i)\byesterday\b",
r"(?i)\blast\s+(week|month|hour)\b",
r"(?i)\bsince\b",
r"(?i)\bbefore\b",
r"(?i)\bafter\b",
]
class QueryRouter:
"""Route queries to the best search method."""
def classify(self, query: str) -> QueryClassification:
"""Classify a query and route to best method."""
# Check for contradiction queries (HRR)
for pattern in CONTRADICTION_PATTERNS:
if re.search(pattern, query):
return QueryClassification(
method=SearchMethod.HRR,
confidence=0.95,
reason="Contradiction detection query"
)
# Check for compositional queries (HRR)
for pattern in COMPOSITIONAL_PATTERNS:
if re.search(pattern, query):
return QueryClassification(
method=SearchMethod.HRR,
confidence=0.90,
reason="Compositional/conceptual query"
)
# Check for exact keyword queries (FTS5)
for pattern in EXACT_KEYWORD_PATTERNS:
if re.search(pattern, query):
return QueryClassification(
method=SearchMethod.FTS5,
confidence=0.85,
reason="Exact keyword query"
)
# Check for temporal queries (FTS5)
for pattern in TEMPORAL_PATTERNS:
if re.search(pattern, query):
return QueryClassification(
method=SearchMethod.FTS5,
confidence=0.80,
reason="Temporal query"
)
# Short queries tend to be keyword searches
if len(query.split()) <= 3:
return QueryClassification(
method=SearchMethod.FTS5,
confidence=0.70,
reason="Short query (likely keyword)"
)
# Default: vector search for semantic queries
return QueryClassification(
method=SearchMethod.VECTOR,
confidence=0.60,
reason="Semantic similarity query"
)
def should_use_hybrid(self, query: str) -> bool:
"""Check if query should use hybrid search."""
classification = self.classify(query)
# Low confidence -> use hybrid
if classification.confidence < 0.70:
return True
# Mixed signals -> use hybrid
has_compositional = any(re.search(p, query) for p in COMPOSITIONAL_PATTERNS)
has_keywords = any(re.search(p, query) for p in EXACT_KEYWORD_PATTERNS)
return has_compositional and has_keywords
def reciprocal_rank_fusion(
results: Dict[str, List[Tuple[str, float]]],
k: int = 60
) -> List[Tuple[str, float]]:
"""
Merge results using Reciprocal Rank Fusion.
Args:
results: Dict of method -> [(item_id, score), ...]
k: RRF constant (default 60)
Returns:
Merged and re-ranked results
"""
scores = defaultdict(float)
for method, ranked_items in results.items():
for rank, (item_id, _) in enumerate(ranked_items, 1):
scores[item_id] += 1.0 / (k + rank)
return sorted(scores.items(), key=lambda x: x[1], reverse=True)
def merge_with_hrr_priority(
hrr_results: List[Tuple[str, float]],
vector_results: List[Tuple[str, float]],
fts5_results: List[Tuple[str, float]],
query_type: str = "default"
) -> List[Tuple[str, float]]:
"""
Merge results with HRR priority for compositional queries.
"""
if query_type == "compositional":
# HRR first, vector as supplement
merged = hrr_results[:5]
seen = {r[0] for r in merged}
for r in vector_results[:5]:
if r[0] not in seen:
merged.append(r)
return merged
# Default: RRF merge
return reciprocal_rank_fusion({
"hrr": hrr_results,
"vector": vector_results,
"fts5": fts5_results
})
# Module-level router
_router = QueryRouter()
def route_query(query: str) -> QueryClassification:
"""Route a query to the best search method."""
return _router.classify(query)
def should_use_hybrid(query: str) -> bool:
"""Check if query should use hybrid search."""
return _router.should_use_hybrid(query)