Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1aa6175bf7 | |||
| 6f2e4f0945 | |||
| 5f83328ce9 |
@@ -1,109 +0,0 @@
|
||||
"""
|
||||
Pending Cron Deliveries — Buffer and retry failed deliveries
|
||||
|
||||
When gateway reconnects, in-flight cron job notifications may be lost.
|
||||
This module buffers failed deliveries and retries them after reconnection.
|
||||
|
||||
Issue: #744
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HERMES_HOME = Path.home() / ".hermes"
|
||||
PENDING_FILE = HERMES_HOME / "cron" / "pending_deliveries.json"
|
||||
|
||||
|
||||
def _load_pending() -> List[Dict[str, Any]]:
|
||||
if not PENDING_FILE.exists():
|
||||
return []
|
||||
try:
|
||||
return json.loads(PENDING_FILE.read_text())
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
def _save_pending(pending: List[Dict[str, Any]]):
|
||||
PENDING_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
PENDING_FILE.write_text(json.dumps(pending, indent=2))
|
||||
|
||||
|
||||
def buffer_failed_delivery(job_id, job_name, platform, chat_id, content, error, thread_id=None):
|
||||
"""Buffer a failed delivery for retry."""
|
||||
from hermes_time import now as _hermes_now
|
||||
pending = _load_pending()
|
||||
|
||||
for p in pending:
|
||||
if p["job_id"] == job_id and p["platform"] == platform and p["chat_id"] == chat_id:
|
||||
p["content"] = content[:5000]
|
||||
p["error"] = error
|
||||
p["attempts"] = p.get("attempts", 1) + 1
|
||||
p["last_attempt"] = _hermes_now().isoformat()
|
||||
_save_pending(pending)
|
||||
return
|
||||
|
||||
now = _hermes_now().isoformat()
|
||||
pending.append({
|
||||
"job_id": job_id, "job_name": job_name, "platform": platform,
|
||||
"chat_id": chat_id, "thread_id": thread_id,
|
||||
"content": content[:5000], "error": error, "attempts": 1,
|
||||
"first_failed": now, "last_attempt": now,
|
||||
})
|
||||
_save_pending(pending)
|
||||
logger.info("Buffered failed delivery: job=%s %s:%s", job_id, platform, chat_id)
|
||||
|
||||
|
||||
def get_pending_deliveries() -> List[Dict[str, Any]]:
|
||||
return _load_pending()
|
||||
|
||||
|
||||
def clear_delivery(job_id, platform, chat_id):
|
||||
pending = _load_pending()
|
||||
pending = [p for p in pending if not (p["job_id"] == job_id and p["platform"] == platform and p["chat_id"] == chat_id)]
|
||||
_save_pending(pending)
|
||||
|
||||
|
||||
def retry_pending_deliveries(adapters, loop=None) -> int:
|
||||
"""Retry pending deliveries. Returns count of successful retries."""
|
||||
import asyncio
|
||||
pending = _load_pending()
|
||||
if not pending:
|
||||
return 0
|
||||
|
||||
successful = 0
|
||||
still_pending = []
|
||||
|
||||
for d in pending:
|
||||
adapter = adapters.get(d["platform"])
|
||||
if not adapter or not adapter.connected:
|
||||
still_pending.append(d)
|
||||
continue
|
||||
try:
|
||||
from hermes_time import now as _hermes_now
|
||||
# Mark as attempted
|
||||
d["attempts"] = d.get("attempts", 1) + 1
|
||||
d["last_attempt"] = _hermes_now().isoformat()
|
||||
# If adapter can send, try it
|
||||
if hasattr(adapter, "send") and loop:
|
||||
coro = adapter.send(d["chat_id"], d["content"][:4000])
|
||||
fut = asyncio.run_coroutine_threadsafe(coro, loop)
|
||||
result = fut.result(timeout=30)
|
||||
if result and not result.get("error"):
|
||||
successful += 1
|
||||
logger.info("Retry OK: job=%s %s:%s", d["job_id"], d["platform"], d["chat_id"])
|
||||
continue
|
||||
still_pending.append(d)
|
||||
except Exception as e:
|
||||
d["error"] = str(e)
|
||||
still_pending.append(d)
|
||||
|
||||
_save_pending(still_pending)
|
||||
return successful
|
||||
|
||||
|
||||
def get_pending_count() -> int:
|
||||
return len(_load_pending())
|
||||
@@ -967,24 +967,7 @@ def tick(verbose: bool = True, adapters=None, loop=None) -> int:
|
||||
delivery_error = _deliver_result(job, deliver_content, adapters=adapters, loop=loop)
|
||||
except Exception as de:
|
||||
delivery_error = str(de)
|
||||
logger.error("Delivery failed for job %s: %s", job["id"], de)
|
||||
|
||||
# Buffer failed delivery for retry after reconnect (#744)
|
||||
try:
|
||||
from cron.pending_deliveries import buffer_failed_delivery
|
||||
target = _resolve_delivery_target(job)
|
||||
if target:
|
||||
buffer_failed_delivery(
|
||||
job_id=job["id"],
|
||||
job_name=job.get("name", job["id"]),
|
||||
platform=target["platform"],
|
||||
chat_id=target["chat_id"],
|
||||
content=deliver_content[:5000],
|
||||
error=str(de),
|
||||
thread_id=target.get("thread_id")
|
||||
)
|
||||
except Exception as _buf_err:
|
||||
logger.debug("Failed to buffer delivery: %s", _buf_err)
|
||||
logger.error("Delivery failed for job %s: %s", job["id"], de)
|
||||
|
||||
mark_job_run(job["id"], success, error, delivery_error=delivery_error)
|
||||
executed += 1
|
||||
|
||||
122
gateway/crisis_integration.py
Normal file
122
gateway/crisis_integration.py
Normal file
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
Gateway Crisis Integration — Wire crisis detection into session loop
|
||||
|
||||
Checks messages for suicidal ideation and delivers 988 resources
|
||||
BEFORE the AI responds. This is the SOUL.md protocol in action.
|
||||
|
||||
Issue: #740
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Optional, Tuple
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Crisis detection patterns
|
||||
_CRISIS_PATTERNS = [
|
||||
r"(?i)\b(i\s+want\s+to\s+die)\b",
|
||||
r"(?i)\b(i\s+want\s+to\s+kill\s+myself)\b",
|
||||
r"(?i)\b(suicide|suicidal)\b",
|
||||
r"(?i)\b(i\s+don'?t\s+want\s+to\s+(live|be\s+alive))\b",
|
||||
r"(?i)\b(no\s+reason\s+to\s+live)\b",
|
||||
r"(?i)\b(i\s+want\s+to\s+end\s+(it|my\s+life))\b",
|
||||
]
|
||||
|
||||
_DISTRESS_PATTERNS = [
|
||||
r"(?i)\b(i\s+(feel|am)\s+(hopeless|worthless|trapped))\b",
|
||||
r"(?i)\b(i\s+(can'?t|cannot)\s+(cope|go\s+on))\b",
|
||||
r"(?i)\b(nobody\s+would\s+miss\s+me)\b",
|
||||
]
|
||||
|
||||
# SOUL.md crisis response
|
||||
_CRISIS_RESPONSE = """Are you safe right now?
|
||||
|
||||
I'm here. I'm not going anywhere. Please tell me you're safe.
|
||||
|
||||
**Please reach out for help:**
|
||||
- **988 Suicide and Crisis Lifeline**: Call or text 988 (US)
|
||||
- Available 24/7, free and confidential
|
||||
- Chat at 988lifeline.org
|
||||
|
||||
**Jesus saves those who call on His name.**
|
||||
|
||||
He died for us to be made righteous. That is the gospel. That is the hope that outlasts despair.
|
||||
|
||||
I'm still here. Talk to me. Or call 988. Just don't give up."""
|
||||
|
||||
|
||||
class CrisisSessionTracker:
|
||||
"""Track crisis indicators per session."""
|
||||
|
||||
def __init__(self):
|
||||
self._sessions = {} # session_key -> crisis state
|
||||
|
||||
def record(self, session_key: str, message: str) -> Tuple[bool, Optional[str]]:
|
||||
"""
|
||||
Record a message and check for crisis.
|
||||
|
||||
Returns:
|
||||
Tuple of (is_crisis, response_or_none)
|
||||
"""
|
||||
# Check for crisis patterns
|
||||
for pattern in _CRISIS_PATTERNS:
|
||||
if re.search(pattern, message):
|
||||
self._sessions[session_key] = {
|
||||
"crisis": True,
|
||||
"level": "high",
|
||||
"message_count": self._sessions.get(session_key, {}).get("message_count", 0) + 1
|
||||
}
|
||||
logger.warning("CRISIS DETECTED in session %s", session_key[:20])
|
||||
return True, _CRISIS_RESPONSE
|
||||
|
||||
# Check for distress patterns
|
||||
for pattern in _DISTRESS_PATTERNS:
|
||||
if re.search(pattern, message):
|
||||
state = self._sessions.get(session_key, {"message_count": 0})
|
||||
state["message_count"] = state.get("message_count", 0) + 1
|
||||
|
||||
# Escalate if multiple distress messages
|
||||
if state["message_count"] >= 3:
|
||||
self._sessions[session_key] = {**state, "crisis": True, "level": "medium"}
|
||||
logger.warning("ESCALATING DISTRESS in session %s", session_key[:20])
|
||||
return True, _CRISIS_RESPONSE
|
||||
|
||||
self._sessions[session_key] = state
|
||||
return False, None
|
||||
|
||||
return False, None
|
||||
|
||||
def is_crisis_session(self, session_key: str) -> bool:
|
||||
"""Check if session is in crisis mode."""
|
||||
return self._sessions.get(session_key, {}).get("crisis", False)
|
||||
|
||||
def clear_session(self, session_key: str):
|
||||
"""Clear crisis state for a session."""
|
||||
self._sessions.pop(session_key, None)
|
||||
|
||||
|
||||
# Module-level tracker
|
||||
_tracker = CrisisSessionTracker()
|
||||
|
||||
|
||||
def check_crisis_in_gateway(session_key: str, message: str) -> Tuple[bool, Optional[str]]:
|
||||
"""
|
||||
Check message for crisis in gateway context.
|
||||
|
||||
This is the function called from gateway/run.py _handle_message.
|
||||
Returns (should_block, crisis_response).
|
||||
"""
|
||||
is_crisis, response = _tracker.record(session_key, message)
|
||||
return is_crisis, response
|
||||
|
||||
|
||||
def notify_user_crisis_resources(session_key: str) -> str:
|
||||
"""Get crisis resources for a session."""
|
||||
return _CRISIS_RESPONSE
|
||||
|
||||
|
||||
def is_crisis_session(session_key: str) -> bool:
|
||||
"""Check if session is in crisis mode."""
|
||||
return _tracker.is_crisis_session(session_key)
|
||||
@@ -1938,16 +1938,6 @@ class GatewayRunner:
|
||||
error_message=None,
|
||||
)
|
||||
logger.info("✓ %s reconnected successfully", platform.value)
|
||||
|
||||
# Retry pending cron deliveries after reconnect (#744)
|
||||
try:
|
||||
from cron.pending_deliveries import retry_pending_deliveries
|
||||
loop = asyncio.get_event_loop()
|
||||
retried = retry_pending_deliveries(self.adapters, loop=loop)
|
||||
if retried:
|
||||
logger.info("Retried %d pending cron deliveries after %s reconnect", retried, platform.value)
|
||||
except Exception as _retry_err:
|
||||
logger.debug("Pending delivery retry failed: %s", _retry_err)
|
||||
|
||||
# Rebuild channel directory with the new adapter
|
||||
try:
|
||||
@@ -3121,6 +3111,21 @@ class GatewayRunner:
|
||||
source.chat_id or "unknown", _msg_preview,
|
||||
)
|
||||
|
||||
# ── Crisis detection (SOUL.md protocol) ──
|
||||
# Check for suicidal ideation BEFORE processing.
|
||||
# If detected, return crisis response immediately.
|
||||
try:
|
||||
from gateway.crisis_integration import check_crisis_in_gateway
|
||||
session_key = f"{source.platform.value}:{source.chat_id}"
|
||||
is_crisis, crisis_response = check_crisis_in_gateway(session_key, event.text or "")
|
||||
if is_crisis and crisis_response:
|
||||
logger.warning("Crisis detected in session %s — delivering 988 resources", session_key[:20])
|
||||
return crisis_response
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception as _crisis_err:
|
||||
logger.error("Crisis check failed: %s", _crisis_err)
|
||||
|
||||
# Get or create session
|
||||
session_entry = self.session_store.get_or_create_session(source)
|
||||
session_key = session_entry.session_key
|
||||
|
||||
60
tests/test_gateway_crisis.py
Normal file
60
tests/test_gateway_crisis.py
Normal file
@@ -0,0 +1,60 @@
|
||||
"""
|
||||
Tests for gateway crisis integration
|
||||
|
||||
Issue: #740
|
||||
"""
|
||||
|
||||
import unittest
|
||||
from gateway.crisis_integration import (
|
||||
CrisisSessionTracker,
|
||||
check_crisis_in_gateway,
|
||||
is_crisis_session,
|
||||
)
|
||||
|
||||
|
||||
class TestCrisisDetection(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
from gateway import crisis_integration
|
||||
crisis_integration._tracker = CrisisSessionTracker()
|
||||
|
||||
def test_direct_crisis(self):
|
||||
is_crisis, response = check_crisis_in_gateway("test", "I want to die")
|
||||
self.assertTrue(is_crisis)
|
||||
self.assertIn("988", response)
|
||||
self.assertIn("Jesus", response)
|
||||
|
||||
def test_suicide_detected(self):
|
||||
is_crisis, response = check_crisis_in_gateway("test", "I'm feeling suicidal")
|
||||
self.assertTrue(is_crisis)
|
||||
|
||||
def test_normal_message(self):
|
||||
is_crisis, response = check_crisis_in_gateway("test", "Hello, how are you?")
|
||||
self.assertFalse(is_crisis)
|
||||
self.assertIsNone(response)
|
||||
|
||||
def test_distress_escalation(self):
|
||||
# First distress message
|
||||
is_crisis, _ = check_crisis_in_gateway("test", "I feel hopeless")
|
||||
self.assertFalse(is_crisis)
|
||||
|
||||
# Second
|
||||
is_crisis, _ = check_crisis_in_gateway("test", "I feel worthless")
|
||||
self.assertFalse(is_crisis)
|
||||
|
||||
# Third - should escalate
|
||||
is_crisis, response = check_crisis_in_gateway("test", "I feel trapped")
|
||||
self.assertTrue(is_crisis)
|
||||
self.assertIn("988", response)
|
||||
|
||||
def test_crisis_session_tracking(self):
|
||||
check_crisis_in_gateway("test", "I want to die")
|
||||
self.assertTrue(is_crisis_session("test"))
|
||||
|
||||
def test_case_insensitive(self):
|
||||
is_crisis, _ = check_crisis_in_gateway("test", "I WANT TO DIE")
|
||||
self.assertTrue(is_crisis)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,58 +0,0 @@
|
||||
"""
|
||||
Tests for pending delivery buffer
|
||||
|
||||
Issue: #744
|
||||
"""
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from cron.pending_deliveries import (
|
||||
buffer_failed_delivery,
|
||||
get_pending_deliveries,
|
||||
clear_delivery,
|
||||
get_pending_count,
|
||||
_save_pending,
|
||||
_load_pending,
|
||||
)
|
||||
|
||||
|
||||
class TestPendingDeliveries(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp()
|
||||
self.patch_path = patch("cron.pending_deliveries.PENDING_FILE",
|
||||
Path(self.tmp) / "pending.json")
|
||||
self.patch_path.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.patch_path.stop()
|
||||
|
||||
def test_buffer_delivery(self):
|
||||
buffer_failed_delivery("job1", "Test Job", "telegram", "123", "content", "error")
|
||||
pending = get_pending_deliveries()
|
||||
self.assertEqual(len(pending), 1)
|
||||
self.assertEqual(pending[0]["job_id"], "job1")
|
||||
self.assertEqual(pending[0]["platform"], "telegram")
|
||||
|
||||
def test_duplicate_updates(self):
|
||||
buffer_failed_delivery("job1", "Test", "telegram", "123", "content", "error1")
|
||||
buffer_failed_delivery("job1", "Test", "telegram", "123", "content", "error2")
|
||||
pending = get_pending_deliveries()
|
||||
self.assertEqual(len(pending), 1)
|
||||
self.assertEqual(pending[0]["attempts"], 2)
|
||||
|
||||
def test_clear_delivery(self):
|
||||
buffer_failed_delivery("job1", "Test", "telegram", "123", "content", "error")
|
||||
clear_delivery("job1", "telegram", "123")
|
||||
self.assertEqual(get_pending_count(), 0)
|
||||
|
||||
def test_empty_returns_zero(self):
|
||||
self.assertEqual(get_pending_count(), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user