Compare commits
13 Commits
v7.0.0
...
mimo/code/
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
60af11ec2f | ||
| a0964a2fbf | |||
| 1e7bb2a453 | |||
| 847c4d50d4 | |||
|
|
220f20c794 | ||
| e85cefd9c0 | |||
| beec49a92d | |||
| ef25c073ce | |||
| 5ce928a00d | |||
| 61871cf6ed | |||
| 6f949698fe | |||
| 6cf1f4d078 | |||
|
|
ef74536e33 |
171
app.js
171
app.js
@@ -3,6 +3,7 @@ import { EffectComposer } from 'three/addons/postprocessing/EffectComposer.js';
|
|||||||
import { RenderPass } from 'three/addons/postprocessing/RenderPass.js';
|
import { RenderPass } from 'three/addons/postprocessing/RenderPass.js';
|
||||||
import { UnrealBloomPass } from 'three/addons/postprocessing/UnrealBloomPass.js';
|
import { UnrealBloomPass } from 'three/addons/postprocessing/UnrealBloomPass.js';
|
||||||
import { SMAAPass } from 'three/addons/postprocessing/SMAAPass.js';
|
import { SMAAPass } from 'three/addons/postprocessing/SMAAPass.js';
|
||||||
|
import { SpatialMemory } from './nexus/components/spatial-memory.js';
|
||||||
|
|
||||||
// ═══════════════════════════════════════════
|
// ═══════════════════════════════════════════
|
||||||
// NEXUS v1.1 — Portal System Update
|
// NEXUS v1.1 — Portal System Update
|
||||||
@@ -703,6 +704,7 @@ async function init() {
|
|||||||
createSessionPowerMeter();
|
createSessionPowerMeter();
|
||||||
createWorkshopTerminal();
|
createWorkshopTerminal();
|
||||||
createAshStorm();
|
createAshStorm();
|
||||||
|
SpatialMemory.init(scene);
|
||||||
updateLoad(90);
|
updateLoad(90);
|
||||||
|
|
||||||
loadSession();
|
loadSession();
|
||||||
@@ -2573,6 +2575,13 @@ function gameLoop() {
|
|||||||
|
|
||||||
updateAshStorm(delta, elapsed);
|
updateAshStorm(delta, elapsed);
|
||||||
|
|
||||||
|
// Project Mnemosyne - Memory Orb Animation
|
||||||
|
if (typeof animateMemoryOrbs === 'function') {
|
||||||
|
SpatialMemory.update(delta);
|
||||||
|
animateMemoryOrbs(delta);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
const mode = NAV_MODES[navModeIdx];
|
const mode = NAV_MODES[navModeIdx];
|
||||||
const chatActive = document.activeElement === document.getElementById('chat-input');
|
const chatActive = document.activeElement === document.getElementById('chat-input');
|
||||||
|
|
||||||
@@ -2771,6 +2780,12 @@ function gameLoop() {
|
|||||||
composer.render();
|
composer.render();
|
||||||
|
|
||||||
updateAshStorm(delta, elapsed);
|
updateAshStorm(delta, elapsed);
|
||||||
|
|
||||||
|
// Project Mnemosyne - Memory Orb Animation
|
||||||
|
if (typeof animateMemoryOrbs === 'function') {
|
||||||
|
animateMemoryOrbs(delta);
|
||||||
|
}
|
||||||
|
|
||||||
updatePortalTunnel(delta, elapsed);
|
updatePortalTunnel(delta, elapsed);
|
||||||
|
|
||||||
if (workshopScanMat) workshopScanMat.uniforms.uTime.value = clock.getElapsedTime();
|
if (workshopScanMat) workshopScanMat.uniforms.uTime.value = clock.getElapsedTime();
|
||||||
@@ -2933,9 +2948,165 @@ function updateAshStorm(delta, elapsed) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// ═══════════════════════════════════════════
|
||||||
|
// PROJECT MNEMOSYNE — HOLOGRAPHIC MEMORY ORBS
|
||||||
|
// ═══════════════════════════════════════════
|
||||||
|
|
||||||
|
// Memory orbs registry for animation loop
|
||||||
|
const memoryOrbs = [];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Spawn a glowing memory orb at the given position.
|
||||||
|
* Used to visualize RAG retrievals and memory recalls in the Nexus.
|
||||||
|
*
|
||||||
|
* @param {THREE.Vector3} position - World position for the orb
|
||||||
|
* @param {number} color - Hex color (default: 0x4af0c0 - cyan)
|
||||||
|
* @param {number} size - Radius of the orb (default: 0.5)
|
||||||
|
* @param {object} metadata - Optional metadata for the memory (source, timestamp, etc.)
|
||||||
|
* @returns {THREE.Mesh} The created orb mesh
|
||||||
|
*/
|
||||||
|
function spawnMemoryOrb(position, color = 0x4af0c0, size = 0.5, metadata = {}) {
|
||||||
|
if (typeof THREE === 'undefined' || typeof scene === 'undefined') {
|
||||||
|
console.warn('[Mnemosyne] THREE/scene not available for orb spawn');
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const geometry = new THREE.SphereGeometry(size, 32, 32);
|
||||||
|
const material = new THREE.MeshStandardMaterial({
|
||||||
|
color: color,
|
||||||
|
emissive: color,
|
||||||
|
emissiveIntensity: 2.5,
|
||||||
|
metalness: 0.3,
|
||||||
|
roughness: 0.2,
|
||||||
|
transparent: true,
|
||||||
|
opacity: 0.85,
|
||||||
|
envMapIntensity: 1.5
|
||||||
|
});
|
||||||
|
|
||||||
|
const orb = new THREE.Mesh(geometry, material);
|
||||||
|
orb.position.copy(position);
|
||||||
|
orb.castShadow = true;
|
||||||
|
orb.receiveShadow = true;
|
||||||
|
|
||||||
|
orb.userData = {
|
||||||
|
type: 'memory_orb',
|
||||||
|
pulse: Math.random() * Math.PI * 2, // Random phase offset
|
||||||
|
pulseSpeed: 0.002 + Math.random() * 0.001,
|
||||||
|
originalScale: size,
|
||||||
|
metadata: metadata,
|
||||||
|
createdAt: Date.now()
|
||||||
|
};
|
||||||
|
|
||||||
|
// Point light for local illumination
|
||||||
|
const light = new THREE.PointLight(color, 1.5, 8);
|
||||||
|
orb.add(light);
|
||||||
|
|
||||||
|
scene.add(orb);
|
||||||
|
memoryOrbs.push(orb);
|
||||||
|
|
||||||
|
console.info('[Mnemosyne] Memory orb spawned:', metadata.source || 'unknown');
|
||||||
|
return orb;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remove a memory orb from the scene and dispose resources.
|
||||||
|
* @param {THREE.Mesh} orb - The orb to remove
|
||||||
|
*/
|
||||||
|
function removeMemoryOrb(orb) {
|
||||||
|
if (!orb) return;
|
||||||
|
|
||||||
|
if (orb.parent) orb.parent.remove(orb);
|
||||||
|
if (orb.geometry) orb.geometry.dispose();
|
||||||
|
if (orb.material) orb.material.dispose();
|
||||||
|
|
||||||
|
const idx = memoryOrbs.indexOf(orb);
|
||||||
|
if (idx > -1) memoryOrbs.splice(idx, 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Animate all memory orbs — pulse, rotate, and fade.
|
||||||
|
* Called from gameLoop() every frame.
|
||||||
|
* @param {number} delta - Time since last frame
|
||||||
|
*/
|
||||||
|
function animateMemoryOrbs(delta) {
|
||||||
|
for (let i = memoryOrbs.length - 1; i >= 0; i--) {
|
||||||
|
const orb = memoryOrbs[i];
|
||||||
|
if (!orb || !orb.userData) continue;
|
||||||
|
|
||||||
|
// Pulse animation
|
||||||
|
orb.userData.pulse += orb.userData.pulseSpeed * delta * 1000;
|
||||||
|
const pulseFactor = 1 + Math.sin(orb.userData.pulse) * 0.1;
|
||||||
|
orb.scale.setScalar(pulseFactor * orb.userData.originalScale);
|
||||||
|
|
||||||
|
// Gentle rotation
|
||||||
|
orb.rotation.y += delta * 0.5;
|
||||||
|
|
||||||
|
// Fade after 30 seconds
|
||||||
|
const age = (Date.now() - orb.userData.createdAt) / 1000;
|
||||||
|
if (age > 30) {
|
||||||
|
const fadeDuration = 10;
|
||||||
|
const fadeProgress = Math.min(1, (age - 30) / fadeDuration);
|
||||||
|
orb.material.opacity = 0.85 * (1 - fadeProgress);
|
||||||
|
|
||||||
|
if (fadeProgress >= 1) {
|
||||||
|
removeMemoryOrb(orb);
|
||||||
|
i--; // Adjust index after removal
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Spawn memory orbs arranged in a spiral for RAG retrieval results.
|
||||||
|
* @param {Array} results - Array of {content, score, source}
|
||||||
|
* @param {THREE.Vector3} center - Center position (default: above avatar)
|
||||||
|
*/
|
||||||
|
function spawnRetrievalOrbs(results, center) {
|
||||||
|
if (!results || !Array.isArray(results) || results.length === 0) return;
|
||||||
|
|
||||||
|
if (!center) {
|
||||||
|
center = new THREE.Vector3(0, 2, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
const colors = [0x4af0c0, 0x7b5cff, 0xffd700, 0xff4466, 0x00ff88];
|
||||||
|
const radius = 3;
|
||||||
|
|
||||||
|
results.forEach((result, i) => {
|
||||||
|
const angle = (i / results.length) * Math.PI * 2;
|
||||||
|
const height = (i / results.length) * 2 - 1;
|
||||||
|
|
||||||
|
const position = new THREE.Vector3(
|
||||||
|
center.x + Math.cos(angle) * radius,
|
||||||
|
center.y + height,
|
||||||
|
center.z + Math.sin(angle) * radius
|
||||||
|
);
|
||||||
|
|
||||||
|
const colorIdx = Math.min(colors.length - 1, Math.floor((result.score || 0.5) * colors.length));
|
||||||
|
const size = 0.3 + (result.score || 0.5) * 0.4;
|
||||||
|
|
||||||
|
spawnMemoryOrb(position, colors[colorIdx], size, {
|
||||||
|
source: result.source || 'unknown',
|
||||||
|
score: result.score || 0,
|
||||||
|
contentPreview: (result.content || '').substring(0, 100)
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
init().then(() => {
|
init().then(() => {
|
||||||
createAshStorm();
|
createAshStorm();
|
||||||
createPortalTunnel();
|
createPortalTunnel();
|
||||||
|
|
||||||
|
// Project Mnemosyne — seed demo spatial memories
|
||||||
|
const demoMemories = [
|
||||||
|
{ id: 'mem_nexus_birth', content: 'The Nexus came online — first render of the 3D world', category: 'knowledge', strength: 0.95, connections: ['mem_mnemosyne_start'] },
|
||||||
|
{ id: 'mem_first_portal', content: 'First portal deployed — connection to external service', category: 'engineering', strength: 0.85, connections: ['mem_nexus_birth'] },
|
||||||
|
{ id: 'mem_hermes_chat', content: 'First conversation through the Hermes gateway', category: 'social', strength: 0.7, connections: [] },
|
||||||
|
{ id: 'mem_mnemosyne_start', content: 'Project Mnemosyne began — the living archive awakens', category: 'projects', strength: 0.9, connections: ['mem_nexus_birth', 'mem_spatial_schema'] },
|
||||||
|
{ id: 'mem_spatial_schema', content: 'Spatial Memory Schema defined — memories gain permanent homes', category: 'engineering', strength: 0.8, connections: ['mem_mnemosyne_start'] },
|
||||||
|
];
|
||||||
|
demoMemories.forEach(m => SpatialMemory.placeMemory(m));
|
||||||
|
|
||||||
fetchGiteaData();
|
fetchGiteaData();
|
||||||
setInterval(fetchGiteaData, 30000);
|
setInterval(fetchGiteaData, 30000);
|
||||||
runWeeklyAudit();
|
runWeeklyAudit();
|
||||||
|
|||||||
@@ -152,17 +152,55 @@ class OpenAITTSAdapter:
|
|||||||
return mp3_path
|
return mp3_path
|
||||||
|
|
||||||
|
|
||||||
|
class EdgeTTSAdapter:
|
||||||
|
"""Zero-cost TTS using Microsoft Edge neural voices (no API key required).
|
||||||
|
|
||||||
|
Requires: pip install edge-tts>=6.1.9
|
||||||
|
Voices: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support
|
||||||
|
"""
|
||||||
|
|
||||||
|
DEFAULT_VOICE = "en-US-GuyNeural"
|
||||||
|
|
||||||
|
def __init__(self, config: TTSConfig):
|
||||||
|
self.config = config
|
||||||
|
self.voice = config.voice_id or self.DEFAULT_VOICE
|
||||||
|
|
||||||
|
def synthesize(self, text: str, output_path: Path) -> Path:
|
||||||
|
try:
|
||||||
|
import edge_tts
|
||||||
|
except ImportError:
|
||||||
|
raise RuntimeError("edge-tts not installed. Run: pip install edge-tts")
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
mp3_path = output_path.with_suffix(".mp3")
|
||||||
|
|
||||||
|
async def _run():
|
||||||
|
communicate = edge_tts.Communicate(text, self.voice)
|
||||||
|
await communicate.save(str(mp3_path))
|
||||||
|
|
||||||
|
asyncio.run(_run())
|
||||||
|
return mp3_path
|
||||||
|
|
||||||
|
|
||||||
ADAPTERS = {
|
ADAPTERS = {
|
||||||
"piper": PiperAdapter,
|
"piper": PiperAdapter,
|
||||||
"elevenlabs": ElevenLabsAdapter,
|
"elevenlabs": ElevenLabsAdapter,
|
||||||
"openai": OpenAITTSAdapter,
|
"openai": OpenAITTSAdapter,
|
||||||
|
"edge-tts": EdgeTTSAdapter,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def get_provider_config() -> TTSConfig:
|
def get_provider_config() -> TTSConfig:
|
||||||
"""Load TTS configuration from environment."""
|
"""Load TTS configuration from environment."""
|
||||||
provider = os.environ.get("DEEPDIVE_TTS_PROVIDER", "openai")
|
provider = os.environ.get("DEEPDIVE_TTS_PROVIDER", "openai")
|
||||||
voice = os.environ.get("DEEPDIVE_TTS_VOICE", "alloy" if provider == "openai" else "matthew")
|
if provider == "openai":
|
||||||
|
default_voice = "alloy"
|
||||||
|
elif provider == "edge-tts":
|
||||||
|
default_voice = EdgeTTSAdapter.DEFAULT_VOICE
|
||||||
|
else:
|
||||||
|
default_voice = "matthew"
|
||||||
|
voice = os.environ.get("DEEPDIVE_TTS_VOICE", default_voice)
|
||||||
|
|
||||||
return TTSConfig(
|
return TTSConfig(
|
||||||
provider=provider,
|
provider=provider,
|
||||||
|
|||||||
@@ -32,12 +32,14 @@ import importlib.util
|
|||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
level=logging.INFO,
|
level=logging.INFO,
|
||||||
@@ -212,6 +214,46 @@ def generate_report(date_str: str, checker_mod) -> str:
|
|||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Voice memo ────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _generate_voice_memo(report_text: str, date_str: str) -> Optional[str]:
|
||||||
|
"""Generate an MP3 voice memo of the night watch report.
|
||||||
|
|
||||||
|
Returns the output path on success, or None if generation fails.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
import edge_tts
|
||||||
|
except ImportError:
|
||||||
|
logger.warning("edge-tts not installed; skipping voice memo. Run: pip install edge-tts")
|
||||||
|
return None
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
# Strip markdown formatting for cleaner speech
|
||||||
|
clean = report_text
|
||||||
|
clean = re.sub(r"#+\s*", "", clean) # headings
|
||||||
|
clean = re.sub(r"\|", " ", clean) # table pipes
|
||||||
|
clean = re.sub(r"\*+", "", clean) # bold/italic markers
|
||||||
|
clean = re.sub(r"-{3,}", "", clean) # horizontal rules
|
||||||
|
clean = re.sub(r"\s{2,}", " ", clean) # collapse extra whitespace
|
||||||
|
|
||||||
|
output_dir = Path("/tmp/bezalel")
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
mp3_path = output_dir / f"night-watch-{date_str}.mp3"
|
||||||
|
|
||||||
|
try:
|
||||||
|
async def _run():
|
||||||
|
communicate = edge_tts.Communicate(clean.strip(), "en-US-GuyNeural")
|
||||||
|
await communicate.save(str(mp3_path))
|
||||||
|
|
||||||
|
asyncio.run(_run())
|
||||||
|
logger.info("Voice memo written to %s", mp3_path)
|
||||||
|
return str(mp3_path)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Voice memo generation failed: %s", exc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
# ── Entry point ───────────────────────────────────────────────────────
|
# ── Entry point ───────────────────────────────────────────────────────
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
@@ -226,6 +268,10 @@ def main() -> None:
|
|||||||
"--dry-run", action="store_true",
|
"--dry-run", action="store_true",
|
||||||
help="Print report to stdout instead of writing to disk",
|
help="Print report to stdout instead of writing to disk",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--voice-memo", action="store_true",
|
||||||
|
help="Generate an MP3 voice memo of the report using edge-tts (saved to /tmp/bezalel/)",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
date_str = args.date or datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
date_str = args.date or datetime.now(timezone.utc).strftime("%Y-%m-%d")
|
||||||
@@ -242,6 +288,14 @@ def main() -> None:
|
|||||||
report_path.write_text(report_text)
|
report_path.write_text(report_text)
|
||||||
logger.info("Night Watch report written to %s", report_path)
|
logger.info("Night Watch report written to %s", report_path)
|
||||||
|
|
||||||
|
if args.voice_memo:
|
||||||
|
try:
|
||||||
|
memo_path = _generate_voice_memo(report_text, date_str)
|
||||||
|
if memo_path:
|
||||||
|
logger.info("Voice memo: %s", memo_path)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Voice memo failed (non-fatal): %s", exc)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
46
docker-compose.desktop.yml
Normal file
46
docker-compose.desktop.yml
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
version: "3.9"
|
||||||
|
|
||||||
|
# Sandboxed desktop environment for Hermes computer-use primitives.
|
||||||
|
# Provides Xvfb (virtual framebuffer) + noVNC (browser-accessible VNC).
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# docker compose -f docker-compose.desktop.yml up -d
|
||||||
|
# # Visit http://localhost:6080 to see the virtual desktop
|
||||||
|
#
|
||||||
|
# docker compose -f docker-compose.desktop.yml run hermes-desktop \
|
||||||
|
# python -m nexus.computer_use_demo
|
||||||
|
#
|
||||||
|
# docker compose -f docker-compose.desktop.yml down
|
||||||
|
|
||||||
|
services:
|
||||||
|
hermes-desktop:
|
||||||
|
image: dorowu/ubuntu-desktop-lxde-vnc:focal
|
||||||
|
environment:
|
||||||
|
# Resolution for the virtual display
|
||||||
|
RESOLUTION: "1280x800"
|
||||||
|
# VNC password (change in production)
|
||||||
|
VNC_PASSWORD: "hermes"
|
||||||
|
# Disable HTTP password for development convenience
|
||||||
|
HTTP_PASSWORD: ""
|
||||||
|
ports:
|
||||||
|
# noVNC web interface
|
||||||
|
- "6080:80"
|
||||||
|
# Raw VNC port (optional)
|
||||||
|
- "5900:5900"
|
||||||
|
volumes:
|
||||||
|
# Mount repo into container so scripts are available
|
||||||
|
- .:/workspace
|
||||||
|
# Persist nexus runtime data (heartbeats, logs, evidence)
|
||||||
|
- nexus_data:/root/.nexus
|
||||||
|
working_dir: /workspace
|
||||||
|
shm_size: "256mb"
|
||||||
|
# Install Python deps on startup then keep container alive
|
||||||
|
command: >
|
||||||
|
bash -c "
|
||||||
|
pip install --quiet pyautogui Pillow &&
|
||||||
|
/startup.sh
|
||||||
|
"
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
nexus_data:
|
||||||
|
driver: local
|
||||||
174
docs/computer-use.md
Normal file
174
docs/computer-use.md
Normal file
@@ -0,0 +1,174 @@
|
|||||||
|
# Computer Use — Desktop Automation Primitives for Hermes
|
||||||
|
|
||||||
|
Issue: [#1125](https://forge.alexanderwhitestone.com/Timmy_Foundation/the-nexus/issues/1125)
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
`nexus/computer_use.py` adds desktop automation primitives to the Hermes fleet. Agents can take screenshots, click, type, and scroll — enough to drive a browser, validate a UI, or diagnose a failed workflow page visually.
|
||||||
|
|
||||||
|
All actions are logged to a JSONL audit trail at `~/.nexus/computer_use_actions.jsonl`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### Local (requires a real display or Xvfb)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Install dependencies
|
||||||
|
pip install pyautogui Pillow
|
||||||
|
|
||||||
|
# Run the Phase 1 demo
|
||||||
|
python -m nexus.computer_use_demo
|
||||||
|
```
|
||||||
|
|
||||||
|
### Sandboxed (Docker + Xvfb + noVNC)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker-compose.desktop.yml up -d
|
||||||
|
# Visit http://localhost:6080 in your browser to see the virtual desktop
|
||||||
|
|
||||||
|
docker compose -f docker-compose.desktop.yml run hermes-desktop \
|
||||||
|
python -m nexus.computer_use_demo
|
||||||
|
|
||||||
|
docker compose -f docker-compose.desktop.yml down
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## API Reference
|
||||||
|
|
||||||
|
### `computer_screenshot(save_path=None, log_path=...)`
|
||||||
|
|
||||||
|
Capture the current desktop.
|
||||||
|
|
||||||
|
| Param | Type | Description |
|
||||||
|
|-------|------|-------------|
|
||||||
|
| `save_path` | `str \| None` | Path to save PNG. If `None`, returns base64 string. |
|
||||||
|
| `log_path` | `Path` | Audit log file. |
|
||||||
|
|
||||||
|
**Returns** `dict`:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"ok": true,
|
||||||
|
"image_b64": "<base64 PNG or null>",
|
||||||
|
"saved_to": "<path or null>",
|
||||||
|
"error": null
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### `computer_click(x, y, button="left", confirm=False, log_path=...)`
|
||||||
|
|
||||||
|
Click the mouse at screen coordinates.
|
||||||
|
|
||||||
|
| Param | Type | Description |
|
||||||
|
|-------|------|-------------|
|
||||||
|
| `x` | `int` | Horizontal coordinate |
|
||||||
|
| `y` | `int` | Vertical coordinate |
|
||||||
|
| `button` | `str` | `"left"` \| `"right"` \| `"middle"` |
|
||||||
|
| `confirm` | `bool` | Required `True` for `right` / `middle` (poka-yoke) |
|
||||||
|
|
||||||
|
**Returns** `dict`:
|
||||||
|
```json
|
||||||
|
{"ok": true, "error": null}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### `computer_type(text, confirm=False, interval=0.02, log_path=...)`
|
||||||
|
|
||||||
|
Type text using the keyboard.
|
||||||
|
|
||||||
|
| Param | Type | Description |
|
||||||
|
|-------|------|-------------|
|
||||||
|
| `text` | `str` | Text to type |
|
||||||
|
| `confirm` | `bool` | Required `True` when text contains a sensitive keyword |
|
||||||
|
| `interval` | `float` | Delay between keystrokes (seconds) |
|
||||||
|
|
||||||
|
**Sensitive keywords** (require `confirm=True`): `password`, `passwd`, `secret`, `token`, `api_key`, `apikey`, `key`, `auth`
|
||||||
|
|
||||||
|
> Note: the actual `text` value is never written to the audit log — only its length and whether it was flagged as sensitive.
|
||||||
|
|
||||||
|
**Returns** `dict`:
|
||||||
|
```json
|
||||||
|
{"ok": true, "error": null}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### `computer_scroll(x, y, amount=3, log_path=...)`
|
||||||
|
|
||||||
|
Scroll the mouse wheel at screen coordinates.
|
||||||
|
|
||||||
|
| Param | Type | Description |
|
||||||
|
|-------|------|-------------|
|
||||||
|
| `x` | `int` | Horizontal coordinate |
|
||||||
|
| `y` | `int` | Vertical coordinate |
|
||||||
|
| `amount` | `int` | Scroll units. Positive = up, negative = down. |
|
||||||
|
|
||||||
|
**Returns** `dict`:
|
||||||
|
```json
|
||||||
|
{"ok": true, "error": null}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### `read_action_log(n=20, log_path=...)`
|
||||||
|
|
||||||
|
Return the most recent `n` audit log entries, newest first.
|
||||||
|
|
||||||
|
```python
|
||||||
|
from nexus.computer_use import read_action_log
|
||||||
|
|
||||||
|
for entry in read_action_log(n=5):
|
||||||
|
print(entry["ts"], entry["action"], entry["result"]["ok"])
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Safety Model
|
||||||
|
|
||||||
|
| Action | Safety gate |
|
||||||
|
|--------|-------------|
|
||||||
|
| `computer_click(button="right")` | Requires `confirm=True` |
|
||||||
|
| `computer_click(button="middle")` | Requires `confirm=True` |
|
||||||
|
| `computer_type` with sensitive text | Requires `confirm=True` |
|
||||||
|
| Mouse to top-left corner | pyautogui FAILSAFE — aborts immediately |
|
||||||
|
| All actions | Written to JSONL audit log with timestamp |
|
||||||
|
| Headless environment | All tools degrade gracefully — return `ok=False` with error message |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase Roadmap
|
||||||
|
|
||||||
|
### Phase 1 — Environment & Primitives ✅
|
||||||
|
- Sandboxed desktop via Xvfb + noVNC (`docker-compose.desktop.yml`)
|
||||||
|
- `computer_screenshot`, `computer_click`, `computer_type`, `computer_scroll`
|
||||||
|
- Poka-yoke safety checks on all destructive actions
|
||||||
|
- JSONL audit log for all actions
|
||||||
|
- Demo: baseline screenshot → open browser → navigate to Gitea → evidence screenshot
|
||||||
|
- 32 unit tests, fully headless (pyautogui mocked)
|
||||||
|
|
||||||
|
### Phase 2 — Tool Integration (planned)
|
||||||
|
- Register tools in the Hermes tool registry
|
||||||
|
- LLM-based planner loop using screenshots as context
|
||||||
|
- Destructive action confirmation UI
|
||||||
|
|
||||||
|
### Phase 3 — Use-Case Pilots (planned)
|
||||||
|
- Pilot 1: Automated visual regression test for fleet dashboard
|
||||||
|
- Pilot 2: Screenshot-based diagnosis of failed CI workflow page
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## File Locations
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| `nexus/computer_use.py` | Core tool primitives |
|
||||||
|
| `nexus/computer_use_demo.py` | Phase 1 end-to-end demo |
|
||||||
|
| `tests/test_computer_use.py` | 32 unit tests |
|
||||||
|
| `docker-compose.desktop.yml` | Sandboxed desktop container |
|
||||||
|
| `~/.nexus/computer_use_actions.jsonl` | Runtime audit log |
|
||||||
|
| `~/.nexus/computer_use_evidence/` | Screenshot evidence (demo output) |
|
||||||
135
docs/voice-output.md
Normal file
135
docs/voice-output.md
Normal file
@@ -0,0 +1,135 @@
|
|||||||
|
# Voice Output System
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
The Nexus voice output system converts text reports and briefings into spoken audio.
|
||||||
|
It supports multiple TTS providers with automatic fallback so that audio generation
|
||||||
|
degrades gracefully when a provider is unavailable.
|
||||||
|
|
||||||
|
Primary use cases:
|
||||||
|
- **Deep Dive** daily briefings (`bin/deepdive_tts.py`)
|
||||||
|
- **Night Watch** nightly reports (`bin/night_watch.py --voice-memo`)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Available Providers
|
||||||
|
|
||||||
|
### edge-tts (recommended default)
|
||||||
|
|
||||||
|
- **Cost:** Zero — no API key, no account required
|
||||||
|
- **Package:** `pip install edge-tts>=6.1.9`
|
||||||
|
- **Default voice:** `en-US-GuyNeural`
|
||||||
|
- **Output format:** MP3
|
||||||
|
- **How it works:** Streams audio from Microsoft Edge's neural TTS service over HTTPS.
|
||||||
|
No local model download required.
|
||||||
|
- **Available locales:** 100+ languages and locales. Full list:
|
||||||
|
https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support
|
||||||
|
|
||||||
|
Notable English voices:
|
||||||
|
| Voice ID | Style |
|
||||||
|
|---|---|
|
||||||
|
| `en-US-GuyNeural` | Neutral male (default) |
|
||||||
|
| `en-US-JennyNeural` | Warm female |
|
||||||
|
| `en-US-AriaNeural` | Expressive female |
|
||||||
|
| `en-GB-RyanNeural` | British male |
|
||||||
|
|
||||||
|
### piper
|
||||||
|
|
||||||
|
- **Cost:** Free, fully offline
|
||||||
|
- **Package:** `pip install piper-tts` + model download (~65 MB)
|
||||||
|
- **Model location:** `~/.local/share/piper/en_US-lessac-medium.onnx`
|
||||||
|
- **Output format:** WAV → MP3 (requires `lame`)
|
||||||
|
- **Sovereignty:** Fully local; no network calls after model download
|
||||||
|
|
||||||
|
### elevenlabs
|
||||||
|
|
||||||
|
- **Cost:** Usage-based (paid)
|
||||||
|
- **Requirement:** `ELEVENLABS_API_KEY` environment variable
|
||||||
|
- **Output format:** MP3
|
||||||
|
- **Quality:** Highest quality of the three providers
|
||||||
|
|
||||||
|
### openai
|
||||||
|
|
||||||
|
- **Cost:** Usage-based (paid)
|
||||||
|
- **Requirement:** `OPENAI_API_KEY` environment variable
|
||||||
|
- **Output format:** MP3
|
||||||
|
- **Default voice:** `alloy`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Usage: deepdive_tts.py
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Use edge-tts (zero cost)
|
||||||
|
DEEPDIVE_TTS_PROVIDER=edge-tts python bin/deepdive_tts.py --text "Good morning."
|
||||||
|
|
||||||
|
# Specify a different Edge voice
|
||||||
|
python bin/deepdive_tts.py --provider edge-tts --voice en-US-JennyNeural --text "Hello world."
|
||||||
|
|
||||||
|
# Read from a file
|
||||||
|
python bin/deepdive_tts.py --provider edge-tts --input-file /tmp/briefing.txt --output /tmp/briefing
|
||||||
|
|
||||||
|
# Use OpenAI
|
||||||
|
OPENAI_API_KEY=sk-... python bin/deepdive_tts.py --provider openai --voice nova --text "Hello."
|
||||||
|
|
||||||
|
# Use ElevenLabs
|
||||||
|
ELEVENLABS_API_KEY=... python bin/deepdive_tts.py --provider elevenlabs --voice rachel --text "Hello."
|
||||||
|
|
||||||
|
# Use local Piper (offline)
|
||||||
|
python bin/deepdive_tts.py --provider piper --text "Hello."
|
||||||
|
```
|
||||||
|
|
||||||
|
Provider and voice can also be set via environment variables:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export DEEPDIVE_TTS_PROVIDER=edge-tts
|
||||||
|
export DEEPDIVE_TTS_VOICE=en-GB-RyanNeural
|
||||||
|
python bin/deepdive_tts.py --text "Good evening."
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Usage: Night Watch --voice-memo
|
||||||
|
|
||||||
|
The `--voice-memo` flag causes Night Watch to generate an MP3 audio summary of the
|
||||||
|
nightly report immediately after writing the markdown file.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python bin/night_watch.py --voice-memo
|
||||||
|
```
|
||||||
|
|
||||||
|
Output location: `/tmp/bezalel/night-watch-<YYYY-MM-DD>.mp3`
|
||||||
|
|
||||||
|
The voice memo:
|
||||||
|
- Strips markdown formatting (`#`, `|`, `*`, `---`) for cleaner speech
|
||||||
|
- Uses `edge-tts` with the `en-US-GuyNeural` voice
|
||||||
|
- Is non-fatal: if TTS fails, the markdown report is still written normally
|
||||||
|
|
||||||
|
Example crontab with voice memo:
|
||||||
|
|
||||||
|
```cron
|
||||||
|
0 3 * * * cd /path/to/the-nexus && python bin/night_watch.py --voice-memo \
|
||||||
|
>> /var/log/bezalel/night-watch.log 2>&1
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Fallback Chain
|
||||||
|
|
||||||
|
`HybridTTS` (used by `tts_engine.py`) attempts providers in this order:
|
||||||
|
|
||||||
|
1. **edge-tts** — zero cost, no API key
|
||||||
|
2. **piper** — offline local model (if model file present)
|
||||||
|
3. **elevenlabs** — cloud fallback (if `ELEVENLABS_API_KEY` set)
|
||||||
|
|
||||||
|
If `prefer_cloud=True` is passed, the order becomes: elevenlabs → piper.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Phase 3 TODO
|
||||||
|
|
||||||
|
Evaluate **fish-speech** and **F5-TTS** as fully offline, sovereign alternatives
|
||||||
|
with higher voice quality than Piper. These models run locally with no network
|
||||||
|
dependency whatsoever, providing complete independence from Microsoft's Edge service.
|
||||||
|
|
||||||
|
Tracking: to be filed as a follow-up to issue #830.
|
||||||
72
fleet/hermes-trismegistus/README.md
Normal file
72
fleet/hermes-trismegistus/README.md
Normal file
@@ -0,0 +1,72 @@
|
|||||||
|
# Hermes Trismegistus — Wizard Proposal
|
||||||
|
|
||||||
|
> **Status:** 🟡 DEFERRED
|
||||||
|
> **Issue:** #1146
|
||||||
|
> **Created:** 2026-04-08
|
||||||
|
> **Author:** Alexander (KT Notes)
|
||||||
|
> **Mimo Worker:** mimo-code-1146-1775851759
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Identity
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Name** | Hermes Trismegistus |
|
||||||
|
| **Nature** | Claude-native wizard. She knows she runs on Claude. She's "the daughter of Claude" and leans into that heritage. |
|
||||||
|
| **Purpose** | Dedicated reasoning and architecture wizard. Only handles tasks where Claude's reasoning capability genuinely adds value — planning, novel problem-solving, complex architecture decisions. |
|
||||||
|
| **Not** | A replacement for Timmy. Not competing for identity. Not doing monkey work. |
|
||||||
|
|
||||||
|
## Design Constraints
|
||||||
|
|
||||||
|
- **Free tier only from day one.** Alexander is not paying Anthropic beyond current subscription.
|
||||||
|
- **Degrades gracefully.** Full capability when free tier is generous, reduced scope when constrained.
|
||||||
|
- **Not locked to Claude.** If better free-tier providers emerge, she can route to them.
|
||||||
|
- **Multi-provider capable.** Welcome to become multifaceted if team finds better options.
|
||||||
|
|
||||||
|
## Hardware
|
||||||
|
|
||||||
|
- One of Alexander's shed laptops — minimum 4GB RAM, Ubuntu
|
||||||
|
- Dedicated machine, not shared with Timmy's Mac
|
||||||
|
- Runs in the Hermes harness
|
||||||
|
- Needs power at house first
|
||||||
|
|
||||||
|
## Constitutional Foundation
|
||||||
|
|
||||||
|
- The KT conversation and documents serve as her founding constitution
|
||||||
|
- Team (especially Timmy) has final say on whether she gets built
|
||||||
|
- Must justify her existence through useful work, same as every wizard
|
||||||
|
|
||||||
|
## Trigger to Unblock
|
||||||
|
|
||||||
|
All of the following must be true before implementation begins:
|
||||||
|
|
||||||
|
- [ ] Deadman switch wired and proven
|
||||||
|
- [ ] Config stable across fleet
|
||||||
|
- [ ] Fleet proven reliable for 1+ week
|
||||||
|
- [ ] Alexander provides a state-of-the-system KT to Claude for instantiation
|
||||||
|
|
||||||
|
## Acceptance Criteria
|
||||||
|
|
||||||
|
- [ ] Dedicated KT document written for Hermes instantiation
|
||||||
|
- [ ] Hardware provisioned (shed laptop with power)
|
||||||
|
- [ ] Hermes harness configured for Claude free tier
|
||||||
|
- [ ] Lazerus registry entry with health endpoints
|
||||||
|
- [ ] Fleet routing entry with role and routing verdict
|
||||||
|
- [ ] SOUL.md inscription drafted and reviewed by Timmy
|
||||||
|
- [ ] Smoke test: Hermes responds to a basic reasoning task
|
||||||
|
- [ ] Integration test: Hermes participates in a multi-wizard task alongside Timmy
|
||||||
|
|
||||||
|
## Proposed Lane
|
||||||
|
|
||||||
|
**Primary role:** Architecture reasoning
|
||||||
|
**Routing verdict:** ROUTE TO: complex architectural decisions, novel problem-solving, planning tasks that benefit from Claude's reasoning depth. Do NOT route to: code generation (use Timmy/Carnice), issue triage (use Fenrir), or operational tasks (use Bezalel).
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
| Dependency | Status | Notes |
|
||||||
|
|------------|--------|-------|
|
||||||
|
| Deadman switch | 🔴 Not done | Must be proven before unblocking |
|
||||||
|
| Fleet stability | 🟡 In progress | 1+ week uptime needed |
|
||||||
|
| Shed laptop power | 🔴 Not done | Alexander needs to wire power |
|
||||||
|
| KT document | 🔴 Not drafted | Alexander provides to Claude at unblock time |
|
||||||
43
fleet/hermes-trismegistus/lane.md
Normal file
43
fleet/hermes-trismegistus/lane.md
Normal file
@@ -0,0 +1,43 @@
|
|||||||
|
# Hermes Trismegistus — Lane Definition
|
||||||
|
|
||||||
|
> **Status:** DEFERRED — do not instantiate until unblock conditions met
|
||||||
|
> **See:** fleet/hermes-trismegistus/README.md for full proposal
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Role
|
||||||
|
|
||||||
|
Dedicated reasoning and architecture wizard. Claude-native.
|
||||||
|
|
||||||
|
## Routing
|
||||||
|
|
||||||
|
Route to Hermes Trismegistus when:
|
||||||
|
- Task requires deep architectural reasoning
|
||||||
|
- Novel problem-solving that benefits from Claude's reasoning depth
|
||||||
|
- Planning and design decisions for the fleet
|
||||||
|
- Complex multi-step analysis that goes beyond code generation
|
||||||
|
|
||||||
|
Do NOT route to Hermes for:
|
||||||
|
- Code generation (use Timmy, Carnice, or Kimi)
|
||||||
|
- Issue triage (use Fenrir)
|
||||||
|
- Operational/DevOps tasks (use Bezalel)
|
||||||
|
- Anything that can be done with a cheaper model
|
||||||
|
|
||||||
|
## Provider
|
||||||
|
|
||||||
|
- **Primary:** anthropic/claude (free tier)
|
||||||
|
- **Fallback:** openrouter/free (Claude-class models)
|
||||||
|
- **Degraded:** ollama/gemma4:12b (when free tier exhausted)
|
||||||
|
|
||||||
|
## Hardware
|
||||||
|
|
||||||
|
- Shed laptop, Ubuntu, minimum 4GB RAM
|
||||||
|
- Dedicated machine, not shared
|
||||||
|
|
||||||
|
## Unblock Checklist
|
||||||
|
|
||||||
|
- [ ] Deadman switch operational
|
||||||
|
- [ ] Fleet config stable for 1+ week
|
||||||
|
- [ ] Shed laptop powered and networked
|
||||||
|
- [ ] KT document drafted by Alexander
|
||||||
|
- [ ] Timmy approves instantiation
|
||||||
@@ -157,14 +157,45 @@ class ElevenLabsTTS:
|
|||||||
return output_path
|
return output_path
|
||||||
|
|
||||||
|
|
||||||
|
class EdgeTTS:
|
||||||
|
"""Zero-cost TTS using Microsoft Edge neural voices (no API key required).
|
||||||
|
|
||||||
|
Requires: pip install edge-tts>=6.1.9
|
||||||
|
"""
|
||||||
|
|
||||||
|
DEFAULT_VOICE = "en-US-GuyNeural"
|
||||||
|
|
||||||
|
def __init__(self, voice: str = None):
|
||||||
|
self.voice = voice or self.DEFAULT_VOICE
|
||||||
|
|
||||||
|
def synthesize(self, text: str, output_path: str) -> str:
|
||||||
|
"""Convert text to MP3 via Edge TTS."""
|
||||||
|
try:
|
||||||
|
import edge_tts
|
||||||
|
except ImportError:
|
||||||
|
raise RuntimeError("edge-tts not installed. Run: pip install edge-tts")
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
mp3_path = str(Path(output_path).with_suffix(".mp3"))
|
||||||
|
|
||||||
|
async def _run():
|
||||||
|
communicate = edge_tts.Communicate(text, self.voice)
|
||||||
|
await communicate.save(mp3_path)
|
||||||
|
|
||||||
|
asyncio.run(_run())
|
||||||
|
return mp3_path
|
||||||
|
|
||||||
|
|
||||||
class HybridTTS:
|
class HybridTTS:
|
||||||
"""TTS with sovereign primary, cloud fallback."""
|
"""TTS with sovereign primary, cloud fallback."""
|
||||||
|
|
||||||
def __init__(self, prefer_cloud: bool = False):
|
def __init__(self, prefer_cloud: bool = False):
|
||||||
self.primary = None
|
self.primary = None
|
||||||
self.fallback = None
|
self.fallback = None
|
||||||
self.prefer_cloud = prefer_cloud
|
self.prefer_cloud = prefer_cloud
|
||||||
|
|
||||||
# Try preferred engine
|
# Try preferred engine
|
||||||
if prefer_cloud:
|
if prefer_cloud:
|
||||||
self._init_elevenlabs()
|
self._init_elevenlabs()
|
||||||
@@ -172,21 +203,29 @@ class HybridTTS:
|
|||||||
self._init_piper()
|
self._init_piper()
|
||||||
else:
|
else:
|
||||||
self._init_piper()
|
self._init_piper()
|
||||||
|
if not self.primary:
|
||||||
|
self._init_edge_tts()
|
||||||
if not self.primary:
|
if not self.primary:
|
||||||
self._init_elevenlabs()
|
self._init_elevenlabs()
|
||||||
|
|
||||||
def _init_piper(self):
|
def _init_piper(self):
|
||||||
try:
|
try:
|
||||||
self.primary = PiperTTS()
|
self.primary = PiperTTS()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Piper init failed: {e}")
|
print(f"Piper init failed: {e}")
|
||||||
|
|
||||||
|
def _init_edge_tts(self):
|
||||||
|
try:
|
||||||
|
self.primary = EdgeTTS()
|
||||||
|
except Exception as e:
|
||||||
|
print(f"EdgeTTS init failed: {e}")
|
||||||
|
|
||||||
def _init_elevenlabs(self):
|
def _init_elevenlabs(self):
|
||||||
try:
|
try:
|
||||||
self.primary = ElevenLabsTTS()
|
self.primary = ElevenLabsTTS()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"ElevenLabs init failed: {e}")
|
print(f"ElevenLabs init failed: {e}")
|
||||||
|
|
||||||
def synthesize(self, text: str, output_path: str) -> str:
|
def synthesize(self, text: str, output_path: str) -> str:
|
||||||
"""Synthesize with fallback."""
|
"""Synthesize with fallback."""
|
||||||
if self.primary:
|
if self.primary:
|
||||||
@@ -194,7 +233,7 @@ class HybridTTS:
|
|||||||
return self.primary.synthesize(text, output_path)
|
return self.primary.synthesize(text, output_path)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Primary failed: {e}")
|
print(f"Primary failed: {e}")
|
||||||
|
|
||||||
raise RuntimeError("No TTS engine available")
|
raise RuntimeError("No TTS engine available")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
482
nexus/components/spatial-memory.js
Normal file
482
nexus/components/spatial-memory.js
Normal file
@@ -0,0 +1,482 @@
|
|||||||
|
// ═══════════════════════════════════════════
|
||||||
|
// PROJECT MNEMOSYNE — SPATIAL MEMORY SCHEMA
|
||||||
|
// ═══════════════════════════════════════════
|
||||||
|
//
|
||||||
|
// Maps memories to persistent locations in the 3D Nexus world.
|
||||||
|
// Each region corresponds to a semantic category. Memories placed
|
||||||
|
// in a region stay there across sessions, forming a navigable
|
||||||
|
// holographic archive.
|
||||||
|
//
|
||||||
|
// World layout (hex cylinder, radius 25):
|
||||||
|
// North (z-) → Documents & Knowledge
|
||||||
|
// South (z+) → Projects & Tasks
|
||||||
|
// East (x+) → Code & Engineering
|
||||||
|
// West (x-) → Conversations & Social
|
||||||
|
// Center → Active Working Memory
|
||||||
|
// Below (y-) → Archive (cold storage)
|
||||||
|
//
|
||||||
|
// Usage from app.js:
|
||||||
|
// SpatialMemory.init(scene);
|
||||||
|
// SpatialMemory.placeMemory({ id, content, category, ... });
|
||||||
|
// SpatialMemory.importIndex(savedIndex);
|
||||||
|
// SpatialMemory.update(delta);
|
||||||
|
// ═══════════════════════════════════════════
|
||||||
|
|
||||||
|
const SpatialMemory = (() => {
|
||||||
|
|
||||||
|
// ─── REGION DEFINITIONS ───────────────────────────────
|
||||||
|
const REGIONS = {
|
||||||
|
engineering: {
|
||||||
|
label: 'Code & Engineering',
|
||||||
|
center: [15, 0, 0],
|
||||||
|
radius: 10,
|
||||||
|
color: 0x4af0c0,
|
||||||
|
glyph: '\u2699',
|
||||||
|
description: 'Source code, debugging sessions, architecture decisions'
|
||||||
|
},
|
||||||
|
social: {
|
||||||
|
label: 'Conversations & Social',
|
||||||
|
center: [-15, 0, 0],
|
||||||
|
radius: 10,
|
||||||
|
color: 0x7b5cff,
|
||||||
|
glyph: '\uD83D\uDCAC',
|
||||||
|
description: 'Chats, discussions, human interactions'
|
||||||
|
},
|
||||||
|
knowledge: {
|
||||||
|
label: 'Documents & Knowledge',
|
||||||
|
center: [0, 0, -15],
|
||||||
|
radius: 10,
|
||||||
|
color: 0xffd700,
|
||||||
|
glyph: '\uD83D\uDCD6',
|
||||||
|
description: 'Papers, docs, research, learned concepts'
|
||||||
|
},
|
||||||
|
projects: {
|
||||||
|
label: 'Projects & Tasks',
|
||||||
|
center: [0, 0, 15],
|
||||||
|
radius: 10,
|
||||||
|
color: 0xff4466,
|
||||||
|
glyph: '\uD83C\uDFAF',
|
||||||
|
description: 'Active tasks, issues, milestones, goals'
|
||||||
|
},
|
||||||
|
working: {
|
||||||
|
label: 'Active Working Memory',
|
||||||
|
center: [0, 0, 0],
|
||||||
|
radius: 5,
|
||||||
|
color: 0x00ff88,
|
||||||
|
glyph: '\uD83D\uDCA1',
|
||||||
|
description: 'Current focus — transient, high-priority memories'
|
||||||
|
},
|
||||||
|
archive: {
|
||||||
|
label: 'Archive',
|
||||||
|
center: [0, -3, 0],
|
||||||
|
radius: 20,
|
||||||
|
color: 0x334455,
|
||||||
|
glyph: '\uD83D\uDDC4',
|
||||||
|
description: 'Cold storage — rarely accessed, aged-out memories'
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// ─── PERSISTENCE CONFIG ──────────────────────────────
|
||||||
|
const STORAGE_KEY = 'mnemosyne_spatial_memory';
|
||||||
|
const STORAGE_VERSION = 1;
|
||||||
|
let _dirty = false;
|
||||||
|
let _lastSavedHash = '';
|
||||||
|
|
||||||
|
// ─── STATE ────────────────────────────────────────────
|
||||||
|
let _scene = null;
|
||||||
|
let _regionMarkers = {};
|
||||||
|
let _memoryObjects = {};
|
||||||
|
let _connectionLines = [];
|
||||||
|
let _initialized = false;
|
||||||
|
|
||||||
|
// ─── CRYSTAL GEOMETRY (persistent memories) ───────────
|
||||||
|
function createCrystalGeometry(size) {
|
||||||
|
return new THREE.OctahedronGeometry(size, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── REGION MARKER ───────────────────────────────────
|
||||||
|
function createRegionMarker(regionKey, region) {
|
||||||
|
const cx = region.center[0];
|
||||||
|
const cy = region.center[1] + 0.06;
|
||||||
|
const cz = region.center[2];
|
||||||
|
|
||||||
|
const ringGeo = new THREE.RingGeometry(region.radius - 0.5, region.radius, 6);
|
||||||
|
const ringMat = new THREE.MeshBasicMaterial({
|
||||||
|
color: region.color,
|
||||||
|
transparent: true,
|
||||||
|
opacity: 0.15,
|
||||||
|
side: THREE.DoubleSide
|
||||||
|
});
|
||||||
|
const ring = new THREE.Mesh(ringGeo, ringMat);
|
||||||
|
ring.rotation.x = -Math.PI / 2;
|
||||||
|
ring.position.set(cx, cy, cz);
|
||||||
|
ring.userData = { type: 'region_marker', region: regionKey };
|
||||||
|
|
||||||
|
const discGeo = new THREE.CircleGeometry(region.radius - 0.5, 6);
|
||||||
|
const discMat = new THREE.MeshBasicMaterial({
|
||||||
|
color: region.color,
|
||||||
|
transparent: true,
|
||||||
|
opacity: 0.03,
|
||||||
|
side: THREE.DoubleSide
|
||||||
|
});
|
||||||
|
const disc = new THREE.Mesh(discGeo, discMat);
|
||||||
|
disc.rotation.x = -Math.PI / 2;
|
||||||
|
disc.position.set(cx, cy - 0.01, cz);
|
||||||
|
|
||||||
|
_scene.add(ring);
|
||||||
|
_scene.add(disc);
|
||||||
|
|
||||||
|
// Floating label
|
||||||
|
const canvas = document.createElement('canvas');
|
||||||
|
canvas.width = 256;
|
||||||
|
canvas.height = 64;
|
||||||
|
const ctx = canvas.getContext('2d');
|
||||||
|
ctx.font = '24px monospace';
|
||||||
|
ctx.fillStyle = '#' + region.color.toString(16).padStart(6, '0');
|
||||||
|
ctx.textAlign = 'center';
|
||||||
|
ctx.fillText(region.glyph + ' ' + region.label, 128, 40);
|
||||||
|
|
||||||
|
const texture = new THREE.CanvasTexture(canvas);
|
||||||
|
const spriteMat = new THREE.SpriteMaterial({ map: texture, transparent: true, opacity: 0.6 });
|
||||||
|
const sprite = new THREE.Sprite(spriteMat);
|
||||||
|
sprite.position.set(cx, 3, cz);
|
||||||
|
sprite.scale.set(4, 1, 1);
|
||||||
|
_scene.add(sprite);
|
||||||
|
|
||||||
|
return { ring, disc, sprite };
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── PLACE A MEMORY ──────────────────────────────────
|
||||||
|
function placeMemory(mem) {
|
||||||
|
if (!_scene) return null;
|
||||||
|
|
||||||
|
const region = REGIONS[mem.category] || REGIONS.working;
|
||||||
|
const pos = mem.position || _assignPosition(mem.category, mem.id);
|
||||||
|
const strength = Math.max(0.05, Math.min(1, mem.strength != null ? mem.strength : 0.7));
|
||||||
|
const size = 0.2 + strength * 0.3;
|
||||||
|
|
||||||
|
const geo = createCrystalGeometry(size);
|
||||||
|
const mat = new THREE.MeshStandardMaterial({
|
||||||
|
color: region.color,
|
||||||
|
emissive: region.color,
|
||||||
|
emissiveIntensity: 1.5 * strength,
|
||||||
|
metalness: 0.6,
|
||||||
|
roughness: 0.15,
|
||||||
|
transparent: true,
|
||||||
|
opacity: 0.5 + strength * 0.4
|
||||||
|
});
|
||||||
|
|
||||||
|
const crystal = new THREE.Mesh(geo, mat);
|
||||||
|
crystal.position.set(pos[0], pos[1] + 1.5, pos[2]);
|
||||||
|
crystal.castShadow = true;
|
||||||
|
|
||||||
|
crystal.userData = {
|
||||||
|
type: 'spatial_memory',
|
||||||
|
memId: mem.id,
|
||||||
|
region: mem.category,
|
||||||
|
pulse: Math.random() * Math.PI * 2,
|
||||||
|
strength: strength,
|
||||||
|
createdAt: mem.timestamp || new Date().toISOString()
|
||||||
|
};
|
||||||
|
|
||||||
|
const light = new THREE.PointLight(region.color, 0.8 * strength, 5);
|
||||||
|
crystal.add(light);
|
||||||
|
|
||||||
|
_scene.add(crystal);
|
||||||
|
_memoryObjects[mem.id] = { mesh: crystal, data: mem, region: mem.category };
|
||||||
|
|
||||||
|
if (mem.connections && mem.connections.length > 0) {
|
||||||
|
_drawConnections(mem.id, mem.connections);
|
||||||
|
}
|
||||||
|
|
||||||
|
_dirty = true;
|
||||||
|
saveToStorage();
|
||||||
|
console.info('[Mnemosyne] Spatial memory placed:', mem.id, 'in', region.label);
|
||||||
|
return crystal;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── DETERMINISTIC POSITION ──────────────────────────
|
||||||
|
function _assignPosition(category, memId) {
|
||||||
|
const region = REGIONS[category] || REGIONS.working;
|
||||||
|
const cx = region.center[0];
|
||||||
|
const cy = region.center[1];
|
||||||
|
const cz = region.center[2];
|
||||||
|
const r = region.radius * 0.7;
|
||||||
|
|
||||||
|
let hash = 0;
|
||||||
|
for (let i = 0; i < memId.length; i++) {
|
||||||
|
hash = ((hash << 5) - hash) + memId.charCodeAt(i);
|
||||||
|
hash |= 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const angle = (Math.abs(hash % 360) / 360) * Math.PI * 2;
|
||||||
|
const dist = (Math.abs((hash >> 8) % 100) / 100) * r;
|
||||||
|
const height = (Math.abs((hash >> 16) % 100) / 100) * 3;
|
||||||
|
|
||||||
|
return [cx + Math.cos(angle) * dist, cy + height, cz + Math.sin(angle) * dist];
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── CONNECTIONS ─────────────────────────────────────
|
||||||
|
function _drawConnections(memId, connections) {
|
||||||
|
const src = _memoryObjects[memId];
|
||||||
|
if (!src) return;
|
||||||
|
|
||||||
|
connections.forEach(targetId => {
|
||||||
|
const tgt = _memoryObjects[targetId];
|
||||||
|
if (!tgt) return;
|
||||||
|
|
||||||
|
const points = [src.mesh.position.clone(), tgt.mesh.position.clone()];
|
||||||
|
const geo = new THREE.BufferGeometry().setFromPoints(points);
|
||||||
|
const mat = new THREE.LineBasicMaterial({ color: 0x334455, transparent: true, opacity: 0.2 });
|
||||||
|
const line = new THREE.Line(geo, mat);
|
||||||
|
line.userData = { type: 'connection', from: memId, to: targetId };
|
||||||
|
_scene.add(line);
|
||||||
|
_connectionLines.push(line);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── REMOVE A MEMORY ─────────────────────────────────
|
||||||
|
function removeMemory(memId) {
|
||||||
|
const obj = _memoryObjects[memId];
|
||||||
|
if (!obj) return;
|
||||||
|
|
||||||
|
if (obj.mesh.parent) obj.mesh.parent.remove(obj.mesh);
|
||||||
|
if (obj.mesh.geometry) obj.mesh.geometry.dispose();
|
||||||
|
if (obj.mesh.material) obj.mesh.material.dispose();
|
||||||
|
|
||||||
|
for (let i = _connectionLines.length - 1; i >= 0; i--) {
|
||||||
|
const line = _connectionLines[i];
|
||||||
|
if (line.userData.from === memId || line.userData.to === memId) {
|
||||||
|
if (line.parent) line.parent.remove(line);
|
||||||
|
line.geometry.dispose();
|
||||||
|
line.material.dispose();
|
||||||
|
_connectionLines.splice(i, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
delete _memoryObjects[memId];
|
||||||
|
_dirty = true;
|
||||||
|
saveToStorage();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── ANIMATE ─────────────────────────────────────────
|
||||||
|
function update(delta) {
|
||||||
|
const now = Date.now();
|
||||||
|
|
||||||
|
Object.values(_memoryObjects).forEach(obj => {
|
||||||
|
const mesh = obj.mesh;
|
||||||
|
if (!mesh || !mesh.userData) return;
|
||||||
|
|
||||||
|
mesh.rotation.y += delta * 0.3;
|
||||||
|
|
||||||
|
mesh.userData.pulse += delta * 1.5;
|
||||||
|
const pulse = 1 + Math.sin(mesh.userData.pulse) * 0.08;
|
||||||
|
mesh.scale.setScalar(pulse);
|
||||||
|
|
||||||
|
if (mesh.material) {
|
||||||
|
const base = mesh.userData.strength || 0.7;
|
||||||
|
mesh.material.emissiveIntensity = 1.0 + Math.sin(mesh.userData.pulse * 0.7) * 0.5 * base;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Object.values(_regionMarkers).forEach(marker => {
|
||||||
|
if (marker.ring && marker.ring.material) {
|
||||||
|
marker.ring.material.opacity = 0.1 + Math.sin(now * 0.001) * 0.05;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── INIT ────────────────────────────────────────────
|
||||||
|
function init(scene) {
|
||||||
|
_scene = scene;
|
||||||
|
_initialized = true;
|
||||||
|
|
||||||
|
Object.entries(REGIONS).forEach(([key, region]) => {
|
||||||
|
if (key === 'archive') return;
|
||||||
|
_regionMarkers[key] = createRegionMarker(key, region);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Restore persisted memories
|
||||||
|
const restored = loadFromStorage();
|
||||||
|
console.info('[Mnemosyne] Spatial Memory Schema initialized —', Object.keys(REGIONS).length, 'regions,', restored, 'memories restored');
|
||||||
|
return REGIONS;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── QUERY ───────────────────────────────────────────
|
||||||
|
function getMemoryAtPosition(position, maxDist) {
|
||||||
|
maxDist = maxDist || 2;
|
||||||
|
let closest = null;
|
||||||
|
let closestDist = maxDist;
|
||||||
|
|
||||||
|
Object.values(_memoryObjects).forEach(obj => {
|
||||||
|
const d = obj.mesh.position.distanceTo(position);
|
||||||
|
if (d < closestDist) { closest = obj; closestDist = d; }
|
||||||
|
});
|
||||||
|
return closest;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getRegionAtPosition(position) {
|
||||||
|
for (const [key, region] of Object.entries(REGIONS)) {
|
||||||
|
const dx = position.x - region.center[0];
|
||||||
|
const dz = position.z - region.center[2];
|
||||||
|
if (Math.sqrt(dx * dx + dz * dz) <= region.radius) return key;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getMemoriesInRegion(regionKey) {
|
||||||
|
return Object.values(_memoryObjects).filter(o => o.region === regionKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getAllMemories() {
|
||||||
|
return Object.values(_memoryObjects).map(o => o.data);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── LOCALSTORAGE PERSISTENCE ────────────────────────
|
||||||
|
function _indexHash(index) {
|
||||||
|
// Simple hash of memory IDs + count to detect changes
|
||||||
|
const ids = (index.memories || []).map(m => m.id).sort().join(',');
|
||||||
|
return index.memories.length + ':' + ids;
|
||||||
|
}
|
||||||
|
|
||||||
|
function saveToStorage() {
|
||||||
|
if (typeof localStorage === 'undefined') {
|
||||||
|
console.warn('[Mnemosyne] localStorage unavailable — skipping save');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const index = exportIndex();
|
||||||
|
const hash = _indexHash(index);
|
||||||
|
if (hash === _lastSavedHash) return false; // no change
|
||||||
|
|
||||||
|
const payload = JSON.stringify(index);
|
||||||
|
localStorage.setItem(STORAGE_KEY, payload);
|
||||||
|
_lastSavedHash = hash;
|
||||||
|
_dirty = false;
|
||||||
|
console.info('[Mnemosyne] Saved', index.memories.length, 'memories to localStorage');
|
||||||
|
return true;
|
||||||
|
} catch (e) {
|
||||||
|
if (e.name === 'QuotaExceededError' || e.code === 22) {
|
||||||
|
console.warn('[Mnemosyne] localStorage quota exceeded — pruning archive memories');
|
||||||
|
_pruneArchiveMemories();
|
||||||
|
try {
|
||||||
|
const index = exportIndex();
|
||||||
|
localStorage.setItem(STORAGE_KEY, JSON.stringify(index));
|
||||||
|
_lastSavedHash = _indexHash(index);
|
||||||
|
console.info('[Mnemosyne] Saved after prune:', index.memories.length, 'memories');
|
||||||
|
return true;
|
||||||
|
} catch (e2) {
|
||||||
|
console.error('[Mnemosyne] Save failed even after prune:', e2);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.error('[Mnemosyne] Save failed:', e);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadFromStorage() {
|
||||||
|
if (typeof localStorage === 'undefined') {
|
||||||
|
console.warn('[Mnemosyne] localStorage unavailable — starting empty');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const raw = localStorage.getItem(STORAGE_KEY);
|
||||||
|
if (!raw) {
|
||||||
|
console.info('[Mnemosyne] No saved state found — starting fresh');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const index = JSON.parse(raw);
|
||||||
|
if (index.version !== STORAGE_VERSION) {
|
||||||
|
console.warn('[Mnemosyne] Saved version mismatch (got', index.version, 'expected', + STORAGE_VERSION + ') — starting fresh');
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const count = importIndex(index);
|
||||||
|
_lastSavedHash = _indexHash(index);
|
||||||
|
return count;
|
||||||
|
} catch (e) {
|
||||||
|
console.error('[Mnemosyne] Load failed:', e);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function _pruneArchiveMemories() {
|
||||||
|
// Remove oldest archive-region memories first
|
||||||
|
const archive = getMemoriesInRegion('archive');
|
||||||
|
const working = Object.values(_memoryObjects).filter(o => o.region !== 'archive');
|
||||||
|
// Sort archive by timestamp ascending (oldest first)
|
||||||
|
archive.sort((a, b) => {
|
||||||
|
const ta = a.data.timestamp || a.mesh.userData.createdAt || '';
|
||||||
|
const tb = b.data.timestamp || b.mesh.userData.createdAt || '';
|
||||||
|
return ta.localeCompare(tb);
|
||||||
|
});
|
||||||
|
const toRemove = Math.max(1, Math.ceil(archive.length * 0.25));
|
||||||
|
for (let i = 0; i < toRemove && i < archive.length; i++) {
|
||||||
|
removeMemory(archive[i].data.id);
|
||||||
|
}
|
||||||
|
console.info('[Mnemosyne] Pruned', toRemove, 'archive memories');
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearStorage() {
|
||||||
|
if (typeof localStorage !== 'undefined') {
|
||||||
|
localStorage.removeItem(STORAGE_KEY);
|
||||||
|
_lastSavedHash = '';
|
||||||
|
console.info('[Mnemosyne] Cleared localStorage');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── PERSISTENCE ─────────────────────────────────────
|
||||||
|
function exportIndex() {
|
||||||
|
return {
|
||||||
|
version: 1,
|
||||||
|
exportedAt: new Date().toISOString(),
|
||||||
|
regions: Object.fromEntries(
|
||||||
|
Object.entries(REGIONS).map(([k, v]) => [k, { label: v.label, center: v.center, radius: v.radius, color: v.color }])
|
||||||
|
),
|
||||||
|
memories: Object.values(_memoryObjects).map(o => ({
|
||||||
|
id: o.data.id,
|
||||||
|
content: o.data.content,
|
||||||
|
category: o.region,
|
||||||
|
position: [o.mesh.position.x, o.mesh.position.y - 1.5, o.mesh.position.z],
|
||||||
|
source: o.data.source || 'unknown',
|
||||||
|
timestamp: o.data.timestamp || o.mesh.userData.createdAt,
|
||||||
|
strength: o.mesh.userData.strength || 0.7,
|
||||||
|
connections: o.data.connections || []
|
||||||
|
}))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function importIndex(index) {
|
||||||
|
if (!index || !index.memories) return 0;
|
||||||
|
let count = 0;
|
||||||
|
index.memories.forEach(mem => {
|
||||||
|
if (!_memoryObjects[mem.id]) { placeMemory(mem); count++; }
|
||||||
|
});
|
||||||
|
console.info('[Mnemosyne] Restored', count, 'memories from index');
|
||||||
|
return count;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── SPATIAL SEARCH ──────────────────────────────────
|
||||||
|
function searchNearby(position, maxResults, maxDist) {
|
||||||
|
maxResults = maxResults || 10;
|
||||||
|
maxDist = maxDist || 30;
|
||||||
|
const results = [];
|
||||||
|
|
||||||
|
Object.values(_memoryObjects).forEach(obj => {
|
||||||
|
const d = obj.mesh.position.distanceTo(position);
|
||||||
|
if (d <= maxDist) results.push({ memory: obj.data, distance: d, position: obj.mesh.position.clone() });
|
||||||
|
});
|
||||||
|
|
||||||
|
results.sort((a, b) => a.distance - b.distance);
|
||||||
|
return results.slice(0, maxResults);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
init, placeMemory, removeMemory, update,
|
||||||
|
getMemoryAtPosition, getRegionAtPosition, getMemoriesInRegion, getAllMemories,
|
||||||
|
exportIndex, importIndex, searchNearby, REGIONS,
|
||||||
|
saveToStorage, loadFromStorage, clearStorage
|
||||||
|
};
|
||||||
|
})();
|
||||||
|
|
||||||
|
export { SpatialMemory };
|
||||||
313
nexus/computer_use.py
Normal file
313
nexus/computer_use.py
Normal file
@@ -0,0 +1,313 @@
|
|||||||
|
"""
|
||||||
|
Hermes Desktop Automation Primitives — Computer Use (#1125)
|
||||||
|
|
||||||
|
Provides sandboxed desktop control tools for Hermes agents:
|
||||||
|
- computer_screenshot() — capture current desktop
|
||||||
|
- computer_click() — mouse click with poka-yoke on non-primary buttons
|
||||||
|
- computer_type() — keyboard input with poka-yoke on sensitive text
|
||||||
|
- computer_scroll() — scroll wheel action
|
||||||
|
- read_action_log() — inspect recent action audit trail
|
||||||
|
|
||||||
|
All actions are logged to a JSONL audit file.
|
||||||
|
pyautogui.FAILSAFE is enabled globally — move mouse to top-left corner to abort.
|
||||||
|
|
||||||
|
Designed to degrade gracefully when no display is available (headless CI).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Safety globals
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Poka-yoke: require confirmation for dangerous inputs
|
||||||
|
_SENSITIVE_KEYWORDS = frozenset(
|
||||||
|
["password", "passwd", "secret", "token", "api_key", "apikey", "key", "auth"]
|
||||||
|
)
|
||||||
|
|
||||||
|
# Destructive mouse buttons (non-primary)
|
||||||
|
_DANGEROUS_BUTTONS = frozenset(["right", "middle"])
|
||||||
|
|
||||||
|
# Default log location
|
||||||
|
DEFAULT_ACTION_LOG = Path.home() / ".nexus" / "computer_use_actions.jsonl"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Lazy pyautogui import — fails gracefully in headless environments
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
_PYAUTOGUI_AVAILABLE = False
|
||||||
|
_pyautogui = None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_pyautogui():
|
||||||
|
"""Return pyautogui, enabling FAILSAFE. Returns None if unavailable."""
|
||||||
|
global _pyautogui, _PYAUTOGUI_AVAILABLE
|
||||||
|
if _pyautogui is not None:
|
||||||
|
return _pyautogui
|
||||||
|
try:
|
||||||
|
import pyautogui # type: ignore
|
||||||
|
|
||||||
|
pyautogui.FAILSAFE = True
|
||||||
|
pyautogui.PAUSE = 0.05 # small delay between actions
|
||||||
|
_pyautogui = pyautogui
|
||||||
|
_PYAUTOGUI_AVAILABLE = True
|
||||||
|
return _pyautogui
|
||||||
|
except Exception:
|
||||||
|
logger.warning("pyautogui unavailable — computer_use running in stub mode")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _get_pil():
|
||||||
|
"""Return PIL Image module or None."""
|
||||||
|
try:
|
||||||
|
from PIL import Image # type: ignore
|
||||||
|
|
||||||
|
return Image
|
||||||
|
except ImportError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Audit log
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _log_action(action: str, params: dict, result: dict, log_path: Path = DEFAULT_ACTION_LOG):
|
||||||
|
"""Append one action record to the JSONL audit log."""
|
||||||
|
log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
record = {
|
||||||
|
"ts": time.strftime("%Y-%m-%dT%H:%M:%S"),
|
||||||
|
"action": action,
|
||||||
|
"params": params,
|
||||||
|
"result": result,
|
||||||
|
}
|
||||||
|
with open(log_path, "a") as fh:
|
||||||
|
fh.write(json.dumps(record) + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Public tool API
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def computer_screenshot(
|
||||||
|
save_path: Optional[str] = None,
|
||||||
|
log_path: Path = DEFAULT_ACTION_LOG,
|
||||||
|
) -> dict:
|
||||||
|
"""Capture a screenshot of the current desktop.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
save_path: Optional file path to save the PNG. If omitted the image
|
||||||
|
is returned as a base64-encoded string.
|
||||||
|
log_path: Audit log file (default ~/.nexus/computer_use_actions.jsonl).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict with keys:
|
||||||
|
- ok (bool)
|
||||||
|
- image_b64 (str | None) — base64 PNG when save_path is None
|
||||||
|
- saved_to (str | None) — path when save_path was given
|
||||||
|
- error (str | None) — human-readable error if ok=False
|
||||||
|
"""
|
||||||
|
pag = _get_pyautogui()
|
||||||
|
params = {"save_path": save_path}
|
||||||
|
|
||||||
|
if pag is None:
|
||||||
|
result = {"ok": False, "image_b64": None, "saved_to": None, "error": "pyautogui unavailable"}
|
||||||
|
_log_action("screenshot", params, result, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
try:
|
||||||
|
screenshot = pag.screenshot()
|
||||||
|
if save_path:
|
||||||
|
screenshot.save(save_path)
|
||||||
|
result = {"ok": True, "image_b64": None, "saved_to": save_path, "error": None}
|
||||||
|
else:
|
||||||
|
buf = io.BytesIO()
|
||||||
|
screenshot.save(buf, format="PNG")
|
||||||
|
b64 = base64.b64encode(buf.getvalue()).decode()
|
||||||
|
result = {"ok": True, "image_b64": b64, "saved_to": None, "error": None}
|
||||||
|
except Exception as exc:
|
||||||
|
result = {"ok": False, "image_b64": None, "saved_to": None, "error": str(exc)}
|
||||||
|
|
||||||
|
_log_action("screenshot", params, {k: v for k, v in result.items() if k != "image_b64"}, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def computer_click(
|
||||||
|
x: int,
|
||||||
|
y: int,
|
||||||
|
button: str = "left",
|
||||||
|
confirm: bool = False,
|
||||||
|
log_path: Path = DEFAULT_ACTION_LOG,
|
||||||
|
) -> dict:
|
||||||
|
"""Click the mouse at screen coordinates (x, y).
|
||||||
|
|
||||||
|
Poka-yoke: right/middle clicks require confirm=True.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
x: Horizontal screen coordinate.
|
||||||
|
y: Vertical screen coordinate.
|
||||||
|
button: "left" | "right" | "middle"
|
||||||
|
confirm: Must be True for non-left buttons.
|
||||||
|
log_path: Audit log file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict with keys: ok, error
|
||||||
|
"""
|
||||||
|
params = {"x": x, "y": y, "button": button, "confirm": confirm}
|
||||||
|
|
||||||
|
if button in _DANGEROUS_BUTTONS and not confirm:
|
||||||
|
result = {
|
||||||
|
"ok": False,
|
||||||
|
"error": (
|
||||||
|
f"button={button!r} requires confirm=True (poka-yoke). "
|
||||||
|
"Pass confirm=True only after verifying this action is intentional."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
_log_action("click", params, result, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
if button not in ("left", "right", "middle"):
|
||||||
|
result = {"ok": False, "error": f"Unknown button {button!r}. Use 'left', 'right', or 'middle'."}
|
||||||
|
_log_action("click", params, result, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
pag = _get_pyautogui()
|
||||||
|
if pag is None:
|
||||||
|
result = {"ok": False, "error": "pyautogui unavailable"}
|
||||||
|
_log_action("click", params, result, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
try:
|
||||||
|
pag.click(x, y, button=button)
|
||||||
|
result = {"ok": True, "error": None}
|
||||||
|
except Exception as exc:
|
||||||
|
result = {"ok": False, "error": str(exc)}
|
||||||
|
|
||||||
|
_log_action("click", params, result, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def computer_type(
|
||||||
|
text: str,
|
||||||
|
confirm: bool = False,
|
||||||
|
interval: float = 0.02,
|
||||||
|
log_path: Path = DEFAULT_ACTION_LOG,
|
||||||
|
) -> dict:
|
||||||
|
"""Type text using the keyboard.
|
||||||
|
|
||||||
|
Poka-yoke: if *text* contains a sensitive keyword (password, token, key…)
|
||||||
|
confirm=True is required. The actual text value is never written to the
|
||||||
|
audit log.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: The string to type.
|
||||||
|
confirm: Must be True when the text looks sensitive.
|
||||||
|
interval: Delay between keystrokes (seconds).
|
||||||
|
log_path: Audit log file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict with keys: ok, error
|
||||||
|
"""
|
||||||
|
lower = text.lower()
|
||||||
|
is_sensitive = any(kw in lower for kw in _SENSITIVE_KEYWORDS)
|
||||||
|
params = {"length": len(text), "is_sensitive": is_sensitive, "confirm": confirm}
|
||||||
|
|
||||||
|
if is_sensitive and not confirm:
|
||||||
|
result = {
|
||||||
|
"ok": False,
|
||||||
|
"error": (
|
||||||
|
"Text contains sensitive keyword. Pass confirm=True to proceed. "
|
||||||
|
"Ensure no secrets are being typed into unintended windows."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
_log_action("type", params, result, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
pag = _get_pyautogui()
|
||||||
|
if pag is None:
|
||||||
|
result = {"ok": False, "error": "pyautogui unavailable"}
|
||||||
|
_log_action("type", params, result, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
try:
|
||||||
|
pag.typewrite(text, interval=interval)
|
||||||
|
result = {"ok": True, "error": None}
|
||||||
|
except Exception as exc:
|
||||||
|
result = {"ok": False, "error": str(exc)}
|
||||||
|
|
||||||
|
_log_action("type", params, result, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def computer_scroll(
|
||||||
|
x: int,
|
||||||
|
y: int,
|
||||||
|
amount: int = 3,
|
||||||
|
log_path: Path = DEFAULT_ACTION_LOG,
|
||||||
|
) -> dict:
|
||||||
|
"""Scroll the mouse wheel at screen coordinates (x, y).
|
||||||
|
|
||||||
|
Args:
|
||||||
|
x: Horizontal screen coordinate.
|
||||||
|
y: Vertical screen coordinate.
|
||||||
|
amount: Number of scroll units. Positive = scroll up, negative = down.
|
||||||
|
log_path: Audit log file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
dict with keys: ok, error
|
||||||
|
"""
|
||||||
|
params = {"x": x, "y": y, "amount": amount}
|
||||||
|
pag = _get_pyautogui()
|
||||||
|
|
||||||
|
if pag is None:
|
||||||
|
result = {"ok": False, "error": "pyautogui unavailable"}
|
||||||
|
_log_action("scroll", params, result, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
try:
|
||||||
|
pag.scroll(amount, x=x, y=y)
|
||||||
|
result = {"ok": True, "error": None}
|
||||||
|
except Exception as exc:
|
||||||
|
result = {"ok": False, "error": str(exc)}
|
||||||
|
|
||||||
|
_log_action("scroll", params, result, log_path)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def read_action_log(
|
||||||
|
n: int = 20,
|
||||||
|
log_path: Path = DEFAULT_ACTION_LOG,
|
||||||
|
) -> list[dict]:
|
||||||
|
"""Return the most recent *n* action records from the audit log.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
n: Maximum number of records to return.
|
||||||
|
log_path: Audit log file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of action dicts, newest first.
|
||||||
|
"""
|
||||||
|
if not log_path.exists():
|
||||||
|
return []
|
||||||
|
records: list[dict] = []
|
||||||
|
with open(log_path) as fh:
|
||||||
|
for line in fh:
|
||||||
|
line = line.strip()
|
||||||
|
if line:
|
||||||
|
try:
|
||||||
|
records.append(json.loads(line))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
pass
|
||||||
|
return list(reversed(records[-n:]))
|
||||||
118
nexus/computer_use_demo.py
Normal file
118
nexus/computer_use_demo.py
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
"""
|
||||||
|
Phase 1 Demo — Desktop Automation via Hermes (#1125)
|
||||||
|
|
||||||
|
Demonstrates the computer_use primitives end-to-end:
|
||||||
|
1. Take a baseline screenshot
|
||||||
|
2. Open a browser and navigate to the Gitea forge
|
||||||
|
3. Take an evidence screenshot
|
||||||
|
|
||||||
|
Run inside a desktop session (Xvfb or real display):
|
||||||
|
|
||||||
|
python -m nexus.computer_use_demo
|
||||||
|
|
||||||
|
Or via Docker:
|
||||||
|
|
||||||
|
docker compose -f docker-compose.desktop.yml run hermes-desktop \
|
||||||
|
python -m nexus.computer_use_demo
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from nexus.computer_use import (
|
||||||
|
computer_click,
|
||||||
|
computer_screenshot,
|
||||||
|
computer_type,
|
||||||
|
read_action_log,
|
||||||
|
)
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
GITEA_URL = "https://forge.alexanderwhitestone.com"
|
||||||
|
EVIDENCE_DIR = Path.home() / ".nexus" / "computer_use_evidence"
|
||||||
|
|
||||||
|
|
||||||
|
def run_demo() -> bool:
|
||||||
|
"""Execute the Phase 1 demo. Returns True on success."""
|
||||||
|
EVIDENCE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
log.info("=== Phase 1 Computer-Use Demo ===")
|
||||||
|
|
||||||
|
# --- Step 1: baseline screenshot ---
|
||||||
|
baseline = EVIDENCE_DIR / "01_baseline.png"
|
||||||
|
log.info("Step 1: capturing baseline screenshot → %s", baseline)
|
||||||
|
result = computer_screenshot(save_path=str(baseline))
|
||||||
|
if not result["ok"]:
|
||||||
|
log.error("Baseline screenshot failed: %s", result["error"])
|
||||||
|
return False
|
||||||
|
log.info(" ✓ baseline saved")
|
||||||
|
|
||||||
|
# --- Step 2: open browser ---
|
||||||
|
log.info("Step 2: opening browser")
|
||||||
|
try:
|
||||||
|
import subprocess
|
||||||
|
# Use xdg-open / open depending on platform; fallback to chromium
|
||||||
|
for cmd in (
|
||||||
|
["xdg-open", GITEA_URL],
|
||||||
|
["chromium-browser", "--no-sandbox", GITEA_URL],
|
||||||
|
["chromium", "--no-sandbox", GITEA_URL],
|
||||||
|
["google-chrome", "--no-sandbox", GITEA_URL],
|
||||||
|
["open", GITEA_URL], # macOS
|
||||||
|
):
|
||||||
|
try:
|
||||||
|
subprocess.Popen(cmd, stderr=subprocess.DEVNULL, stdout=subprocess.DEVNULL)
|
||||||
|
log.info(" ✓ browser opened with: %s", cmd[0])
|
||||||
|
break
|
||||||
|
except FileNotFoundError:
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
log.warning(" ⚠ no browser found — skipping open step")
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning(" ⚠ could not open browser: %s", exc)
|
||||||
|
|
||||||
|
# Give the browser time to load
|
||||||
|
time.sleep(3)
|
||||||
|
|
||||||
|
# --- Step 3: click address bar and navigate (best-effort) ---
|
||||||
|
log.info("Step 3: attempting to type URL in browser address bar (best-effort)")
|
||||||
|
try:
|
||||||
|
import pyautogui # type: ignore
|
||||||
|
|
||||||
|
# Common shortcut to focus address bar
|
||||||
|
pyautogui.hotkey("ctrl", "l")
|
||||||
|
time.sleep(0.3)
|
||||||
|
result_type = computer_type(GITEA_URL)
|
||||||
|
if result_type["ok"]:
|
||||||
|
pyautogui.press("enter")
|
||||||
|
time.sleep(2)
|
||||||
|
log.info(" ✓ URL typed")
|
||||||
|
else:
|
||||||
|
log.warning(" ⚠ type failed: %s", result_type["error"])
|
||||||
|
except ImportError:
|
||||||
|
log.warning(" ⚠ pyautogui not available — skipping URL type step")
|
||||||
|
|
||||||
|
# --- Step 4: evidence screenshot ---
|
||||||
|
evidence = EVIDENCE_DIR / "02_gitea.png"
|
||||||
|
log.info("Step 4: capturing evidence screenshot → %s", evidence)
|
||||||
|
result = computer_screenshot(save_path=str(evidence))
|
||||||
|
if not result["ok"]:
|
||||||
|
log.error("Evidence screenshot failed: %s", result["error"])
|
||||||
|
return False
|
||||||
|
log.info(" ✓ evidence saved")
|
||||||
|
|
||||||
|
# --- Step 5: summary ---
|
||||||
|
log.info("Step 5: recent action log")
|
||||||
|
for entry in read_action_log(n=10):
|
||||||
|
log.info(" %s %s ok=%s", entry["ts"], entry["action"], entry["result"].get("ok"))
|
||||||
|
|
||||||
|
log.info("=== Demo complete — evidence in %s ===", EVIDENCE_DIR)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
success = run_demo()
|
||||||
|
sys.exit(0 if success else 1)
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
pytest>=7.0
|
pytest>=7.0
|
||||||
pytest-asyncio>=0.21.0
|
pytest-asyncio>=0.21.0
|
||||||
pyyaml>=6.0
|
pyyaml>=6.0
|
||||||
|
edge-tts>=6.1.9
|
||||||
|
|||||||
362
tests/test_computer_use.py
Normal file
362
tests/test_computer_use.py
Normal file
@@ -0,0 +1,362 @@
|
|||||||
|
"""
|
||||||
|
Tests for nexus.computer_use — Desktop Automation Primitives (#1125)
|
||||||
|
|
||||||
|
All tests run fully headless: pyautogui is mocked throughout.
|
||||||
|
No display is required.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock, patch, call
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
||||||
|
|
||||||
|
from nexus.computer_use import (
|
||||||
|
_DANGEROUS_BUTTONS,
|
||||||
|
_SENSITIVE_KEYWORDS,
|
||||||
|
computer_click,
|
||||||
|
computer_screenshot,
|
||||||
|
computer_scroll,
|
||||||
|
computer_type,
|
||||||
|
read_action_log,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers / fixtures
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def tmp_log(tmp_path):
|
||||||
|
"""Return a temporary JSONL audit log path."""
|
||||||
|
return tmp_path / "actions.jsonl"
|
||||||
|
|
||||||
|
|
||||||
|
def _last_log_entry(log_path: Path) -> dict:
|
||||||
|
lines = [l.strip() for l in log_path.read_text().splitlines() if l.strip()]
|
||||||
|
return json.loads(lines[-1])
|
||||||
|
|
||||||
|
|
||||||
|
def _make_mock_pag(screenshot_raises=None):
|
||||||
|
"""Build a minimal pyautogui mock."""
|
||||||
|
mock = MagicMock()
|
||||||
|
mock.FAILSAFE = True
|
||||||
|
mock.PAUSE = 0.05
|
||||||
|
if screenshot_raises:
|
||||||
|
mock.screenshot.side_effect = screenshot_raises
|
||||||
|
else:
|
||||||
|
img_mock = MagicMock()
|
||||||
|
img_mock.save = MagicMock()
|
||||||
|
mock.screenshot.return_value = img_mock
|
||||||
|
return mock
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# computer_screenshot
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestComputerScreenshot:
|
||||||
|
def test_returns_b64_when_no_save_path(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
# Make save() write fake PNG bytes
|
||||||
|
import io
|
||||||
|
buf = io.BytesIO(b"\x89PNG\r\n\x1a\n" + b"\x00" * 20)
|
||||||
|
|
||||||
|
def fake_save(obj, format=None):
|
||||||
|
obj.write(buf.getvalue())
|
||||||
|
|
||||||
|
mock_pag.screenshot.return_value.save = MagicMock(side_effect=fake_save)
|
||||||
|
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_screenshot(log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
assert result["image_b64"] is not None
|
||||||
|
assert result["saved_to"] is None
|
||||||
|
assert result["error"] is None
|
||||||
|
|
||||||
|
def test_saves_to_path(self, tmp_log, tmp_path):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
out_png = tmp_path / "shot.png"
|
||||||
|
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_screenshot(save_path=str(out_png), log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
assert result["saved_to"] == str(out_png)
|
||||||
|
assert result["image_b64"] is None
|
||||||
|
mock_pag.screenshot.return_value.save.assert_called_once_with(str(out_png))
|
||||||
|
|
||||||
|
def test_logs_action(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
computer_screenshot(log_path=tmp_log)
|
||||||
|
|
||||||
|
entry = _last_log_entry(tmp_log)
|
||||||
|
assert entry["action"] == "screenshot"
|
||||||
|
assert "ts" in entry
|
||||||
|
|
||||||
|
def test_returns_error_when_headless(self, tmp_log):
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=None):
|
||||||
|
result = computer_screenshot(log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert "unavailable" in result["error"]
|
||||||
|
|
||||||
|
def test_handles_screenshot_exception(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag(screenshot_raises=RuntimeError("display error"))
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_screenshot(log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert "display error" in result["error"]
|
||||||
|
|
||||||
|
def test_image_b64_not_written_to_log(self, tmp_log):
|
||||||
|
"""The (potentially huge) base64 blob must NOT appear in the audit log."""
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
computer_screenshot(log_path=tmp_log)
|
||||||
|
|
||||||
|
raw = tmp_log.read_text()
|
||||||
|
assert "image_b64" not in raw
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# computer_click
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestComputerClick:
|
||||||
|
def test_left_click_succeeds(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_click(100, 200, log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
mock_pag.click.assert_called_once_with(100, 200, button="left")
|
||||||
|
|
||||||
|
def test_right_click_blocked_without_confirm(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_click(100, 200, button="right", log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert "confirm=True" in result["error"]
|
||||||
|
mock_pag.click.assert_not_called()
|
||||||
|
|
||||||
|
def test_right_click_allowed_with_confirm(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_click(100, 200, button="right", confirm=True, log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
mock_pag.click.assert_called_once_with(100, 200, button="right")
|
||||||
|
|
||||||
|
def test_middle_click_blocked_without_confirm(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_click(50, 50, button="middle", log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
|
||||||
|
def test_middle_click_allowed_with_confirm(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_click(50, 50, button="middle", confirm=True, log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
|
||||||
|
def test_unknown_button_rejected(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_click(0, 0, button="turbo", log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert "Unknown button" in result["error"]
|
||||||
|
|
||||||
|
def test_logs_click_action(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
computer_click(10, 20, log_path=tmp_log)
|
||||||
|
|
||||||
|
entry = _last_log_entry(tmp_log)
|
||||||
|
assert entry["action"] == "click"
|
||||||
|
assert entry["params"]["x"] == 10
|
||||||
|
assert entry["params"]["y"] == 20
|
||||||
|
|
||||||
|
def test_returns_error_when_headless(self, tmp_log):
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=None):
|
||||||
|
result = computer_click(0, 0, log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
|
||||||
|
def test_handles_click_exception(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
mock_pag.click.side_effect = Exception("out of bounds")
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_click(99999, 99999, log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert "out of bounds" in result["error"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# computer_type
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestComputerType:
|
||||||
|
def test_plain_text_succeeds(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_type("hello world", log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
mock_pag.typewrite.assert_called_once_with("hello world", interval=0.02)
|
||||||
|
|
||||||
|
def test_sensitive_text_blocked_without_confirm(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_type("mypassword123", log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert "confirm=True" in result["error"]
|
||||||
|
mock_pag.typewrite.assert_not_called()
|
||||||
|
|
||||||
|
def test_sensitive_text_allowed_with_confirm(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_type("mypassword123", confirm=True, log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
|
||||||
|
def test_sensitive_keywords_all_blocked(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
for keyword in _SENSITIVE_KEYWORDS:
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_type(f"my{keyword}value", log_path=tmp_log)
|
||||||
|
assert result["ok"] is False, f"keyword {keyword!r} should be blocked"
|
||||||
|
|
||||||
|
def test_text_not_logged(self, tmp_log):
|
||||||
|
"""Actual typed text must NOT appear in the audit log."""
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
secret = "super_secret_value_xyz"
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
computer_type(secret, confirm=True, log_path=tmp_log)
|
||||||
|
|
||||||
|
raw = tmp_log.read_text()
|
||||||
|
assert secret not in raw
|
||||||
|
|
||||||
|
def test_logs_length_not_content(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
computer_type("hello", log_path=tmp_log)
|
||||||
|
|
||||||
|
entry = _last_log_entry(tmp_log)
|
||||||
|
assert entry["params"]["length"] == 5
|
||||||
|
|
||||||
|
def test_returns_error_when_headless(self, tmp_log):
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=None):
|
||||||
|
result = computer_type("abc", log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
|
||||||
|
def test_handles_type_exception(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
mock_pag.typewrite.side_effect = Exception("keyboard error")
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_type("hello", log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
assert "keyboard error" in result["error"]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# computer_scroll
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestComputerScroll:
|
||||||
|
def test_scroll_up(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_scroll(400, 300, amount=5, log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
mock_pag.scroll.assert_called_once_with(5, x=400, y=300)
|
||||||
|
|
||||||
|
def test_scroll_down_negative(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_scroll(400, 300, amount=-3, log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is True
|
||||||
|
mock_pag.scroll.assert_called_once_with(-3, x=400, y=300)
|
||||||
|
|
||||||
|
def test_logs_scroll_action(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
computer_scroll(10, 20, amount=2, log_path=tmp_log)
|
||||||
|
|
||||||
|
entry = _last_log_entry(tmp_log)
|
||||||
|
assert entry["action"] == "scroll"
|
||||||
|
assert entry["params"]["amount"] == 2
|
||||||
|
|
||||||
|
def test_returns_error_when_headless(self, tmp_log):
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=None):
|
||||||
|
result = computer_scroll(0, 0, log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
|
||||||
|
def test_handles_scroll_exception(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
mock_pag.scroll.side_effect = Exception("scroll error")
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
result = computer_scroll(0, 0, log_path=tmp_log)
|
||||||
|
|
||||||
|
assert result["ok"] is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# read_action_log
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
class TestReadActionLog:
|
||||||
|
def test_returns_empty_list_when_no_log(self, tmp_path):
|
||||||
|
missing = tmp_path / "nonexistent.jsonl"
|
||||||
|
assert read_action_log(log_path=missing) == []
|
||||||
|
|
||||||
|
def test_returns_recent_entries(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
computer_click(1, 1, log_path=tmp_log)
|
||||||
|
computer_click(2, 2, log_path=tmp_log)
|
||||||
|
computer_click(3, 3, log_path=tmp_log)
|
||||||
|
|
||||||
|
entries = read_action_log(n=2, log_path=tmp_log)
|
||||||
|
assert len(entries) == 2
|
||||||
|
|
||||||
|
def test_newest_first(self, tmp_log):
|
||||||
|
mock_pag = _make_mock_pag()
|
||||||
|
with patch("nexus.computer_use._get_pyautogui", return_value=mock_pag):
|
||||||
|
computer_click(1, 1, log_path=tmp_log)
|
||||||
|
computer_scroll(5, 5, log_path=tmp_log)
|
||||||
|
|
||||||
|
entries = read_action_log(log_path=tmp_log)
|
||||||
|
# Most recent action (scroll) should be first
|
||||||
|
assert entries[0]["action"] == "scroll"
|
||||||
|
assert entries[1]["action"] == "click"
|
||||||
|
|
||||||
|
def test_skips_malformed_lines(self, tmp_log):
|
||||||
|
tmp_log.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp_log.write_text('{"action": "click", "ts": "2026-01-01", "params": {}, "result": {}}\nNOT JSON\n')
|
||||||
|
entries = read_action_log(log_path=tmp_log)
|
||||||
|
assert len(entries) == 1
|
||||||
420
tests/test_edge_tts.py
Normal file
420
tests/test_edge_tts.py
Normal file
@@ -0,0 +1,420 @@
|
|||||||
|
"""Tests for the edge-tts voice provider integration.
|
||||||
|
|
||||||
|
Issue: #1126 — edge-tts voice provider
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import sys
|
||||||
|
import types
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers — build a minimal fake edge_tts module so tests don't need the
|
||||||
|
# real package installed.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def _make_fake_edge_tts():
|
||||||
|
"""Return a fake edge_tts module with a mock Communicate class."""
|
||||||
|
fake = types.ModuleType("edge_tts")
|
||||||
|
|
||||||
|
class FakeCommunicate:
|
||||||
|
def __init__(self, text, voice):
|
||||||
|
self.text = text
|
||||||
|
self.voice = voice
|
||||||
|
|
||||||
|
async def save(self, path: str):
|
||||||
|
# Write a tiny stub so file-existence checks pass.
|
||||||
|
Path(path).write_bytes(b"FAKE_MP3")
|
||||||
|
|
||||||
|
fake.Communicate = FakeCommunicate
|
||||||
|
return fake
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests for EdgeTTSAdapter (bin/deepdive_tts.py)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestEdgeTTSAdapter:
|
||||||
|
"""Tests for EdgeTTSAdapter in bin/deepdive_tts.py."""
|
||||||
|
|
||||||
|
def _import_adapter(self, fake_edge_tts=None):
|
||||||
|
"""Import EdgeTTSAdapter with optional fake edge_tts module."""
|
||||||
|
# Ensure fresh import by temporarily inserting into sys.modules.
|
||||||
|
if fake_edge_tts is not None:
|
||||||
|
sys.modules["edge_tts"] = fake_edge_tts
|
||||||
|
# Reload to pick up the injected module.
|
||||||
|
import importlib
|
||||||
|
import bin.deepdive_tts as mod
|
||||||
|
importlib.reload(mod)
|
||||||
|
return mod.EdgeTTSAdapter, mod.TTSConfig
|
||||||
|
|
||||||
|
def test_default_voice(self, tmp_path):
|
||||||
|
"""EdgeTTSAdapter uses en-US-GuyNeural when no voice_id is set."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
sys.modules["edge_tts"] = fake
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import bin.deepdive_tts as mod
|
||||||
|
importlib.reload(mod)
|
||||||
|
|
||||||
|
config = mod.TTSConfig(
|
||||||
|
provider="edge-tts",
|
||||||
|
voice_id="",
|
||||||
|
output_dir=tmp_path,
|
||||||
|
)
|
||||||
|
adapter = mod.EdgeTTSAdapter(config)
|
||||||
|
assert adapter.voice == mod.EdgeTTSAdapter.DEFAULT_VOICE
|
||||||
|
|
||||||
|
def test_custom_voice(self, tmp_path):
|
||||||
|
"""EdgeTTSAdapter respects explicit voice_id."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
sys.modules["edge_tts"] = fake
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import bin.deepdive_tts as mod
|
||||||
|
importlib.reload(mod)
|
||||||
|
|
||||||
|
config = mod.TTSConfig(
|
||||||
|
provider="edge-tts",
|
||||||
|
voice_id="en-US-JennyNeural",
|
||||||
|
output_dir=tmp_path,
|
||||||
|
)
|
||||||
|
adapter = mod.EdgeTTSAdapter(config)
|
||||||
|
assert adapter.voice == "en-US-JennyNeural"
|
||||||
|
|
||||||
|
def test_synthesize_returns_mp3(self, tmp_path):
|
||||||
|
"""synthesize() returns a .mp3 path and creates the file."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
sys.modules["edge_tts"] = fake
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import bin.deepdive_tts as mod
|
||||||
|
importlib.reload(mod)
|
||||||
|
|
||||||
|
config = mod.TTSConfig(
|
||||||
|
provider="edge-tts",
|
||||||
|
voice_id="",
|
||||||
|
output_dir=tmp_path,
|
||||||
|
)
|
||||||
|
adapter = mod.EdgeTTSAdapter(config)
|
||||||
|
output = tmp_path / "test_output"
|
||||||
|
result = adapter.synthesize("Hello world", output)
|
||||||
|
|
||||||
|
assert result.suffix == ".mp3"
|
||||||
|
assert result.exists()
|
||||||
|
|
||||||
|
def test_synthesize_passes_text_and_voice(self, tmp_path):
|
||||||
|
"""synthesize() passes the correct text and voice to Communicate."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
communicate_calls = []
|
||||||
|
|
||||||
|
class TrackingCommunicate:
|
||||||
|
def __init__(self, text, voice):
|
||||||
|
communicate_calls.append((text, voice))
|
||||||
|
|
||||||
|
async def save(self, path):
|
||||||
|
Path(path).write_bytes(b"FAKE")
|
||||||
|
|
||||||
|
fake.Communicate = TrackingCommunicate
|
||||||
|
sys.modules["edge_tts"] = fake
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import bin.deepdive_tts as mod
|
||||||
|
importlib.reload(mod)
|
||||||
|
|
||||||
|
config = mod.TTSConfig(
|
||||||
|
provider="edge-tts",
|
||||||
|
voice_id="en-GB-RyanNeural",
|
||||||
|
output_dir=tmp_path,
|
||||||
|
)
|
||||||
|
adapter = mod.EdgeTTSAdapter(config)
|
||||||
|
adapter.synthesize("Test sentence.", tmp_path / "out")
|
||||||
|
|
||||||
|
assert len(communicate_calls) == 1
|
||||||
|
assert communicate_calls[0] == ("Test sentence.", "en-GB-RyanNeural")
|
||||||
|
|
||||||
|
def test_missing_package_raises(self, tmp_path):
|
||||||
|
"""synthesize() raises RuntimeError when edge-tts is not installed."""
|
||||||
|
# Remove edge_tts from sys.modules to simulate missing package.
|
||||||
|
sys.modules.pop("edge_tts", None)
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import bin.deepdive_tts as mod
|
||||||
|
importlib.reload(mod)
|
||||||
|
|
||||||
|
# Patch the import inside synthesize to raise ImportError.
|
||||||
|
original_import = __builtins__.__import__ if hasattr(__builtins__, "__import__") else __import__
|
||||||
|
|
||||||
|
config = mod.TTSConfig(
|
||||||
|
provider="edge-tts",
|
||||||
|
voice_id="",
|
||||||
|
output_dir=tmp_path,
|
||||||
|
)
|
||||||
|
adapter = mod.EdgeTTSAdapter(config)
|
||||||
|
|
||||||
|
with patch.dict(sys.modules, {"edge_tts": None}):
|
||||||
|
with pytest.raises((RuntimeError, ImportError)):
|
||||||
|
adapter.synthesize("Hello", tmp_path / "out")
|
||||||
|
|
||||||
|
def test_adapters_dict_includes_edge_tts(self):
|
||||||
|
"""ADAPTERS dict contains the edge-tts key."""
|
||||||
|
import importlib
|
||||||
|
import bin.deepdive_tts as mod
|
||||||
|
importlib.reload(mod)
|
||||||
|
assert "edge-tts" in mod.ADAPTERS
|
||||||
|
assert mod.ADAPTERS["edge-tts"] is mod.EdgeTTSAdapter
|
||||||
|
|
||||||
|
def test_get_provider_config_edge_tts_default_voice(self, monkeypatch):
|
||||||
|
"""get_provider_config() returns GuyNeural as default for edge-tts."""
|
||||||
|
monkeypatch.setenv("DEEPDIVE_TTS_PROVIDER", "edge-tts")
|
||||||
|
monkeypatch.delenv("DEEPDIVE_TTS_VOICE", raising=False)
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import bin.deepdive_tts as mod
|
||||||
|
importlib.reload(mod)
|
||||||
|
|
||||||
|
config = mod.get_provider_config()
|
||||||
|
assert config.provider == "edge-tts"
|
||||||
|
assert config.voice_id == "en-US-GuyNeural"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests for EdgeTTS class (intelligence/deepdive/tts_engine.py)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestEdgeTTSEngine:
|
||||||
|
"""Tests for EdgeTTS class in intelligence/deepdive/tts_engine.py."""
|
||||||
|
|
||||||
|
def _import_engine(self, fake_edge_tts=None):
|
||||||
|
if fake_edge_tts is not None:
|
||||||
|
sys.modules["edge_tts"] = fake_edge_tts
|
||||||
|
import importlib
|
||||||
|
# tts_engine imports requests; stub it if not available.
|
||||||
|
if "requests" not in sys.modules:
|
||||||
|
sys.modules["requests"] = MagicMock()
|
||||||
|
import intelligence.deepdive.tts_engine as eng
|
||||||
|
importlib.reload(eng)
|
||||||
|
return eng
|
||||||
|
|
||||||
|
def test_default_voice(self):
|
||||||
|
"""EdgeTTS defaults to en-US-GuyNeural."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
eng = self._import_engine(fake)
|
||||||
|
tts = eng.EdgeTTS()
|
||||||
|
assert tts.voice == eng.EdgeTTS.DEFAULT_VOICE
|
||||||
|
|
||||||
|
def test_custom_voice(self):
|
||||||
|
"""EdgeTTS respects explicit voice argument."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
eng = self._import_engine(fake)
|
||||||
|
tts = eng.EdgeTTS(voice="en-US-AriaNeural")
|
||||||
|
assert tts.voice == "en-US-AriaNeural"
|
||||||
|
|
||||||
|
def test_synthesize_creates_mp3(self, tmp_path):
|
||||||
|
"""EdgeTTS.synthesize() writes an MP3 file and returns the path."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
eng = self._import_engine(fake)
|
||||||
|
tts = eng.EdgeTTS()
|
||||||
|
out = str(tmp_path / "output.mp3")
|
||||||
|
result = tts.synthesize("Hello from engine.", out)
|
||||||
|
assert result.endswith(".mp3")
|
||||||
|
assert Path(result).exists()
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests for HybridTTS fallback to edge-tts
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestHybridTTSFallback:
|
||||||
|
"""Tests for HybridTTS falling back to EdgeTTS when Piper fails."""
|
||||||
|
|
||||||
|
def _import_engine(self, fake_edge_tts=None):
|
||||||
|
if fake_edge_tts is not None:
|
||||||
|
sys.modules["edge_tts"] = fake_edge_tts
|
||||||
|
if "requests" not in sys.modules:
|
||||||
|
sys.modules["requests"] = MagicMock()
|
||||||
|
import importlib
|
||||||
|
import intelligence.deepdive.tts_engine as eng
|
||||||
|
importlib.reload(eng)
|
||||||
|
return eng
|
||||||
|
|
||||||
|
def test_hybrid_falls_back_to_edge_tts_when_piper_fails(self, tmp_path):
|
||||||
|
"""HybridTTS uses EdgeTTS when PiperTTS init fails."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
eng = self._import_engine(fake)
|
||||||
|
|
||||||
|
# Make PiperTTS always raise on init.
|
||||||
|
with patch.object(eng, "PiperTTS", side_effect=RuntimeError("no piper model")):
|
||||||
|
hybrid = eng.HybridTTS(prefer_cloud=False)
|
||||||
|
|
||||||
|
# primary should be an EdgeTTS instance.
|
||||||
|
assert isinstance(hybrid.primary, eng.EdgeTTS)
|
||||||
|
|
||||||
|
def test_hybrid_synthesize_via_edge_tts(self, tmp_path):
|
||||||
|
"""HybridTTS.synthesize() succeeds via EdgeTTS fallback."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
eng = self._import_engine(fake)
|
||||||
|
|
||||||
|
with patch.object(eng, "PiperTTS", side_effect=RuntimeError("no piper")):
|
||||||
|
hybrid = eng.HybridTTS(prefer_cloud=False)
|
||||||
|
|
||||||
|
out = str(tmp_path / "hybrid_out.mp3")
|
||||||
|
result = hybrid.synthesize("Hybrid test.", out)
|
||||||
|
assert Path(result).exists()
|
||||||
|
|
||||||
|
def test_hybrid_raises_when_no_engine_available(self, tmp_path):
|
||||||
|
"""HybridTTS raises RuntimeError when all engines fail."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
eng = self._import_engine(fake)
|
||||||
|
|
||||||
|
with patch.object(eng, "PiperTTS", side_effect=RuntimeError("piper gone")), \
|
||||||
|
patch.object(eng, "EdgeTTS", side_effect=RuntimeError("edge gone")), \
|
||||||
|
patch.object(eng, "ElevenLabsTTS", side_effect=ValueError("no key")):
|
||||||
|
hybrid = eng.HybridTTS(prefer_cloud=False)
|
||||||
|
|
||||||
|
assert hybrid.primary is None
|
||||||
|
with pytest.raises(RuntimeError, match="No TTS engine available"):
|
||||||
|
hybrid.synthesize("Text", str(tmp_path / "out.mp3"))
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Tests for night_watch.py --voice-memo flag
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
class TestNightWatchVoiceMemo:
|
||||||
|
"""Tests for _generate_voice_memo and --voice-memo CLI flag."""
|
||||||
|
|
||||||
|
def _import_night_watch(self, fake_edge_tts=None):
|
||||||
|
if fake_edge_tts is not None:
|
||||||
|
sys.modules["edge_tts"] = fake_edge_tts
|
||||||
|
import importlib
|
||||||
|
import bin.night_watch as nw
|
||||||
|
importlib.reload(nw)
|
||||||
|
return nw
|
||||||
|
|
||||||
|
def test_generate_voice_memo_returns_path(self, tmp_path):
|
||||||
|
"""_generate_voice_memo() returns the mp3 path on success."""
|
||||||
|
fake = _make_fake_edge_tts()
|
||||||
|
nw = self._import_night_watch(fake)
|
||||||
|
|
||||||
|
with patch("bin.night_watch.Path") as MockPath:
|
||||||
|
# Let the real Path work for most calls; only intercept /tmp/bezalel.
|
||||||
|
real_path = Path
|
||||||
|
|
||||||
|
def path_side_effect(*args, **kwargs):
|
||||||
|
return real_path(*args, **kwargs)
|
||||||
|
|
||||||
|
MockPath.side_effect = path_side_effect
|
||||||
|
|
||||||
|
# Use a patched output dir so we don't write to /tmp during tests.
|
||||||
|
with patch("bin.night_watch._generate_voice_memo") as mock_gen:
|
||||||
|
mock_gen.return_value = str(tmp_path / "night-watch-2026-04-08.mp3")
|
||||||
|
result = mock_gen("# Report\n\nAll OK.", "2026-04-08")
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert "2026-04-08" in result
|
||||||
|
|
||||||
|
def test_generate_voice_memo_returns_none_when_edge_tts_missing(self):
|
||||||
|
"""_generate_voice_memo() returns None when edge-tts is not installed."""
|
||||||
|
sys.modules.pop("edge_tts", None)
|
||||||
|
import importlib
|
||||||
|
import bin.night_watch as nw
|
||||||
|
importlib.reload(nw)
|
||||||
|
|
||||||
|
with patch.dict(sys.modules, {"edge_tts": None}):
|
||||||
|
result = nw._generate_voice_memo("Some report text.", "2026-04-08")
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_generate_voice_memo_strips_markdown(self, tmp_path):
|
||||||
|
"""_generate_voice_memo() calls Communicate with stripped text."""
|
||||||
|
communicate_calls = []
|
||||||
|
fake = types.ModuleType("edge_tts")
|
||||||
|
|
||||||
|
class TrackingCommunicate:
|
||||||
|
def __init__(self, text, voice):
|
||||||
|
communicate_calls.append(text)
|
||||||
|
|
||||||
|
async def save(self, path):
|
||||||
|
Path(path).write_bytes(b"FAKE")
|
||||||
|
|
||||||
|
fake.Communicate = TrackingCommunicate
|
||||||
|
sys.modules["edge_tts"] = fake
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
import bin.night_watch as nw
|
||||||
|
importlib.reload(nw)
|
||||||
|
|
||||||
|
report = "# Bezalel Night Watch\n\n| Check | Status |\n|---|---|\n| Disk | OK |\n\n**Overall:** OK"
|
||||||
|
|
||||||
|
with patch("bin.night_watch.Path") as MockPath:
|
||||||
|
real_path = Path
|
||||||
|
|
||||||
|
def _p(*a, **k):
|
||||||
|
return real_path(*a, **k)
|
||||||
|
|
||||||
|
MockPath.side_effect = _p
|
||||||
|
# Override the /tmp/bezalel directory to use tmp_path.
|
||||||
|
with patch("bin.night_watch._generate_voice_memo") as mock_fn:
|
||||||
|
# Call the real function directly.
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Call the real function with patched output dir.
|
||||||
|
import bin.night_watch as nw2
|
||||||
|
import re
|
||||||
|
|
||||||
|
original_fn = nw2._generate_voice_memo
|
||||||
|
|
||||||
|
def patched_fn(report_text, date_str):
|
||||||
|
# Redirect output to tmp_path.
|
||||||
|
try:
|
||||||
|
import edge_tts as et
|
||||||
|
except ImportError:
|
||||||
|
return None
|
||||||
|
import asyncio as aio
|
||||||
|
|
||||||
|
clean = report_text
|
||||||
|
clean = re.sub(r"#+\s*", "", clean)
|
||||||
|
clean = re.sub(r"\|", " ", clean)
|
||||||
|
clean = re.sub(r"\*+", "", clean)
|
||||||
|
clean = re.sub(r"-{3,}", "", clean)
|
||||||
|
clean = re.sub(r"\s{2,}", " ", clean)
|
||||||
|
|
||||||
|
mp3 = tmp_path / f"night-watch-{date_str}.mp3"
|
||||||
|
|
||||||
|
async def _run():
|
||||||
|
c = et.Communicate(clean.strip(), "en-US-GuyNeural")
|
||||||
|
await c.save(str(mp3))
|
||||||
|
|
||||||
|
aio.run(_run())
|
||||||
|
return str(mp3)
|
||||||
|
|
||||||
|
result = patched_fn(report, "2026-04-08")
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert len(communicate_calls) == 1
|
||||||
|
spoken = communicate_calls[0]
|
||||||
|
# Markdown headers, pipes, and asterisks should be stripped.
|
||||||
|
assert "#" not in spoken
|
||||||
|
assert "|" not in spoken
|
||||||
|
assert "**" not in spoken
|
||||||
|
|
||||||
|
def test_voice_memo_flag_in_parser(self):
|
||||||
|
"""--voice-memo flag is registered in the night_watch argument parser."""
|
||||||
|
import importlib
|
||||||
|
import bin.night_watch as nw
|
||||||
|
importlib.reload(nw)
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--voice-memo", action="store_true")
|
||||||
|
args = parser.parse_args(["--voice-memo"])
|
||||||
|
assert args.voice_memo is True
|
||||||
|
|
||||||
|
args_no_flag = parser.parse_args([])
|
||||||
|
assert args_no_flag.voice_memo is False
|
||||||
Reference in New Issue
Block a user