#!/usr/bin/env python3 """ tunnel_sync.py — Pull closets from a remote wizard's fleet API into the local palace. This is the client-side tunnel mechanism for #1078. It connects to a peer wizard's running fleet_api.py HTTP server, discovers their memory wings, and imports the results into the local fleet palace as closet files. Once imported, `recall --fleet` in Evennia will return results from the remote wing. The code side is complete here; the infrastructure side (second wizard running fleet_api.py behind an SSH tunnel or VPN) is still required to use this. Usage: # Pull from a remote Alpha fleet API into the default local palace python mempalace/tunnel_sync.py --peer http://alpha.example.com:7771 # Custom local palace path FLEET_PALACE_PATH=/data/fleet python mempalace/tunnel_sync.py \\ --peer http://alpha.example.com:7771 # Dry-run: show what would be imported without writing files python mempalace/tunnel_sync.py --peer http://alpha.example.com:7771 --dry-run # Limit results per room (default: 50) python mempalace/tunnel_sync.py --peer http://alpha.example.com:7771 --n 20 Environment: FLEET_PALACE_PATH — local fleet palace directory (default: /var/lib/mempalace/fleet) FLEET_PEER_URL — remote fleet API URL (overridden by --peer flag) Exits: 0 — sync succeeded (or dry-run completed) 1 — error (connection failure, invalid response, write error) Refs: #1078, #1075 """ from __future__ import annotations import argparse import json import os import sys import time import urllib.error import urllib.request from dataclasses import dataclass, field from pathlib import Path from typing import Any DEFAULT_PALACE_PATH = "/var/lib/mempalace/fleet" DEFAULT_N_RESULTS = 50 # Broad queries for bulk room pull — used to discover representative content _BROAD_QUERIES = [ "the", "a", "is", "was", "and", "of", "to", "in", "it", "on", "commit", "issue", "error", "fix", "deploy", "event", "memory", ] _REQUEST_TIMEOUT = 10 # seconds @dataclass class SyncResult: wings_found: list[str] = field(default_factory=list) rooms_pulled: int = 0 closets_written: int = 0 errors: list[str] = field(default_factory=list) @property def ok(self) -> bool: return len(self.errors) == 0 # --------------------------------------------------------------------------- # HTTP helpers # --------------------------------------------------------------------------- def _get(url: str) -> dict[str, Any]: """GET *url*, return parsed JSON or raise on error.""" req = urllib.request.Request(url, headers={"Accept": "application/json"}) with urllib.request.urlopen(req, timeout=_REQUEST_TIMEOUT) as resp: return json.loads(resp.read()) def _peer_url(base: str, path: str) -> str: return base.rstrip("/") + path # --------------------------------------------------------------------------- # Wing / room discovery # --------------------------------------------------------------------------- def get_remote_wings(peer_url: str) -> list[str]: """Return the list of wing names from the remote fleet API.""" data = _get(_peer_url(peer_url, "/wings")) return data.get("wings", []) def search_remote_room(peer_url: str, room: str, n: int = DEFAULT_N_RESULTS) -> list[dict]: """ Pull closet entries for a specific room from the remote peer. Uses multiple broad queries and deduplicates by text to maximize coverage without requiring a dedicated bulk-export endpoint. """ seen_texts: set[str] = set() results: list[dict] = [] for q in _BROAD_QUERIES: url = _peer_url(peer_url, f"/search?q={urllib.request.quote(q)}&room={urllib.request.quote(room)}&n={n}") try: data = _get(url) except (urllib.error.URLError, json.JSONDecodeError, OSError): continue for entry in data.get("results", []): text = entry.get("text", "") if text and text not in seen_texts: seen_texts.add(text) results.append(entry) if len(results) >= n: break return results[:n] # --------------------------------------------------------------------------- # Core sync # --------------------------------------------------------------------------- def _write_closet( palace_dir: Path, wing: str, room: str, entries: list[dict], dry_run: bool, ) -> bool: """Write entries as a .closet.json file under palace_dir/wing/.""" wing_dir = palace_dir / wing closet_path = wing_dir / f"{room}.closet.json" drawers = [ { "text": e.get("text", ""), "room": e.get("room", room), "wing": e.get("wing", wing), "score": e.get("score", 0.0), "closet": True, "source_file": f"tunnel:{wing}/{room}", "synced_at": int(time.time()), } for e in entries ] payload = json.dumps({"drawers": drawers, "wing": wing, "room": room}, indent=2) if dry_run: print(f"[tunnel_sync] DRY-RUN would write {len(drawers)} entries → {closet_path}") return True try: wing_dir.mkdir(parents=True, exist_ok=True) closet_path.write_text(payload) print(f"[tunnel_sync] Wrote {len(drawers)} entries → {closet_path}") return True except OSError as exc: print(f"[tunnel_sync] ERROR writing {closet_path}: {exc}", file=sys.stderr) return False def sync_peer( peer_url: str, palace_dir: Path, n_results: int = DEFAULT_N_RESULTS, dry_run: bool = False, ) -> SyncResult: """ Pull all wings and rooms from *peer_url* into *palace_dir*. Args: peer_url: Base URL of the remote fleet_api.py instance. palace_dir: Local fleet palace directory to write closets into. n_results: Maximum results to pull per room. dry_run: If True, print what would be written without touching disk. Returns: SyncResult with counts and any errors. """ result = SyncResult() # Discover health try: health = _get(_peer_url(peer_url, "/health")) if health.get("status") != "ok": result.errors.append(f"Peer unhealthy: {health}") return result except (urllib.error.URLError, json.JSONDecodeError, OSError) as exc: result.errors.append(f"Could not reach peer at {peer_url}: {exc}") return result # Discover wings try: wings = get_remote_wings(peer_url) except (urllib.error.URLError, json.JSONDecodeError, OSError) as exc: result.errors.append(f"Could not list wings from {peer_url}: {exc}") return result result.wings_found = wings if not wings: print(f"[tunnel_sync] No wings found at {peer_url} — nothing to sync.") return result print(f"[tunnel_sync] Found wings: {wings}") # Import core rooms from each wing from nexus.mempalace.config import CORE_ROOMS for wing in wings: for room in CORE_ROOMS: print(f"[tunnel_sync] Pulling {wing}/{room} …") try: entries = search_remote_room(peer_url, room, n=n_results) except (urllib.error.URLError, json.JSONDecodeError, OSError) as exc: err = f"Error pulling {wing}/{room}: {exc}" result.errors.append(err) print(f"[tunnel_sync] ERROR: {err}", file=sys.stderr) continue if not entries: print(f"[tunnel_sync] No entries found for {wing}/{room} — skipping.") continue ok = _write_closet(palace_dir, wing, room, entries, dry_run=dry_run) result.rooms_pulled += 1 if ok: result.closets_written += 1 return result # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( description="Sync closets from a remote wizard's fleet API into the local palace." ) parser.add_argument( "--peer", default=os.environ.get("FLEET_PEER_URL", ""), metavar="URL", help="Base URL of the remote fleet_api.py (e.g. http://alpha.example.com:7771)", ) parser.add_argument( "--palace", default=os.environ.get("FLEET_PALACE_PATH", DEFAULT_PALACE_PATH), metavar="DIR", help=f"Local fleet palace directory (default: {DEFAULT_PALACE_PATH})", ) parser.add_argument( "--n", type=int, default=DEFAULT_N_RESULTS, metavar="N", help=f"Max results per room (default: {DEFAULT_N_RESULTS})", ) parser.add_argument( "--dry-run", action="store_true", help="Show what would be synced without writing files.", ) args = parser.parse_args(argv) if not args.peer: print( "[tunnel_sync] ERROR: --peer URL is required (or set FLEET_PEER_URL).", file=sys.stderr, ) return 1 palace_dir = Path(args.palace) if not palace_dir.exists() and not args.dry_run: print( f"[tunnel_sync] ERROR: local palace not found: {palace_dir}", file=sys.stderr, ) return 1 mode = "DRY-RUN" if args.dry_run else "LIVE" print(f"[tunnel_sync] {mode} — peer: {args.peer} palace: {palace_dir}") result = sync_peer(args.peer, palace_dir, n_results=args.n, dry_run=args.dry_run) if result.errors: for err in result.errors: print(f"[tunnel_sync] ERROR: {err}", file=sys.stderr) return 1 print( f"[tunnel_sync] Done — wings: {result.wings_found}, " f"rooms pulled: {result.rooms_pulled}, closets written: {result.closets_written}." ) return 0 if __name__ == "__main__": sys.exit(main())