the-nexus/mempalace/tunnel_sync.py

#!/usr/bin/env python3
"""
tunnel_sync.py — Pull closets from a remote wizard's fleet API into the local palace.

This is the client-side tunnel mechanism for #1078.  It connects to a peer
wizard's running fleet_api.py HTTP server, discovers their memory wings, and
imports the results into the local fleet palace as closet files.  Once imported,
`recall <query> --fleet` in Evennia will return results from the remote wing.

The code side is complete here; the infrastructure side (second wizard running
fleet_api.py behind an SSH tunnel or VPN) is still required to use this.

Usage:
    # Pull from a remote Alpha fleet API into the default local palace
    python mempalace/tunnel_sync.py --peer http://alpha.example.com:7771

    # Custom local palace path
    FLEET_PALACE_PATH=/data/fleet python mempalace/tunnel_sync.py \\
        --peer http://alpha.example.com:7771

    # Dry-run: show what would be imported without writing files
    python mempalace/tunnel_sync.py --peer http://alpha.example.com:7771 --dry-run

    # Limit results per room (default: 50)
    python mempalace/tunnel_sync.py --peer http://alpha.example.com:7771 --n 20

Environment:
    FLEET_PALACE_PATH — local fleet palace directory (default: /var/lib/mempalace/fleet)
    FLEET_PEER_URL    — remote fleet API URL (overridden by --peer flag)

Exits:
    0 — sync succeeded (or dry-run completed)
    1 — error (connection failure, invalid response, write error)

Refs: #1078, #1075
"""

from __future__ import annotations

import argparse
import json
import os
import sys
import time
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any

DEFAULT_PALACE_PATH = "/var/lib/mempalace/fleet"
DEFAULT_N_RESULTS = 50
# Broad queries for bulk room pull — used to discover representative content
_BROAD_QUERIES = [
    "the", "a", "is", "was", "and", "of", "to", "in", "it", "on",
    "commit", "issue", "error", "fix", "deploy", "event", "memory",
]
_REQUEST_TIMEOUT = 10  # seconds


@dataclass
class SyncResult:
    wings_found: list[str] = field(default_factory=list)
    rooms_pulled: int = 0
    closets_written: int = 0
    errors: list[str] = field(default_factory=list)

    @property
    def ok(self) -> bool:
        return len(self.errors) == 0


# ---------------------------------------------------------------------------
# HTTP helpers
# ---------------------------------------------------------------------------

def _get(url: str) -> dict[str, Any]:
    """GET *url*, return parsed JSON or raise on error."""
    req = urllib.request.Request(url, headers={"Accept": "application/json"})
    with urllib.request.urlopen(req, timeout=_REQUEST_TIMEOUT) as resp:
        return json.loads(resp.read())


def _peer_url(base: str, path: str) -> str:
    return base.rstrip("/") + path


# ---------------------------------------------------------------------------
# Wing / room discovery
# ---------------------------------------------------------------------------

def get_remote_wings(peer_url: str) -> list[str]:
    """Return the list of wing names from the remote fleet API."""
    data = _get(_peer_url(peer_url, "/wings"))
    return data.get("wings", [])


def search_remote_room(peer_url: str, room: str, n: int = DEFAULT_N_RESULTS) -> list[dict]:
    """
    Pull closet entries for a specific room from the remote peer.

    Uses multiple broad queries and deduplicates by text to maximize coverage
    without requiring a dedicated bulk-export endpoint.
    """
    seen_texts: set[str] = set()
    results: list[dict] = []

    for q in _BROAD_QUERIES:
        url = _peer_url(peer_url, f"/search?q={urllib.request.quote(q)}&room={urllib.request.quote(room)}&n={n}")
        try:
            data = _get(url)
        except (urllib.error.URLError, json.JSONDecodeError, OSError):
            continue

        for entry in data.get("results", []):
            text = entry.get("text", "")
            if text and text not in seen_texts:
                seen_texts.add(text)
                results.append(entry)

        if len(results) >= n:
            break

    return results[:n]


# ---------------------------------------------------------------------------
# Core sync
# ---------------------------------------------------------------------------

def _write_closet(
    palace_dir: Path,
    wing: str,
    room: str,
    entries: list[dict],
    dry_run: bool,
) -> bool:
    """Write entries as a .closet.json file under palace_dir/wing/."""
    wing_dir = palace_dir / wing
    closet_path = wing_dir / f"{room}.closet.json"

    drawers = [
        {
            "text": e.get("text", ""),
            "room": e.get("room", room),
            "wing": e.get("wing", wing),
            "score": e.get("score", 0.0),
            "closet": True,
            "source_file": f"tunnel:{wing}/{room}",
            "synced_at": int(time.time()),
        }
        for e in entries
    ]

    payload = json.dumps({"drawers": drawers, "wing": wing, "room": room}, indent=2)

    if dry_run:
        print(f"[tunnel_sync] DRY-RUN would write {len(drawers)} entries → {closet_path}")
        return True

    try:
        wing_dir.mkdir(parents=True, exist_ok=True)
        closet_path.write_text(payload)
        print(f"[tunnel_sync] Wrote {len(drawers)} entries → {closet_path}")
        return True
    except OSError as exc:
        print(f"[tunnel_sync] ERROR writing {closet_path}: {exc}", file=sys.stderr)
        return False


def sync_peer(
    peer_url: str,
    palace_dir: Path,
    n_results: int = DEFAULT_N_RESULTS,
    dry_run: bool = False,
) -> SyncResult:
    """
    Pull all wings and rooms from *peer_url* into *palace_dir*.

    Args:
        peer_url: Base URL of the remote fleet_api.py instance.
        palace_dir: Local fleet palace directory to write closets into.
        n_results: Maximum results to pull per room.
        dry_run: If True, print what would be written without touching disk.

    Returns:
        SyncResult with counts and any errors.
    """
    result = SyncResult()

    # Discover health
    try:
        health = _get(_peer_url(peer_url, "/health"))
        if health.get("status") != "ok":
            result.errors.append(f"Peer unhealthy: {health}")
            return result
    except (urllib.error.URLError, json.JSONDecodeError, OSError) as exc:
        result.errors.append(f"Could not reach peer at {peer_url}: {exc}")
        return result

    # Discover wings
    try:
        wings = get_remote_wings(peer_url)
    except (urllib.error.URLError, json.JSONDecodeError, OSError) as exc:
        result.errors.append(f"Could not list wings from {peer_url}: {exc}")
        return result

    result.wings_found = wings
    if not wings:
        print(f"[tunnel_sync] No wings found at {peer_url} — nothing to sync.")
        return result

    print(f"[tunnel_sync] Found wings: {wings}")

    # Import core rooms from each wing
    from nexus.mempalace.config import CORE_ROOMS

    for wing in wings:
        for room in CORE_ROOMS:
            print(f"[tunnel_sync] Pulling {wing}/{room} …")
            try:
                entries = search_remote_room(peer_url, room, n=n_results)
            except (urllib.error.URLError, json.JSONDecodeError, OSError) as exc:
                err = f"Error pulling {wing}/{room}: {exc}"
                result.errors.append(err)
                print(f"[tunnel_sync] ERROR: {err}", file=sys.stderr)
                continue

            if not entries:
                print(f"[tunnel_sync] No entries found for {wing}/{room} — skipping.")
                continue

            ok = _write_closet(palace_dir, wing, room, entries, dry_run=dry_run)
            result.rooms_pulled += 1
            if ok:
                result.closets_written += 1

    return result


# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------

def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        description="Sync closets from a remote wizard's fleet API into the local palace."
    )
    parser.add_argument(
        "--peer",
        default=os.environ.get("FLEET_PEER_URL", ""),
        metavar="URL",
        help="Base URL of the remote fleet_api.py (e.g. http://alpha.example.com:7771)",
    )
    parser.add_argument(
        "--palace",
        default=os.environ.get("FLEET_PALACE_PATH", DEFAULT_PALACE_PATH),
        metavar="DIR",
        help=f"Local fleet palace directory (default: {DEFAULT_PALACE_PATH})",
    )
    parser.add_argument(
        "--n",
        type=int,
        default=DEFAULT_N_RESULTS,
        metavar="N",
        help=f"Max results per room (default: {DEFAULT_N_RESULTS})",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Show what would be synced without writing files.",
    )
    args = parser.parse_args(argv)

    if not args.peer:
        print(
            "[tunnel_sync] ERROR: --peer URL is required (or set FLEET_PEER_URL).",
            file=sys.stderr,
        )
        return 1

    palace_dir = Path(args.palace)
    if not palace_dir.exists() and not args.dry_run:
        print(
            f"[tunnel_sync] ERROR: local palace not found: {palace_dir}",
            file=sys.stderr,
        )
        return 1

    mode = "DRY-RUN" if args.dry_run else "LIVE"
    print(f"[tunnel_sync] {mode} — peer: {args.peer}  palace: {palace_dir}")

    result = sync_peer(args.peer, palace_dir, n_results=args.n, dry_run=args.dry_run)

    if result.errors:
        for err in result.errors:
            print(f"[tunnel_sync] ERROR: {err}", file=sys.stderr)
        return 1

    print(
        f"[tunnel_sync] Done — wings: {result.wings_found}, "
        f"rooms pulled: {result.rooms_pulled}, closets written: {result.closets_written}."
    )
    return 0


if __name__ == "__main__":
    sys.exit(main())