the-nexus/mimo-swarm/scripts/mimo-dispatcher.py

#!/usr/bin/env python3
"""
Mimo Swarm Dispatcher — The Brain

Scans Gitea for open issues, claims them atomically via labels,
routes to lanes, and spawns one-shot mimo-v2-pro workers.
No new issues created. No duplicate claims. No bloat.
"""

import glob
import json
import os
import sys
import time
import subprocess
import urllib.request
import urllib.error
from datetime import datetime, timezone, timedelta

# ── Config ──────────────────────────────────────────────────────────────

GITEA_URL = "https://forge.alexanderwhitestone.com"
TOKEN_FILE = os.path.expanduser("~/.config/gitea/token")
STATE_DIR = os.path.expanduser("~/.hermes/mimo-swarm/state")
LOG_DIR = os.path.expanduser("~/.hermes/mimo-swarm/logs")
WORKER_SCRIPT = os.path.expanduser("~/.hermes/mimo-swarm/scripts/mimo-worker.sh")

# FOCUS MODE: all workers on ONE repo, deep polish
FOCUS_MODE = True
FOCUS_REPO = "Timmy_Foundation/the-nexus"
FOCUS_BUILD_CMD = "npm run build"  # validation command before PR
FOCUS_BUILD_DIR = None  # set to repo root after clone, auto-detected

# Lane caps (in focus mode, all lanes get more)
if FOCUS_MODE:
    MAX_WORKERS_PER_LANE = {"CODE": 15, "BUILD": 8, "RESEARCH": 5, "CREATE": 7}
else:
    MAX_WORKERS_PER_LANE = {"CODE": 10, "BUILD": 5, "RESEARCH": 5, "CREATE": 5}

CLAIM_TIMEOUT_MINUTES = 30
CLAIM_LABEL = "mimo-claimed"
MAX_QUEUE_DEPTH = 10  # Don't dispatch if queue already has this many prompts
CLAIM_COMMENT = "/claim"
DONE_COMMENT = "/done"
ABANDON_COMMENT = "/abandon"

# Lane detection from issue labels
LANE_MAP = {
    "CODE": ["bug", "fix", "defect", "error", "harness", "config", "ci", "devops",
             "critical", "p0", "p1", "backend", "api", "integration", "refactor"],
    "BUILD": ["feature", "enhancement", "build", "ui", "frontend", "game", "tool",
              "project", "deploy", "infrastructure"],
    "RESEARCH": ["research", "investigate", "spike", "audit", "analysis", "study",
                 "benchmark", "evaluate", "explore"],
    "CREATE": ["content", "creative", "write", "docs", "documentation", "story",
               "narrative", "design", "art", "media"],
}

# Priority repos (serve first) — ordered by backlog richness
PRIORITY_REPOS = [
    "Timmy_Foundation/the-nexus",
    "Timmy_Foundation/hermes-agent",
    "Timmy_Foundation/timmy-home",
    "Timmy_Foundation/timmy-config",
    "Timmy_Foundation/the-beacon",
    "Timmy_Foundation/the-testament",
    "Rockachopa/hermes-config",
    "Timmy/claw-agent",
    "replit/timmy-tower",
    "Timmy_Foundation/fleet-ops",
    "Timmy_Foundation/forge-log",
]

# Priority tags — issues with these labels get served FIRST regardless of lane
PRIORITY_TAGS = ["mnemosyne", "p0", "p1", "critical"]


# ── Helpers ─────────────────────────────────────────────────────────────

def load_token():
    with open(TOKEN_FILE) as f:
        return f.read().strip()


def api_get(path, token):
    """GET request to Gitea API."""
    url = f"{GITEA_URL}/api/v1{path}"
    req = urllib.request.Request(url, headers={
        "Authorization": f"token {token}",
        "Accept": "application/json",
    })
    try:
        with urllib.request.urlopen(req, timeout=30) as resp:
            return json.loads(resp.read())
    except urllib.error.HTTPError as e:
        if e.code == 404:
            return None
        raise


def api_post(path, token, data):
    """POST request to Gitea API."""
    url = f"{GITEA_URL}/api/v1{path}"
    body = json.dumps(data).encode()
    req = urllib.request.Request(url, data=body, headers={
        "Authorization": f"token {token}",
        "Content-Type": "application/json",
    }, method="POST")
    try:
        with urllib.request.urlopen(req, timeout=30) as resp:
            return json.loads(resp.read())
    except urllib.error.HTTPError as e:
        body = e.read().decode() if e.fp else ""
        log(f"  API error {e.code}: {body[:200]}")
        return None


def api_delete(path, token):
    """DELETE request to Gitea API."""
    url = f"{GITEA_URL}/api/v1{path}"
    req = urllib.request.Request(url, headers={
        "Authorization": f"token {token}",
    }, method="DELETE")
    try:
        with urllib.request.urlopen(req, timeout=30) as resp:
            return resp.status
    except urllib.error.HTTPError as e:
        return e.code


def log(msg):
    ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    line = f"[{ts}] {msg}"
    print(line)
    log_file = os.path.join(LOG_DIR, f"dispatcher-{datetime.now().strftime('%Y%m%d')}.log")
    with open(log_file, "a") as f:
        f.write(line + "\n")


def load_state():
    """Load dispatcher state (active claims)."""
    state_file = os.path.join(STATE_DIR, "dispatcher.json")
    if os.path.exists(state_file):
        with open(state_file) as f:
            return json.load(f)
    return {"active_claims": {}, "stats": {"total_dispatched": 0, "total_released": 0, "total_prs": 0}}


def save_state(state):
    state_file = os.path.join(STATE_DIR, "dispatcher.json")
    with open(state_file, "w") as f:
        json.dump(state, f, indent=2)


# ── Issue Analysis ──────────────────────────────────────────────────────

def get_repos(token):
    """Get all accessible repos (excluding archived)."""
    repos = []
    page = 1
    while True:
        data = api_get(f"/repos/search?limit=50&page={page}&sort=updated", token)
        if not data or not data.get("data"):
            break
        # Filter out archived repos
        active = [r for r in data["data"] if not r.get("archived", False)]
        repos.extend(active)
        page += 1
        if len(data["data"]) < 50:
            break
    return repos


def get_open_issues(repo_full_name, token):
    """Get open issues for a repo (not PRs)."""
    issues = []
    page = 1
    while True:
        data = api_get(f"/repos/{repo_full_name}/issues?state=open&limit=50&page={page}", token)
        if not data:
            break
        # Filter out pull requests
        real_issues = [i for i in data if not i.get("pull_request")]
        issues.extend(real_issues)
        page += 1
        if len(data) < 50:
            break
    return issues


# Pre-fetched PR references (set by dispatch function before loop)
_PR_REFS = set()
_CLAIMED_COMMENTS = set()


def prefetch_pr_refs(repo_name, token):
    """Fetch all open PRs once and build a set of issue numbers they reference."""
    global _PR_REFS
    _PR_REFS = set()
    prs = api_get(f"/repos/{repo_name}/pulls?state=open&limit=100", token)
    if prs:
        for pr in prs:
            body = pr.get("body", "") or ""
            head = pr.get("head", {}).get("ref", "")
            # Extract issue numbers from body (Closes #NNN) and branch (issue-NNN)
            import re
            for match in re.finditer(r'#(\d+)', body):
                _PR_REFS.add(int(match.group(1)))
            for match in re.finditer(r'issue-(\d+)', head):
                _PR_REFS.add(int(match.group(1)))


def is_claimed(issue, repo_name, token):
    """Check if issue is claimed (has mimo-claimed label or existing PR). NO extra API calls."""
    labels = [l["name"] for l in issue.get("labels", [])]
    if CLAIM_LABEL in labels:
        return True

    # Check pre-fetched PR refs (no API call)
    if issue["number"] in _PR_REFS:
        return True

    # Skip comment check for speed — label is the primary mechanism
    return False


def priority_score(issue):
    """Score an issue's priority. Higher = serve first."""
    score = 0
    labels = [l["name"].lower() for l in issue.get("labels", [])]
    title = issue.get("title", "").lower()

    # Mnemosyne gets absolute priority — check title AND labels
    if "mnemosyne" in title or any("mnemosyne" in l for l in labels):
        score += 300

    # Priority tags boost
    for tag in PRIORITY_TAGS:
        if tag in labels or f"[{tag}]" in title:
            score += 100

    # Older issues get slight boost (clear backlog)
    created = issue.get("created_at", "")
    if created:
        try:
            created_dt = datetime.fromisoformat(created.replace("Z", "+00:00"))
            age_days = (datetime.now(timezone.utc) - created_dt).days
            score += min(age_days, 30)  # Cap at 30 days
        except:
            pass

    return score


def detect_lane(issue):
    """Detect which lane an issue belongs to based on labels."""
    labels = [l["name"].lower() for l in issue.get("labels", [])]

    for lane, keywords in LANE_MAP.items():
        for label in labels:
            if label in keywords:
                return lane

    # Check title for keywords
    title = issue.get("title", "").lower()
    for lane, keywords in LANE_MAP.items():
        for kw in keywords:
            if kw in title:
                return lane

    return "CODE"  # Default


def count_active_in_lane(state, lane):
    """Count currently active workers in a lane."""
    count = 0
    for claim in state["active_claims"].values():
        if claim.get("lane") == lane:
            count += 1
    return count


# ── Claiming ────────────────────────────────────────────────────────────

def claim_issue(issue, repo_name, lane, token):
    """Claim an issue: add label + comment."""
    repo = repo_name
    num = issue["number"]

    # Add mimo-claimed label
    api_post(f"/repos/{repo}/issues/{num}/labels", token, {"labels": [CLAIM_LABEL]})

    # Add /claim comment
    comment_body = f"/claim — mimo-v2-pro [{lane}] lane. Branch: `mimo/{lane.lower()}/issue-{num}`"
    api_post(f"/repos/{repo}/issues/{num}/comments", token, {"body": comment_body})

    log(f"  CLAIMED #{num} in {repo} [{lane}]")


def release_issue(issue, repo_name, reason, token):
    """Release a claim: remove label, add /done or /abandon comment."""
    repo = repo_name
    num = issue["number"]

    # Remove mimo-claimed label
    labels = [l["name"] for l in issue.get("labels", [])]
    if CLAIM_LABEL in labels:
        api_delete(f"/repos/{repo}/issues/{num}/labels/{CLAIM_LABEL}", token)

    # Add completion comment
    comment = f"{ABANDON_COMMENT} — {reason}" if reason != "done" else f"{DONE_COMMENT} — completed by mimo-v2-pro"
    api_post(f"/repos/{repo}/issues/{num}/comments", token, {"body": comment})

    log(f"  RELEASED #{num} in {repo}: {reason}")


# ── Worker Spawning ─────────────────────────────────────────────────────

def spawn_worker(issue, repo_name, lane, token):
    """Spawn a one-shot mimo worker for an issue."""
    repo = repo_name
    num = issue["number"]
    title = issue["title"]
    body = issue.get("body", "")[:2000]  # Truncate long bodies
    labels = [l["name"] for l in issue.get("labels", [])]

    # Build worker prompt
    worker_id = f"mimo-{lane.lower()}-{num}-{int(time.time())}"

    prompt = build_worker_prompt(repo, num, title, body, labels, lane, worker_id)

    # Write prompt to temp file for the cron job to pick up
    prompt_file = os.path.join(STATE_DIR, f"prompt-{worker_id}.txt")
    with open(prompt_file, "w") as f:
        f.write(prompt)

    log(f"  SPAWNING worker {worker_id} for #{num} [{lane}]")
    return worker_id


def build_worker_prompt(repo, num, title, body, labels, lane, worker_id):
    """Build the prompt for a mimo worker. Focus-mode aware with build validation."""

    lane_instructions = {
        "CODE": """You are a coding worker. Fix bugs, implement features, refactor code.
- Read existing code BEFORE writing anything
- Match the code style of the file you're editing
- If Three.js code: use the existing patterns in the codebase
- If config/infra: be precise, check existing values first""",
        "BUILD": """You are a builder. Create new functionality, UI components, tools.
- Study the existing architecture before building
- Create complete, working implementations — no stubs
- For UI: match the existing visual style
- For APIs: follow the existing route patterns""",
        "RESEARCH": """You are a researcher. Investigate the issue thoroughly.
- Read all relevant code and documentation
- Document findings in a markdown file: FINDINGS-issue-{num}.md
- Include: what you found, what's broken, recommended fix, effort estimate
- Create a summary PR with the findings document""",
        "CREATE": """You are a creative worker. Write content, documentation, design.
- Quality over quantity — one excellent asset beats five mediocre ones
- Match the existing tone and style of the project
- For docs: include code examples where relevant""",
    }

    clone_url = f"{GITEA_URL}/{repo}.git"
    branch = f"mimo/{lane.lower()}/issue-{num}"

    focus_section = ""
    if FOCUS_MODE and repo == FOCUS_REPO:
        focus_section = f"""
## FOCUS MODE — THIS IS THE NEXUS
The Nexus is a Three.js 3D world — Timmy's sovereign home on the web.
Tech stack: vanilla JS, Three.js, WebSocket, HTML/CSS.
Entry point: app.js (root) or public/nexus/app.js
The world features: nebula skybox, portals, memory crystals, batcave terminal.

IMPORTANT: After implementing, you MUST validate:
1. cd /tmp/{worker_id}
2. Check for syntax errors: node --check *.js (if JS files changed)
3. If package.json exists: npm install --legacy-peer-deps && npm run build
4. If build fails: FIX IT before pushing. No broken builds.
5. If no build command exists: just validate syntax on changed files
"""

    return f"""You are a mimo-v2-pro swarm worker. {lane_instructions.get(lane, lane_instructions["CODE"])}

## ISSUE
Repository: {repo}
Issue: #{num}
Title: {title}
Labels: {', '.join(labels)}

Description:
{body}
{focus_section}
## WORKFLOW
1. Clone: git clone {clone_url} /tmp/{worker_id} 2>/dev/null || (cd /tmp/{worker_id} && git fetch origin && git checkout main && git pull)
2. cd /tmp/{worker_id}
3. Create branch: git checkout -b {branch}
4. READ THE CODE. Understand the architecture before writing anything.
5. Implement the fix/feature/solution.
6. BUILD VALIDATION:
   - Syntax check: node --check <file>.js for any JS changed
   - If package.json exists: npm install --legacy-peer-deps 2>/dev/null && npm run build 2>&1
   - If build fails: FIX THE BUILD. No broken PRs.
   - Ensure git diff shows meaningful changes (>0 lines)
7. Commit: git add -A && git commit -m "fix: {title} (closes #{num})"
8. Push: git push origin {branch}
9. Create PR via API:
   curl -s -X POST '{GITEA_URL}/api/v1/repos/{repo}/pulls' \\
     -H 'Authorization: token $(cat ~/.config/gitea/token)' \\
     -H 'Content-Type: application/json' \\
     -d '{{"title":"fix: {title}","head":"{branch}","base":"main","body":"Closes #{num}\\n\\nAutomated by mimo-v2-pro swarm.\\n\\n## Changes\\n- [describe what you changed]\\n\\n## Validation\\n- [x] Syntax check passed\\n- [x] Build passes (if applicable)"}}'

## HARD RULES
- NEVER exit without committing. Even partial progress must be committed.
- NEVER create new issues. Only work on issue #{num}.
- NEVER push to main. Only push to your branch.
- NEVER push a broken build. Fix it or abandon with clear notes.
- If too complex: commit WIP, push, PR body says "WIP — needs human review"
- If build fails and you can't fix: commit anyway, push, PR body says "Build failed — needs human fix"

Worker: {worker_id}
"""


# ── Main ────────────────────────────────────────────────────────────────

def dispatch(token):
    """Main dispatch loop."""
    state = load_state()
    dispatched = 0

    log("=" * 60)
    log("MIMO DISPATCHER — scanning for work")

    # Clean stale claims first
    stale = []
    for claim_id, claim in list(state["active_claims"].items()):
        started = datetime.fromisoformat(claim["started"])
        age = datetime.now(timezone.utc) - started
        if age > timedelta(minutes=CLAIM_TIMEOUT_MINUTES):
            stale.append(claim_id)

    for claim_id in stale:
        claim = state["active_claims"].pop(claim_id)
        log(f"  EXPIRED claim: {claim['repo']}#{claim['issue']} [{claim['lane']}]")
        state["stats"]["total_released"] += 1

    # Prefetch PR refs once (avoids N API calls in is_claimed)
    target_repo = FOCUS_REPO if FOCUS_MODE else PRIORITY_REPOS[0]
    prefetch_pr_refs(target_repo, token)
    log(f"  Prefetched {len(_PR_REFS)} PR references")

    # Check queue depth — don't pile up if workers haven't caught up
    pending_prompts = len(glob.glob(os.path.join(STATE_DIR, "prompt-*.txt")))
    if pending_prompts >= MAX_QUEUE_DEPTH:
        log(f"  QUEUE THROTTLE: {pending_prompts} prompts pending (max {MAX_QUEUE_DEPTH}) — skipping dispatch")
        save_state(state)
        return 0

    # FOCUS MODE: scan only the focus repo. FIREHOSE: scan all.
    if FOCUS_MODE:
        ordered = [FOCUS_REPO]
        log(f"  FOCUS MODE: targeting {FOCUS_REPO} only")
    else:
        repos = get_repos(token)
        repo_names = [r["full_name"] for r in repos]
        ordered = []
        for pr in PRIORITY_REPOS:
            if pr in repo_names:
                ordered.append(pr)
        for rn in repo_names:
            if rn not in ordered:
                ordered.append(rn)

    # Scan each repo and collect all issues for priority sorting
    all_issues = []
    for repo_name in ordered[:20 if not FOCUS_MODE else 1]:
        issues = get_open_issues(repo_name, token)
        for issue in issues:
            issue["_repo_name"] = repo_name  # Tag with repo
            all_issues.append(issue)

    # Sort by priority score (highest first)
    all_issues.sort(key=priority_score, reverse=True)

    for issue in all_issues:
        repo_name = issue["_repo_name"]

        # Skip if already claimed in state
        claim_key = f"{repo_name}#{issue['number']}"
        if claim_key in state["active_claims"]:
            continue

        # Skip if claimed in Gitea
        if is_claimed(issue, repo_name, token):
            continue

        # Detect lane
        lane = detect_lane(issue)

        # Check lane capacity
        active_in_lane = count_active_in_lane(state, lane)
        max_in_lane = MAX_WORKERS_PER_LANE.get(lane, 1)

        if active_in_lane >= max_in_lane:
            continue  # Lane full, skip

        # Claim and spawn
        claim_issue(issue, repo_name, lane, token)
        worker_id = spawn_worker(issue, repo_name, lane, token)

        state["active_claims"][claim_key] = {
            "repo": repo_name,
            "issue": issue["number"],
            "lane": lane,
            "worker_id": worker_id,
            "started": datetime.now(timezone.utc).isoformat(),
        }
        state["stats"]["total_dispatched"] += 1
        dispatched += 1

        max_dispatch = 35 if FOCUS_MODE else 25
        if dispatched >= max_dispatch:
            break

    save_state(state)

    # Summary
    active = len(state["active_claims"])
    log(f"Dispatch complete: {dispatched} new, {active} active, {state['stats']['total_dispatched']} total dispatched")
    log(f"Active by lane: CODE={count_active_in_lane(state,'CODE')}, BUILD={count_active_in_lane(state,'BUILD')}, RESEARCH={count_active_in_lane(state,'RESEARCH')}, CREATE={count_active_in_lane(state,'CREATE')}")

    return dispatched


if __name__ == "__main__":
    token = load_token()
    dispatched = dispatch(token)
    sys.exit(0 if dispatched >= 0 else 1)