Wire orchestrator pipe into task runner + pipe-verifying integration tests (#134)

This commit is contained in:
Alexander Whitestone
2026-03-06 01:20:14 -05:00
committed by GitHub
parent d10cff333a
commit 87dc5eadfe
14 changed files with 2617 additions and 0 deletions

View File

View File

@@ -0,0 +1,195 @@
"""Paperclip bridge — CEO-level orchestration logic.
Timmy acts as the CEO: reviews issues, delegates to agents, tracks goals,
and approves/rejects work. All business logic lives here; routes stay thin.
"""
from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional
from config import settings
from integrations.paperclip.client import PaperclipClient, paperclip
from integrations.paperclip.models import (
CreateIssueRequest,
PaperclipAgent,
PaperclipGoal,
PaperclipIssue,
PaperclipStatusResponse,
UpdateIssueRequest,
)
logger = logging.getLogger(__name__)
class PaperclipBridge:
"""Bidirectional bridge between Timmy and Paperclip.
Timmy is the CEO — he creates issues, delegates to agents via wakeup,
reviews results, and manages the company's goals.
"""
def __init__(self, client: Optional[PaperclipClient] = None):
self.client = client or paperclip
# ── status / health ──────────────────────────────────────────────────
async def get_status(self) -> PaperclipStatusResponse:
"""Return integration status for the dashboard."""
if not settings.paperclip_enabled:
return PaperclipStatusResponse(
enabled=False,
paperclip_url=settings.paperclip_url,
)
connected = await self.client.healthy()
agent_count = 0
issue_count = 0
error = None
if connected:
try:
agents = await self.client.list_agents()
agent_count = len(agents)
issues = await self.client.list_issues()
issue_count = len(issues)
except Exception as exc:
error = str(exc)
else:
error = "Cannot reach Paperclip server"
return PaperclipStatusResponse(
enabled=True,
connected=connected,
paperclip_url=settings.paperclip_url,
company_id=settings.paperclip_company_id,
agent_count=agent_count,
issue_count=issue_count,
error=error,
)
# ── CEO actions: issue management ────────────────────────────────────
async def create_and_assign(
self,
title: str,
description: str = "",
assignee_id: Optional[str] = None,
priority: Optional[str] = None,
wake: bool = True,
) -> Optional[PaperclipIssue]:
"""Create an issue and optionally assign + wake an agent.
This is the primary CEO action: decide what needs doing, create
the ticket, assign it to the right agent, and kick off execution.
"""
req = CreateIssueRequest(
title=title,
description=description,
priority=priority,
assignee_id=assignee_id,
)
issue = await self.client.create_issue(req)
if not issue:
logger.error("Failed to create issue: %s", title)
return None
logger.info("Created issue %s: %s", issue.id, title)
if assignee_id and wake:
result = await self.client.wake_agent(assignee_id, issue_id=issue.id)
if result:
logger.info("Woke agent %s for issue %s", assignee_id, issue.id)
else:
logger.warning("Failed to wake agent %s", assignee_id)
return issue
async def delegate_issue(
self,
issue_id: str,
agent_id: str,
message: Optional[str] = None,
) -> bool:
"""Assign an existing issue to an agent and wake them."""
updated = await self.client.update_issue(
issue_id,
UpdateIssueRequest(assignee_id=agent_id),
)
if not updated:
return False
if message:
await self.client.add_comment(issue_id, f"[CEO] {message}")
await self.client.wake_agent(agent_id, issue_id=issue_id)
return True
async def review_issue(
self,
issue_id: str,
) -> Dict[str, Any]:
"""Gather all context for CEO review of an issue."""
issue = await self.client.get_issue(issue_id)
comments = await self.client.list_comments(issue_id)
return {
"issue": issue.model_dump() if issue else None,
"comments": [c.model_dump() for c in comments],
}
async def close_issue(self, issue_id: str, comment: Optional[str] = None) -> bool:
"""Close an issue as the CEO."""
if comment:
await self.client.add_comment(issue_id, f"[CEO] {comment}")
result = await self.client.update_issue(
issue_id,
UpdateIssueRequest(status="done"),
)
return result is not None
# ── CEO actions: team management ─────────────────────────────────────
async def get_team(self) -> List[PaperclipAgent]:
"""Get the full agent roster."""
return await self.client.list_agents()
async def get_org_chart(self) -> Optional[Dict[str, Any]]:
"""Get the organizational hierarchy."""
return await self.client.get_org()
# ── CEO actions: goal management ─────────────────────────────────────
async def list_goals(self) -> List[PaperclipGoal]:
return await self.client.list_goals()
async def set_goal(self, title: str, description: str = "") -> Optional[PaperclipGoal]:
return await self.client.create_goal(title, description)
# ── CEO actions: approvals ───────────────────────────────────────────
async def pending_approvals(self) -> List[Dict[str, Any]]:
return await self.client.list_approvals()
async def approve(self, approval_id: str, comment: str = "") -> bool:
result = await self.client.approve(approval_id, comment)
return result is not None
async def reject(self, approval_id: str, comment: str = "") -> bool:
result = await self.client.reject(approval_id, comment)
return result is not None
# ── CEO actions: monitoring ──────────────────────────────────────────
async def active_runs(self) -> List[Dict[str, Any]]:
"""Get currently running heartbeat executions."""
return await self.client.list_heartbeat_runs()
async def cancel_run(self, run_id: str) -> bool:
result = await self.client.cancel_run(run_id)
return result is not None
# Module-level singleton
bridge = PaperclipBridge()

View File

@@ -0,0 +1,308 @@
"""Paperclip AI API client.
Async HTTP client for communicating with a remote Paperclip server.
All methods degrade gracefully — log the error, return a fallback, never crash.
Paperclip API is mounted at ``/api`` and uses ``local_trusted`` mode on the
VPS, so the board actor is implicit. When the server sits behind an nginx
auth-gate the client authenticates with Basic-auth on the first request and
re-uses the session cookie thereafter.
"""
from __future__ import annotations
import base64
import logging
from typing import Any, Dict, List, Optional
import httpx
from config import settings
from integrations.paperclip.models import (
AddCommentRequest,
CreateIssueRequest,
PaperclipAgent,
PaperclipComment,
PaperclipGoal,
PaperclipIssue,
UpdateIssueRequest,
)
logger = logging.getLogger(__name__)
class PaperclipClient:
"""Thin async wrapper around the Paperclip REST API.
All public methods return typed results on success or ``None`` / ``[]``
on failure so callers never need to handle exceptions.
"""
def __init__(
self,
base_url: Optional[str] = None,
api_key: Optional[str] = None,
timeout: int = 30,
):
self._base_url = (base_url or settings.paperclip_url).rstrip("/")
self._api_key = api_key or settings.paperclip_api_key
self._timeout = timeout or settings.paperclip_timeout
self._client: Optional[httpx.AsyncClient] = None
# ── lifecycle ────────────────────────────────────────────────────────
def _get_client(self) -> httpx.AsyncClient:
if self._client is None or self._client.is_closed:
headers: Dict[str, str] = {"Accept": "application/json"}
if self._api_key:
headers["Authorization"] = f"Bearer {self._api_key}"
self._client = httpx.AsyncClient(
base_url=self._base_url,
headers=headers,
timeout=self._timeout,
)
return self._client
async def close(self) -> None:
if self._client and not self._client.is_closed:
await self._client.aclose()
# ── helpers ──────────────────────────────────────────────────────────
async def _get(self, path: str, params: Optional[Dict] = None) -> Optional[Any]:
try:
resp = await self._get_client().get(path, params=params)
resp.raise_for_status()
return resp.json()
except Exception as exc:
logger.warning("Paperclip GET %s failed: %s", path, exc)
return None
async def _post(self, path: str, json: Optional[Dict] = None) -> Optional[Any]:
try:
resp = await self._get_client().post(path, json=json)
resp.raise_for_status()
return resp.json()
except Exception as exc:
logger.warning("Paperclip POST %s failed: %s", path, exc)
return None
async def _patch(self, path: str, json: Optional[Dict] = None) -> Optional[Any]:
try:
resp = await self._get_client().patch(path, json=json)
resp.raise_for_status()
return resp.json()
except Exception as exc:
logger.warning("Paperclip PATCH %s failed: %s", path, exc)
return None
async def _delete(self, path: str) -> bool:
try:
resp = await self._get_client().delete(path)
resp.raise_for_status()
return True
except Exception as exc:
logger.warning("Paperclip DELETE %s failed: %s", path, exc)
return False
# ── health ───────────────────────────────────────────────────────────
async def healthy(self) -> bool:
"""Quick connectivity check."""
data = await self._get("/api/health")
return data is not None
# ── companies ────────────────────────────────────────────────────────
async def list_companies(self) -> List[Dict[str, Any]]:
data = await self._get("/api/companies")
return data if isinstance(data, list) else []
# ── agents ───────────────────────────────────────────────────────────
async def list_agents(self, company_id: Optional[str] = None) -> List[PaperclipAgent]:
cid = company_id or settings.paperclip_company_id
if not cid:
logger.warning("paperclip_company_id not set — cannot list agents")
return []
data = await self._get(f"/api/companies/{cid}/agents")
if not isinstance(data, list):
return []
return [PaperclipAgent(**a) for a in data]
async def get_agent(self, agent_id: str) -> Optional[PaperclipAgent]:
data = await self._get(f"/api/agents/{agent_id}")
return PaperclipAgent(**data) if data else None
async def wake_agent(
self,
agent_id: str,
issue_id: Optional[str] = None,
message: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
"""Trigger a heartbeat wake for an agent."""
body: Dict[str, Any] = {}
if issue_id:
body["issueId"] = issue_id
if message:
body["message"] = message
return await self._post(f"/api/agents/{agent_id}/wakeup", json=body)
async def get_org(self, company_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
cid = company_id or settings.paperclip_company_id
if not cid:
return None
return await self._get(f"/api/companies/{cid}/org")
# ── issues (tickets) ─────────────────────────────────────────────────
async def list_issues(
self,
company_id: Optional[str] = None,
status: Optional[str] = None,
) -> List[PaperclipIssue]:
cid = company_id or settings.paperclip_company_id
if not cid:
return []
params: Dict[str, str] = {}
if status:
params["status"] = status
data = await self._get(f"/api/companies/{cid}/issues", params=params)
if not isinstance(data, list):
return []
return [PaperclipIssue(**i) for i in data]
async def get_issue(self, issue_id: str) -> Optional[PaperclipIssue]:
data = await self._get(f"/api/issues/{issue_id}")
return PaperclipIssue(**data) if data else None
async def create_issue(
self,
req: CreateIssueRequest,
company_id: Optional[str] = None,
) -> Optional[PaperclipIssue]:
cid = company_id or settings.paperclip_company_id
if not cid:
logger.warning("paperclip_company_id not set — cannot create issue")
return None
data = await self._post(
f"/api/companies/{cid}/issues",
json=req.model_dump(exclude_none=True),
)
return PaperclipIssue(**data) if data else None
async def update_issue(
self,
issue_id: str,
req: UpdateIssueRequest,
) -> Optional[PaperclipIssue]:
data = await self._patch(
f"/api/issues/{issue_id}",
json=req.model_dump(exclude_none=True),
)
return PaperclipIssue(**data) if data else None
async def delete_issue(self, issue_id: str) -> bool:
return await self._delete(f"/api/issues/{issue_id}")
# ── issue comments ───────────────────────────────────────────────────
async def list_comments(self, issue_id: str) -> List[PaperclipComment]:
data = await self._get(f"/api/issues/{issue_id}/comments")
if not isinstance(data, list):
return []
return [PaperclipComment(**c) for c in data]
async def add_comment(
self,
issue_id: str,
content: str,
) -> Optional[PaperclipComment]:
data = await self._post(
f"/api/issues/{issue_id}/comments",
json={"content": content},
)
return PaperclipComment(**data) if data else None
# ── issue workflow ───────────────────────────────────────────────────
async def checkout_issue(self, issue_id: str) -> Optional[Dict[str, Any]]:
"""Assign an issue to Timmy (checkout)."""
body: Dict[str, Any] = {}
if settings.paperclip_agent_id:
body["agentId"] = settings.paperclip_agent_id
return await self._post(f"/api/issues/{issue_id}/checkout", json=body)
async def release_issue(self, issue_id: str) -> Optional[Dict[str, Any]]:
"""Release a checked-out issue."""
return await self._post(f"/api/issues/{issue_id}/release")
# ── goals ────────────────────────────────────────────────────────────
async def list_goals(self, company_id: Optional[str] = None) -> List[PaperclipGoal]:
cid = company_id or settings.paperclip_company_id
if not cid:
return []
data = await self._get(f"/api/companies/{cid}/goals")
if not isinstance(data, list):
return []
return [PaperclipGoal(**g) for g in data]
async def create_goal(
self,
title: str,
description: str = "",
company_id: Optional[str] = None,
) -> Optional[PaperclipGoal]:
cid = company_id or settings.paperclip_company_id
if not cid:
return None
data = await self._post(
f"/api/companies/{cid}/goals",
json={"title": title, "description": description},
)
return PaperclipGoal(**data) if data else None
# ── heartbeat runs ───────────────────────────────────────────────────
async def list_heartbeat_runs(
self,
company_id: Optional[str] = None,
) -> List[Dict[str, Any]]:
cid = company_id or settings.paperclip_company_id
if not cid:
return []
data = await self._get(f"/api/companies/{cid}/heartbeat-runs")
return data if isinstance(data, list) else []
async def get_run_events(self, run_id: str) -> List[Dict[str, Any]]:
data = await self._get(f"/api/heartbeat-runs/{run_id}/events")
return data if isinstance(data, list) else []
async def cancel_run(self, run_id: str) -> Optional[Dict[str, Any]]:
return await self._post(f"/api/heartbeat-runs/{run_id}/cancel")
# ── approvals ────────────────────────────────────────────────────────
async def list_approvals(self, company_id: Optional[str] = None) -> List[Dict[str, Any]]:
cid = company_id or settings.paperclip_company_id
if not cid:
return []
data = await self._get(f"/api/companies/{cid}/approvals")
return data if isinstance(data, list) else []
async def approve(self, approval_id: str, comment: str = "") -> Optional[Dict[str, Any]]:
body: Dict[str, Any] = {}
if comment:
body["comment"] = comment
return await self._post(f"/api/approvals/{approval_id}/approve", json=body)
async def reject(self, approval_id: str, comment: str = "") -> Optional[Dict[str, Any]]:
body: Dict[str, Any] = {}
if comment:
body["comment"] = comment
return await self._post(f"/api/approvals/{approval_id}/reject", json=body)
# Module-level singleton
paperclip = PaperclipClient()

View File

@@ -0,0 +1,120 @@
"""Pydantic models for Paperclip AI API objects."""
from __future__ import annotations
from datetime import datetime
from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Field
# ── Inbound: Paperclip → Timmy ──────────────────────────────────────────────
class PaperclipIssue(BaseModel):
"""A ticket/issue in Paperclip's task system."""
id: str
title: str
description: str = ""
status: str = "open"
priority: Optional[str] = None
assignee_id: Optional[str] = None
project_id: Optional[str] = None
labels: List[str] = Field(default_factory=list)
created_at: Optional[str] = None
updated_at: Optional[str] = None
class PaperclipComment(BaseModel):
"""A comment on a Paperclip issue."""
id: str
issue_id: str
content: str
author: Optional[str] = None
created_at: Optional[str] = None
class PaperclipAgent(BaseModel):
"""An agent in the Paperclip org chart."""
id: str
name: str
role: str = ""
status: str = "active"
adapter_type: Optional[str] = None
company_id: Optional[str] = None
class PaperclipGoal(BaseModel):
"""A company goal in Paperclip."""
id: str
title: str
description: str = ""
status: str = "active"
company_id: Optional[str] = None
class HeartbeatRun(BaseModel):
"""A heartbeat execution run."""
id: str
agent_id: str
status: str
issue_id: Optional[str] = None
started_at: Optional[str] = None
finished_at: Optional[str] = None
# ── Outbound: Timmy → Paperclip ─────────────────────────────────────────────
class CreateIssueRequest(BaseModel):
"""Request to create a new issue in Paperclip."""
title: str
description: str = ""
priority: Optional[str] = None
assignee_id: Optional[str] = None
project_id: Optional[str] = None
labels: List[str] = Field(default_factory=list)
class UpdateIssueRequest(BaseModel):
"""Request to update an existing issue."""
title: Optional[str] = None
description: Optional[str] = None
status: Optional[str] = None
priority: Optional[str] = None
assignee_id: Optional[str] = None
class AddCommentRequest(BaseModel):
"""Request to add a comment to an issue."""
content: str
class WakeAgentRequest(BaseModel):
"""Request to wake an agent via heartbeat."""
issue_id: Optional[str] = None
message: Optional[str] = None
# ── API route models ─────────────────────────────────────────────────────────
class PaperclipStatusResponse(BaseModel):
"""Response for GET /api/paperclip/status."""
enabled: bool
connected: bool = False
paperclip_url: str = ""
company_id: str = ""
agent_count: int = 0
issue_count: int = 0
error: Optional[str] = None

View File

@@ -0,0 +1,214 @@
"""Paperclip task runner — automated issue processing loop.
Timmy grabs open issues assigned to him, processes each one, posts a
completion comment, marks the issue done, and creates a recursive
follow-up task for himself.
Green-path workflow:
1. Poll Paperclip for open issues assigned to Timmy
2. Check out the first issue in queue
3. Process it (delegate to orchestrator via execute_task)
4. Post completion comment with the result
5. Mark the issue done
6. Create a follow-up task for himself (recursive musing)
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any, Callable, Coroutine, Dict, List, Optional, Protocol, runtime_checkable
from config import settings
from integrations.paperclip.bridge import PaperclipBridge, bridge as default_bridge
from integrations.paperclip.models import PaperclipIssue
logger = logging.getLogger(__name__)
@runtime_checkable
class Orchestrator(Protocol):
"""Anything with an ``execute_task`` matching Timmy's orchestrator."""
async def execute_task(
self, task_id: str, description: str, context: dict
) -> Any: ...
def _wrap_orchestrator(orch: Orchestrator) -> Callable:
"""Adapt an orchestrator's execute_task to the process_fn signature."""
async def _process(task_id: str, description: str, context: Dict) -> str:
raw = await orch.execute_task(task_id, description, context)
# execute_task may return str or dict — normalise to str
if isinstance(raw, dict):
return raw.get("result", str(raw))
return str(raw)
return _process
class TaskRunner:
"""Autonomous task loop: grab → process → complete → follow-up.
Wire an *orchestrator* (anything with ``execute_task``) and the runner
pushes issues through the real agent pipe. Falls back to a plain
``process_fn`` callable or a no-op default.
The runner operates on a single cycle via ``run_once`` (testable) or
continuously via ``start`` with ``paperclip_poll_interval``.
"""
def __init__(
self,
bridge: Optional[PaperclipBridge] = None,
orchestrator: Optional[Orchestrator] = None,
process_fn: Optional[Callable[[str, str, Dict], Coroutine[Any, Any, str]]] = None,
):
self.bridge = bridge or default_bridge
self.orchestrator = orchestrator
# Priority: explicit process_fn > orchestrator wrapper > default
if process_fn:
self._process_fn = process_fn
elif orchestrator:
self._process_fn = _wrap_orchestrator(orchestrator)
else:
self._process_fn = None
self._running = False
# ── single cycle ──────────────────────────────────────────────────
async def grab_next_task(self) -> Optional[PaperclipIssue]:
"""Grab the first open issue assigned to Timmy."""
agent_id = settings.paperclip_agent_id
if not agent_id:
logger.warning("paperclip_agent_id not set — cannot grab tasks")
return None
issues = await self.bridge.client.list_issues(status="open")
# Filter to issues assigned to Timmy, take the first one
for issue in issues:
if issue.assignee_id == agent_id:
return issue
return None
async def process_task(self, issue: PaperclipIssue) -> str:
"""Process an issue: check out, run through the orchestrator, return result."""
# Check out the issue so others know we're working on it
await self.bridge.client.checkout_issue(issue.id)
context = {
"issue_id": issue.id,
"title": issue.title,
"priority": issue.priority,
"labels": issue.labels,
}
if self._process_fn:
result = await self._process_fn(issue.id, issue.description or issue.title, context)
else:
result = f"Processed task: {issue.title}"
return result
async def complete_task(self, issue: PaperclipIssue, result: str) -> bool:
"""Post completion comment and mark issue done."""
# Post the result as a comment
await self.bridge.client.add_comment(
issue.id,
f"[Timmy] Task completed.\n\n{result}",
)
# Mark the issue as done
return await self.bridge.close_issue(issue.id, comment=None)
async def create_follow_up(self, original: PaperclipIssue, result: str) -> Optional[PaperclipIssue]:
"""Create a recursive follow-up task for Timmy.
Timmy muses about task automation and writes a follow-up issue
assigned to himself — the recursive self-improvement loop.
"""
follow_up_title = f"Follow-up: {original.title}"
follow_up_description = (
f"Automated follow-up from completed task '{original.title}' "
f"(issue {original.id}).\n\n"
f"Previous result:\n{result}\n\n"
"Review the outcome and determine if further action is needed. "
"Muse about task automation improvements and recursive self-improvement."
)
return await self.bridge.create_and_assign(
title=follow_up_title,
description=follow_up_description,
assignee_id=settings.paperclip_agent_id,
priority=original.priority,
wake=False, # Don't wake immediately — let the next poll pick it up
)
async def run_once(self) -> Optional[Dict[str, Any]]:
"""Execute one full cycle of the green-path workflow.
Returns a summary dict on success, None if no work found.
"""
# Step 1: Grab next task
issue = await self.grab_next_task()
if not issue:
logger.debug("No tasks in queue for Timmy")
return None
logger.info("Grabbed task %s: %s", issue.id, issue.title)
# Step 2: Process the task
result = await self.process_task(issue)
logger.info("Processed task %s", issue.id)
# Step 3: Complete it
completed = await self.complete_task(issue, result)
if not completed:
logger.warning("Failed to mark task %s as done", issue.id)
# Step 4: Create follow-up
follow_up = await self.create_follow_up(issue, result)
follow_up_id = follow_up.id if follow_up else None
if follow_up:
logger.info("Created follow-up %s for task %s", follow_up.id, issue.id)
return {
"original_issue_id": issue.id,
"original_title": issue.title,
"result": result,
"completed": completed,
"follow_up_issue_id": follow_up_id,
}
# ── continuous loop ───────────────────────────────────────────────
async def start(self) -> None:
"""Run the task loop continuously using paperclip_poll_interval."""
interval = settings.paperclip_poll_interval
if interval <= 0:
logger.info("Task runner disabled (poll_interval=%d)", interval)
return
self._running = True
logger.info("Task runner started (poll every %ds)", interval)
while self._running:
try:
await self.run_once()
except Exception as exc:
logger.error("Task runner cycle failed: %s", exc)
await asyncio.sleep(interval)
def stop(self) -> None:
"""Signal the loop to stop."""
self._running = False
logger.info("Task runner stopping")
# Module-level singleton
task_runner = TaskRunner()