kimi/Timmy-time-dashboard
Archived
1
0
Fork 0
forked from Rockachopa/Timmy-time-dashboard
Code Pull Requests Activity

feat: task queue system with startup drain and backlogging (#76)

Browse Source 
* feat: add task queue system for Timmy - all work goes through the queue

- Add queue position tracking to task_queue models with task_type field
- Add TaskProcessor class that consumes tasks from queue one at a time
- Modify chat route to queue all messages for async processing
- Chat responses get 'high' priority to jump ahead of thought tasks
- Add queue status API endpoints for position polling
- Update UI to show queue position (x/y) and current task banner
- Replace thinking loop with task-based approach - thoughts are queued tasks
- Push responses to user via WebSocket instead of immediate HTTP response
- Add database migrations for existing tables

* feat: Timmy drains task queue on startup, backlogs unhandleable tasks

On spin-up, Timmy now iterates through all pending/approved tasks
immediately instead of waiting for the polling loop. Tasks without a
registered handler or with permanent errors are moved to a new
BACKLOGGED status with a reason, keeping the queue clear for work
Timmy can actually do.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Alexander Payne <apayne@MM.local>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Alexander Whitestone
2026-02-27 01:52:42 -05:00
committed by GitHub
parent 849b5b1a8d
commit 5b6d33e05a
12 changed files with 1286 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

237
src/swarm/task_queue/models.py
View File

@@ -24,6 +24,7 @@ class TaskStatus(str, Enum):
COMPLETED = "completed"
VETOED = "vetoed"
FAILED = "failed"
BACKLOGGED = "backlogged"
class TaskPriority(str, Enum):
@@ -47,6 +48,7 @@ class QueueTask:
id: str = field(default_factory=lambda: str(uuid.uuid4()))
title: str = ""
description: str = ""
task_type: str = "chat_response" # chat_response, thought, internal, external
assigned_to: str = "timmy"
created_by: str = "user"
status: TaskStatus = TaskStatus.PENDING_APPROVAL
@@ -64,6 +66,8 @@ class QueueTask:
updated_at: str = field(
default_factory=lambda: datetime.now(timezone.utc).isoformat()
)
queue_position: Optional[int] = None # Position in queue when created
backlog_reason: Optional[str] = None # Why the task was backlogged
# ── Auto-Approve Rules ──────────────────────────────────────────────────
@@ -97,6 +101,7 @@ def should_auto_approve(task: QueueTask) -> bool:
# ── Database ─────────────────────────────────────────────────────────────
def _get_conn() -> sqlite3.Connection:
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(DB_PATH))
@@ -107,6 +112,7 @@ def _get_conn() -> sqlite3.Connection:
id TEXT PRIMARY KEY,
title TEXT NOT NULL,
description TEXT DEFAULT '',
task_type TEXT DEFAULT 'chat_response',
assigned_to TEXT DEFAULT 'timmy',
created_by TEXT DEFAULT 'user',
status TEXT DEFAULT 'pending_approval',
@@ -119,18 +125,31 @@ def _get_conn() -> sqlite3.Connection:
created_at TEXT NOT NULL,
started_at TEXT,
completed_at TEXT,
updated_at TEXT NOT NULL
updated_at TEXT NOT NULL,
queue_position INTEGER,
backlog_reason TEXT
)
""")
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_tq_status ON task_queue(status)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_tq_priority ON task_queue(priority)"
)
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_tq_created ON task_queue(created_at)"
)
conn.execute("CREATE INDEX IF NOT EXISTS idx_tq_status ON task_queue(status)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_tq_priority ON task_queue(priority)")
conn.execute("CREATE INDEX IF NOT EXISTS idx_tq_created ON task_queue(created_at)")
# Migrate existing tables - add new columns if they don't exist
try:
conn.execute(
"ALTER TABLE task_queue ADD COLUMN task_type TEXT DEFAULT 'chat_response'"
)
except sqlite3.OperationalError:
pass # Column already exists
try:
conn.execute("ALTER TABLE task_queue ADD COLUMN queue_position INTEGER")
except sqlite3.OperationalError:
pass # Column already exists
try:
conn.execute("ALTER TABLE task_queue ADD COLUMN backlog_reason TEXT")
except sqlite3.OperationalError:
pass # Column already exists
conn.commit()
return conn
@@ -146,6 +165,7 @@ def _row_to_task(row: sqlite3.Row) -> QueueTask:
id=d["id"],
title=d["title"],
description=d.get("description", ""),
task_type=d.get("task_type", "chat_response"),
assigned_to=d.get("assigned_to", "timmy"),
created_by=d.get("created_by", "user"),
status=TaskStatus(d["status"]),
@@ -159,11 +179,14 @@ def _row_to_task(row: sqlite3.Row) -> QueueTask:
started_at=d.get("started_at"),
completed_at=d.get("completed_at"),
updated_at=d["updated_at"],
queue_position=d.get("queue_position"),
backlog_reason=d.get("backlog_reason"),
)
# ── CRUD ─────────────────────────────────────────────────────────────────
def create_task(
title: str,
description: str = "",
@@ -174,12 +197,18 @@ def create_task(
auto_approve: bool = False,
parent_task_id: Optional[str] = None,
steps: Optional[list] = None,
task_type: str = "chat_response",
) -> QueueTask:
"""Create a new task in the queue."""
now = datetime.now(timezone.utc).isoformat()
# Calculate queue position - count tasks ahead in queue (pending or approved)
queue_position = get_queue_position_ahead(assigned_to)
task = QueueTask(
title=title,
description=description,
task_type=task_type,
assigned_to=assigned_to,
created_by=created_by,
status=TaskStatus.PENDING_APPROVAL,
@@ -190,6 +219,7 @@ def create_task(
steps=steps or [],
created_at=now,
updated_at=now,
queue_position=queue_position,
)
# Check auto-approve
@@ -200,17 +230,31 @@ def create_task(
conn = _get_conn()
conn.execute(
"""INSERT INTO task_queue
(id, title, description, assigned_to, created_by, status, priority,
(id, title, description, task_type, assigned_to, created_by, status, priority,
requires_approval, auto_approve, parent_task_id, result, steps,
created_at, started_at, completed_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
created_at, started_at, completed_at, updated_at, queue_position,
backlog_reason)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
task.id, task.title, task.description, task.assigned_to,
task.created_by, task.status.value, task.priority.value,
int(task.requires_approval), int(task.auto_approve),
task.parent_task_id, task.result, json.dumps(task.steps),
task.created_at, task.started_at, task.completed_at,
task.id,
task.title,
task.description,
task.task_type,
task.assigned_to,
task.created_by,
task.status.value,
task.priority.value,
int(task.requires_approval),
int(task.auto_approve),
task.parent_task_id,
task.result,
json.dumps(task.steps),
task.created_at,
task.started_at,
task.completed_at,
task.updated_at,
task.queue_position,
task.backlog_reason,
),
)
conn.commit()
@@ -220,9 +264,7 @@ def create_task(
def get_task(task_id: str) -> Optional[QueueTask]:
conn = _get_conn()
row = conn.execute(
"SELECT * FROM task_queue WHERE id = ?", (task_id,)
).fetchone()
row = conn.execute("SELECT * FROM task_queue WHERE id = ?", (task_id,)).fetchone()
conn.close()
return _row_to_task(row) if row else None
@@ -264,6 +306,7 @@ def update_task_status(
task_id: str,
new_status: TaskStatus,
result: Optional[str] = None,
backlog_reason: Optional[str] = None,
) -> Optional[QueueTask]:
now = datetime.now(timezone.utc).isoformat()
conn = _get_conn()
@@ -282,6 +325,10 @@ def update_task_status(
updates.append("result = ?")
params.append(result)
if backlog_reason is not None:
updates.append("backlog_reason = ?")
params.append(backlog_reason)
params.append(task_id)
conn.execute(
f"UPDATE task_queue SET {', '.join(updates)} WHERE id = ?",
@@ -289,9 +336,7 @@ def update_task_status(
)
conn.commit()
row = conn.execute(
"SELECT * FROM task_queue WHERE id = ?", (task_id,)
).fetchone()
row = conn.execute("SELECT * FROM task_queue WHERE id = ?", (task_id,)).fetchone()
conn.close()
return _row_to_task(row) if row else None
@@ -330,9 +375,7 @@ def update_task(
)
conn.commit()
row = conn.execute(
"SELECT * FROM task_queue WHERE id = ?", (task_id,)
).fetchone()
row = conn.execute("SELECT * FROM task_queue WHERE id = ?", (task_id,)).fetchone()
conn.close()
return _row_to_task(row) if row else None
@@ -369,6 +412,90 @@ def get_pending_count() -> int:
return row["cnt"] if row else 0
def get_queue_position_ahead(assigned_to: str) -> int:
"""Get count of tasks ahead of new tasks for a given assignee.
Counts tasks that are pending_approval or approved (waiting to be processed).
"""
conn = _get_conn()
row = conn.execute(
"""SELECT COUNT(*) as cnt FROM task_queue
WHERE assigned_to = ? AND status IN ('pending_approval', 'approved', 'running')
AND created_at < datetime('now')""",
(assigned_to,),
).fetchone()
conn.close()
return row["cnt"] if row else 0
def get_queue_status_for_task(task_id: str) -> dict:
"""Get queue position info for a specific task."""
task = get_task(task_id)
if not task:
return {"error": "Task not found"}
conn = _get_conn()
# Count tasks ahead of this one (created earlier, not completed)
ahead = conn.execute(
"""SELECT COUNT(*) as cnt FROM task_queue
WHERE assigned_to = ? AND status NOT IN ('completed', 'failed', 'vetoed')
AND created_at < ?""",
(task.assigned_to, task.created_at),
).fetchone()
total = conn.execute(
"""SELECT COUNT(*) as cnt FROM task_queue
WHERE assigned_to = ? AND status NOT IN ('completed', 'failed', 'vetoed')""",
(task.assigned_to,),
).fetchone()
conn.close()
position = ahead["cnt"] + 1 if ahead else 1
total_count = total["cnt"] if total else 1
return {
"task_id": task_id,
"position": position,
"total": total_count,
"percent_ahead": int((ahead["cnt"] / total_count * 100))
if total_count > 0
else 0,
}
def get_current_task_for_agent(assigned_to: str) -> Optional[QueueTask]:
"""Get the currently running task for an agent."""
conn = _get_conn()
row = conn.execute(
"""SELECT * FROM task_queue
WHERE assigned_to = ? AND status = 'running'
ORDER BY started_at DESC LIMIT 1""",
(assigned_to,),
).fetchone()
conn.close()
return _row_to_task(row) if row else None
def get_next_pending_task(assigned_to: str) -> Optional[QueueTask]:
"""Get the next pending/approved task for an agent to work on."""
conn = _get_conn()
row = conn.execute(
"""SELECT * FROM task_queue
WHERE assigned_to = ? AND status IN ('approved', 'pending_approval')
ORDER BY
CASE priority
WHEN 'urgent' THEN 1
WHEN 'high' THEN 2
WHEN 'normal' THEN 3
WHEN 'low' THEN 4
END,
created_at ASC
LIMIT 1""",
(assigned_to,),
).fetchone()
conn.close()
return _row_to_task(row) if row else None
def get_task_summary_for_briefing() -> dict:
"""Get task stats for the morning briefing."""
counts = get_counts_by_status()
@@ -377,6 +504,10 @@ def get_task_summary_for_briefing() -> dict:
failed = conn.execute(
"SELECT title, result FROM task_queue WHERE status = 'failed' ORDER BY updated_at DESC LIMIT 5"
).fetchall()
# Backlogged tasks
backlogged = conn.execute(
"SELECT title, backlog_reason FROM task_queue WHERE status = 'backlogged' ORDER BY updated_at DESC LIMIT 5"
).fetchall()
conn.close()
return {
@@ -385,6 +516,56 @@ def get_task_summary_for_briefing() -> dict:
"completed": counts.get("completed", 0),
"failed": counts.get("failed", 0),
"vetoed": counts.get("vetoed", 0),
"backlogged": counts.get("backlogged", 0),
"total": sum(counts.values()),
"recent_failures": [{"title": r["title"], "result": r["result"]} for r in failed],
"recent_failures": [
{"title": r["title"], "result": r["result"]} for r in failed
],
"recent_backlogged": [
{"title": r["title"], "reason": r["backlog_reason"]} for r in backlogged
],
}
def list_backlogged_tasks(
assigned_to: Optional[str] = None, limit: int = 50
) -> list[QueueTask]:
"""List all backlogged tasks, optionally filtered by assignee."""
conn = _get_conn()
if assigned_to:
rows = conn.execute(
"""SELECT * FROM task_queue WHERE status = 'backlogged' AND assigned_to = ?
ORDER BY priority, created_at ASC LIMIT ?""",
(assigned_to, limit),
).fetchall()
else:
rows = conn.execute(
"""SELECT * FROM task_queue WHERE status = 'backlogged'
ORDER BY priority, created_at ASC LIMIT ?""",
(limit,),
).fetchall()
conn.close()
return [_row_to_task(r) for r in rows]
def get_all_actionable_tasks(assigned_to: str) -> list[QueueTask]:
"""Get all tasks that should be processed on startup — approved or auto-approved pending.
Returns tasks ordered by priority then creation time (urgent first, oldest first).
"""
conn = _get_conn()
rows = conn.execute(
"""SELECT * FROM task_queue
WHERE assigned_to = ? AND status IN ('approved', 'pending_approval')
ORDER BY
CASE priority
WHEN 'urgent' THEN 1
WHEN 'high' THEN 2
WHEN 'normal' THEN 3
WHEN 'low' THEN 4
END,
created_at ASC""",
(assigned_to,),
).fetchall()
conn.close()
return [_row_to_task(r) for r in rows]