timmy-talking-turd/src/inference-queue.js

export class OverloadError extends Error {
  constructor(message = 'Timmy is busy right now. Your journal still works — try again shortly or continue manually.') {
    super(message);
    this.name = 'OverloadError';
  }
}

function normalizeReason(reason) {
  if (reason instanceof Error) return reason;
  const text = String(reason ?? '').trim().slice(0, 200);
  return new Error(text || 'Request cancelled.');
}

function expiredMessage() {
  return 'Timmy is busy right now and your request timed out waiting. Your journal still works — try again shortly or continue manually.';
}

export function createInferenceQueue({
  concurrency = 1,
  maxQueueDepth = 0,
  requestTimeoutMs = 60_000,
  now = () => Date.now(),
} = {}) {
  let active = 0;
  const waiting = [];
  const handles = new Map();

  function expireStale(nowMs) {
    while (waiting.length) {
      if (nowMs - waiting[0].enqueuedAt <= requestTimeoutMs) break;
      const expired = waiting.shift();
      expired.settled = true;
      expired.abort(new OverloadError(expiredMessage()));
    }
  }

  function pump(nowMs = now()) {
    expireStale(nowMs);
    while (active < concurrency && waiting.length) {
      const entry = waiting.shift();
      if (entry.settled) continue;
      active += 1;
      entry.grant();
    }
  }

  function release() {
    active -= 1;
    pump();
  }

  async function admit(record) {
    const { controller } = record;
    if (controller.signal.aborted) throw normalizeReason(controller.signal.reason);
    expireStale(now());
    if (active < concurrency) {
      active += 1;
      return;
    }
    if (waiting.length >= maxQueueDepth) throw new OverloadError();
    await new Promise((resolve, reject) => {
      const entry = {
        enqueuedAt: now(),
        settled: false,
        grant: resolve,
        abort: reject,
      };
      record.entry = entry;
      waiting.push(entry);
    });
    record.entry = null;
  }

  function run(task) {
    const controller = new AbortController();
    const record = { controller, entry: null };
    let resolveOutcome;
    let rejectOutcome;
    // Deliberately not an async function: the caller must receive the very
    // promise registered in `handles`, or cancellation lookups would target
    // a different object than the one they hold.
    const promise = new Promise((resolve, reject) => {
      resolveOutcome = resolve;
      rejectOutcome = reject;
    });
    handles.set(promise, record);
    (async () => {
      let holdsSlot = false;
      try {
        await admit(record);
        holdsSlot = true;
        // The task body may not have started even though a slot is held; an
        // external cancel that raced ahead must still stop it here.
        if (controller.signal.aborted) throw normalizeReason(controller.signal.reason);
        resolveOutcome(await task(controller.signal));
      } catch (error) {
        rejectOutcome(error instanceof Error ? error : normalizeReason(error));
      } finally {
        if (holdsSlot) release();
      }
      // Drop the handle one microtask after the caller-visible promise
      // settles, so a same-tick external cancel still finds it.
      promise.then(
        () => handles.delete(promise),
        () => handles.delete(promise),
      );
    })();
    promise.catch(() => {});
    return promise;
  }

  function cancel(handle, reason) {
    const record = handles.get(handle);
    if (!record) return false;
    handles.delete(handle);
    const normalized = normalizeReason(reason);
    const entry = record.entry;
    if (entry && !entry.settled) {
      entry.settled = true;
      const index = waiting.indexOf(entry);
      if (index >= 0) waiting.splice(index, 1);
      entry.abort(normalized);
    }
    record.controller.abort(normalized);
    return true;
  }

  pump();
  return { run, cancel };
}