timmy-talking-turd/scripts/record_worker_benchmark.mjs

#!/usr/bin/env node
// Record the target-GPU cost of the pinned worker (issue #15).
//
// This script measures; it never invents. If no GPU can be probed, if the pinned model is not
// loaded, or if the cold start cannot be observed, it exits non-zero and writes no receipt at all,
// because a receipt with zeroed or assumed numbers is worse than no receipt.
import { spawn } from 'node:child_process';
import { readFile, writeFile } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';
import path from 'node:path';

const LOOPBACK = /^(127\.0\.0\.1|localhost|::1)$/;

function parseArgs(argv) {
  const options = {
    out: '',
    host: '127.0.0.1',
    port: '8080',
    alias: 'SmolVLM2-2.2B-Instruct',
    gpuProbe: 'nvidia-smi',
    warmSamples: 20,
    startCommand: '',
    readyTimeoutSeconds: 900,
  };
  for (let index = 0; index < argv.length; index += 1) {
    const [flag, inline] = argv[index].split('=');
    const value = inline ?? argv[index + 1];
    const consume = inline === undefined ? 2 : 1;
    if (flag === '--out') options.out = value;
    else if (flag === '--host') options.host = value;
    else if (flag === '--port') options.port = value;
    else if (flag === '--alias') options.alias = value;
    else if (flag === '--gpu-probe') options.gpuProbe = value;
    else if (flag === '--warm-samples') options.warmSamples = Number(value);
    else if (flag === '--start-command') options.startCommand = value;
    else if (flag === '--ready-timeout') options.readyTimeoutSeconds = Number(value);
    else fail(`Unknown argument: ${flag}`);
    index += consume - 1;
  }
  return options;
}

function fail(message, code = 64) {
  process.stderr.write(`${message}\n`);
  process.exit(code);
}

const reject = message => {
  process.stderr.write(`${message}\n`);
  process.exit(1);
};

const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));

async function modelsServed({ host, port, alias }) {
  try {
    const response = await fetch(`http://${host}:${port}/v1/models`, { signal: AbortSignal.timeout(3000) });
    if (!response.ok) return false;
    const body = await response.json();
    const ids = Array.isArray(body?.data) ? body.data.map(item => item?.id).filter(Boolean) : [];
    return ids.some(id => id === alias || id.includes(alias));
  } catch {
    return false;
  }
}

function runCommand(command, args) {
  return new Promise(resolve => {
    const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
    let stdout = '';
    let stderr = '';
    child.stdout.on('data', chunk => { stdout += chunk; });
    child.stderr.on('data', chunk => { stderr += chunk; });
    child.on('error', error => resolve({ code: 127, stdout, stderr: String(error.message) }));
    child.on('close', code => resolve({ code, stdout, stderr }));
  });
}

async function probeGpu(probeCommand) {
  const result = await runCommand(probeCommand, [
    '--query-gpu=name,memory.total,memory.used',
    '--format=csv,noheader,nounits',
  ]);
  if (result.code !== 0) {
    reject(`No GPU could be probed with ${probeCommand}: ${result.stderr.trim() || `exit ${result.code}`}. A benchmark receipt needs a real device, so nothing was written.`);
  }
  const [name, total, used] = result.stdout.trim().split('\n')[0].split(',').map(part => part.trim());
  if (!name || !/^\d+$/.test(total) || !/^\d+$/.test(used)) {
    reject(`The GPU probe returned unreadable output (${JSON.stringify(result.stdout.trim())}); refusing to record VRAM it cannot read.`);
  }
  return { name, vramTotalMiB: Number(total), vramUsedMiB: Number(used) };
}

function percentile(sorted, fraction) {
  const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(fraction * sorted.length) - 1));
  return sorted[index];
}

const options = parseArgs(process.argv.slice(2));
if (!options.out) fail('--out is required so the receipt has a destination');
if (!LOOPBACK.test(options.host)) fail(`The worker is a loopback service; refusing to benchmark ${options.host}.`);
if (!Number.isInteger(options.warmSamples) || options.warmSamples < 1) fail('--warm-samples must be a positive integer');

const receipts = JSON.parse(await readFile(fileURLToPath(new URL('../deploy/worker/receipts.json', import.meta.url)), 'utf8'));

// The device is probed first: nothing is timed until we know the numbers have somewhere to be real.
const gpu = await probeGpu(options.gpuProbe);

if (!(await modelsServed(options))) {
  reject(`${options.alias} is not loaded on ${options.host}:${options.port} yet, so there is no ready worker to time. Start the worker and wait for the post-load gate.`);
}

let coldStartMs = null;
if (options.startCommand) {
  const started = Date.now();
  const started_process = await runCommand('bash', ['-c', options.startCommand]);
  if (started_process.code !== 0) {
    reject(`The cold-start command failed (exit ${started_process.code}): ${started_process.stderr.trim()}`);
  }
  const deadline = Date.now() + options.readyTimeoutSeconds * 1000;
  let ready = false;
  while (Date.now() < deadline) {
    if (await modelsServed(options)) { ready = true; break; }
    await sleep(250);
  }
  if (!ready) reject(`The worker did not become ready within ${options.readyTimeoutSeconds}s, so the cold start could not be measured.`);
  coldStartMs = Date.now() - started;
}

const samples = [];
for (let index = 0; index < options.warmSamples; index += 1) {
  const started = process.hrtime.bigint();
  const ok = await modelsServed(options);
  const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6;
  if (!ok) reject('The worker stopped answering during the warm pass; the receipt would be partial, so nothing was written.');
  samples.push(Math.round(elapsedMs * 1000) / 1000);
}
const sorted = [...samples].sort((left, right) => left - right);

const receipt = {
  schemaVersion: 1,
  status: 'measured_on_target_gpu',
  measuredAt: new Date().toISOString(),
  gpu,
  coldStartMs,
  warmLatencyMs: {
    samples: sorted.length,
    min: sorted[0],
    p50: percentile(sorted, 0.5),
    p95: percentile(sorted, 0.95),
    max: sorted[sorted.length - 1],
  },
  worker: { baseUrl: `http://${options.host}:${options.port}/v1`, alias: options.alias },
  pins: {
    llamaCppCommit: receipts.llamaCpp.commit,
    modelSha256: receipts.models.map(model => model.sha256),
  },
};

await writeFile(path.resolve(options.out), `${JSON.stringify(receipt, null, 2)}\n`, 'utf8');
process.stdout.write(`${path.resolve(options.out)}\n`);