timmy-talking-turd/tests/model-metrics.test.js

import test from 'node:test';
import assert from 'node:assert/strict';

import { createModelMetrics } from '../src/model-metrics.js';

// Issue #19: the worker has to be operable without the metrics becoming a second copy of the
// journal. These tests are the boundary: what a metric sample may say, and what it may never say.

test('abstention rate is computed from the model answers that came back at all', async () => {
  const metrics = createModelMetrics({ now: () => 1_000 });

  metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 });
  metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 1_100 });
  metrics.recordAnalyze({ outcome: 'abstained', durationMs: 700 });
  metrics.recordAnalyze({ outcome: 'abstained', durationMs: 800 });

  const snapshot = metrics.snapshot();

  assert.equal(snapshot.model.outcomes.suggestion, 2);
  assert.equal(snapshot.model.outcomes.abstained, 2);
  assert.equal(snapshot.model.abstentionRate, 0.5);
});

test('a label that is not an enumerated outcome is counted, never copied into the snapshot', async () => {
  const metrics = createModelMetrics({ now: () => 1_000 });

  // The worst case for a metrics boundary: a caller-supplied string arrives where a label belongs.
  metrics.recordAnalyze({ outcome: 'Type 4, urgent, 2026-11-01, milkY4B2cQ9w==', durationMs: 900 });

  const snapshot = metrics.snapshot();
  const serialized = JSON.stringify(snapshot);

  assert.deepEqual(Object.keys(snapshot.model.outcomes).sort(), ['abstained', 'failure', 'suggestion']);
  assert.doesNotMatch(serialized, /urgent/i);
  assert.doesNotMatch(serialized, /milkY4B2cQ9w/);
  assert.equal(snapshot.model.outcomes.failure, 1, 'an unrecognised outcome is one failed answer');
});

test('a failed request is classified and kept out of the abstention denominator', async () => {
  const metrics = createModelMetrics({ now: () => 1_000 });

  metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 });
  metrics.recordAnalyze({ outcome: 'failure', failureClass: 'provider_unreachable', durationMs: 2_000 });
  metrics.recordAnalyze({ outcome: 'failure', failureClass: 'provider_unreachable', durationMs: 2_100 });
  metrics.recordAnalyze({ outcome: 'abstained', durationMs: 800 });

  const snapshot = metrics.snapshot();

  assert.equal(snapshot.model.failureClasses.provider_unreachable, 2);
  assert.equal(snapshot.model.abstentionRate, 0.5, 'a failed answer is not an abstention');
});

test('a failure class outside the enumerated set collapses instead of being echoed', async () => {
  const metrics = createModelMetrics({ now: () => 1_000 });

  metrics.recordAnalyze({ outcome: 'failure', failureClass: 'note:blood in stool, image 9f2a', durationMs: 2_000 });

  const snapshot = metrics.snapshot();

  assert.equal(snapshot.model.failureClasses.unexpected, 1);
  assert.doesNotMatch(JSON.stringify(snapshot), /blood in stool/);
});

test('analysis latency is reported as warm work, with counts and a bounded histogram', async () => {
  const metrics = createModelMetrics({ now: () => 1_000 });

  metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 });
  metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 1_500 });
  metrics.recordAnalyze({ outcome: 'abstained', durationMs: 6_000 });
  // A refused request never waited on the model, so it must not be sampled as latency.
  metrics.recordAnalyze({ outcome: 'failure', failureClass: 'ingress_refused', durationMs: 5 });

  const latency = metrics.snapshot().model.latency;

  assert.equal(latency.count, 3, 'latency counts model work only');
  assert.equal(latency.maxMs, 6_000);
  assert.equal(latency.meanMs, 2_800);
  // Fixed bucket edges keep the snapshot a histogram of numbers, not a list of timings.
  assert.deepEqual(latency.bucketsMs, { '1000': 1, '2000': 2, '5000': 2, '15000': 3, '60000': 3 });
});

test('queue depth is the in-flight model work, and a released slot is never counted twice', async () => {
  const metrics = createModelMetrics({ now: () => 1_000, limit: 2 });

  const first = metrics.enter();
  const second = metrics.enter();
  assert.equal(metrics.snapshot().queue.inFlight, 2);
  assert.equal(metrics.snapshot().queue.atCapacity, true);

  metrics.leave(first);
  metrics.leave(first); // a double release must not free a slot that is still in use
  const queue = metrics.snapshot().queue;

  assert.equal(queue.inFlight, 1, 'the second request still holds its slot');
  assert.equal(queue.atCapacity, false);
  assert.equal(queue.accepted, 2);

  metrics.leave(second);
  assert.equal(metrics.snapshot().queue.inFlight, 0);
  assert.equal(metrics.snapshot().queue.accepted, 2, 'a completed request is still an accepted request');
});

test('readiness is reported as a phase, and the cold model load is measured once', async () => {
  let clock = 10_000;
  const metrics = createModelMetrics({ now: () => clock });

  metrics.recordReadiness({ phase: 'loading', ready: false });
  clock = 74_000; // 64s of weights loading
  metrics.recordReadiness({ phase: 'ready', ready: true });
  clock = 90_000;
  metrics.recordReadiness({ phase: 'offline', ready: false });

  const readiness = metrics.snapshot().model.readiness;

  assert.equal(readiness.phase, 'offline');
  assert.equal(readiness.ready, false);
  assert.equal(readiness.coldLoadMs, 64_000, 'the cold load is the first time the model became resident');
});

test('a readiness phase outside the enumerated set is reported as checking', async () => {
  const metrics = createModelMetrics({ now: () => 1_000 });

  metrics.recordReadiness({ phase: 'user says: bloody stool', ready: true });

  const snapshot = metrics.snapshot();
  assert.equal(snapshot.model.readiness.phase, 'checking');
  assert.doesNotMatch(JSON.stringify(snapshot), /bloody stool/);
});

test('the snapshot keeps the app, queue, and model boundaries separate', async () => {
  let clock = 1_000;
  const metrics = createModelMetrics({ now: () => clock, limit: 2 });

  // One request is refused at the app boundary and never takes a model slot; the other is admitted,
  // answered, and released. Each of the three boundaries should account for exactly that.
  const slot = metrics.enter();
  metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 800 });
  metrics.leave(slot);
  metrics.recordAnalyze({ outcome: 'failure', failureClass: 'rate_limited', durationMs: 0 });
  clock = 41_000;

  const snapshot = metrics.snapshot();

  assert.deepEqual(Object.keys(snapshot).sort(), ['app', 'generatedAt', 'model', 'queue', 'schema']);
  assert.equal(snapshot.app.uptimeMs, 40_000);
  assert.equal(snapshot.app.analyzeRequests, 2, 'the app boundary counts every request it saw');
  assert.equal(snapshot.app.refusedBeforeModel, 1, 'a refusal that never reached the queue is an app fact');
  assert.equal(snapshot.queue.accepted, 1, 'the queue boundary counts what it admitted, not what it answered');
  assert.equal(snapshot.queue.inFlight, 0, 'a released slot is not in flight');
  assert.equal(snapshot.model.outcomes.suggestion, 1, 'the model boundary counts answers only');
});