import test from 'node:test'; import assert from 'node:assert/strict'; import { createModelMetrics } from '../src/model-metrics.js'; // Issue #19: the worker has to be operable without the metrics becoming a second copy of the // journal. These tests are the boundary: what a metric sample may say, and what it may never say. test('abstention rate is computed from the model answers that came back at all', async () => { const metrics = createModelMetrics({ now: () => 1_000 }); metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 }); metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 1_100 }); metrics.recordAnalyze({ outcome: 'abstained', durationMs: 700 }); metrics.recordAnalyze({ outcome: 'abstained', durationMs: 800 }); const snapshot = metrics.snapshot(); assert.equal(snapshot.model.outcomes.suggestion, 2); assert.equal(snapshot.model.outcomes.abstained, 2); assert.equal(snapshot.model.abstentionRate, 0.5); }); test('a label that is not an enumerated outcome is counted, never copied into the snapshot', async () => { const metrics = createModelMetrics({ now: () => 1_000 }); // The worst case for a metrics boundary: a caller-supplied string arrives where a label belongs. metrics.recordAnalyze({ outcome: 'Type 4, urgent, 2026-11-01, milkY4B2cQ9w==', durationMs: 900 }); const snapshot = metrics.snapshot(); const serialized = JSON.stringify(snapshot); assert.deepEqual(Object.keys(snapshot.model.outcomes).sort(), ['abstained', 'failure', 'suggestion']); assert.doesNotMatch(serialized, /urgent/i); assert.doesNotMatch(serialized, /milkY4B2cQ9w/); assert.equal(snapshot.model.outcomes.failure, 1, 'an unrecognised outcome is one failed answer'); }); test('a failed request is classified and kept out of the abstention denominator', async () => { const metrics = createModelMetrics({ now: () => 1_000 }); metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 }); metrics.recordAnalyze({ outcome: 'failure', failureClass: 'provider_unreachable', durationMs: 2_000 }); metrics.recordAnalyze({ outcome: 'failure', failureClass: 'provider_unreachable', durationMs: 2_100 }); metrics.recordAnalyze({ outcome: 'abstained', durationMs: 800 }); const snapshot = metrics.snapshot(); assert.equal(snapshot.model.failureClasses.provider_unreachable, 2); assert.equal(snapshot.model.abstentionRate, 0.5, 'a failed answer is not an abstention'); }); test('a failure class outside the enumerated set collapses instead of being echoed', async () => { const metrics = createModelMetrics({ now: () => 1_000 }); metrics.recordAnalyze({ outcome: 'failure', failureClass: 'note:blood in stool, image 9f2a', durationMs: 2_000 }); const snapshot = metrics.snapshot(); assert.equal(snapshot.model.failureClasses.unexpected, 1); assert.doesNotMatch(JSON.stringify(snapshot), /blood in stool/); }); test('analysis latency is reported as warm work, with counts and a bounded histogram', async () => { const metrics = createModelMetrics({ now: () => 1_000 }); metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 }); metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 1_500 }); metrics.recordAnalyze({ outcome: 'abstained', durationMs: 6_000 }); // A refused request never waited on the model, so it must not be sampled as latency. metrics.recordAnalyze({ outcome: 'failure', failureClass: 'ingress_refused', durationMs: 5 }); const latency = metrics.snapshot().model.latency; assert.equal(latency.count, 3, 'latency counts model work only'); assert.equal(latency.maxMs, 6_000); assert.equal(latency.meanMs, 2_800); // Fixed bucket edges keep the snapshot a histogram of numbers, not a list of timings. assert.deepEqual(latency.bucketsMs, { '1000': 1, '2000': 2, '5000': 2, '15000': 3, '60000': 3 }); }); test('queue depth is the in-flight model work, and a released slot is never counted twice', async () => { const metrics = createModelMetrics({ now: () => 1_000, limit: 2 }); const first = metrics.enter(); const second = metrics.enter(); assert.equal(metrics.snapshot().queue.inFlight, 2); assert.equal(metrics.snapshot().queue.atCapacity, true); metrics.leave(first); metrics.leave(first); // a double release must not free a slot that is still in use const queue = metrics.snapshot().queue; assert.equal(queue.inFlight, 1, 'the second request still holds its slot'); assert.equal(queue.atCapacity, false); assert.equal(queue.accepted, 2); metrics.leave(second); assert.equal(metrics.snapshot().queue.inFlight, 0); assert.equal(metrics.snapshot().queue.accepted, 2, 'a completed request is still an accepted request'); }); test('readiness is reported as a phase, and the cold model load is measured once', async () => { let clock = 10_000; const metrics = createModelMetrics({ now: () => clock }); metrics.recordReadiness({ phase: 'loading', ready: false }); clock = 74_000; // 64s of weights loading metrics.recordReadiness({ phase: 'ready', ready: true }); clock = 90_000; metrics.recordReadiness({ phase: 'offline', ready: false }); const readiness = metrics.snapshot().model.readiness; assert.equal(readiness.phase, 'offline'); assert.equal(readiness.ready, false); assert.equal(readiness.coldLoadMs, 64_000, 'the cold load is the first time the model became resident'); }); test('a readiness phase outside the enumerated set is reported as checking', async () => { const metrics = createModelMetrics({ now: () => 1_000 }); metrics.recordReadiness({ phase: 'user says: bloody stool', ready: true }); const snapshot = metrics.snapshot(); assert.equal(snapshot.model.readiness.phase, 'checking'); assert.doesNotMatch(JSON.stringify(snapshot), /bloody stool/); }); test('the snapshot keeps the app, queue, and model boundaries separate', async () => { let clock = 1_000; const metrics = createModelMetrics({ now: () => clock, limit: 2 }); // One request is refused at the app boundary and never takes a model slot; the other is admitted, // answered, and released. Each of the three boundaries should account for exactly that. const slot = metrics.enter(); metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 800 }); metrics.leave(slot); metrics.recordAnalyze({ outcome: 'failure', failureClass: 'rate_limited', durationMs: 0 }); clock = 41_000; const snapshot = metrics.snapshot(); assert.deepEqual(Object.keys(snapshot).sort(), ['app', 'generatedAt', 'model', 'queue', 'schema']); assert.equal(snapshot.app.uptimeMs, 40_000); assert.equal(snapshot.app.analyzeRequests, 2, 'the app boundary counts every request it saw'); assert.equal(snapshot.app.refusedBeforeModel, 1, 'a refusal that never reached the queue is an app fact'); assert.equal(snapshot.queue.accepted, 1, 'the queue boundary counts what it admitted, not what it answered'); assert.equal(snapshot.queue.inFlight, 0, 'a released slot is not in flight'); assert.equal(snapshot.model.outcomes.suggestion, 1, 'the model boundary counts answers only'); });