Timmy
e08b537866
Quality gates / quality (pull_request) Waiting to run
The counters had an endpoint and no way to read them without jq. Adds a pure formatter plus a CLI that prints the app, queue, and model boundaries. The formatter is a function of the snapshot and the three frozen vocabularies, so a caller-supplied failure label cannot reach the terminal: unknown classes are simply not printed. Refusal paths are explicit — 404 means the route is off because no token is configured, 403 means the token was rejected — and the token is never echoed in the report or an error line. RED: tests/model-metrics-report-format.test.js failed with ERR_MODULE_NOT_FOUND before src/model-metrics-report.js existed; the refusal-path test failed with a generic 'answered 404' before the 404 branch was written. Tests: npm test 298/298, npm run test:metrics-report 8/8, syntax and diff gates clean, real-Chromium ui acceptance green with the privacy disclosure visible.
83 lines
3.2 KiB
JavaScript
83 lines
3.2 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { formatOperatorReport } from '../src/model-metrics-report.js';
|
|
import { createModelMetrics } from '../src/model-metrics.js';
|
|
|
|
function snapshot(overrides = {}) {
|
|
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') });
|
|
metrics.recordReadiness({ phase: 'ready', ready: true });
|
|
return { ...metrics.snapshot(), ...overrides };
|
|
}
|
|
|
|
test('the report names the app, queue, and model boundaries in that order', () => {
|
|
const report = formatOperatorReport(snapshot());
|
|
|
|
assert.match(report, /^APP$/m);
|
|
assert.match(report, /^QUEUE$/m);
|
|
assert.match(report, /^MODEL$/m);
|
|
assert.ok(report.indexOf('APP') < report.indexOf('QUEUE'), 'app boundary comes before queue');
|
|
assert.ok(report.indexOf('QUEUE') < report.indexOf('MODEL'), 'queue boundary comes before model');
|
|
});
|
|
|
|
test('the report prints the readiness phase the worker actually reported', () => {
|
|
const report = formatOperatorReport(snapshot({
|
|
model: { ...snapshot().model, readiness: { phase: 'loading', ready: false, coldLoadMs: null } },
|
|
}));
|
|
|
|
assert.match(report, /readiness\s+loading/);
|
|
// A loading worker is not ready, and the report must not imply AI is available.
|
|
assert.doesNotMatch(report, /readiness\s+ready/);
|
|
});
|
|
|
|
test('an abstention rate with no answers is absence, never zero percent', () => {
|
|
const report = formatOperatorReport(snapshot());
|
|
|
|
assert.match(report, /abstention\s+no answers yet/i);
|
|
assert.doesNotMatch(report, /abstention\s+0(\.0)?%/);
|
|
});
|
|
|
|
test('an abstention rate is rendered from answers only, so failures cannot dilute it', () => {
|
|
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') });
|
|
metrics.recordReadiness({ phase: 'ready', ready: true });
|
|
metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 });
|
|
metrics.recordAnalyze({ outcome: 'abstained', durationMs: 1200 });
|
|
metrics.recordAnalyze({ outcome: 'failure', failureClass: 'timeout', durationMs: 30_000 });
|
|
|
|
const report = formatOperatorReport(metrics.snapshot());
|
|
|
|
assert.match(report, /abstention\s+50\.0%/);
|
|
assert.match(report, /failure\s+timeout\s+1|timeout 1/i);
|
|
});
|
|
|
|
test('the report renders only the enumerated failure vocabulary, never a caller-supplied label', () => {
|
|
const base = snapshot();
|
|
const report = formatOperatorReport({
|
|
...base,
|
|
model: {
|
|
...base.model,
|
|
failureClasses: {
|
|
...base.model.failureClasses,
|
|
'note: watery stool at 06:40': 3,
|
|
'user@example.com': 1,
|
|
},
|
|
},
|
|
});
|
|
|
|
assert.doesNotMatch(report, /watery stool/);
|
|
assert.doesNotMatch(report, /user@example\.com/);
|
|
// The closed set is still fully accounted for, so the extra keys are visibly absent rather than
|
|
// silently folded into a bucket the operator would read as something else.
|
|
assert.match(report, /unexpected\s+0/);
|
|
});
|
|
|
|
test('the report shows queue occupancy against the configured ceiling', () => {
|
|
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z'), limit: 3 });
|
|
metrics.enter();
|
|
metrics.enter();
|
|
|
|
const report = formatOperatorReport(metrics.snapshot());
|
|
|
|
assert.match(report, /in flight\s+2 \/ 3/);
|
|
assert.match(report, /at capacity\s+no/);
|
|
});
|