Timmy Release
ef83ad93bb
Quality gates / quality (push) Waiting to run
The release video proved the disclosure sentence but never exercised the operator surface it describes. This records the real scripts/model_metrics_report.mjs against the recording's own server, shows the counters that run produced, and proves a wrong token is refused instead of answered. The first run of that CLI printed 'failure ingress_refused0': the longest failure label exactly filled the 22-character column, so label and count rendered as one broken token. LABEL_WIDTH is now 30, and a format test fails if any row loses the gap between its label and its value.
98 lines
4.1 KiB
JavaScript
98 lines
4.1 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { formatOperatorReport } from '../src/model-metrics-report.js';
|
|
import { createModelMetrics } from '../src/model-metrics.js';
|
|
|
|
function snapshot(overrides = {}) {
|
|
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') });
|
|
metrics.recordReadiness({ phase: 'ready', ready: true });
|
|
return { ...metrics.snapshot(), ...overrides };
|
|
}
|
|
|
|
test('the report names the app, queue, and model boundaries in that order', () => {
|
|
const report = formatOperatorReport(snapshot());
|
|
|
|
assert.match(report, /^APP$/m);
|
|
assert.match(report, /^QUEUE$/m);
|
|
assert.match(report, /^MODEL$/m);
|
|
assert.ok(report.indexOf('APP') < report.indexOf('QUEUE'), 'app boundary comes before queue');
|
|
assert.ok(report.indexOf('QUEUE') < report.indexOf('MODEL'), 'queue boundary comes before model');
|
|
});
|
|
|
|
test('the report prints the readiness phase the worker actually reported', () => {
|
|
const report = formatOperatorReport(snapshot({
|
|
model: { ...snapshot().model, readiness: { phase: 'loading', ready: false, coldLoadMs: null } },
|
|
}));
|
|
|
|
assert.match(report, /readiness\s+loading/);
|
|
// A loading worker is not ready, and the report must not imply AI is available.
|
|
assert.doesNotMatch(report, /readiness\s+ready/);
|
|
});
|
|
|
|
test('an abstention rate with no answers is absence, never zero percent', () => {
|
|
const report = formatOperatorReport(snapshot());
|
|
|
|
assert.match(report, /abstention\s+no answers yet/i);
|
|
assert.doesNotMatch(report, /abstention\s+0(\.0)?%/);
|
|
});
|
|
|
|
test('an abstention rate is rendered from answers only, so failures cannot dilute it', () => {
|
|
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') });
|
|
metrics.recordReadiness({ phase: 'ready', ready: true });
|
|
metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 });
|
|
metrics.recordAnalyze({ outcome: 'abstained', durationMs: 1200 });
|
|
metrics.recordAnalyze({ outcome: 'failure', failureClass: 'timeout', durationMs: 30_000 });
|
|
|
|
const report = formatOperatorReport(metrics.snapshot());
|
|
|
|
assert.match(report, /abstention\s+50\.0%/);
|
|
assert.match(report, /failure\s+timeout\s+1|timeout 1/i);
|
|
});
|
|
|
|
test('the report renders only the enumerated failure vocabulary, never a caller-supplied label', () => {
|
|
const base = snapshot();
|
|
const report = formatOperatorReport({
|
|
...base,
|
|
model: {
|
|
...base.model,
|
|
failureClasses: {
|
|
...base.model.failureClasses,
|
|
'note: watery stool at 06:40': 3,
|
|
'user@example.com': 1,
|
|
},
|
|
},
|
|
});
|
|
|
|
assert.doesNotMatch(report, /watery stool/);
|
|
assert.doesNotMatch(report, /user@example\.com/);
|
|
// The closed set is still fully accounted for, so the extra keys are visibly absent rather than
|
|
// silently folded into a bucket the operator would read as something else.
|
|
assert.match(report, /unexpected\s+0/);
|
|
});
|
|
|
|
test('the report shows queue occupancy against the configured ceiling', () => {
|
|
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z'), limit: 3 });
|
|
metrics.enter();
|
|
metrics.enter();
|
|
|
|
const report = formatOperatorReport(metrics.snapshot());
|
|
|
|
assert.match(report, /in flight\s+2 \/ 3/);
|
|
assert.match(report, /at capacity\s+no/);
|
|
});
|
|
|
|
test('every row keeps a visible gap between its label and its count', () => {
|
|
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') });
|
|
// The longest label is the one that used to run together with its value: a column that exactly fits
|
|
// the text reads as `failure ingress_refused0`, one broken token instead of a metric and its count.
|
|
metrics.recordAnalyze({ outcome: 'failure', failureClass: 'ingress_refused', durationMs: 5 });
|
|
const report = formatOperatorReport(metrics.snapshot());
|
|
|
|
for (const line of report.split('\n')) {
|
|
if (!/^ {2}\S/.test(line)) continue;
|
|
assert.match(line, /^ {2}\S.*?\s{2,}\S/, `row has no gap between label and value: ${JSON.stringify(line)}`);
|
|
}
|
|
assert.match(report, /failure ingress_refused\s+1/);
|
|
assert.match(report, /failure provider_unreachable\s+0/);
|
|
});
|