Timmy
d547b4887b
All checks were successful
Quality gates / quality (pull_request) Successful in 2m9s
Strict vertical RED-GREEN TDD across eight review blockers: - drill baseline: requires every required boundary check explicitly passing plus zero alerts of any severity before anything is announced or flipped; unknown, degraded, malformed, missing-boundary, and backlog-warning baselines exit 2 with switch count zero, and stdout stays empty whenever no drill actually ran - manual fallback: deterministic truthful states — app-down, local-journal, none-required, and new app-state-unknown (availability null) so unknown telemetry can never render as proven availability or 'no degradation'; runbook gains a state table with exact wording - exact schema types: schemaVersion must be JSON number 1; counters and latency must be finite bounded integers (0..1e6, 0..3600000); generatedAtUtc must be canonical real UTC surviving round-trip; releaseTag/commit keep exact safe types; coercions reject - control-safety: controlSafe strips LF/CR too, so hostile paths can no longer forge terminal lines (regression test ships a path embedding LF + forged PAGE text); read errors use fixed wording instead of echoing attacker-derived message bytes - malformed checks reject the whole evidence object instead of silently disappearing beside healthy checks; forbidden top-level keys now fail closed rather than being ignored - status/failureClass consistency: pass/unknown checks carrying any failureClass reject the file, so all-pass evidence can never page worker.outage (fail/degraded may stay classless, preserving the closed vocabulary and nullable schema) - sanitization idempotence: omitted latency stays omitted on a second pass; explicit null latency rejects since the schema forbids it - removed the always-false queue-depth tautology in the shared fixture and made the runbook contract test verify each rule's real numeric threshold against DEFAULT_ALERT_RULES Preserved: closed failure-class vocabulary, queue depth 9/10/11 edges, raw loopback-origin validation, app-down fallback, missing-boundary rejection, transition semantics, and every independent drill case.
292 lines
13 KiB
JavaScript
292 lines
13 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
|
|
import { sanitizeEvidence, buildDashboard, evaluateAlerts, DEFAULT_ALERT_RULES } from '../src/release-observability.js';
|
|
|
|
const healthyEvidence = () => sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'pass', counters: { ok: 40, fail: 0 } },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 0, ok: 30, fail: 0 } },
|
|
{ id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 900, counters: { ok: 25, fail: 0, abstain: 3 } },
|
|
],
|
|
}).evidence;
|
|
|
|
test('dashboard separates app, api, queue, and model boundaries', () => {
|
|
const dashboard = buildDashboard(healthyEvidence());
|
|
assert.equal(dashboard.ok, true);
|
|
assert.deepEqual(Object.keys(dashboard.boundaries).sort(), ['api', 'app', 'model', 'queue']);
|
|
assert.equal(dashboard.boundaries.app.pass, 1);
|
|
assert.equal(dashboard.boundaries.model.pass, 1);
|
|
assert.equal(dashboard.boundaries.model.latencyMs, 900);
|
|
assert.equal(dashboard.totals.checks, 4);
|
|
assert.equal(dashboard.totals.fail, 0);
|
|
assert.deepEqual(dashboard.failureClasses, {});
|
|
assert.equal(dashboard.identity.releaseTag, 'daily-2026-08-22.1');
|
|
assert.equal(dashboard.identity.commit, 'ca31e6d38bec649407f63880504554c59f2878ae');
|
|
});
|
|
|
|
test('dashboard summarizes privacy-safe failure classes without any payload text', () => {
|
|
const evidence = sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass' },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'degraded', failureClass: 'vision.timeout' },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'pass' },
|
|
{ id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'model.error' },
|
|
{ id: 'model.inference.warmup', boundary: 'model', status: 'fail', failureClass: 'model.error' },
|
|
],
|
|
}).evidence;
|
|
const dashboard = buildDashboard(evidence);
|
|
assert.deepEqual(dashboard.failureClasses, { 'vision.timeout': 1, 'model.error': 2 });
|
|
assert.equal(dashboard.totals.fail, 2);
|
|
assert.equal(dashboard.totals.degraded, 1);
|
|
assert.equal(JSON.stringify(dashboard).includes('note'), false);
|
|
});
|
|
|
|
test('simulated worker outage fires exactly one actionable alert with owner, threshold, and runbook', () => {
|
|
const evidence = sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 11 },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable', counters: { depth: 17 } },
|
|
{ id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'worker.unavailable', counters: { timeout: 4 } },
|
|
],
|
|
}).evidence;
|
|
|
|
const dashboard = buildDashboard(evidence);
|
|
const alerts = evaluateAlerts(dashboard, DEFAULT_ALERT_RULES);
|
|
|
|
assert.equal(alerts.length, 1, `expected exactly one alert, got ${JSON.stringify(alerts)}`);
|
|
const alert = alerts[0];
|
|
assert.equal(alert.id, 'worker.outage');
|
|
assert.equal(alert.severity, 'page');
|
|
assert.equal(alert.owner, 'release-operator');
|
|
assert.match(alert.runbook, /docs\/RELEASE-OBSERVABILITY\.md#simulate-a-worker-outage/);
|
|
assert.match(alert.message, /worker\.unavailable/);
|
|
assert.match(alert.message, /daily-2026-08-22\.1/);
|
|
assert.equal(alert.count, 3);
|
|
assert.equal(alert.threshold, 1);
|
|
});
|
|
|
|
test('healthy evidence raises no alerts and every default rule carries owner, threshold, and runbook', () => {
|
|
assert.deepEqual(evaluateAlerts(buildDashboard(healthyEvidence()), DEFAULT_ALERT_RULES), []);
|
|
for (const rule of DEFAULT_ALERT_RULES) {
|
|
assert.ok(rule.owner && rule.owner.length > 0);
|
|
assert.ok(Number.isInteger(rule.threshold) && rule.threshold >= 1);
|
|
assert.match(rule.runbook, /^docs\/RELEASE-OBSERVABILITY\.md#/);
|
|
assert.ok(['page', 'warn'].includes(rule.severity));
|
|
}
|
|
});
|
|
|
|
test('manual fallback stays available whenever any non-app check is unhealthy', () => {
|
|
const healthy = buildDashboard(healthyEvidence());
|
|
assert.deepEqual(healthy.manualFallback, { available: true, reason: 'none-required' });
|
|
|
|
const outage = buildDashboard(sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass' },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'pass' },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable' },
|
|
{ id: 'model.inference', boundary: 'model', status: 'pass' },
|
|
],
|
|
}).evidence);
|
|
assert.deepEqual(outage.manualFallback, {
|
|
available: true,
|
|
reason: 'local-journal',
|
|
});
|
|
});
|
|
|
|
test('manual fallback is unavailable only when the app boundary itself is down', () => {
|
|
const evidence = sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'fail' },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'pass' },
|
|
{ id: 'model.inference', boundary: 'model', status: 'pass' },
|
|
],
|
|
}).evidence;
|
|
const dashboard = buildDashboard(evidence);
|
|
assert.deepEqual(dashboard.manualFallback, {
|
|
available: false,
|
|
reason: 'app-down',
|
|
});
|
|
});
|
|
|
|
test('manualFallback state is a deterministic function of boundary health alone', () => {
|
|
const cases = [
|
|
{ statuses: ['pass', 'pass', 'pass', 'pass'], expected: { available: true, reason: 'none-required' } },
|
|
{ statuses: ['degraded', 'pass', 'pass', 'pass'], expected: { available: true, reason: 'local-journal' } },
|
|
{ statuses: ['pass', 'unknown', 'pass', 'pass'], expected: { available: null, reason: 'app-state-unknown' } },
|
|
{ statuses: ['fail', 'fail', 'fail', 'fail'], expected: { available: false, reason: 'app-down' } },
|
|
];
|
|
for (const { statuses, expected } of cases) {
|
|
const evidence = sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: statuses[0] },
|
|
{ id: 'api.analyze', boundary: 'api', status: statuses[1] },
|
|
{ id: 'queue.depth', boundary: 'queue', status: statuses[2] },
|
|
{ id: 'model.inference', boundary: 'model', status: statuses[3] },
|
|
],
|
|
}).evidence;
|
|
assert.deepEqual(buildDashboard(evidence).manualFallback, expected, `statuses ${statuses}`);
|
|
}
|
|
});
|
|
|
|
const backlogEvidence = depth => sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass' },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'pass' },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'degraded', counters: { depth } },
|
|
{ id: 'model.inference', boundary: 'model', status: 'pass' },
|
|
],
|
|
}).evidence;
|
|
|
|
test('queue.depth counter survives sanitization so queue.backlog can fire', () => {
|
|
const dashboard = buildDashboard(backlogEvidence(10));
|
|
assert.equal(dashboard.boundaries.queue.counters.depth, 10);
|
|
});
|
|
|
|
test('queue.backlog stays silent at depth 9 and fires at exactly depth 10', () => {
|
|
assert.deepEqual(evaluateAlerts(buildDashboard(backlogEvidence(9)), DEFAULT_ALERT_RULES), []);
|
|
const atThreshold = evaluateAlerts(buildDashboard(backlogEvidence(10)), DEFAULT_ALERT_RULES);
|
|
assert.equal(atThreshold.length, 1);
|
|
assert.equal(atThreshold[0].id, 'queue.backlog');
|
|
assert.equal(atThreshold[0].severity, 'warn');
|
|
assert.equal(atThreshold[0].count, 10);
|
|
assert.equal(atThreshold[0].threshold, 10);
|
|
});
|
|
|
|
test('queue.backlog reports observed count above threshold and sums across queue checks', () => {
|
|
const evidence = sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass' },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'pass' },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'degraded', counters: { depth: 5 } },
|
|
{ id: 'queue.depth.sidecar', boundary: 'queue', status: 'pass', counters: { depth: 6 } },
|
|
{ id: 'model.inference', boundary: 'model', status: 'pass' },
|
|
],
|
|
}).evidence;
|
|
const alerts = evaluateAlerts(buildDashboard(evidence), DEFAULT_ALERT_RULES);
|
|
const backlog = alerts.filter(alert => alert.id === 'queue.backlog');
|
|
assert.equal(backlog.length, 1);
|
|
assert.equal(backlog[0].count, 11);
|
|
});
|
|
|
|
test('queue.backlog is suppressed while worker.outage pages because outage depth is residual', () => {
|
|
const evidence = sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass' },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable', counters: { depth: 17 } },
|
|
{ id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'worker.unavailable' },
|
|
],
|
|
}).evidence;
|
|
const alerts = evaluateAlerts(buildDashboard(evidence), DEFAULT_ALERT_RULES);
|
|
assert.equal(alerts.length, 1, `expected only the outage page, got ${JSON.stringify(alerts)}`);
|
|
assert.equal(alerts[0].id, 'worker.outage');
|
|
});
|
|
|
|
test('queue.bound depth counter above the bounded ceiling cannot smuggle into the dashboard', () => {
|
|
const evidence = sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass' },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'pass' },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'degraded', counters: { depth: 10_000_000 } },
|
|
{ id: 'model.inference', boundary: 'model', status: 'pass' },
|
|
],
|
|
}).evidence;
|
|
assert.equal(evidence, null, 'unbounded depth must fail closed at sanitization');
|
|
});
|
|
|
|
test('sanitizer fails closed when any app, api, queue, or model boundary is entirely missing', () => {
|
|
const fullChecks = [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass' },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'pass' },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'pass' },
|
|
{ id: 'model.inference', boundary: 'model', status: 'pass' },
|
|
];
|
|
for (const missing of ['app', 'api', 'queue', 'model']) {
|
|
const evidence = sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: fullChecks.filter(check => check.boundary !== missing),
|
|
});
|
|
assert.equal(evidence.ok, false, `missing ${missing} boundary must fail closed`);
|
|
}
|
|
});
|
|
|
|
test('unknown statuses surface as a warn telemetry.gap alert instead of failing open', () => {
|
|
const evidence = sanitizeEvidence({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass' },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'unknown' },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'unknown' },
|
|
{ id: 'model.inference', boundary: 'model', status: 'pass' },
|
|
],
|
|
}).evidence;
|
|
assert.ok(evidence, 'unknown is an allowed status and must not fail sanitization');
|
|
const dashboard = buildDashboard(evidence);
|
|
assert.equal(dashboard.boundaries.api.unknown, 1);
|
|
const alerts = evaluateAlerts(dashboard, DEFAULT_ALERT_RULES);
|
|
assert.equal(alerts.length, 1, `expected exactly the telemetry.gap alert, got ${JSON.stringify(alerts)}`);
|
|
const gap = alerts[0];
|
|
assert.equal(gap.id, 'telemetry.gap');
|
|
assert.equal(gap.severity, 'warn');
|
|
assert.equal(gap.owner, 'release-operator');
|
|
assert.equal(gap.threshold, 1);
|
|
assert.equal(gap.count, 2);
|
|
assert.match(gap.runbook, /docs\/RELEASE-OBSERVABILITY\.md#/);
|
|
assert.match(gap.message, /telemetry\.gap/);
|
|
});
|
|
|
|
test('healthy evidence raises no telemetry.gap alert', () => {
|
|
const alerts = evaluateAlerts(buildDashboard(healthyEvidence()), DEFAULT_ALERT_RULES);
|
|
assert.equal(alerts.some(alert => alert.id === 'telemetry.gap'), false);
|
|
});
|