import test from 'node:test'; import assert from 'node:assert/strict'; import { sanitizeEvidence, buildDashboard, evaluateAlerts, DEFAULT_ALERT_RULES } from '../src/release-observability.js'; const healthyEvidence = () => sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 }, { id: 'api.analyze', boundary: 'api', status: 'pass', counters: { ok: 40, fail: 0 } }, { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 0 } === undefined ? {} : { ok: 30, fail: 0 } }, { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 900, counters: { ok: 25, fail: 0, abstain: 3 } }, ], }).evidence; test('dashboard separates app, api, queue, and model boundaries', () => { const dashboard = buildDashboard(healthyEvidence()); assert.equal(dashboard.ok, true); assert.deepEqual(Object.keys(dashboard.boundaries).sort(), ['api', 'app', 'model', 'queue']); assert.equal(dashboard.boundaries.app.pass, 1); assert.equal(dashboard.boundaries.model.pass, 1); assert.equal(dashboard.boundaries.model.latencyMs, 900); assert.equal(dashboard.totals.checks, 4); assert.equal(dashboard.totals.fail, 0); assert.deepEqual(dashboard.failureClasses, {}); assert.equal(dashboard.identity.releaseTag, 'daily-2026-08-22.1'); assert.equal(dashboard.identity.commit, 'ca31e6d38bec649407f63880504554c59f2878ae'); }); test('dashboard summarizes privacy-safe failure classes without any payload text', () => { const evidence = sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass' }, { id: 'api.analyze', boundary: 'api', status: 'degraded', failureClass: 'vision.timeout' }, { id: 'queue.depth', boundary: 'queue', status: 'pass' }, { id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'model.error' }, { id: 'model.inference.warmup', boundary: 'model', status: 'fail', failureClass: 'model.error' }, ], }).evidence; const dashboard = buildDashboard(evidence); assert.deepEqual(dashboard.failureClasses, { 'vision.timeout': 1, 'model.error': 2 }); assert.equal(dashboard.totals.fail, 2); assert.equal(dashboard.totals.degraded, 1); assert.equal(JSON.stringify(dashboard).includes('note'), false); }); test('simulated worker outage fires exactly one actionable alert with owner, threshold, and runbook', () => { const evidence = sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 11 }, { id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' }, { id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable', counters: { depth: 17 } }, { id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'worker.unavailable', counters: { timeout: 4 } }, ], }).evidence; const dashboard = buildDashboard(evidence); const alerts = evaluateAlerts(dashboard, DEFAULT_ALERT_RULES); assert.equal(alerts.length, 1, `expected exactly one alert, got ${JSON.stringify(alerts)}`); const alert = alerts[0]; assert.equal(alert.id, 'worker.outage'); assert.equal(alert.severity, 'page'); assert.equal(alert.owner, 'release-operator'); assert.match(alert.runbook, /docs\/RELEASE-OBSERVABILITY\.md#simulate-a-worker-outage/); assert.match(alert.message, /worker\.unavailable/); assert.match(alert.message, /daily-2026-08-22\.1/); assert.equal(alert.count, 3); assert.equal(alert.threshold, 1); }); test('healthy evidence raises no alerts and every default rule carries owner, threshold, and runbook', () => { assert.deepEqual(evaluateAlerts(buildDashboard(healthyEvidence()), DEFAULT_ALERT_RULES), []); for (const rule of DEFAULT_ALERT_RULES) { assert.ok(rule.owner && rule.owner.length > 0); assert.ok(Number.isInteger(rule.threshold) && rule.threshold >= 1); assert.match(rule.runbook, /^docs\/RELEASE-OBSERVABILITY\.md#/); assert.ok(['page', 'warn'].includes(rule.severity)); } }); test('manual fallback stays available whenever any non-app check is unhealthy', () => { const healthy = buildDashboard(healthyEvidence()); assert.deepEqual(healthy.manualFallback, { available: true, reason: 'none-required' }); const outage = buildDashboard(sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass' }, { id: 'api.analyze', boundary: 'api', status: 'pass' }, { id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable' }, { id: 'model.inference', boundary: 'model', status: 'pass' }, ], }).evidence); assert.deepEqual(outage.manualFallback, { available: true, reason: 'local-journal', }); }); test('manual fallback is unavailable only when the app boundary itself is down', () => { const evidence = sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'fail' }, { id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' }, { id: 'queue.depth', boundary: 'queue', status: 'pass' }, { id: 'model.inference', boundary: 'model', status: 'pass' }, ], }).evidence; const dashboard = buildDashboard(evidence); assert.deepEqual(dashboard.manualFallback, { available: false, reason: 'app-down', }); }); test('manualFallback state is a deterministic function of boundary health alone', () => { const cases = [ { statuses: ['pass', 'pass', 'pass', 'pass'], expected: { available: true, reason: 'none-required' } }, { statuses: ['degraded', 'pass', 'pass', 'pass'], expected: { available: true, reason: 'local-journal' } }, { statuses: ['pass', 'unknown', 'pass', 'pass'], expected: { available: true, reason: 'none-required' } }, { statuses: ['fail', 'fail', 'fail', 'fail'], expected: { available: false, reason: 'app-down' } }, ]; for (const { statuses, expected } of cases) { const evidence = sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: statuses[0] }, { id: 'api.analyze', boundary: 'api', status: statuses[1] }, { id: 'queue.depth', boundary: 'queue', status: statuses[2] }, { id: 'model.inference', boundary: 'model', status: statuses[3] }, ], }).evidence; assert.deepEqual(buildDashboard(evidence).manualFallback, expected, `statuses ${statuses}`); } }); const backlogEvidence = depth => sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass' }, { id: 'api.analyze', boundary: 'api', status: 'pass' }, { id: 'queue.depth', boundary: 'queue', status: 'degraded', counters: { depth } }, { id: 'model.inference', boundary: 'model', status: 'pass' }, ], }).evidence; test('queue.depth counter survives sanitization so queue.backlog can fire', () => { const dashboard = buildDashboard(backlogEvidence(10)); assert.equal(dashboard.boundaries.queue.counters.depth, 10); }); test('queue.backlog stays silent at depth 9 and fires at exactly depth 10', () => { assert.deepEqual(evaluateAlerts(buildDashboard(backlogEvidence(9)), DEFAULT_ALERT_RULES), []); const atThreshold = evaluateAlerts(buildDashboard(backlogEvidence(10)), DEFAULT_ALERT_RULES); assert.equal(atThreshold.length, 1); assert.equal(atThreshold[0].id, 'queue.backlog'); assert.equal(atThreshold[0].severity, 'warn'); assert.equal(atThreshold[0].count, 10); assert.equal(atThreshold[0].threshold, 10); }); test('queue.backlog reports observed count above threshold and sums across queue checks', () => { const evidence = sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass' }, { id: 'api.analyze', boundary: 'api', status: 'pass' }, { id: 'queue.depth', boundary: 'queue', status: 'degraded', counters: { depth: 5 } }, { id: 'queue.depth.sidecar', boundary: 'queue', status: 'pass', counters: { depth: 6 } }, { id: 'model.inference', boundary: 'model', status: 'pass' }, ], }).evidence; const alerts = evaluateAlerts(buildDashboard(evidence), DEFAULT_ALERT_RULES); const backlog = alerts.filter(alert => alert.id === 'queue.backlog'); assert.equal(backlog.length, 1); assert.equal(backlog[0].count, 11); }); test('queue.backlog is suppressed while worker.outage pages because outage depth is residual', () => { const evidence = sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass' }, { id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' }, { id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable', counters: { depth: 17 } }, { id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'worker.unavailable' }, ], }).evidence; const alerts = evaluateAlerts(buildDashboard(evidence), DEFAULT_ALERT_RULES); assert.equal(alerts.length, 1, `expected only the outage page, got ${JSON.stringify(alerts)}`); assert.equal(alerts[0].id, 'worker.outage'); }); test('queue.bound depth counter above the bounded ceiling cannot smuggle into the dashboard', () => { const evidence = sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass' }, { id: 'api.analyze', boundary: 'api', status: 'pass' }, { id: 'queue.depth', boundary: 'queue', status: 'degraded', counters: { depth: 10_000_000 } }, { id: 'model.inference', boundary: 'model', status: 'pass' }, ], }).evidence; assert.equal(evidence, null, 'unbounded depth must fail closed at sanitization'); }); test('sanitizer fails closed when any app, api, queue, or model boundary is entirely missing', () => { const fullChecks = [ { id: 'app.healthz', boundary: 'app', status: 'pass' }, { id: 'api.analyze', boundary: 'api', status: 'pass' }, { id: 'queue.depth', boundary: 'queue', status: 'pass' }, { id: 'model.inference', boundary: 'model', status: 'pass' }, ]; for (const missing of ['app', 'api', 'queue', 'model']) { const evidence = sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: fullChecks.filter(check => check.boundary !== missing), }); assert.equal(evidence.ok, false, `missing ${missing} boundary must fail closed`); } }); test('unknown statuses surface as a warn telemetry.gap alert instead of failing open', () => { const evidence = sanitizeEvidence({ schemaVersion: 1, releaseTag: 'daily-2026-08-22.1', commit: 'ca31e6d38bec649407f63880504554c59f2878ae', generatedAtUtc: '2026-08-22T12:00:00Z', checks: [ { id: 'app.healthz', boundary: 'app', status: 'pass' }, { id: 'api.analyze', boundary: 'api', status: 'unknown' }, { id: 'queue.depth', boundary: 'queue', status: 'unknown' }, { id: 'model.inference', boundary: 'model', status: 'pass' }, ], }).evidence; assert.ok(evidence, 'unknown is an allowed status and must not fail sanitization'); const dashboard = buildDashboard(evidence); assert.equal(dashboard.boundaries.api.unknown, 1); const alerts = evaluateAlerts(dashboard, DEFAULT_ALERT_RULES); assert.equal(alerts.length, 1, `expected exactly the telemetry.gap alert, got ${JSON.stringify(alerts)}`); const gap = alerts[0]; assert.equal(gap.id, 'telemetry.gap'); assert.equal(gap.severity, 'warn'); assert.equal(gap.owner, 'release-operator'); assert.equal(gap.threshold, 1); assert.equal(gap.count, 2); assert.match(gap.runbook, /docs\/RELEASE-OBSERVABILITY\.md#/); assert.match(gap.message, /telemetry\.gap/); }); test('healthy evidence raises no telemetry.gap alert', () => { const alerts = evaluateAlerts(buildDashboard(healthyEvidence()), DEFAULT_ALERT_RULES); assert.equal(alerts.some(alert => alert.id === 'telemetry.gap'), false); });