timmy-talking-turd/tests/release-dashboard.test.js

import test from 'node:test';
import assert from 'node:assert/strict';

import { sanitizeEvidence, buildDashboard, evaluateAlerts, DEFAULT_ALERT_RULES } from '../src/release-observability.js';

const healthyEvidence = () => sanitizeEvidence({
  schemaVersion: 1,
  releaseTag: 'daily-2026-08-22.1',
  commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
  generatedAtUtc: '2026-08-22T12:00:00Z',
  checks: [
    { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 },
    { id: 'api.analyze', boundary: 'api', status: 'pass', counters: { ok: 40, fail: 0 } },
    { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 0 } === undefined ? {} : { ok: 30, fail: 0 } },
    { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 900, counters: { ok: 25, fail: 0, abstain: 3 } },
  ],
}).evidence;

test('dashboard separates app, api, queue, and model boundaries', () => {
  const dashboard = buildDashboard(healthyEvidence());
  assert.equal(dashboard.ok, true);
  assert.deepEqual(Object.keys(dashboard.boundaries).sort(), ['api', 'app', 'model', 'queue']);
  assert.equal(dashboard.boundaries.app.pass, 1);
  assert.equal(dashboard.boundaries.model.pass, 1);
  assert.equal(dashboard.boundaries.model.latencyMs, 900);
  assert.equal(dashboard.totals.checks, 4);
  assert.equal(dashboard.totals.fail, 0);
  assert.deepEqual(dashboard.failureClasses, {});
  assert.equal(dashboard.identity.releaseTag, 'daily-2026-08-22.1');
  assert.equal(dashboard.identity.commit, 'ca31e6d38bec649407f63880504554c59f2878ae');
});

test('dashboard summarizes privacy-safe failure classes without any payload text', () => {
  const evidence = sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'app.healthz', boundary: 'app', status: 'pass' },
      { id: 'api.analyze', boundary: 'api', status: 'degraded', failureClass: 'vision.timeout' },
      { id: 'queue.depth', boundary: 'queue', status: 'pass' },
      { id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'model.error' },
      { id: 'model.inference.warmup', boundary: 'model', status: 'fail', failureClass: 'model.error' },
    ],
  }).evidence;
  const dashboard = buildDashboard(evidence);
  assert.deepEqual(dashboard.failureClasses, { 'vision.timeout': 1, 'model.error': 2 });
  assert.equal(dashboard.totals.fail, 2);
  assert.equal(dashboard.totals.degraded, 1);
  assert.equal(JSON.stringify(dashboard).includes('note'), false);
});

test('simulated worker outage fires exactly one actionable alert with owner, threshold, and runbook', () => {
  const evidence = sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 11 },
      { id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' },
      { id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable', counters: { depth: 17 } },
      { id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'worker.unavailable', counters: { timeout: 4 } },
    ],
  }).evidence;

  const dashboard = buildDashboard(evidence);
  const alerts = evaluateAlerts(dashboard, DEFAULT_ALERT_RULES);

  assert.equal(alerts.length, 1, `expected exactly one alert, got ${JSON.stringify(alerts)}`);
  const alert = alerts[0];
  assert.equal(alert.id, 'worker.outage');
  assert.equal(alert.severity, 'page');
  assert.equal(alert.owner, 'release-operator');
  assert.match(alert.runbook, /docs\/RELEASE-OBSERVABILITY\.md#simulate-a-worker-outage/);
  assert.match(alert.message, /worker\.unavailable/);
  assert.match(alert.message, /daily-2026-08-22\.1/);
  assert.equal(alert.count, 3);
  assert.equal(alert.threshold, 1);
});

test('healthy evidence raises no alerts and every default rule carries owner, threshold, and runbook', () => {
  assert.deepEqual(evaluateAlerts(buildDashboard(healthyEvidence()), DEFAULT_ALERT_RULES), []);
  for (const rule of DEFAULT_ALERT_RULES) {
    assert.ok(rule.owner && rule.owner.length > 0);
    assert.ok(Number.isInteger(rule.threshold) && rule.threshold >= 1);
    assert.match(rule.runbook, /^docs\/RELEASE-OBSERVABILITY\.md#/);
    assert.ok(['page', 'warn'].includes(rule.severity));
  }
});

test('manual fallback stays available whenever any non-app check is unhealthy', () => {
  const healthy = buildDashboard(healthyEvidence());
  assert.deepEqual(healthy.manualFallback, { available: true, reason: 'none-required' });

  const outage = buildDashboard(sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'app.healthz', boundary: 'app', status: 'pass' },
      { id: 'api.analyze', boundary: 'api', status: 'pass' },
      { id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable' },
      { id: 'model.inference', boundary: 'model', status: 'pass' },
    ],
  }).evidence);
  assert.deepEqual(outage.manualFallback, {
    available: true,
    reason: 'local-journal',
  });
});

test('manual fallback is unavailable only when the app boundary itself is down', () => {
  const evidence = sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'app.healthz', boundary: 'app', status: 'fail' },
      { id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' },
      { id: 'queue.depth', boundary: 'queue', status: 'pass' },
      { id: 'model.inference', boundary: 'model', status: 'pass' },
    ],
  }).evidence;
  const dashboard = buildDashboard(evidence);
  assert.deepEqual(dashboard.manualFallback, {
    available: false,
    reason: 'app-down',
  });
});

test('manualFallback state is a deterministic function of boundary health alone', () => {
  const cases = [
    { statuses: ['pass', 'pass', 'pass', 'pass'], expected: { available: true, reason: 'none-required' } },
    { statuses: ['degraded', 'pass', 'pass', 'pass'], expected: { available: true, reason: 'local-journal' } },
    { statuses: ['pass', 'unknown', 'pass', 'pass'], expected: { available: true, reason: 'none-required' } },
    { statuses: ['fail', 'fail', 'fail', 'fail'], expected: { available: false, reason: 'app-down' } },
  ];
  for (const { statuses, expected } of cases) {
    const evidence = sanitizeEvidence({
      schemaVersion: 1,
      releaseTag: 'daily-2026-08-22.1',
      commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
      generatedAtUtc: '2026-08-22T12:00:00Z',
      checks: [
        { id: 'app.healthz', boundary: 'app', status: statuses[0] },
        { id: 'api.analyze', boundary: 'api', status: statuses[1] },
        { id: 'queue.depth', boundary: 'queue', status: statuses[2] },
        { id: 'model.inference', boundary: 'model', status: statuses[3] },
      ],
    }).evidence;
    assert.deepEqual(buildDashboard(evidence).manualFallback, expected, `statuses ${statuses}`);
  }
});

const backlogEvidence = depth => sanitizeEvidence({
  schemaVersion: 1,
  releaseTag: 'daily-2026-08-22.1',
  commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
  generatedAtUtc: '2026-08-22T12:00:00Z',
  checks: [
    { id: 'app.healthz', boundary: 'app', status: 'pass' },
    { id: 'api.analyze', boundary: 'api', status: 'pass' },
    { id: 'queue.depth', boundary: 'queue', status: 'degraded', counters: { depth } },
    { id: 'model.inference', boundary: 'model', status: 'pass' },
  ],
}).evidence;

test('queue.depth counter survives sanitization so queue.backlog can fire', () => {
  const dashboard = buildDashboard(backlogEvidence(10));
  assert.equal(dashboard.boundaries.queue.counters.depth, 10);
});

test('queue.backlog stays silent at depth 9 and fires at exactly depth 10', () => {
  assert.deepEqual(evaluateAlerts(buildDashboard(backlogEvidence(9)), DEFAULT_ALERT_RULES), []);
  const atThreshold = evaluateAlerts(buildDashboard(backlogEvidence(10)), DEFAULT_ALERT_RULES);
  assert.equal(atThreshold.length, 1);
  assert.equal(atThreshold[0].id, 'queue.backlog');
  assert.equal(atThreshold[0].severity, 'warn');
  assert.equal(atThreshold[0].count, 10);
  assert.equal(atThreshold[0].threshold, 10);
});

test('queue.backlog reports observed count above threshold and sums across queue checks', () => {
  const evidence = sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'app.healthz', boundary: 'app', status: 'pass' },
      { id: 'api.analyze', boundary: 'api', status: 'pass' },
      { id: 'queue.depth', boundary: 'queue', status: 'degraded', counters: { depth: 5 } },
      { id: 'queue.depth.sidecar', boundary: 'queue', status: 'pass', counters: { depth: 6 } },
      { id: 'model.inference', boundary: 'model', status: 'pass' },
    ],
  }).evidence;
  const alerts = evaluateAlerts(buildDashboard(evidence), DEFAULT_ALERT_RULES);
  const backlog = alerts.filter(alert => alert.id === 'queue.backlog');
  assert.equal(backlog.length, 1);
  assert.equal(backlog[0].count, 11);
});

test('queue.backlog is suppressed while worker.outage pages because outage depth is residual', () => {
  const evidence = sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'app.healthz', boundary: 'app', status: 'pass' },
      { id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' },
      { id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable', counters: { depth: 17 } },
      { id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'worker.unavailable' },
    ],
  }).evidence;
  const alerts = evaluateAlerts(buildDashboard(evidence), DEFAULT_ALERT_RULES);
  assert.equal(alerts.length, 1, `expected only the outage page, got ${JSON.stringify(alerts)}`);
  assert.equal(alerts[0].id, 'worker.outage');
});

test('queue.bound depth counter above the bounded ceiling cannot smuggle into the dashboard', () => {
  const evidence = sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'app.healthz', boundary: 'app', status: 'pass' },
      { id: 'api.analyze', boundary: 'api', status: 'pass' },
      { id: 'queue.depth', boundary: 'queue', status: 'degraded', counters: { depth: 10_000_000 } },
      { id: 'model.inference', boundary: 'model', status: 'pass' },
    ],
  }).evidence;
  assert.equal(evidence, null, 'unbounded depth must fail closed at sanitization');
});

test('sanitizer fails closed when any app, api, queue, or model boundary is entirely missing', () => {
  const fullChecks = [
    { id: 'app.healthz', boundary: 'app', status: 'pass' },
    { id: 'api.analyze', boundary: 'api', status: 'pass' },
    { id: 'queue.depth', boundary: 'queue', status: 'pass' },
    { id: 'model.inference', boundary: 'model', status: 'pass' },
  ];
  for (const missing of ['app', 'api', 'queue', 'model']) {
    const evidence = sanitizeEvidence({
      schemaVersion: 1,
      releaseTag: 'daily-2026-08-22.1',
      commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
      generatedAtUtc: '2026-08-22T12:00:00Z',
      checks: fullChecks.filter(check => check.boundary !== missing),
    });
    assert.equal(evidence.ok, false, `missing ${missing} boundary must fail closed`);
  }
});

test('unknown statuses surface as a warn telemetry.gap alert instead of failing open', () => {
  const evidence = sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'app.healthz', boundary: 'app', status: 'pass' },
      { id: 'api.analyze', boundary: 'api', status: 'unknown' },
      { id: 'queue.depth', boundary: 'queue', status: 'unknown' },
      { id: 'model.inference', boundary: 'model', status: 'pass' },
    ],
  }).evidence;
  assert.ok(evidence, 'unknown is an allowed status and must not fail sanitization');
  const dashboard = buildDashboard(evidence);
  assert.equal(dashboard.boundaries.api.unknown, 1);
  const alerts = evaluateAlerts(dashboard, DEFAULT_ALERT_RULES);
  assert.equal(alerts.length, 1, `expected exactly the telemetry.gap alert, got ${JSON.stringify(alerts)}`);
  const gap = alerts[0];
  assert.equal(gap.id, 'telemetry.gap');
  assert.equal(gap.severity, 'warn');
  assert.equal(gap.owner, 'release-operator');
  assert.equal(gap.threshold, 1);
  assert.equal(gap.count, 2);
  assert.match(gap.runbook, /docs\/RELEASE-OBSERVABILITY\.md#/);
  assert.match(gap.message, /telemetry\.gap/);
});

test('healthy evidence raises no telemetry.gap alert', () => {
  const alerts = evaluateAlerts(buildDashboard(healthyEvidence()), DEFAULT_ALERT_RULES);
  assert.equal(alerts.some(alert => alert.id === 'telemetry.gap'), false);
});