timmy-talking-turd/tests/release-dashboard.test.js

import test from 'node:test';
import assert from 'node:assert/strict';

import { sanitizeEvidence, buildDashboard, evaluateAlerts, DEFAULT_ALERT_RULES } from '../src/release-observability.js';

const healthyEvidence = () => sanitizeEvidence({
  schemaVersion: 1,
  releaseTag: 'daily-2026-08-22.1',
  commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
  generatedAtUtc: '2026-08-22T12:00:00Z',
  checks: [
    { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 },
    { id: 'api.analyze', boundary: 'api', status: 'pass', counters: { ok: 40, fail: 0 } },
    { id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { depth: 0 } === undefined ? {} : { ok: 30, fail: 0 } },
    { id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 900, counters: { ok: 25, fail: 0, abstain: 3 } },
  ],
}).evidence;

test('dashboard separates app, api, queue, and model boundaries', () => {
  const dashboard = buildDashboard(healthyEvidence());
  assert.equal(dashboard.ok, true);
  assert.deepEqual(Object.keys(dashboard.boundaries).sort(), ['api', 'app', 'model', 'queue']);
  assert.equal(dashboard.boundaries.app.pass, 1);
  assert.equal(dashboard.boundaries.model.pass, 1);
  assert.equal(dashboard.boundaries.model.latencyMs, 900);
  assert.equal(dashboard.totals.checks, 4);
  assert.equal(dashboard.totals.fail, 0);
  assert.deepEqual(dashboard.failureClasses, {});
  assert.equal(dashboard.identity.releaseTag, 'daily-2026-08-22.1');
  assert.equal(dashboard.identity.commit, 'ca31e6d38bec649407f63880504554c59f2878ae');
});

test('dashboard summarizes privacy-safe failure classes without any payload text', () => {
  const evidence = sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'api.analyze', boundary: 'api', status: 'degraded', failureClass: 'vision.timeout' },
      { id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'model.error' },
      { id: 'model.inference.warmup', boundary: 'model', status: 'fail', failureClass: 'model.error' },
    ],
  }).evidence;
  const dashboard = buildDashboard(evidence);
  assert.deepEqual(dashboard.failureClasses, { 'vision.timeout': 1, 'model.error': 2 });
  assert.equal(dashboard.totals.fail, 2);
  assert.equal(dashboard.totals.degraded, 1);
  assert.equal(JSON.stringify(dashboard).includes('note'), false);
});

test('simulated worker outage fires exactly one actionable alert with owner, threshold, and runbook', () => {
  const evidence = sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 11 },
      { id: 'api.analyze', boundary: 'api', status: 'fail', failureClass: 'worker.unavailable' },
      { id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable', counters: { depth: 17 } },
      { id: 'model.inference', boundary: 'model', status: 'fail', failureClass: 'worker.unavailable', counters: { timeout: 4 } },
    ],
  }).evidence;

  const dashboard = buildDashboard(evidence);
  const alerts = evaluateAlerts(dashboard, DEFAULT_ALERT_RULES);

  assert.equal(alerts.length, 1, `expected exactly one alert, got ${JSON.stringify(alerts)}`);
  const alert = alerts[0];
  assert.equal(alert.id, 'worker.outage');
  assert.equal(alert.severity, 'page');
  assert.equal(alert.owner, 'release-operator');
  assert.match(alert.runbook, /docs\/RELEASE-OBSERVABILITY\.md#simulate-a-worker-outage/);
  assert.match(alert.message, /worker\.unavailable/);
  assert.match(alert.message, /daily-2026-08-22\.1/);
  assert.equal(alert.count, 3);
  assert.equal(alert.threshold, 1);
});

test('healthy evidence raises no alerts and every default rule carries owner, threshold, and runbook', () => {
  assert.deepEqual(evaluateAlerts(buildDashboard(healthyEvidence()), DEFAULT_ALERT_RULES), []);
  for (const rule of DEFAULT_ALERT_RULES) {
    assert.ok(rule.owner && rule.owner.length > 0);
    assert.ok(Number.isInteger(rule.threshold) && rule.threshold >= 1);
    assert.match(rule.runbook, /^docs\/RELEASE-OBSERVABILITY\.md#/);
    assert.ok(['page', 'warn'].includes(rule.severity));
  }
});

test('manual fallback stays available whenever any non-app check is unhealthy', () => {
  const healthy = buildDashboard(healthyEvidence());
  assert.deepEqual(healthy.manualFallback, { available: true, reason: 'none-required' });

  const outage = buildDashboard(sanitizeEvidence({
    schemaVersion: 1,
    releaseTag: 'daily-2026-08-22.1',
    commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
    generatedAtUtc: '2026-08-22T12:00:00Z',
    checks: [
      { id: 'app.healthz', boundary: 'app', status: 'pass' },
      { id: 'queue.depth', boundary: 'queue', status: 'fail', failureClass: 'worker.unavailable' },
    ],
  }).evidence);
  assert.deepEqual(outage.manualFallback, {
    available: true,
    reason: 'local-journal',
  });
});