feat(ops): release observability dashboard and incident-response flow
Quality gates / quality (pull_request) Successful in 1m44s
Quality gates / quality (pull_request) Successful in 1m44s
Implements issue #41 acceptance criteria without touching the live host: - src/release-observability.js: sanitizeEvidence() allowlists bounded, privacy-safe evidence (release tag, commit, UTC time, per-check id/boundary/status/failure-class/latency/counters) and drops session identifiers, credentials, environment dumps, photo payloads, base64, image hashes, note text, emails, and any oversized/suspicious value. - buildDashboard(): app/api/queue/model boundary rollups, failure-class counts, manual-fallback state; evaluateAlerts() with owner, threshold, severity, and runbook anchor per rule. - scripts/release_dashboard.mjs: local operator CLI over sanitized evidence files or a loopback drill fixture; never contacts a live host and exits nonzero without echoing rejected input. - docs/RELEASE-OBSERVABILITY.md: alert inventory, evidence schema, simulated worker-outage drill, incident flow, privacy boundary. - Tests: sanitizer hostile-payload coverage, dashboard/alert rules, end-to-end loopback outage drill asserting exactly one actionable page alert plus graceful manual fallback, runbook/package contract.
This commit is contained in:
1 parent
47294a98aa
commit
ad8c8a9f4c
8 files changed
+844
-2
No files matched your search
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env node
|
||||
// Local operator surface for release observability (issue #41).
|
||||
// Consumes only sanitized evidence files or a loopback drill fixture.
|
||||
// Never contacts a live host, never accepts credentials, never prints secrets.
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { sanitizeEvidence, buildDashboard, evaluateAlerts } from '../src/release-observability.js';
|
||||
|
||||
function argValue(args, flag) {
|
||||
const index = args.indexOf(flag);
|
||||
if (index === -1) return null;
|
||||
return args[index + 1] || null;
|
||||
}
|
||||
|
||||
async function loadEvidence(source) {
|
||||
if (source.drillOrigin) {
|
||||
const response = await fetch(`${source.drillOrigin.replace(/\/$/, '')}/api/drill/checks`, { signal: AbortSignal.timeout(5_000) });
|
||||
if (!response.ok) throw new Error(`drill fixture returned ${response.status}`);
|
||||
const body = await response.json();
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
releaseTag: 'daily-2026-08-22.1',
|
||||
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
||||
generatedAtUtc: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'),
|
||||
checks: Array.isArray(body.checks) ? body.checks : [],
|
||||
};
|
||||
}
|
||||
const raw = JSON.parse(await readFile(source.evidencePath, 'utf8'));
|
||||
return raw;
|
||||
}
|
||||
|
||||
function renderDashboard(dashboard, alerts) {
|
||||
const lines = [];
|
||||
lines.push('RELEASE OBSERVABILITY DASHBOARD');
|
||||
lines.push(`release ${dashboard.identity.releaseTag} · commit ${dashboard.identity.commit.slice(0, 12)} · evidence ${dashboard.identity.generatedAtUtc}`);
|
||||
lines.push('');
|
||||
for (const boundary of ['app', 'api', 'queue', 'model']) {
|
||||
const stats = dashboard.boundaries[boundary];
|
||||
const latency = stats.latencyMs == null ? '' : ` · max latency ${stats.latencyMs}ms`;
|
||||
const counterBits = Object.keys(stats.counters).sort()
|
||||
.map(key => `${key}=${stats.counters[key]}`)
|
||||
.join(' ');
|
||||
const counterText = counterBits ? ` · ${counterBits}` : '';
|
||||
lines.push(`${boundary.padEnd(6)}${stats.pass}/${stats.checks} pass · fail ${stats.fail} · degraded ${stats.degraded}${latency}${counterText}`);
|
||||
}
|
||||
lines.push('');
|
||||
const failureEntries = Object.entries(dashboard.failureClasses);
|
||||
if (failureEntries.length === 0) {
|
||||
lines.push('failure classes: none');
|
||||
} else {
|
||||
for (const [failureClass, count] of failureEntries.sort((a, b) => b[1] - a[1])) {
|
||||
lines.push(`failure class ${failureClass}: ${count}`);
|
||||
}
|
||||
}
|
||||
lines.push('');
|
||||
lines.push(`alerts (${alerts.length})`);
|
||||
for (const alert of alerts) {
|
||||
lines.push(`[${alert.severity.toUpperCase()}] ${alert.id} — owner ${alert.owner}, observed ${alert.count} >= threshold ${alert.threshold}`);
|
||||
lines.push(` ${alert.message}`);
|
||||
lines.push(` runbook: ${alert.runbook}`);
|
||||
}
|
||||
if (alerts.length === 0) lines.push('no alert conditions met');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.includes('--self-check')) {
|
||||
console.log('release-observability module self-check: exports OK');
|
||||
return 0;
|
||||
}
|
||||
|
||||
const drillOrigin = argValue(args, '--drill-origin');
|
||||
const evidencePath = argValue(args, '--evidence');
|
||||
if (Boolean(drillOrigin) === Boolean(evidencePath)) {
|
||||
console.error('usage: node scripts/release_dashboard.mjs (--evidence <sanitized-evidence.json> | --drill-origin <http://127.0.0.1:port>)');
|
||||
return 2;
|
||||
}
|
||||
|
||||
let rawEvidence;
|
||||
try {
|
||||
rawEvidence = await loadEvidence({ drillOrigin, evidencePath });
|
||||
} catch (error) {
|
||||
console.error(`release_dashboard: cannot read sanitized evidence (${error.message}). No contents are echoed.`);
|
||||
return 2;
|
||||
}
|
||||
|
||||
const validated = sanitizeEvidence(rawEvidence);
|
||||
if (!validated.ok) {
|
||||
console.error('release_dashboard: evidence failed sanitization (schema mismatch, forbidden fields, or unbounded values). Nothing was rendered.');
|
||||
return 2;
|
||||
}
|
||||
|
||||
const dashboard = buildDashboard(validated.evidence);
|
||||
const alerts = evaluateAlerts(dashboard);
|
||||
console.log(renderDashboard(dashboard, alerts));
|
||||
console.log('');
|
||||
console.log(`manual fallback: ${dashboard.manualFallback.available ? 'available' : 'unavailable'} — ${fallbackReason(dashboard.manualFallback.reason)}`);
|
||||
|
||||
if (drillOrigin) return runDrill({ dashboard, alerts, drillOrigin });
|
||||
return alerts.some(alert => alert.severity === 'page') ? 1 : 0;
|
||||
}
|
||||
|
||||
function fallbackReason(reason) {
|
||||
if (reason === 'local-journal') return 'local journal remains usable';
|
||||
return 'no outage detected; fallback not required';
|
||||
}
|
||||
|
||||
async function runDrill({ dashboard, alerts, drillOrigin }) {
|
||||
console.log('');
|
||||
console.log('WORKER OUTAGE DRILL (simulated fixture on loopback; no live host is contacted)');
|
||||
const preDrillAlerts = evaluateAlerts(buildDashboard(sanitizeEvidence({
|
||||
schemaVersion: 1,
|
||||
releaseTag: dashboard.identity.releaseTag,
|
||||
commit: dashboard.identity.commit,
|
||||
generatedAtUtc: dashboard.identity.generatedAtUtc,
|
||||
checks: [{ id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 }],
|
||||
}).evidence));
|
||||
console.log(`pre-drill: ${preDrillAlerts.length} alerts`);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${drillOrigin.replace(/\/$/, '')}/drill/outage`, { method: 'POST', signal: AbortSignal.timeout(5_000) });
|
||||
if (!response.ok) throw new Error(`fixture returned ${response.status}`);
|
||||
} catch (error) {
|
||||
console.error(`release_dashboard: drill switch failed (${error.message})`);
|
||||
return 2;
|
||||
}
|
||||
|
||||
const postRaw = await loadEvidence({ drillOrigin });
|
||||
const postValidated = sanitizeEvidence(postRaw);
|
||||
if (!postValidated.ok) {
|
||||
console.error('release_dashboard: post-outage evidence failed sanitization.');
|
||||
return 2;
|
||||
}
|
||||
const postDashboard = buildDashboard(postValidated.evidence);
|
||||
const postAlerts = evaluateAlerts(postDashboard);
|
||||
const pageAlerts = postAlerts.filter(alert => alert.severity === 'page');
|
||||
|
||||
console.log(`post-outage: exactly 1 alert expected, found ${pageAlerts.length}`);
|
||||
for (const alert of pageAlerts) {
|
||||
console.log(` ${alert.severity.toUpperCase()} ${alert.message.replace(/^PAGE /, '')}`);
|
||||
console.log(` owner ${alert.owner} · threshold ${alert.threshold} · runbook ${alert.runbook}`);
|
||||
}
|
||||
console.log(`manual fallback: ${postDashboard.manualFallback.available ? 'AVAILABLE' : 'UNAVAILABLE'} — local journal remains usable`);
|
||||
const pass = pageAlerts.length === 1
|
||||
&& pageAlerts[0].id === 'worker.outage'
|
||||
&& postDashboard.manualFallback.available === true;
|
||||
console.log(pass ? 'DRILL PASS' : 'DRILL FAIL');
|
||||
return pass ? 0 : 1;
|
||||
}
|
||||
|
||||
process.exitCode = await main();
|
||||
Reference in new issue
Block a user