fix: close hostile-review blockers in release observability
Quality gates / quality (pull_request) Successful in 2m9s
Quality gates / quality (pull_request) Successful in 2m9s
Strict vertical RED-GREEN TDD across eight review blockers: - drill baseline: requires every required boundary check explicitly passing plus zero alerts of any severity before anything is announced or flipped; unknown, degraded, malformed, missing-boundary, and backlog-warning baselines exit 2 with switch count zero, and stdout stays empty whenever no drill actually ran - manual fallback: deterministic truthful states — app-down, local-journal, none-required, and new app-state-unknown (availability null) so unknown telemetry can never render as proven availability or 'no degradation'; runbook gains a state table with exact wording - exact schema types: schemaVersion must be JSON number 1; counters and latency must be finite bounded integers (0..1e6, 0..3600000); generatedAtUtc must be canonical real UTC surviving round-trip; releaseTag/commit keep exact safe types; coercions reject - control-safety: controlSafe strips LF/CR too, so hostile paths can no longer forge terminal lines (regression test ships a path embedding LF + forged PAGE text); read errors use fixed wording instead of echoing attacker-derived message bytes - malformed checks reject the whole evidence object instead of silently disappearing beside healthy checks; forbidden top-level keys now fail closed rather than being ignored - status/failureClass consistency: pass/unknown checks carrying any failureClass reject the file, so all-pass evidence can never page worker.outage (fail/degraded may stay classless, preserving the closed vocabulary and nullable schema) - sanitization idempotence: omitted latency stays omitted on a second pass; explicit null latency rejects since the schema forbids it - removed the always-false queue-depth tautology in the shared fixture and made the runbook contract test verify each rule's real numeric threshold against DEFAULT_ALERT_RULES Preserved: closed failure-class vocabulary, queue depth 9/10/11 edges, raw loopback-origin validation, app-down fallback, missing-boundary rejection, transition semantics, and every independent drill case.
This commit is contained in:
1 parent
9c9286b59f
commit
d547b4887b
7 files changed
+477
-142
No files matched your search
@@ -11,12 +11,14 @@ function argValue(args, flag) {
|
||||
return args[index + 1] || null;
|
||||
}
|
||||
|
||||
// Terminal safety: strip C0/C1 control characters (newline excepted) from any
|
||||
// dynamically produced text before it reaches stdout/stderr, so hostile file
|
||||
// paths or fixture payloads can never smuggle ANSI escapes into a terminal.
|
||||
// Terminal safety: strip C0/C1 control characters — newline and carriage
|
||||
// return included — from any dynamically produced text before it reaches
|
||||
// stdout/stderr. Hostile file paths or fixture payloads can therefore never
|
||||
// forge additional terminal lines (e.g. a fake "PAGE worker.outage"): the
|
||||
// only line boundaries in CLI output are the ones this program authors.
|
||||
function controlSafe(text) {
|
||||
return String(text)
|
||||
.replace(/[\u0000-\u0009\u000B-\u001F\u007F-\u009F]/g, '');
|
||||
.replace(/[\u0000-\u001F\u007F-\u009F]/g, '');
|
||||
}
|
||||
|
||||
async function loadEvidence(source) {
|
||||
@@ -92,7 +94,7 @@ async function main() {
|
||||
try {
|
||||
validatedOrigin = validateLoopbackOrigin(drillOrigin);
|
||||
} catch (error) {
|
||||
console.error(error.message);
|
||||
console.error(controlSafe(error.message));
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
@@ -100,11 +102,19 @@ async function main() {
|
||||
let rawEvidence;
|
||||
try {
|
||||
rawEvidence = await loadEvidence({ drillOrigin: validatedOrigin, evidencePath });
|
||||
} catch (error) {
|
||||
console.error(`release_dashboard: cannot read sanitized evidence (${controlSafe(error.message)}). No contents are echoed.`);
|
||||
} catch {
|
||||
// Fixed wording on purpose: interpolating filesystem error messages would
|
||||
// echo attacker-controlled path bytes (even sanitized) into the terminal.
|
||||
console.error('release_dashboard: cannot read sanitized evidence (unreadable file or invalid JSON). No contents are echoed.');
|
||||
return 2;
|
||||
}
|
||||
|
||||
// Drill mode owns its whole flow: the baseline gate must speak with one
|
||||
// voice ("not a genuine drill baseline") whether the fixture served
|
||||
// unsanitizable evidence or merely unhealthy checks, and nothing may be
|
||||
// rendered or flipped until the baseline proves genuinely healthy.
|
||||
if (drillOrigin) return runDrill({ drillOrigin: validatedOrigin });
|
||||
|
||||
const validated = sanitizeEvidence(rawEvidence);
|
||||
if (!validated.ok) {
|
||||
console.error('release_dashboard: evidence failed sanitization (schema mismatch, forbidden fields, or unbounded values). Nothing was rendered.');
|
||||
@@ -115,16 +125,29 @@ async function main() {
|
||||
const alerts = evaluateAlerts(dashboard);
|
||||
console.log(renderDashboard(dashboard, alerts));
|
||||
console.log('');
|
||||
console.log(`manual fallback: ${dashboard.manualFallback.available ? 'available' : 'unavailable'} — ${fallbackReason(dashboard.manualFallback.reason)}`);
|
||||
console.log(manualFallbackLine(dashboard.manualFallback));
|
||||
|
||||
if (drillOrigin) return runDrill({ drillOrigin: validatedOrigin });
|
||||
return alerts.some(alert => alert.severity === 'page') ? 1 : 0;
|
||||
}
|
||||
|
||||
function fallbackReason(reason) {
|
||||
if (reason === 'local-journal') return 'local journal remains usable';
|
||||
if (reason === 'app-down') return 'app boundary is down; local journal cannot be served';
|
||||
return 'no degradation detected; fallback not required';
|
||||
// Truthful operator wording: the manual-fallback line is a safety statement,
|
||||
// so each deterministic state says exactly what is known. Availability is
|
||||
// only ever claimed from explicit passes; an unknown app boundary reads as
|
||||
// unknown and demands hand verification — never as quiet health.
|
||||
const FALLBACK_PHRASES = {
|
||||
'none-required': 'all checks explicitly passing; no degradation observed; fallback not required',
|
||||
'local-journal': 'degraded elsewhere; local journal remains usable',
|
||||
'app-down': 'app boundary is down; local journal cannot be served',
|
||||
'app-state-unknown': 'app state unknown; verify the local journal by hand before relying on it',
|
||||
};
|
||||
|
||||
function manualFallbackLine(fallback, { shout = false } = {}) {
|
||||
const label = fallback.available === null
|
||||
? (shout ? 'UNKNOWN' : 'unknown')
|
||||
: fallback.available
|
||||
? (shout ? 'AVAILABLE' : 'available')
|
||||
: (shout ? 'UNAVAILABLE' : 'unavailable');
|
||||
return `manual fallback: ${label} — ${FALLBACK_PHRASES[fallback.reason] ?? fallback.reason}`;
|
||||
}
|
||||
|
||||
// A drill may only ever talk to a throwaway fixture on 127.0.0.1. The origin
|
||||
@@ -167,37 +190,47 @@ function validateLoopbackOrigin(rawOrigin) {
|
||||
}
|
||||
|
||||
async function runDrill({ drillOrigin }) {
|
||||
console.log('');
|
||||
console.log('WORKER OUTAGE DRILL (simulated fixture on loopback; no live host is contacted)');
|
||||
let origin;
|
||||
try {
|
||||
origin = validateLoopbackOrigin(drillOrigin);
|
||||
} catch (error) {
|
||||
console.error(error.message);
|
||||
console.error(controlSafe(error.message));
|
||||
return 2;
|
||||
}
|
||||
|
||||
// Genuine baseline: read the fixture's own current state. A drill may only
|
||||
// start from proven health — never from an assumed or already-outaged state.
|
||||
// Genuine baseline: read the fixture's own current state BEFORE anything is
|
||||
// announced or flipped. A drill may only start from proven health — every
|
||||
// required boundary check must explicitly pass and zero alerts may fire.
|
||||
// Unknown, degraded, malformed, missing-boundary, or warning baselines are
|
||||
// refused here, so stdout stays empty whenever no drill actually ran.
|
||||
const preRaw = await loadEvidence({ drillOrigin });
|
||||
const preValidated = sanitizeEvidence(preRaw);
|
||||
if (!preValidated.ok) {
|
||||
console.error('release_dashboard: pre-drill fixture evidence failed sanitization. Nothing was rendered.');
|
||||
console.error('release_dashboard: fixture evidence is unsanitizable: not a genuine drill baseline (schema mismatch, forbidden fields, or unbounded values). Nothing will be rendered or flipped.');
|
||||
return 2;
|
||||
}
|
||||
const preAlerts = evaluateAlerts(buildDashboard(preValidated.evidence));
|
||||
const prePages = preAlerts.filter(alert => alert.severity === 'page');
|
||||
if (prePages.length > 0) {
|
||||
console.error(`release_dashboard: fixture is not healthy before the drill (${prePages.length} page alert[s]); refusing to flip an already-down worker.`);
|
||||
const preDashboard = buildDashboard(preValidated.evidence);
|
||||
const preAlerts = evaluateAlerts(preDashboard);
|
||||
const allExplicitlyPassing = Object.values(preDashboard.boundaries).every(
|
||||
boundary => boundary.checks > 0 && boundary.pass === boundary.checks,
|
||||
);
|
||||
if (!allExplicitlyPassing || preAlerts.length > 0) {
|
||||
console.error('release_dashboard: fixture is not healthy before the drill: not a genuine drill baseline (every required boundary check must explicitly pass with zero alerts). Refusing to flip an already-down or uncertain worker.');
|
||||
return 2;
|
||||
}
|
||||
|
||||
console.log('');
|
||||
console.log('WORKER OUTAGE DRILL (simulated fixture on loopback; no live host is contacted)');
|
||||
console.log('pre-drill baseline: every required boundary check explicitly passing; alerts: none');
|
||||
console.log('pre-drill: healthy (0 page alerts)');
|
||||
|
||||
try {
|
||||
const response = await fetch(`${origin}/drill/outage`, { method: 'POST', signal: AbortSignal.timeout(5_000) });
|
||||
if (!response.ok) throw new Error(`fixture returned ${response.status}`);
|
||||
} catch (error) {
|
||||
console.error(`release_dashboard: drill switch failed (${error.message})`);
|
||||
} catch {
|
||||
// Fixed wording on purpose: fetch failures can embed attacker-chosen URL
|
||||
// text; echoing it (even sanitized) has no operational value.
|
||||
console.error('release_dashboard: drill switch failed. The fixture switch was not verifiably flipped.');
|
||||
return 2;
|
||||
}
|
||||
|
||||
@@ -221,7 +254,7 @@ async function runDrill({ drillOrigin }) {
|
||||
console.log(` ${alert.severity.toUpperCase()} ${alert.message.replace(/^PAGE /, '')}`);
|
||||
console.log(` owner ${alert.owner} · threshold ${alert.threshold} · runbook ${alert.runbook}`);
|
||||
}
|
||||
console.log(`manual fallback: ${postDashboard.manualFallback.available ? 'AVAILABLE' : 'UNAVAILABLE'} — ${fallbackReason(postDashboard.manualFallback.reason)}`);
|
||||
console.log(manualFallbackLine(postDashboard.manualFallback, { shout: true }));
|
||||
const pass = pageAlerts.length === 1
|
||||
&& pageAlerts[0].id === 'worker.outage'
|
||||
&& postDashboard.manualFallback.available === true;
|
||||
|
||||
Reference in new issue
Block a user