Timmy
d547b4887b
All checks were successful
Quality gates / quality (pull_request) Successful in 2m9s
Strict vertical RED-GREEN TDD across eight review blockers: - drill baseline: requires every required boundary check explicitly passing plus zero alerts of any severity before anything is announced or flipped; unknown, degraded, malformed, missing-boundary, and backlog-warning baselines exit 2 with switch count zero, and stdout stays empty whenever no drill actually ran - manual fallback: deterministic truthful states — app-down, local-journal, none-required, and new app-state-unknown (availability null) so unknown telemetry can never render as proven availability or 'no degradation'; runbook gains a state table with exact wording - exact schema types: schemaVersion must be JSON number 1; counters and latency must be finite bounded integers (0..1e6, 0..3600000); generatedAtUtc must be canonical real UTC surviving round-trip; releaseTag/commit keep exact safe types; coercions reject - control-safety: controlSafe strips LF/CR too, so hostile paths can no longer forge terminal lines (regression test ships a path embedding LF + forged PAGE text); read errors use fixed wording instead of echoing attacker-derived message bytes - malformed checks reject the whole evidence object instead of silently disappearing beside healthy checks; forbidden top-level keys now fail closed rather than being ignored - status/failureClass consistency: pass/unknown checks carrying any failureClass reject the file, so all-pass evidence can never page worker.outage (fail/degraded may stay classless, preserving the closed vocabulary and nullable schema) - sanitization idempotence: omitted latency stays omitted on a second pass; explicit null latency rejects since the schema forbids it - removed the always-false queue-depth tautology in the shared fixture and made the runbook contract test verify each rule's real numeric threshold against DEFAULT_ALERT_RULES Preserved: closed failure-class vocabulary, queue depth 9/10/11 edges, raw loopback-origin validation, app-down fallback, missing-boundary rejection, transition semantics, and every independent drill case.
230 lines
10 KiB
JavaScript
230 lines
10 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
|
|
import { sanitizeEvidence } from '../src/release-observability.js';
|
|
|
|
const validEvidence = () => ({
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass', latencyMs: 12 },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'pass', counters: { ok: 40, fail: 1 } },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'pass', counters: { ok: 30, fail: 0 } },
|
|
{ id: 'model.inference', boundary: 'model', status: 'pass', latencyMs: 900, counters: { ok: 25, fail: 0, abstain: 3 } },
|
|
],
|
|
});
|
|
|
|
test('sanitizer keeps bounded sanitized release evidence intact', () => {
|
|
const result = sanitizeEvidence(validEvidence());
|
|
assert.equal(result.ok, true);
|
|
assert.deepEqual(result.evidence, {
|
|
schemaVersion: 1,
|
|
releaseTag: 'daily-2026-08-22.1',
|
|
commit: 'ca31e6d38bec649407f63880504554c59f2878ae',
|
|
generatedAtUtc: '2026-08-22T12:00:00Z',
|
|
checks: [
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass', failureClass: null, latencyMs: 12, counters: {} },
|
|
{ id: 'api.analyze', boundary: 'api', status: 'pass', failureClass: null, counters: { ok: 40, fail: 1 } },
|
|
{ id: 'queue.depth', boundary: 'queue', status: 'pass', failureClass: null, counters: { ok: 30, fail: 0 } },
|
|
{ id: 'model.inference', boundary: 'model', status: 'pass', failureClass: null, latencyMs: 900, counters: { ok: 25, fail: 0, abstain: 3 } },
|
|
],
|
|
});
|
|
});
|
|
|
|
test('sanitizer fail-closes evidence carrying session identifiers, photo payloads, credentials, and free-text notes', () => {
|
|
// Assembled at runtime so no contiguous private-key marker ever lands in Git history.
|
|
const privateKeyFixture = `-----BEGIN ${'OPENSSH'} PRIVATE KEY${'-----'}`;
|
|
// Forbidden-shaped fields are a rejection of the whole evidence object, not
|
|
// a silent drop: censoring a hostile check would dress the gap up as health.
|
|
const hostile = {
|
|
...validEvidence(),
|
|
sessionToken: 'sess_live_abc123',
|
|
adminCookie: 'timmy_agent=secret-cookie-value',
|
|
environmentDump: { NODE_ENV: 'production', SECRET_TOKEN: 'must-not-leak' },
|
|
operatorEmail: 'someone@example.com',
|
|
credentialFile: privateKeyFixture,
|
|
photos: [{ dataBase64: 'aGVsbG8gd29ybGQgaGVsbG8gd29ybGQ=' }],
|
|
};
|
|
assert.equal(sanitizeEvidence(hostile).ok, false, 'forbidden top-level fields must reject the whole evidence object');
|
|
|
|
const leakyCheck = {
|
|
id: 'leak',
|
|
boundary: 'app',
|
|
status: 'pass',
|
|
noteText: 'patient said blood at 3am, see photo hash e3b0c442',
|
|
imageHash: 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
|
|
base64Payload: 'aGVsbG8=',
|
|
authHeader: `Bearer ${'sk-'}live-abcdef1234567890`,
|
|
};
|
|
const withLeakyCheck = validEvidence();
|
|
withLeakyCheck.checks = [...validEvidence().checks, leakyCheck];
|
|
const result = sanitizeEvidence(withLeakyCheck);
|
|
assert.equal(result.ok, false, 'a check carrying forbidden or free-text fields must reject the whole evidence object');
|
|
});
|
|
|
|
test('sanitizer fail-closes evidence carrying hostile failure-class payloads', () => {
|
|
const hostileFailureClasses = [
|
|
'worker.unavailable\nsecond line',
|
|
'worker.unavailable\r\ncarriage',
|
|
'\x1b[31mANSI-red',
|
|
'worker.unavailable\x1b[0m',
|
|
'bell\x07class',
|
|
'tab\tseparated',
|
|
'DROP TABLE users',
|
|
'patient reported blood at 3am',
|
|
'sess_live_abc123def456',
|
|
'-----BEGIN OPENSSH PRIVATE KEY-----',
|
|
'a@b.example.com',
|
|
'.leading.dot',
|
|
'trailing.dot.',
|
|
'double..dot',
|
|
'-leading-dash.d',
|
|
'd.trailing-dash-',
|
|
'-'.repeat(65),
|
|
42,
|
|
{},
|
|
[],
|
|
];
|
|
for (const failureClass of hostileFailureClasses) {
|
|
const evidence = validEvidence();
|
|
evidence.checks = [{ id: 'app.healthz', boundary: 'app', status: 'fail', failureClass }];
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(result.ok, false, `expected rejection for failureClass ${JSON.stringify(failureClass)}`);
|
|
}
|
|
});
|
|
|
|
test('sanitizer accepts only the exact privacy-safe failure-class vocabulary in slug grammar', () => {
|
|
const baseChecks = validEvidence().checks;
|
|
for (const failureClass of ['worker.unavailable', 'vision.timeout', 'model.error']) {
|
|
const evidence = validEvidence();
|
|
evidence.checks = [
|
|
...baseChecks.filter(check => check.boundary !== 'api'),
|
|
{ id: 'api.analyze', boundary: 'api', status: 'degraded', failureClass },
|
|
];
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(result.ok, true, `vocabulary class ${failureClass} must survive sanitization`);
|
|
assert.equal(result.evidence.checks.some(check => check.failureClass === failureClass), true);
|
|
}
|
|
for (const failureClass of ['made.up.class', 'worker.somethingelse', 'VISION.TIMEOUT', 'vision..timeout']) {
|
|
const evidence = validEvidence();
|
|
evidence.checks = [
|
|
...baseChecks.filter(check => check.boundary !== 'api'),
|
|
{ id: 'api.analyze', boundary: 'api', status: 'degraded', failureClass },
|
|
];
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(result.ok, false, `non-vocabulary class ${failureClass} must be rejected`);
|
|
}
|
|
});
|
|
|
|
test('schemaVersion must be the JSON number 1 and nothing else', () => {
|
|
for (const hostileVersion of ['1', true, false, null, 1.5, 0, 2, [1], { value: 1 }]) {
|
|
const evidence = { ...validEvidence(), schemaVersion: hostileVersion };
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(result.ok, false, `schemaVersion ${JSON.stringify(hostileVersion)} must be rejected`);
|
|
}
|
|
});
|
|
|
|
test('counters and latency must be actual JSON numbers, never strings, booleans, nulls, or fractions', () => {
|
|
for (const counters of [{ ok: '40' }, { ok: true }, { fail: null }, { depth: 10.5 }, { depth: -1 }, { retry: Infinity }]) {
|
|
const evidence = validEvidence();
|
|
evidence.checks[0] = { ...evidence.checks[0], counters };
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(result.ok, false, `counters ${JSON.stringify(counters)} must be rejected`);
|
|
}
|
|
for (const latencyMs of ['12', '', true, false, null, 12.5, -1, Infinity]) {
|
|
const evidence = validEvidence();
|
|
evidence.checks[0] = { ...evidence.checks[0], latencyMs };
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(result.ok, false, `latencyMs ${JSON.stringify(latencyMs)} must be rejected`);
|
|
}
|
|
});
|
|
|
|
test('generatedAtUtc must be canonical real UTC that survives a Date round-trip', () => {
|
|
const invalidTimestamps = [
|
|
'2026-02-30T12:00:00Z',
|
|
'2026-08-22T25:00:00Z',
|
|
'2026-08-22T12:61:00Z',
|
|
'2026-13-01T00:00:00Z',
|
|
'2026-08-22T12:00:00+00:00',
|
|
'2026-08-22t12:00:00z',
|
|
'2026-08-22 12:00:00Z',
|
|
'not-a-timestamp',
|
|
1234567890,
|
|
true,
|
|
null,
|
|
];
|
|
for (const generatedAtUtc of invalidTimestamps) {
|
|
const evidence = { ...validEvidence(), generatedAtUtc };
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(result.ok, false, `generatedAtUtc ${JSON.stringify(generatedAtUtc)} must be rejected`);
|
|
}
|
|
});
|
|
|
|
test('releaseTag and commit must have their exact safe types', () => {
|
|
for (const releaseTag of [42, true, null, {}, [], '']) {
|
|
const evidence = { ...validEvidence(), releaseTag };
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(result.ok, false, `releaseTag ${JSON.stringify(releaseTag)} must be rejected`);
|
|
}
|
|
for (const commit of [0xca31e6d38bec, true, null, {}, []]) {
|
|
const evidence = { ...validEvidence(), commit };
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(result.ok, false, `commit ${JSON.stringify(commit)} must be rejected`);
|
|
}
|
|
});
|
|
|
|
test('one malformed check rejects the entire evidence object instead of vanishing beside healthy checks', () => {
|
|
const malformedChecks = [
|
|
{ boundary: 'app', status: 'pass' },
|
|
{ id: 'BAD_ID', boundary: 'app', status: 'pass' },
|
|
{ id: 42, boundary: 'app', status: 'pass' },
|
|
{ id: 'app.healthz', status: 'pass' },
|
|
{ id: 'app.healthz', boundary: 'bus', status: 'pass' },
|
|
{ id: 'app.healthz', boundary: 42, status: 'pass' },
|
|
{ id: 'app.healthz', boundary: 'app' },
|
|
{ id: 'app.healthz', boundary: 'app', status: 'healthy' },
|
|
{ id: 'app.healthz', boundary: 'app', status: 42 },
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass', counters: 'many' },
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass', counters: [] },
|
|
{ id: 'app.healthz', boundary: 'app', status: 'pass', failureClass: 42 },
|
|
];
|
|
for (const malformed of malformedChecks) {
|
|
const evidence = validEvidence();
|
|
evidence.checks = [...validEvidence().checks, malformed];
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(result.ok, false, `malformed check ${JSON.stringify(malformed)} must reject the whole evidence object`);
|
|
}
|
|
});
|
|
|
|
test('a passing or unknown check can never carry a failureClass, so all-pass evidence can never page worker.outage', () => {
|
|
const baseChecks = validEvidence().checks;
|
|
for (const status of ['pass', 'unknown']) {
|
|
for (const failureClass of ['worker.unavailable', 'model.error', 'vision.timeout']) {
|
|
const evidence = validEvidence();
|
|
evidence.checks = baseChecks.map(check =>
|
|
check.boundary === 'app' ? { ...check, status, failureClass } : check,
|
|
);
|
|
const result = sanitizeEvidence(evidence);
|
|
assert.equal(
|
|
result.ok,
|
|
false,
|
|
`status ${status} carrying failureClass ${failureClass} must reject the whole evidence object`,
|
|
);
|
|
}
|
|
}
|
|
});
|
|
|
|
test('sanitization is idempotent: re-sanitizing sanitized evidence changes nothing and omits nullable latency', () => {
|
|
const first = sanitizeEvidence(validEvidence());
|
|
assert.equal(first.ok, true);
|
|
assert.equal(Object.hasOwn(first.evidence.checks[0], 'latencyMs'), true, 'present latency must stay present');
|
|
assert.equal(Object.hasOwn(first.evidence.checks[1], 'latencyMs'), false, 'omitted latency must stay omitted');
|
|
|
|
const second = sanitizeEvidence(first.evidence);
|
|
assert.equal(second.ok, true, 'sanitized evidence must survive a second sanitization pass');
|
|
assert.deepEqual(second.evidence, first.evidence, 'second pass must not rewrite omitted latency to a different shape');
|
|
assert.equal(JSON.stringify(second.evidence).includes('"latencyMs":0'), false, 'no zero-latency fabrication');
|
|
});
|