From e08b537866adf03cd19c159382e758408bbfedf0 Mon Sep 17 00:00:00 2001 From: Timmy Date: Mon, 2 Nov 2026 10:56:16 +0000 Subject: [PATCH] feat(metrics): render the operator health snapshot as a report (#19) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The counters had an endpoint and no way to read them without jq. Adds a pure formatter plus a CLI that prints the app, queue, and model boundaries. The formatter is a function of the snapshot and the three frozen vocabularies, so a caller-supplied failure label cannot reach the terminal: unknown classes are simply not printed. Refusal paths are explicit — 404 means the route is off because no token is configured, 403 means the token was rejected — and the token is never echoed in the report or an error line. RED: tests/model-metrics-report-format.test.js failed with ERR_MODULE_NOT_FOUND before src/model-metrics-report.js existed; the refusal-path test failed with a generic 'answered 404' before the 404 branch was written. Tests: npm test 298/298, npm run test:metrics-report 8/8, syntax and diff gates clean, real-Chromium ui acceptance green with the privacy disclosure visible. --- docs/MODEL-METRICS.md | 12 +++ package.json | 7 +- scripts/model_metrics_report.mjs | 88 +++++++++++++++++++ src/model-metrics-report.js | 74 ++++++++++++++++ tests/model-metrics-report-format.test.js | 82 +++++++++++++++++ tests/model-metrics-report.acceptance.test.js | 51 +++++++++++ 6 files changed, 311 insertions(+), 3 deletions(-) create mode 100644 scripts/model_metrics_report.mjs create mode 100644 src/model-metrics-report.js create mode 100644 tests/model-metrics-report-format.test.js diff --git a/docs/MODEL-METRICS.md b/docs/MODEL-METRICS.md index 9e7ee01..00cef94 100644 --- a/docs/MODEL-METRICS.md +++ b/docs/MODEL-METRICS.md @@ -80,6 +80,18 @@ by feeding hostile strings into every label position, and curl -sS -H "Authorization: Bearer $TIMMY_METRICS_TOKEN" http://127.0.0.1:4173/api/metrics | jq . ``` +For a quick look that does not need `jq`, the report renders the same snapshot as three labelled blocks: + +```bash +TIMMY_METRICS_TOKEN=... node scripts/model_metrics_report.mjs --url http://127.0.0.1:4173 +``` + +Exit codes are the point of the command as much as the numbers: `0` means a snapshot was rendered, +`1` means the server answered non-200 (404 says the route is disabled because no token is configured, +403 says the supplied token was rejected), `2` means the command itself was called wrongly or the +server could not be reached. The token is read from the environment or `--token` and is never echoed +in the report or in an error line. + What the numbers mean to an operator: | Signal | Reading | diff --git a/package.json b/package.json index 9c61034..e3ef900 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "private": true, "type": "module", "scripts": { - "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js tests/training-contribution.test.js tests/portability.test.js tests/abstention-policy.test.js tests/release-security-gate.test.js tests/model-metrics.test.js tests/metrics-endpoint.acceptance.test.js tests/image-ingress.test.js tests/rate-limit.test.js tests/http-body.test.js tests/analyze-ingress.acceptance.test.js", + "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js tests/training-contribution.test.js tests/portability.test.js tests/abstention-policy.test.js tests/release-security-gate.test.js tests/model-metrics.test.js tests/metrics-endpoint.acceptance.test.js tests/image-ingress.test.js tests/rate-limit.test.js tests/http-body.test.js tests/analyze-ingress.acceptance.test.js tests/model-metrics-report-format.test.js tests/model-metrics-report.acceptance.test.js", "test:ui": "node tests/ui.acceptance.mjs", "test:photo": "node tests/photo-first.acceptance.mjs", "test:mobile-capture": "node tests/mobile-capture.acceptance.mjs", @@ -19,7 +19,7 @@ "test:grounded-chat": "node tests/grounded-chat.acceptance.mjs", "test:photo-lifecycle": "node tests/photo-lifecycle.acceptance.mjs", "test:training-contribution": "node tests/training-contribution.acceptance.mjs", - "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check src/training-contribution.js && node --check src/abstention-policy.js && node --check src/release-security-gate.js && node --check scripts/security_attack_suite.mjs && node --check tests/security-attack.acceptance.mjs && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/corpus_manifest.py scripts/review_corpus.py scripts/dataset_card.py scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py scripts/calibration_report.py scripts/operator_backup.py && node --check scripts/record_worker_benchmark.mjs && node --check scripts/abstention_policy_receipt.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/training-contribution.acceptance.mjs && node --check tests/portability.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh && node --check src/model-metrics.js && node --check src/image-ingress.js && node --check src/rate-limit.js && node --check src/http-body.js && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py && node --check tests/ingress-capture.acceptance.mjs && node --check tests/analyze-ingress.acceptance.test.js", + "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check src/training-contribution.js && node --check src/abstention-policy.js && node --check src/release-security-gate.js && node --check scripts/security_attack_suite.mjs && node --check tests/security-attack.acceptance.mjs && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/corpus_manifest.py scripts/review_corpus.py scripts/dataset_card.py scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py scripts/calibration_report.py scripts/operator_backup.py && node --check scripts/record_worker_benchmark.mjs && node --check scripts/abstention_policy_receipt.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/training-contribution.acceptance.mjs && node --check tests/portability.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh && node --check src/model-metrics.js && node --check src/model-metrics-report.js && node --check scripts/model_metrics_report.mjs && node --check src/image-ingress.js && node --check src/rate-limit.js && node --check src/http-body.js && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py && node --check tests/ingress-capture.acceptance.mjs && node --check tests/analyze-ingress.acceptance.test.js", "check:diff": "bash scripts/check_diff.sh", "start": "node server.mjs", "test:worker-readiness": "node tests/worker-readiness.acceptance.mjs", @@ -31,7 +31,8 @@ "test:dataset-card": "python3 tests/dataset-card.test.py", "test:backup": "python3 tests/backup-restore.test.py", "test:runbook": "python3 tests/production-runbook.test.py", - "test:security-attack": "node tests/security-attack.acceptance.mjs" + "test:security-attack": "node tests/security-attack.acceptance.mjs", + "test:metrics-report": "node --test tests/model-metrics-report-format.test.js tests/model-metrics-report.acceptance.test.js" }, "devDependencies": { "playwright": "^1.62.1" diff --git a/scripts/model_metrics_report.mjs b/scripts/model_metrics_report.mjs new file mode 100644 index 0000000..4a1eaf9 --- /dev/null +++ b/scripts/model_metrics_report.mjs @@ -0,0 +1,88 @@ +#!/usr/bin/env node +// Operator report for the privacy-safe model counters (issue #19). +// +// It reads one snapshot from a running Timmy server and prints it. The token is a credential for the +// operator surface, so it is read from the environment or argv and never echoed — not in errors, not +// in the report, not in a stack trace. +import { formatOperatorReport } from '../src/model-metrics-report.js'; + +const USAGE = `Usage: node scripts/model_metrics_report.mjs [options] + + --url base URL of the Timmy server (default http://127.0.0.1:4173) + --token operator token (default: $TIMMY_METRICS_TOKEN) + --json print the raw snapshot instead of the text report + --timeout-ms request deadline (default 5000) + --help show this text +`; + +function parseArgs(argv) { + const options = { + url: process.env.TIMMY_METRICS_URL || 'http://127.0.0.1:4173', + token: process.env.TIMMY_METRICS_TOKEN || '', + json: false, + timeoutMs: 5_000, + help: false, + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--help' || arg === '-h') { + options.help = true; + } else if (arg === '--json') { + options.json = true; + } else if (arg === '--url') { + options.url = argv[++index] ?? ''; + } else if (arg === '--token') { + options.token = argv[++index] ?? ''; + } else if (arg === '--timeout-ms') { + options.timeoutMs = Number(argv[++index]); + } else { + throw new Error(`unknown option: ${arg}`); + } + } + return options; +} + +async function main(argv) { + const options = parseArgs(argv); + if (options.help) { + process.stdout.write(USAGE); + return 0; + } + if (!options.token) { + throw new Error('operator token required: pass --token or set TIMMY_METRICS_TOKEN'); + } + const endpoint = new URL('api/metrics', options.url.endsWith('/') ? options.url : `${options.url}/`); + let response; + try { + response = await fetch(endpoint, { + headers: { authorization: `Bearer ${options.token}`, accept: 'application/json' }, + signal: AbortSignal.timeout(Number.isFinite(options.timeoutMs) ? options.timeoutMs : 5_000), + }); + } catch (error) { + process.stderr.write(`Could not read ${endpoint.origin}/api/metrics: ${error?.message || 'request failed'}\n`); + return 1; + } + if (!response.ok) { + if (response.status === 404) { + // The route is absent rather than guarded: that is a deployment fact, not a credential problem. + process.stderr.write('Metrics are disabled on this server: the route only exists when TIMMY_METRICS_TOKEN is set.\n'); + return 1; + } + if (response.status === 403) { + process.stderr.write('The server rejected the operator token.\n'); + return 1; + } + process.stderr.write(`The server answered ${response.status} for /api/metrics.\n`); + return 1; + } + const snapshot = await response.json(); + process.stdout.write(options.json ? `${JSON.stringify(snapshot, null, 2)}\n` : formatOperatorReport(snapshot)); + return 0; +} + +main(process.argv.slice(2)).then(code => { + process.exitCode = code; +}).catch(error => { + process.stderr.write(`${error?.message || 'report failed'}\n`); + process.exitCode = 2; +}); diff --git a/src/model-metrics-report.js b/src/model-metrics-report.js new file mode 100644 index 0000000..d55ef12 --- /dev/null +++ b/src/model-metrics-report.js @@ -0,0 +1,74 @@ +// Renders the operator snapshot from `src/model-metrics.js` as a plain-text report. +// +// This module is a pure function of the snapshot, on purpose: the report is built from numbers and +// from the three frozen vocabularies, so there is no path by which request content could reach the +// terminal even if the counters were compromised. Anything outside the enumerated vocabularies is +// not printed — an unknown failure class is visible as a missing row, never as a new label. +import { FAILURE_CLASSES, OUTCOMES } from './model-metrics.js'; + +const LABEL_WIDTH = 22; + +function row(label, value) { + return ` ${String(label).padEnd(LABEL_WIDTH)}${value}`; +} + +function duration(value) { + if (!Number.isFinite(value) || value < 0) return 'n/a'; + if (value < 1_000) return `${Math.round(value)}ms`; + const seconds = value / 1_000; + if (seconds < 60) return `${seconds.toFixed(1)}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m ${Math.round(seconds % 60)}s`; + return `${Math.floor(minutes / 60)}h ${minutes % 60}m`; +} + +function count(value) { + return Number.isFinite(value) ? String(value) : '0'; +} + +function readinessLine(model) { + const readiness = model?.readiness ?? {}; + const phase = typeof readiness.phase === 'string' && readiness.phase ? readiness.phase : 'checking'; + return readiness.coldLoadMs === null || readiness.coldLoadMs === undefined + ? phase + : `${phase} · cold load ${duration(readiness.coldLoadMs)}`; +} + +function abstentionLine(model) { + const rate = model?.abstentionRate; + const answered = (model?.outcomes?.suggestion ?? 0) + (model?.outcomes?.abstained ?? 0); + if (rate === null || rate === undefined || answered === 0) return 'no answers yet'; + return `${(rate * 100).toFixed(1)}% of ${answered} answer${answered === 1 ? '' : 's'}`; +} + +export function formatOperatorReport(snapshot) { + if (!snapshot || typeof snapshot !== 'object') throw new Error('a metrics snapshot is required'); + const app = snapshot.app ?? {}; + const queue = snapshot.queue ?? {}; + const model = snapshot.model ?? {}; + const latency = model.latency ?? {}; + const outcomes = OUTCOMES.map(name => `${name} ${count(model.outcomes?.[name])}`).join(' · '); + + const lines = [ + `Timmy model health · ${snapshot.generatedAt ?? 'unknown time'} · schema ${snapshot.schema ?? '?'}`, + '', + 'APP', + row('uptime', duration(app.uptimeMs)), + row('analyze requests', count(app.analyzeRequests)), + row('refused before model', count(app.refusedBeforeModel)), + '', + 'QUEUE', + row('in flight', `${count(queue.inFlight)} / ${count(queue.limit)}`), + row('accepted', count(queue.accepted)), + row('at capacity', queue.atCapacity === true ? 'yes' : 'no'), + '', + 'MODEL', + row('readiness', readinessLine(model)), + row('outcomes', outcomes), + row('abstention', abstentionLine(model)), + row('latency', `mean ${duration(latency.meanMs)} · max ${duration(latency.maxMs)} · samples ${count(latency.count)}`), + ...FAILURE_CLASSES.map(name => row(`failure ${name}`, count(model.failureClasses?.[name]))), + ]; + + return `${lines.join('\n')}\n`; +} diff --git a/tests/model-metrics-report-format.test.js b/tests/model-metrics-report-format.test.js new file mode 100644 index 0000000..c1fb3f9 --- /dev/null +++ b/tests/model-metrics-report-format.test.js @@ -0,0 +1,82 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { formatOperatorReport } from '../src/model-metrics-report.js'; +import { createModelMetrics } from '../src/model-metrics.js'; + +function snapshot(overrides = {}) { + const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') }); + metrics.recordReadiness({ phase: 'ready', ready: true }); + return { ...metrics.snapshot(), ...overrides }; +} + +test('the report names the app, queue, and model boundaries in that order', () => { + const report = formatOperatorReport(snapshot()); + + assert.match(report, /^APP$/m); + assert.match(report, /^QUEUE$/m); + assert.match(report, /^MODEL$/m); + assert.ok(report.indexOf('APP') < report.indexOf('QUEUE'), 'app boundary comes before queue'); + assert.ok(report.indexOf('QUEUE') < report.indexOf('MODEL'), 'queue boundary comes before model'); +}); + +test('the report prints the readiness phase the worker actually reported', () => { + const report = formatOperatorReport(snapshot({ + model: { ...snapshot().model, readiness: { phase: 'loading', ready: false, coldLoadMs: null } }, + })); + + assert.match(report, /readiness\s+loading/); + // A loading worker is not ready, and the report must not imply AI is available. + assert.doesNotMatch(report, /readiness\s+ready/); +}); + +test('an abstention rate with no answers is absence, never zero percent', () => { + const report = formatOperatorReport(snapshot()); + + assert.match(report, /abstention\s+no answers yet/i); + assert.doesNotMatch(report, /abstention\s+0(\.0)?%/); +}); + +test('an abstention rate is rendered from answers only, so failures cannot dilute it', () => { + const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') }); + metrics.recordReadiness({ phase: 'ready', ready: true }); + metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 }); + metrics.recordAnalyze({ outcome: 'abstained', durationMs: 1200 }); + metrics.recordAnalyze({ outcome: 'failure', failureClass: 'timeout', durationMs: 30_000 }); + + const report = formatOperatorReport(metrics.snapshot()); + + assert.match(report, /abstention\s+50\.0%/); + assert.match(report, /failure\s+timeout\s+1|timeout 1/i); +}); + +test('the report renders only the enumerated failure vocabulary, never a caller-supplied label', () => { + const base = snapshot(); + const report = formatOperatorReport({ + ...base, + model: { + ...base.model, + failureClasses: { + ...base.model.failureClasses, + 'note: watery stool at 06:40': 3, + 'user@example.com': 1, + }, + }, + }); + + assert.doesNotMatch(report, /watery stool/); + assert.doesNotMatch(report, /user@example\.com/); + // The closed set is still fully accounted for, so the extra keys are visibly absent rather than + // silently folded into a bucket the operator would read as something else. + assert.match(report, /unexpected\s+0/); +}); + +test('the report shows queue occupancy against the configured ceiling', () => { + const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z'), limit: 3 }); + metrics.enter(); + metrics.enter(); + + const report = formatOperatorReport(metrics.snapshot()); + + assert.match(report, /in flight\s+2 \/ 3/); + assert.match(report, /at capacity\s+no/); +}); diff --git a/tests/model-metrics-report.acceptance.test.js b/tests/model-metrics-report.acceptance.test.js index b0940bc..a081a51 100644 --- a/tests/model-metrics-report.acceptance.test.js +++ b/tests/model-metrics-report.acceptance.test.js @@ -87,3 +87,54 @@ test('the operator report renders the app, queue, and model boundaries from a li await provider.close(); } }); + +async function startServer(env) { + const port = await freePort(); + const child = spawn(process.execPath, ['server.mjs'], { + cwd: repoRoot, + env: { ...process.env, HOST: '127.0.0.1', PORT: String(port), ...env }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const base = `http://127.0.0.1:${port}`; + const deadline = Date.now() + 15_000; + for (;;) { + try { + const response = await fetch(`${base}/api/healthz`); + if (response.ok) break; + } catch { /* not up yet */ } + if (Date.now() > deadline) throw new Error('server did not start'); + await new Promise(resolve => setTimeout(resolve, 100)); + } + return { base, stop: () => child.kill('SIGKILL') }; +} + +test('the report refuses a missing or wrong operator token without echoing it', async () => { + const tokenless = await startServer({ TIMMY_METRICS_ENABLED: undefined, TIMMY_METRICS_TOKEN: '' }); + try { + await assert.rejects( + execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', '--url', tokenless.base, '--token', 'x'], { cwd: repoRoot }), + error => { + assert.notEqual(error.code, 0); + assert.match(error.stderr, /disabled/i, 'a server with no metrics route must say so, not "unauthorized"'); + return true; + }, + ); + } finally { + tokenless.stop(); + } + + const guarded = await startServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN }); + try { + await assert.rejects( + execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', '--url', guarded.base, '--token', 'not-the-token'], { cwd: repoRoot }), + error => { + assert.notEqual(error.code, 0); + assert.match(error.stderr, /rejected/i); + assert.doesNotMatch(`${error.stderr}${error.stdout}`, /not-the-token/, 'the rejected token must never be echoed'); + return true; + }, + ); + } finally { + guarded.stop(); + } +});