feat(metrics): render the operator health snapshot as a report (#19)

The counters had an endpoint and no way to read them without jq. Adds a pure
formatter plus a CLI that prints the app, queue, and model boundaries.

The formatter is a function of the snapshot and the three frozen vocabularies,
so a caller-supplied failure label cannot reach the terminal: unknown classes
are simply not printed. Refusal paths are explicit — 404 means the route is off
because no token is configured, 403 means the token was rejected — and the token
is never echoed in the report or an error line.

RED: tests/model-metrics-report-format.test.js failed with ERR_MODULE_NOT_FOUND
before src/model-metrics-report.js existed; the refusal-path test failed with a
generic 'answered 404' before the 404 branch was written.

Tests: npm test 298/298, npm run test:metrics-report 8/8, syntax and diff gates
clean, real-Chromium ui acceptance green with the privacy disclosure visible.

docs/MODEL-METRICS.md

@@ -80,6 +80,18 @@ by feeding hostile strings into every label position, and
 curl -sS -H "Authorization: Bearer $TIMMY_METRICS_TOKEN" http://127.0.0.1:4173/api/metrics | jq .
 ```
 
+For a quick look that does not need `jq`, the report renders the same snapshot as three labelled blocks:
+
+```bash
+TIMMY_METRICS_TOKEN=... node scripts/model_metrics_report.mjs --url http://127.0.0.1:4173
+```
+
+Exit codes are the point of the command as much as the numbers: `0` means a snapshot was rendered,
+`1` means the server answered non-200 (404 says the route is disabled because no token is configured,
+403 says the supplied token was rejected), `2` means the command itself was called wrongly or the
+server could not be reached. The token is read from the environment or `--token` and is never echoed
+in the report or in an error line.
+
 What the numbers mean to an operator:
 
 | Signal | Reading |

package.json

@@ -4,7 +4,7 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js tests/training-contribution.test.js tests/portability.test.js tests/abstention-policy.test.js tests/release-security-gate.test.js tests/model-metrics.test.js tests/metrics-endpoint.acceptance.test.js tests/image-ingress.test.js tests/rate-limit.test.js tests/http-body.test.js tests/analyze-ingress.acceptance.test.js",
+    "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js tests/training-contribution.test.js tests/portability.test.js tests/abstention-policy.test.js tests/release-security-gate.test.js tests/model-metrics.test.js tests/metrics-endpoint.acceptance.test.js tests/image-ingress.test.js tests/rate-limit.test.js tests/http-body.test.js tests/analyze-ingress.acceptance.test.js tests/model-metrics-report-format.test.js tests/model-metrics-report.acceptance.test.js",
     "test:ui": "node tests/ui.acceptance.mjs",
     "test:photo": "node tests/photo-first.acceptance.mjs",
     "test:mobile-capture": "node tests/mobile-capture.acceptance.mjs",
@@ -19,7 +19,7 @@
     "test:grounded-chat": "node tests/grounded-chat.acceptance.mjs",
     "test:photo-lifecycle": "node tests/photo-lifecycle.acceptance.mjs",
     "test:training-contribution": "node tests/training-contribution.acceptance.mjs",
-    "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check src/training-contribution.js && node --check src/abstention-policy.js && node --check src/release-security-gate.js && node --check scripts/security_attack_suite.mjs && node --check tests/security-attack.acceptance.mjs && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/corpus_manifest.py scripts/review_corpus.py scripts/dataset_card.py scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py scripts/calibration_report.py scripts/operator_backup.py && node --check scripts/record_worker_benchmark.mjs && node --check scripts/abstention_policy_receipt.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/training-contribution.acceptance.mjs && node --check tests/portability.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh && node --check src/model-metrics.js && node --check src/image-ingress.js && node --check src/rate-limit.js && node --check src/http-body.js && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py && node --check tests/ingress-capture.acceptance.mjs && node --check tests/analyze-ingress.acceptance.test.js",
+    "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check src/training-contribution.js && node --check src/abstention-policy.js && node --check src/release-security-gate.js && node --check scripts/security_attack_suite.mjs && node --check tests/security-attack.acceptance.mjs && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/corpus_manifest.py scripts/review_corpus.py scripts/dataset_card.py scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py scripts/calibration_report.py scripts/operator_backup.py && node --check scripts/record_worker_benchmark.mjs && node --check scripts/abstention_policy_receipt.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/training-contribution.acceptance.mjs && node --check tests/portability.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh && node --check src/model-metrics.js && node --check src/model-metrics-report.js && node --check scripts/model_metrics_report.mjs && node --check src/image-ingress.js && node --check src/rate-limit.js && node --check src/http-body.js && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py && node --check tests/ingress-capture.acceptance.mjs && node --check tests/analyze-ingress.acceptance.test.js",
     "check:diff": "bash scripts/check_diff.sh",
     "start": "node server.mjs",
     "test:worker-readiness": "node tests/worker-readiness.acceptance.mjs",
@@ -31,7 +31,8 @@
     "test:dataset-card": "python3 tests/dataset-card.test.py",
     "test:backup": "python3 tests/backup-restore.test.py",
     "test:runbook": "python3 tests/production-runbook.test.py",
-    "test:security-attack": "node tests/security-attack.acceptance.mjs"
+    "test:security-attack": "node tests/security-attack.acceptance.mjs",
+    "test:metrics-report": "node --test tests/model-metrics-report-format.test.js tests/model-metrics-report.acceptance.test.js"
   },
   "devDependencies": {
     "playwright": "^1.62.1"

scripts/model_metrics_report.mjs

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+// Operator report for the privacy-safe model counters (issue #19).
+//
+// It reads one snapshot from a running Timmy server and prints it. The token is a credential for the
+// operator surface, so it is read from the environment or argv and never echoed — not in errors, not
+// in the report, not in a stack trace.
+import { formatOperatorReport } from '../src/model-metrics-report.js';
+
+const USAGE = `Usage: node scripts/model_metrics_report.mjs [options]
+
+  --url <origin>        base URL of the Timmy server (default http://127.0.0.1:4173)
+  --token <token>       operator token (default: $TIMMY_METRICS_TOKEN)
+  --json                print the raw snapshot instead of the text report
+  --timeout-ms <ms>     request deadline (default 5000)
+  --help                show this text
+`;
+
+function parseArgs(argv) {
+  const options = {
+    url: process.env.TIMMY_METRICS_URL || 'http://127.0.0.1:4173',
+    token: process.env.TIMMY_METRICS_TOKEN || '',
+    json: false,
+    timeoutMs: 5_000,
+    help: false,
+  };
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (arg === '--help' || arg === '-h') {
+      options.help = true;
+    } else if (arg === '--json') {
+      options.json = true;
+    } else if (arg === '--url') {
+      options.url = argv[++index] ?? '';
+    } else if (arg === '--token') {
+      options.token = argv[++index] ?? '';
+    } else if (arg === '--timeout-ms') {
+      options.timeoutMs = Number(argv[++index]);
+    } else {
+      throw new Error(`unknown option: ${arg}`);
+    }
+  }
+  return options;
+}
+
+async function main(argv) {
+  const options = parseArgs(argv);
+  if (options.help) {
+    process.stdout.write(USAGE);
+    return 0;
+  }
+  if (!options.token) {
+    throw new Error('operator token required: pass --token or set TIMMY_METRICS_TOKEN');
+  }
+  const endpoint = new URL('api/metrics', options.url.endsWith('/') ? options.url : `${options.url}/`);
+  let response;
+  try {
+    response = await fetch(endpoint, {
+      headers: { authorization: `Bearer ${options.token}`, accept: 'application/json' },
+      signal: AbortSignal.timeout(Number.isFinite(options.timeoutMs) ? options.timeoutMs : 5_000),
+    });
+  } catch (error) {
+    process.stderr.write(`Could not read ${endpoint.origin}/api/metrics: ${error?.message || 'request failed'}\n`);
+    return 1;
+  }
+  if (!response.ok) {
+    if (response.status === 404) {
+      // The route is absent rather than guarded: that is a deployment fact, not a credential problem.
+      process.stderr.write('Metrics are disabled on this server: the route only exists when TIMMY_METRICS_TOKEN is set.\n');
+      return 1;
+    }
+    if (response.status === 403) {
+      process.stderr.write('The server rejected the operator token.\n');
+      return 1;
+    }
+    process.stderr.write(`The server answered ${response.status} for /api/metrics.\n`);
+    return 1;
+  }
+  const snapshot = await response.json();
+  process.stdout.write(options.json ? `${JSON.stringify(snapshot, null, 2)}\n` : formatOperatorReport(snapshot));
+  return 0;
+}
+
+main(process.argv.slice(2)).then(code => {
+  process.exitCode = code;
+}).catch(error => {
+  process.stderr.write(`${error?.message || 'report failed'}\n`);
+  process.exitCode = 2;
+});

src/model-metrics-report.js

@@ -0,0 +1,74 @@
+// Renders the operator snapshot from `src/model-metrics.js` as a plain-text report.
+//
+// This module is a pure function of the snapshot, on purpose: the report is built from numbers and
+// from the three frozen vocabularies, so there is no path by which request content could reach the
+// terminal even if the counters were compromised. Anything outside the enumerated vocabularies is
+// not printed — an unknown failure class is visible as a missing row, never as a new label.
+import { FAILURE_CLASSES, OUTCOMES } from './model-metrics.js';
+
+const LABEL_WIDTH = 22;
+
+function row(label, value) {
+  return `  ${String(label).padEnd(LABEL_WIDTH)}${value}`;
+}
+
+function duration(value) {
+  if (!Number.isFinite(value) || value < 0) return 'n/a';
+  if (value < 1_000) return `${Math.round(value)}ms`;
+  const seconds = value / 1_000;
+  if (seconds < 60) return `${seconds.toFixed(1)}s`;
+  const minutes = Math.floor(seconds / 60);
+  if (minutes < 60) return `${minutes}m ${Math.round(seconds % 60)}s`;
+  return `${Math.floor(minutes / 60)}h ${minutes % 60}m`;
+}
+
+function count(value) {
+  return Number.isFinite(value) ? String(value) : '0';
+}
+
+function readinessLine(model) {
+  const readiness = model?.readiness ?? {};
+  const phase = typeof readiness.phase === 'string' && readiness.phase ? readiness.phase : 'checking';
+  return readiness.coldLoadMs === null || readiness.coldLoadMs === undefined
+    ? phase
+    : `${phase} · cold load ${duration(readiness.coldLoadMs)}`;
+}
+
+function abstentionLine(model) {
+  const rate = model?.abstentionRate;
+  const answered = (model?.outcomes?.suggestion ?? 0) + (model?.outcomes?.abstained ?? 0);
+  if (rate === null || rate === undefined || answered === 0) return 'no answers yet';
+  return `${(rate * 100).toFixed(1)}% of ${answered} answer${answered === 1 ? '' : 's'}`;
+}
+
+export function formatOperatorReport(snapshot) {
+  if (!snapshot || typeof snapshot !== 'object') throw new Error('a metrics snapshot is required');
+  const app = snapshot.app ?? {};
+  const queue = snapshot.queue ?? {};
+  const model = snapshot.model ?? {};
+  const latency = model.latency ?? {};
+  const outcomes = OUTCOMES.map(name => `${name} ${count(model.outcomes?.[name])}`).join(' · ');
+
+  const lines = [
+    `Timmy model health · ${snapshot.generatedAt ?? 'unknown time'} · schema ${snapshot.schema ?? '?'}`,
+    '',
+    'APP',
+    row('uptime', duration(app.uptimeMs)),
+    row('analyze requests', count(app.analyzeRequests)),
+    row('refused before model', count(app.refusedBeforeModel)),
+    '',
+    'QUEUE',
+    row('in flight', `${count(queue.inFlight)} / ${count(queue.limit)}`),
+    row('accepted', count(queue.accepted)),
+    row('at capacity', queue.atCapacity === true ? 'yes' : 'no'),
+    '',
+    'MODEL',
+    row('readiness', readinessLine(model)),
+    row('outcomes', outcomes),
+    row('abstention', abstentionLine(model)),
+    row('latency', `mean ${duration(latency.meanMs)} · max ${duration(latency.maxMs)} · samples ${count(latency.count)}`),
+    ...FAILURE_CLASSES.map(name => row(`failure ${name}`, count(model.failureClasses?.[name]))),
+  ];
+
+  return `${lines.join('\n')}\n`;
+}

tests/model-metrics-report-format.test.js

@@ -0,0 +1,82 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { formatOperatorReport } from '../src/model-metrics-report.js';
+import { createModelMetrics } from '../src/model-metrics.js';
+
+function snapshot(overrides = {}) {
+  const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') });
+  metrics.recordReadiness({ phase: 'ready', ready: true });
+  return { ...metrics.snapshot(), ...overrides };
+}
+
+test('the report names the app, queue, and model boundaries in that order', () => {
+  const report = formatOperatorReport(snapshot());
+
+  assert.match(report, /^APP$/m);
+  assert.match(report, /^QUEUE$/m);
+  assert.match(report, /^MODEL$/m);
+  assert.ok(report.indexOf('APP') < report.indexOf('QUEUE'), 'app boundary comes before queue');
+  assert.ok(report.indexOf('QUEUE') < report.indexOf('MODEL'), 'queue boundary comes before model');
+});
+
+test('the report prints the readiness phase the worker actually reported', () => {
+  const report = formatOperatorReport(snapshot({
+    model: { ...snapshot().model, readiness: { phase: 'loading', ready: false, coldLoadMs: null } },
+  }));
+
+  assert.match(report, /readiness\s+loading/);
+  // A loading worker is not ready, and the report must not imply AI is available.
+  assert.doesNotMatch(report, /readiness\s+ready/);
+});
+
+test('an abstention rate with no answers is absence, never zero percent', () => {
+  const report = formatOperatorReport(snapshot());
+
+  assert.match(report, /abstention\s+no answers yet/i);
+  assert.doesNotMatch(report, /abstention\s+0(\.0)?%/);
+});
+
+test('an abstention rate is rendered from answers only, so failures cannot dilute it', () => {
+  const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') });
+  metrics.recordReadiness({ phase: 'ready', ready: true });
+  metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 });
+  metrics.recordAnalyze({ outcome: 'abstained', durationMs: 1200 });
+  metrics.recordAnalyze({ outcome: 'failure', failureClass: 'timeout', durationMs: 30_000 });
+
+  const report = formatOperatorReport(metrics.snapshot());
+
+  assert.match(report, /abstention\s+50\.0%/);
+  assert.match(report, /failure\s+timeout\s+1|timeout 1/i);
+});
+
+test('the report renders only the enumerated failure vocabulary, never a caller-supplied label', () => {
+  const base = snapshot();
+  const report = formatOperatorReport({
+    ...base,
+    model: {
+      ...base.model,
+      failureClasses: {
+        ...base.model.failureClasses,
+        'note: watery stool at 06:40': 3,
+        'user@example.com': 1,
+      },
+    },
+  });
+
+  assert.doesNotMatch(report, /watery stool/);
+  assert.doesNotMatch(report, /user@example\.com/);
+  // The closed set is still fully accounted for, so the extra keys are visibly absent rather than
+  // silently folded into a bucket the operator would read as something else.
+  assert.match(report, /unexpected\s+0/);
+});
+
+test('the report shows queue occupancy against the configured ceiling', () => {
+  const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z'), limit: 3 });
+  metrics.enter();
+  metrics.enter();
+
+  const report = formatOperatorReport(metrics.snapshot());
+
+  assert.match(report, /in flight\s+2 \/ 3/);
+  assert.match(report, /at capacity\s+no/);
+});

tests/model-metrics-report.acceptance.test.js

@@ -87,3 +87,54 @@ test('the operator report renders the app, queue, and model boundaries from a li
     await provider.close();
   }
 });
+
+async function startServer(env) {
+  const port = await freePort();
+  const child = spawn(process.execPath, ['server.mjs'], {
+    cwd: repoRoot,
+    env: { ...process.env, HOST: '127.0.0.1', PORT: String(port), ...env },
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+  const base = `http://127.0.0.1:${port}`;
+  const deadline = Date.now() + 15_000;
+  for (;;) {
+    try {
+      const response = await fetch(`${base}/api/healthz`);
+      if (response.ok) break;
+    } catch { /* not up yet */ }
+    if (Date.now() > deadline) throw new Error('server did not start');
+    await new Promise(resolve => setTimeout(resolve, 100));
+  }
+  return { base, stop: () => child.kill('SIGKILL') };
+}
+
+test('the report refuses a missing or wrong operator token without echoing it', async () => {
+  const tokenless = await startServer({ TIMMY_METRICS_ENABLED: undefined, TIMMY_METRICS_TOKEN: '' });
+  try {
+    await assert.rejects(
+      execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', '--url', tokenless.base, '--token', 'x'], { cwd: repoRoot }),
+      error => {
+        assert.notEqual(error.code, 0);
+        assert.match(error.stderr, /disabled/i, 'a server with no metrics route must say so, not "unauthorized"');
+        return true;
+      },
+    );
+  } finally {
+    tokenless.stop();
+  }
+
+  const guarded = await startServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN });
+  try {
+    await assert.rejects(
+      execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', '--url', guarded.base, '--token', 'not-the-token'], { cwd: repoRoot }),
+      error => {
+        assert.notEqual(error.code, 0);
+        assert.match(error.stderr, /rejected/i);
+        assert.doesNotMatch(`${error.stderr}${error.stdout}`, /not-the-token/, 'the rejected token must never be echoed');
+        return true;
+      },
+    );
+  } finally {
+    guarded.stop();
+  }
+});