timmy-talking-turd/src/model-metrics.js

// Privacy-safe model health metrics for issue #19.
//
// The point of this module is the boundary it does not cross: a metric sample is assembled from
// enumerated outcomes and numbers, never from a request. No note text, no photo hash, no image
// bytes, and no caller identity can reach a label, because the only strings this module stores are
// members of the frozen lists below. Anything else is coerced to a bucket instead of copied.

export const OUTCOMES = Object.freeze(['suggestion', 'abstained', 'failure']);

// Failure classes are the operator's vocabulary for "what went wrong". They are chosen here, not
// observed from an error, so an upstream message can never become a metric label.
export const FAILURE_CLASSES = Object.freeze([
  'ingress_refused',
  'rate_limited',
  'disabled',
  'provider_unreachable',
  'provider_rejected',
  'schema_invalid',
  'timeout',
  'unexpected',
]);

// Readiness is the same vocabulary the browser already uses for the AI affordance, so a metric and
// the screen can never disagree about why AI is withheld.
export const READINESS_PHASES = Object.freeze(['checking', 'disabled', 'offline', 'loading', 'ready']);

// Failure classes that are decided before a request can occupy a queue slot or wait on the model.
const REFUSED_BEFORE_MODEL = Object.freeze(['ingress_refused', 'rate_limited', 'disabled']);

export function createModelMetrics({ now = () => Date.now(), limit = 4 } = {}) {
  const outcomes = { suggestion: 0, abstained: 0, failure: 0 };
  const failureClasses = Object.fromEntries(FAILURE_CLASSES.map(name => [name, 0]));
  // The queue boundary is tracked by opaque tickets, not by request identity: the only thing that
  // survives a request here is the count of slots in use.
  const inUse = new Set();
  let accepted = 0;
  const startedAt = now();
  let readiness = { phase: 'checking', ready: false, coldLoadMs: null };
  let analyzeRequests = 0;
  let refusedBeforeModel = 0;
  // Latency is a histogram, never a list of individual timings: a bucket count says how the worker
  // behaves without carrying a per-request record of when someone photographed their stool.
  const bucketEdgesMs = Object.freeze([1_000, 2_000, 5_000, 15_000, 60_000]);
  const bucketsMs = Object.fromEntries(bucketEdgesMs.map(edge => [String(edge), 0]));
  let latencyCount = 0;
  let latencySumMs = 0;
  let latencyMaxMs = 0;

  function abstentionRate() {
    const answered = outcomes.suggestion + outcomes.abstained;
    return answered === 0 ? null : outcomes.abstained / answered;
  }

  function recordLatency(durationMs) {
    if (!Number.isFinite(durationMs) || durationMs < 0) return;
    latencyCount += 1;
    latencySumMs += durationMs;
    latencyMaxMs = Math.max(latencyMaxMs, durationMs);
    // Edges are cumulative on purpose: `bucketsMs['5000']` answers "how many samples were at or
    // below five seconds", so an operator can read a percentile without this module storing the
    // individual timings that produced it.
    for (const edge of bucketEdgesMs) {
      if (durationMs <= edge) bucketsMs[String(edge)] += 1;
    }
    // Anything slower than the last edge stays counted in the total but out of the histogram, so a
    // hanging request cannot invent a bucket.
  }

  return {
    enter() {
      const ticket = Symbol('model-slot');
      inUse.add(ticket);
      accepted += 1;
      return ticket;
    },
    leave(ticket) {
      // Deleting a ticket that is already gone is a no-op, so a double release cannot free a slot
      // that a different request still holds.
      inUse.delete(ticket);
    },
    recordReadiness({ phase, ready }) {
      const next = READINESS_PHASES.includes(phase) ? phase : 'checking';
      // A worker can flap between ready and offline. Only the first time the model became resident
      // is a cold load; every later observation is just the current phase.
      const coldLoadMs = readiness.coldLoadMs === null && next === 'ready' && ready === true
        ? Math.max(0, now() - startedAt)
        : readiness.coldLoadMs;
      readiness = { phase: next, ready: ready === true, coldLoadMs };
    },
    recordAnalyze({ outcome, failureClass, durationMs }) {
      // The label is looked up, never stored: an unrecognised outcome becomes `failure` so the
      // counters stay a closed set no matter what a caller sends.
      const bucket = OUTCOMES.includes(outcome) ? outcome : 'failure';
      outcomes[bucket] += 1;
      analyzeRequests += 1;
      if (bucket === 'failure') {
        failureClasses[FAILURE_CLASSES.includes(failureClass) ? failureClass : 'unexpected'] += 1;
        // A refusal decided before any model work (bad bytes, the per-device budget, or the disabled
        // switch) is an app fact: it never occupied a queue slot and never waited on the model.
        if (REFUSED_BEFORE_MODEL.includes(failureClass)) {
          refusedBeforeModel += 1;
        } else {
          recordLatency(durationMs);
        }
      } else {
        recordLatency(durationMs);
      }
    },
    snapshot() {
      return {
        schema: 1,
        generatedAt: new Date(now()).toISOString(),
        app: {
          uptimeMs: Math.max(0, now() - startedAt),
          analyzeRequests,
          refusedBeforeModel,
        },
        queue: {
          inFlight: inUse.size,
          limit,
          atCapacity: inUse.size >= limit,
          accepted,
        },
        model: {
          outcomes: { ...outcomes },
          failureClasses: { ...failureClasses },
          abstentionRate: abstentionRate(),
          readiness: { ...readiness },
          latency: {
            count: latencyCount,
            meanMs: latencyCount === 0 ? null : Math.round(latencySumMs / latencyCount),
            maxMs: latencyMaxMs,
            bucketsMs: { ...bucketsMs },
          },
        },
      };
    },
  };
}