feat: pin the self-hosted worker container and gate readiness on the loaded model (#15)

A worker that answers /v1/models has not necessarily loaded its weights, so the app
was offering an AI control that could only fail after the upload. Readiness now
passes only once the pinned model is loaded, and the not-loaded state reaches manual
Bristol entry in one tap. Verified against the real pinned llama.cpp build: it serves
503 while loading and the gate reports the distinct not-loaded state.

Adds the reproducible container recipe (digest-pinned bases, pinned llama.cpp commit,
hash-verified read-only weights, unprivileged, loopback-only publish), one shared
post-load readiness gate used by the host bootstrap and the image healthcheck, and a
target-GPU benchmark that refuses to write a receipt it did not measure.

Refs #15

scripts/bootstrap_selfhost_smolvlm.sh

@@ -137,6 +137,10 @@ case "${1:-}" in
   start)
     start_worker
     ;;
+  ready)
+    # Post-load readiness (issue #15): delegate so the host and the container gate cannot drift.
+    exec "$(dirname "$0")/worker_ready.sh" --host "$HOST" --port "$PORT" --alias SmolVLM2-2.2B-Instruct "${@:2}"
+    ;;
   health)
     health_worker
     ;;
@@ -144,7 +148,7 @@ case "${1:-}" in
     stop_worker
     ;;
   *)
-    printf 'Usage: %s {install|start|health|stop|receipt|verify}\n' "$0" >&2
+    printf 'Usage: %s {install|start|health|ready|stop|receipt|verify}\n' "$0" >&2
     exit 64
     ;;
 esac

scripts/build_release.py

@@ -39,6 +39,7 @@ DEMO_REQUIRED_CAPTIONS = (
     "Urgent language is intercepted deterministically before Hermes",
     "An out-of-range answer fails closed: nothing is applied and no provider detail reaches you",
     "A filled-in field that is out of range still fails closed, even when the other field came back blank",
+    "Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident",
 )
 
 
@@ -136,6 +137,7 @@ def main() -> int:
             run(["npm", "run", "test:provisional"], tree)
             run(["npm", "run", "test:followup"], tree)
             run(["npm", "run", "test:vision-refusal"], tree)
+            run(["npm", "run", "test:worker-readiness"], tree)
             demo_raw = release_dir / f"timmy-talking-turd-{version}-demo.raw.webm"
             caption_receipt = release_dir / f"timmy-talking-turd-{version}-demo-captions.json"
             demo_env = dict(server_env)
@@ -177,10 +179,14 @@ def main() -> int:
         if not contact_sheet.exists() or contact_sheet.stat().st_size == 0:
             raise SystemExit("Release demo contact sheet was not produced")
         run(["npm", "audit", "--audit-level=high"], tree)
-        for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs"):
+        for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "scripts/record_worker_benchmark.mjs", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs", "tests/worker-readiness.acceptance.mjs"):
             run(["node", "--check", file], tree)
         run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree)
         run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree)
+        run(["bash", "-n", "scripts/worker_ready.sh"], tree)
+        run(["bash", "-n", "deploy/worker/entrypoint.sh"], tree)
+        run(["bash", "-n", "deploy/worker/build_image.sh"], tree)
+        run(["bash", "-n", "deploy/worker/run_worker_container.sh"], tree)
         run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py", "scripts/build_vision_corpus.py"], tree)
         run(["git", "diff", "--check", commit], tree)
 
@@ -209,6 +215,9 @@ def main() -> int:
             "- AI suggestions stay provisional: suggested, confirmed, and corrected states are distinct and screen-reader described; correcting one clears the AI-confirmed state without losing user context\n"
             "- Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions (difficulty passing, soreness, visible blood, truly black/tarry versus dark brown, hydration); mild reports get non-diagnostic self-care, and a confirmed red-flag answer lands on the same deterministic escalation path as manual logging\n"
             "- Vision corpus: a committed CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) proves every reply is schema-valid or fails closed; a schema-valid reply with unreadable fields abstains instead of erroring, and unparsable or out-of-range replies return plain copy with nothing applied — including a reply that fills in one field and leaves the other blank, where an out-of-range value still fails closed instead of being reported as an unreadable photo (issue #73)\n"
+            "- Self-hosted worker container: a digest-pinned image that builds the pinned llama.cpp revision, mounts hash-verified weights read-only, runs unprivileged with no credential, and is published on loopback only (issue #15)\n"
+            "- Post-load readiness: the app offers AI only once the pinned model is actually loaded, so a reachable-but-unloaded worker falls back to manual logging in one tap instead of failing after the upload (issue #15)\n"
+            "- Target-GPU benchmark: scripts/record_worker_benchmark.mjs records cold start, warm latency, and VRAM on the target GPU and refuses to write a receipt it did not measure; no GPU measurement is bundled with this release\n"
             "- Safety: suggestions are observable-field assistance, never diagnosis; user confirmation is required\n"
             "- Known limitation: open-weight VLM acceptance is proven, Bristol accuracy is not clinically validated\n"
             "- Package excludes model weights, medical-image corpora, raw research pages, generated screenshots, and video production assets\n"
@@ -265,6 +274,8 @@ def main() -> int:
             "provisional_suggestion_state": "passed",
             "hard_stool_followup": "passed",
             "vision_corpus_refusal": "passed",
+            "worker_container_receipts": "passed",
+            "worker_post_load_readiness": "passed",
             "dependency_audit_high": "passed",
             "syntax": "passed",
             "secret_scan": "passed",

scripts/record_release_demo.mjs

@@ -49,6 +49,9 @@ const errors = [];
 page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); });
 page.on('pageerror', error => errors.push(error.message));
 let visionOffline = false;
+// Reachable is not loaded: the worker can answer /v1/models before its pinned model is resident,
+// and the recording shows the app withholding AI in exactly that state (issue #15).
+let modelLoaded = true;
 await page.route('**/api/vision-status', route => route.fulfill({
   status: 200,
   contentType: 'application/json',
@@ -59,7 +62,7 @@ await page.route('**/api/vision-status', route => route.fulfill({
     model: 'SmolVLM2-2.2B-Instruct',
     // The second pass flips the app's own readiness contract so no AI control is offered at all.
     providerReady: !visionOffline,
-    modelSeen: !visionOffline,
+    modelSeen: !visionOffline && modelLoaded,
   }),
 }));
 // The happy path keeps one deterministic suggestion. The refusal scene swaps this for a real
@@ -231,6 +234,38 @@ await tap('#discard-draft', 500);
 await caption('Start over clears the kept draft — nothing lingers silently', 1500);
 await tap('#close-sheet', 450);
 
+// Post-load readiness pass (issue #15): the worker answers, but its pinned model is not resident
+// yet. The app must withhold AI instead of offering a control that fails after the upload. Every
+// claim below is asserted in the browser, so the recording cannot show a state the app does not have.
+modelLoaded = false;
+visionOffline = false;
+await caption('Now the same worker while its pinned model is still loading', 1700);
+await tap('[data-scan]', 550);
+await page.getByText(/model is not loaded yet/i).first().waitFor();
+if (await page.getByText(/Self-hosted model ready/i).count() !== 0) {
+  throw new Error('A reachable worker without its loaded model must never be reported ready');
+}
+const loadingAmber = await page.locator('.model-status.loading').first().evaluate(el => getComputedStyle(el).backgroundColor);
+if (loadingAmber !== 'rgb(253, 243, 216)') throw new Error('The not-loaded state must be visually distinct from ready and offline');
+await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
+await page.getByText(/Ready for your review/i).waitFor();
+if (await page.locator('#analyze-photo').count() !== 0) throw new Error('No AI control may exist before the model is loaded');
+if (await page.locator('.consent-card').count() !== 0) throw new Error('No consent may be requested before the model is loaded');
+await caption('Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident', 2900);
+await tap('#manual-from-scan', 500);
+await page.getByText('Pick the closest form').waitFor();
+await caption('One tap still reaches manual Bristol entry while the model finishes loading', 2000);
+await tap('#close-sheet', 450);
+modelLoaded = true;
+await caption('Once the pinned model is loaded, the same screen offers the AI analysis again', 2200);
+await tap('[data-scan]', 550);
+await page.getByText(/Self-hosted model ready/i).waitFor();
+await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
+await page.getByText(/Ready for your review/i).waitFor();
+if (await page.locator('.consent-card').isVisible() !== true) throw new Error('The AI path must return once the model is loaded');
+if (await page.locator('#analyze-photo').count() !== 1) throw new Error('The analyze control must return once the model is loaded');
+await tap('#close-sheet', 450);
+
 // Corpus refusal pass (issue #18): the same photo flow, fed the answers Timmy has to refuse.
 visionOffline = false;
 await caption('The corpus also replays the answers Timmy has to refuse', 1700);

scripts/record_worker_benchmark.mjs

@@ -0,0 +1,161 @@
+#!/usr/bin/env node
+// Record the target-GPU cost of the pinned worker (issue #15).
+//
+// This script measures; it never invents. If no GPU can be probed, if the pinned model is not
+// loaded, or if the cold start cannot be observed, it exits non-zero and writes no receipt at all,
+// because a receipt with zeroed or assumed numbers is worse than no receipt.
+import { spawn } from 'node:child_process';
+import { readFile, writeFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+
+const LOOPBACK = /^(127\.0\.0\.1|localhost|::1)$/;
+
+function parseArgs(argv) {
+  const options = {
+    out: '',
+    host: '127.0.0.1',
+    port: '8080',
+    alias: 'SmolVLM2-2.2B-Instruct',
+    gpuProbe: 'nvidia-smi',
+    warmSamples: 20,
+    startCommand: '',
+    readyTimeoutSeconds: 900,
+  };
+  for (let index = 0; index < argv.length; index += 1) {
+    const [flag, inline] = argv[index].split('=');
+    const value = inline ?? argv[index + 1];
+    const consume = inline === undefined ? 2 : 1;
+    if (flag === '--out') options.out = value;
+    else if (flag === '--host') options.host = value;
+    else if (flag === '--port') options.port = value;
+    else if (flag === '--alias') options.alias = value;
+    else if (flag === '--gpu-probe') options.gpuProbe = value;
+    else if (flag === '--warm-samples') options.warmSamples = Number(value);
+    else if (flag === '--start-command') options.startCommand = value;
+    else if (flag === '--ready-timeout') options.readyTimeoutSeconds = Number(value);
+    else fail(`Unknown argument: ${flag}`);
+    index += consume - 1;
+  }
+  return options;
+}
+
+function fail(message, code = 64) {
+  process.stderr.write(`${message}\n`);
+  process.exit(code);
+}
+
+const reject = message => {
+  process.stderr.write(`${message}\n`);
+  process.exit(1);
+};
+
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+async function modelsServed({ host, port, alias }) {
+  try {
+    const response = await fetch(`http://${host}:${port}/v1/models`, { signal: AbortSignal.timeout(3000) });
+    if (!response.ok) return false;
+    const body = await response.json();
+    const ids = Array.isArray(body?.data) ? body.data.map(item => item?.id).filter(Boolean) : [];
+    return ids.some(id => id === alias || id.includes(alias));
+  } catch {
+    return false;
+  }
+}
+
+function runCommand(command, args) {
+  return new Promise(resolve => {
+    const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', chunk => { stdout += chunk; });
+    child.stderr.on('data', chunk => { stderr += chunk; });
+    child.on('error', error => resolve({ code: 127, stdout, stderr: String(error.message) }));
+    child.on('close', code => resolve({ code, stdout, stderr }));
+  });
+}
+
+async function probeGpu(probeCommand) {
+  const result = await runCommand(probeCommand, [
+    '--query-gpu=name,memory.total,memory.used',
+    '--format=csv,noheader,nounits',
+  ]);
+  if (result.code !== 0) {
+    reject(`No GPU could be probed with ${probeCommand}: ${result.stderr.trim() || `exit ${result.code}`}. A benchmark receipt needs a real device, so nothing was written.`);
+  }
+  const [name, total, used] = result.stdout.trim().split('\n')[0].split(',').map(part => part.trim());
+  if (!name || !/^\d+$/.test(total) || !/^\d+$/.test(used)) {
+    reject(`The GPU probe returned unreadable output (${JSON.stringify(result.stdout.trim())}); refusing to record VRAM it cannot read.`);
+  }
+  return { name, vramTotalMiB: Number(total), vramUsedMiB: Number(used) };
+}
+
+function percentile(sorted, fraction) {
+  const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(fraction * sorted.length) - 1));
+  return sorted[index];
+}
+
+const options = parseArgs(process.argv.slice(2));
+if (!options.out) fail('--out is required so the receipt has a destination');
+if (!LOOPBACK.test(options.host)) fail(`The worker is a loopback service; refusing to benchmark ${options.host}.`);
+if (!Number.isInteger(options.warmSamples) || options.warmSamples < 1) fail('--warm-samples must be a positive integer');
+
+const receipts = JSON.parse(await readFile(fileURLToPath(new URL('../deploy/worker/receipts.json', import.meta.url)), 'utf8'));
+
+// The device is probed first: nothing is timed until we know the numbers have somewhere to be real.
+const gpu = await probeGpu(options.gpuProbe);
+
+if (!(await modelsServed(options))) {
+  reject(`${options.alias} is not loaded on ${options.host}:${options.port} yet, so there is no ready worker to time. Start the worker and wait for the post-load gate.`);
+}
+
+let coldStartMs = null;
+if (options.startCommand) {
+  const started = Date.now();
+  const started_process = await runCommand('bash', ['-c', options.startCommand]);
+  if (started_process.code !== 0) {
+    reject(`The cold-start command failed (exit ${started_process.code}): ${started_process.stderr.trim()}`);
+  }
+  const deadline = Date.now() + options.readyTimeoutSeconds * 1000;
+  let ready = false;
+  while (Date.now() < deadline) {
+    if (await modelsServed(options)) { ready = true; break; }
+    await sleep(250);
+  }
+  if (!ready) reject(`The worker did not become ready within ${options.readyTimeoutSeconds}s, so the cold start could not be measured.`);
+  coldStartMs = Date.now() - started;
+}
+
+const samples = [];
+for (let index = 0; index < options.warmSamples; index += 1) {
+  const started = process.hrtime.bigint();
+  const ok = await modelsServed(options);
+  const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6;
+  if (!ok) reject('The worker stopped answering during the warm pass; the receipt would be partial, so nothing was written.');
+  samples.push(Math.round(elapsedMs * 1000) / 1000);
+}
+const sorted = [...samples].sort((left, right) => left - right);
+
+const receipt = {
+  schemaVersion: 1,
+  status: 'measured_on_target_gpu',
+  measuredAt: new Date().toISOString(),
+  gpu,
+  coldStartMs,
+  warmLatencyMs: {
+    samples: sorted.length,
+    min: sorted[0],
+    p50: percentile(sorted, 0.5),
+    p95: percentile(sorted, 0.95),
+    max: sorted[sorted.length - 1],
+  },
+  worker: { baseUrl: `http://${options.host}:${options.port}/v1`, alias: options.alias },
+  pins: {
+    llamaCppCommit: receipts.llamaCpp.commit,
+    modelSha256: receipts.models.map(model => model.sha256),
+  },
+};
+
+await writeFile(path.resolve(options.out), `${JSON.stringify(receipt, null, 2)}\n`, 'utf8');
+process.stdout.write(`${path.resolve(options.out)}\n`);

scripts/worker_ready.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Post-load readiness gate for Timmy's self-hosted vision worker (issue #15).
+#
+# A worker that answers on its port is not necessarily a worker that can answer: llama.cpp opens
+# its HTTP listener before the model finishes loading and returns an error until the weights are
+# resident. Readiness therefore passes only when the pinned model alias is actually served.
+#
+# Exit codes:
+#   0  ready — the pinned model is loaded and being served
+#   2  the worker is not reachable
+#   3  the worker is reachable but the pinned model is not loaded yet
+#  64  usage error (including a non-loopback probe host)
+set -uo pipefail
+
+HOST="${TIMMY_MODEL_HOST:-127.0.0.1}"
+PORT="${TIMMY_MODEL_PORT:-8080}"
+ALIAS="${TIMMY_MODEL_ALIAS:-SmolVLM2-2.2B-Instruct}"
+WAIT=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --host) HOST="${2:-}"; shift 2 ;;
+    --port) PORT="${2:-}"; shift 2 ;;
+    --alias) ALIAS="${2:-}"; shift 2 ;;
+    --wait) WAIT="${2:-}"; shift 2 ;;
+    *)
+      printf 'Usage: %s [--host H] [--port P] [--alias A] [--wait SECONDS]\n' "$0" >&2
+      exit 64
+      ;;
+  esac
+done
+
+# The worker is a private, loopback-only service. Probing anything else would be a mistake about
+# exposure, so refuse rather than answer about a host this gate is not responsible for.
+if [[ ! "$HOST" =~ ^(127\.0\.0\.1|localhost|::1)$ ]]; then
+  printf 'Readiness must be probed on loopback, got %s\n' "$HOST" >&2
+  exit 64
+fi
+
+deadline=$(( $(date +%s) + WAIT ))
+outcome='unreachable'
+while :; do
+  body="$(curl --silent --max-time 3 "http://$HOST:$PORT/v1/models" 2>/dev/null)"
+  if [[ $? -eq 0 && -n "$body" ]]; then
+    if printf '%s' "$body" | grep -q -- "$ALIAS"; then
+      printf 'Worker ready: %s is loaded on http://%s:%s/v1\n' "$ALIAS" "$HOST" "$PORT"
+      exit 0
+    fi
+    outcome='not-loaded'
+    printf 'Worker reachable on http://%s:%s/v1 but %s is not loaded yet.\n' "$HOST" "$PORT" "$ALIAS" >&2
+  else
+    outcome='unreachable'
+    printf 'Worker not reachable on http://%s:%s/v1.\n' "$HOST" "$PORT" >&2
+  fi
+  [[ "$(date +%s)" -ge "$deadline" ]] && break
+  sleep 1
+done
+
+[[ "$outcome" == 'not-loaded' ]] && exit 3
+exit 2