diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6d8dc2d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +# The worker image is built from source; weights, releases, and private data never enter it. +.git/ +node_modules/ +models/ +*.gguf +artifacts/ +video/ +research/source-pages/ +__pycache__/ +*.pyc +.env +.env.* diff --git a/.gitea/workflows/quality.yml b/.gitea/workflows/quality.yml index 15dfdb5..7b1967f 100644 --- a/.gitea/workflows/quality.yml +++ b/.gitea/workflows/quality.yml @@ -56,6 +56,7 @@ jobs: npm run test:provisional npm run test:followup npm run test:vision-refusal + npm run test:worker-readiness - name: Dependency audit run: npm audit --audit-level=high - name: Syntax checks diff --git a/app.js b/app.js index 0830a20..6e9a12e 100644 --- a/app.js +++ b/app.js @@ -176,6 +176,9 @@ async function loadVisionStatus(){ } function visionStatusHtml(){ if(!visionStatus)return '
◌ Checking the vision worker…
'; + // Reachable is not loaded (#15): a worker answering before its pinned model is loaded must not be + // shown as ready, because the AI controls are withheld until the model can actually answer. + if(visionStatus.providerReady&&visionStatus.modelSeen!==true)return '
◌ Vision worker up · the model is not loaded yet · manual logging is available
'; if(visionStatus.profile==='selfhost'&&visionStatus.providerReady)return `
● Self-hosted model ready · ${esc(visionStatus.model)}
`; if(visionStatus.providerReady)return `
● Vision provider ready · ${esc(visionStatus.model)}
`; return '
○ Vision worker offline · manual logging is still available
'; @@ -192,7 +195,7 @@ function showPhotoFirst(mode='pick',error=''){ photoFirstMode=mode; document.querySelector('.sheet-backdrop')?.remove();const wrap=document.createElement('div');wrap.className='sheet-backdrop';wrap.innerHTML=``;document.body.append(wrap);document.querySelector('#close-sheet').onclick=()=>wrap.remove();wrap.onclick=e=>{if(e.target===wrap)wrap.remove()}; document.querySelectorAll('#camera-photo,#gallery-photo').forEach(file=>{file.onchange=handleAiPhoto;file.addEventListener('cancel',()=>showPhotoFirst('pick','Camera or photo picker closed. If permission was denied, allow camera access in browser settings, choose from the gallery, or continue without AI.'))}); - const consent=document.querySelector('#ai-consent'),analyze=document.querySelector('#analyze-photo');if(consent&&analyze)consent.onchange=()=>analyze.disabled=!consent.checked||visionStatus?.providerReady===false;if(analyze)analyze.onclick=runAiAnalysis; + const consent=document.querySelector('#ai-consent'),analyze=document.querySelector('#analyze-photo');if(consent&&analyze)consent.onchange=()=>analyze.disabled=!consent.checked||!visionReadiness(visionStatus).ready;if(analyze)analyze.onclick=runAiAnalysis; document.querySelector('#retake-photo')?.addEventListener('click',()=>{form.photoDataUrl='';photoHint='';resetSuggestion();showPhotoFirst('pick')}); document.querySelector('#manual-from-scan')?.addEventListener('click',()=>{resetSuggestion();showLogStep(1)}); document.querySelector('#use-suggestion')?.addEventListener('click',()=>{suggestionState=confirmSuggestion(suggestionState);form=mergeVisualSuggestion(form,aiSuggestion);form.typeChosen=true;showLogStep(1)}); diff --git a/artifacts/worker-model-loaded-mobile.png b/artifacts/worker-model-loaded-mobile.png new file mode 100644 index 0000000..3af6520 Binary files /dev/null and b/artifacts/worker-model-loaded-mobile.png differ diff --git a/artifacts/worker-model-loading-mobile.png b/artifacts/worker-model-loading-mobile.png new file mode 100644 index 0000000..5b0e88d Binary files /dev/null and b/artifacts/worker-model-loading-mobile.png differ diff --git a/deploy/worker/Dockerfile b/deploy/worker/Dockerfile new file mode 100644 index 0000000..a328f9b --- /dev/null +++ b/deploy/worker/Dockerfile @@ -0,0 +1,67 @@ +# Timmy self-hosted vision worker (issue #15). +# +# Reproducible by construction: the llama.cpp revision and both model hashes are pinned here and in +# deploy/worker/receipts.json, and tests/worker-container.test.js fails if either drifts from +# scripts/bootstrap_selfhost_smolvlm.sh on the host. +# +# The weights are deliberately NOT baked into the image. They are mounted read-only at /models and +# hash-verified against the pinned receipts by the entrypoint before the server starts. + +ARG LLAMA_CPP_COMMIT=6d05498314db1b57f81c271080018aa2d0b89be9 +ARG MODEL_SHA256=0cf76814555b8665149075b74ab6b5c1d428ea1d3d01c1918c12012e8d7c9f58 +ARG MMPROJ_SHA256=ae07ea1facd07dd3230c4483b63e8cda96c6944ad2481f33d531f79e892dd024 + +# Digest-pinned bases, recorded in deploy/worker/receipts.json. A tag alone can be repointed, so +# the digest is what makes this build reproducible. +FROM nvidia/cuda:12.4.1-devel-ubuntu22.04@sha256:da6791294b0b04d7e65d87b7451d6f2390b4d36225ab0701ee7dfec5769829f5 AS build + +ARG LLAMA_CPP_COMMIT +RUN apt-get update \ + && apt-get install --yes --no-install-recommends git ca-certificates cmake build-essential \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +RUN git init llama.cpp \ + && cd llama.cpp \ + && git remote add origin https://github.com/ggml-org/llama.cpp \ + && git fetch --depth 1 origin "${LLAMA_CPP_COMMIT}" \ + && git checkout --detach FETCH_HEAD +RUN cmake -S llama.cpp -B llama.cpp/build -DCMAKE_BUILD_TYPE=Release -DLLAMA_CURL=OFF -DGGML_CUDA=ON \ + && cmake --build llama.cpp/build --parallel "$(nproc)" --target llama-server + +FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04@sha256:517da2300c184c9999ec203c2665244bdebd3578d12fcc7065e83667932643d9 AS runtime + +ARG MODEL_SHA256 +ARG MMPROJ_SHA256 + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /models + +COPY --from=build /src/llama.cpp/build/bin/ /usr/local/bin/ +COPY deploy/worker/entrypoint.sh /usr/local/bin/timmy-worker-entrypoint +COPY scripts/worker_ready.sh /usr/local/bin/timmy-worker-ready +RUN chmod 0755 /usr/local/bin/timmy-worker-entrypoint /usr/local/bin/timmy-worker-ready && ldconfig + +# No credential is configured: the worker is a private service the app reaches over host loopback. +# 0.0.0.0 here is the address inside this container's own network namespace; the run recipe decides +# exposure and publishes on 127.0.0.1 only. +ENV TIMMY_MODEL_DIR=/models \ + TIMMY_MODEL_FILE=/models/SmolVLM2-2.2B-Instruct-Q4_K_M.gguf \ + TIMMY_MMPROJ_FILE=/models/mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf \ + TIMMY_MODEL_SHA256=${MODEL_SHA256} \ + TIMMY_MMPROJ_SHA256=${MMPROJ_SHA256} \ + TIMMY_MODEL_ALIAS=SmolVLM2-2.2B-Instruct \ + TIMMY_MODEL_HOST=0.0.0.0 \ + TIMMY_MODEL_PORT=8080 + +USER 10001:10001 +EXPOSE 8080 + +# Health is the post-load gate: the worker is not healthy until the pinned model is being served, +# so an orchestrator cannot route to a worker that cannot answer. start-period covers a cold load. +HEALTHCHECK --interval=30s --timeout=5s --start-period=600s --retries=3 \ + CMD ["/usr/local/bin/timmy-worker-ready", "--host", "127.0.0.1", "--port", "8080", "--alias", "SmolVLM2-2.2B-Instruct"] + +ENTRYPOINT ["/usr/local/bin/timmy-worker-entrypoint"] diff --git a/deploy/worker/build_image.sh b/deploy/worker/build_image.sh new file mode 100755 index 0000000..ffaf3b1 --- /dev/null +++ b/deploy/worker/build_image.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Build the pinned Timmy worker image (issue #15). +# +# Every pin is checked before docker is invoked, so an unpinned or floating reference fails here +# rather than producing an image nobody can reproduce. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +RECEIPTS="$REPO_ROOT/deploy/worker/receipts.json" +IMAGE="${TIMMY_WORKER_IMAGE:-timmy-vision-worker}" +TAG="${TIMMY_WORKER_TAG:-$(date -u +%Y%m%d)}" +PUBLISH_HOST="${TIMMY_WORKER_PUBLISH_HOST:-127.0.0.1}" +PORT="${TIMMY_MODEL_PORT:-8080}" + +receipt_field() { + python3 - "$RECEIPTS" "$1" <<'PY' +import json +import sys +document = json.load(open(sys.argv[1])) +value = document +for part in sys.argv[2].split('.'): + value = value[int(part)] if part.isdigit() else value[part] +print(value) +PY +} + +# An explicit value still wins, but an empty one is treated as an invalid pin rather than a default. +LLAMA_CPP_COMMIT="${TIMMY_LLAMA_CPP_COMMIT-$(receipt_field llamaCpp.commit)}" +MODEL_SHA256="${TIMMY_MODEL_SHA256-$(receipt_field models.0.sha256)}" +MMPROJ_SHA256="${TIMMY_MMPROJ_SHA256-$(receipt_field models.1.sha256)}" +MODEL_FILE="$(receipt_field models.0.file)" +MMPROJ_FILE="$(receipt_field models.1.file)" +BUILD_BASE="$(receipt_field baseImages.build.reference)@$(receipt_field baseImages.build.digest)" +RUNTIME_BASE="$(receipt_field baseImages.runtime.reference)@$(receipt_field baseImages.runtime.digest)" + +if [[ ! "$LLAMA_CPP_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then + printf 'Refusing to build: the llama.cpp pin must be a 40-hex commit, got %s (floating refs are not reproducible).\n' "${LLAMA_CPP_COMMIT:-}" >&2 + exit 64 +fi +for pin in "$MODEL_SHA256" "$MMPROJ_SHA256"; do + if [[ ! "$pin" =~ ^[0-9a-f]{64}$ ]]; then + printf 'Refusing to build: every model hash must be a pinned sha256, got %s.\n' "${pin:-}" >&2 + exit 64 + fi +done +for base in "$BUILD_BASE" "$RUNTIME_BASE"; do + if [[ ! "$base" =~ @sha256:[0-9a-f]{64}$ ]]; then + printf 'Refusing to build: base images must be digest-pinned, got %s.\n' "$base" >&2 + exit 64 + fi +done + +if [[ ! "$PUBLISH_HOST" =~ ^(127\.0\.0\.1|::1|localhost)$ ]]; then + printf 'Refusing to publish the worker outside loopback (127.0.0.1): got %s.\n' "$PUBLISH_HOST" >&2 + exit 64 +fi + +print_plan() { + printf 'image: %s:%s\n' "$IMAGE" "$TAG" + printf 'llama.cpp commit: %s\n' "$LLAMA_CPP_COMMIT" + printf '%s sha256: %s\n' "$MODEL_FILE" "$MODEL_SHA256" + printf '%s sha256: %s\n' "$MMPROJ_FILE" "$MMPROJ_SHA256" + printf 'base image (build): %s\n' "$BUILD_BASE" + printf 'base image (runtime): %s\n' "$RUNTIME_BASE" + printf 'bind address: %s:%s\n' "$PUBLISH_HOST" "$PORT" + printf 'publish: %s:%s\n' "$PUBLISH_HOST" "$PORT" + printf 'docker build --file deploy/worker/Dockerfile --build-arg LLAMA_CPP_COMMIT=%s --build-arg MODEL_SHA256=%s --build-arg MMPROJ_SHA256=%s --tag %s:%s .\n' \ + "$LLAMA_CPP_COMMIT" "$MODEL_SHA256" "$MMPROJ_SHA256" "$IMAGE" "$TAG" +} + +if [[ "${1:-}" == '--print-plan' ]]; then + print_plan + exit 0 +fi + +print_plan +cd "$REPO_ROOT" +docker build \ + --file deploy/worker/Dockerfile \ + --build-arg "LLAMA_CPP_COMMIT=$LLAMA_CPP_COMMIT" \ + --build-arg "MODEL_SHA256=$MODEL_SHA256" \ + --build-arg "MMPROJ_SHA256=$MMPROJ_SHA256" \ + --tag "$IMAGE:$TAG" \ + . +printf 'Built %s:%s from pinned receipts. Run it with deploy/worker/run_worker_container.sh\n' "$IMAGE" "$TAG" diff --git a/deploy/worker/entrypoint.sh b/deploy/worker/entrypoint.sh new file mode 100755 index 0000000..d0f17f8 --- /dev/null +++ b/deploy/worker/entrypoint.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Start the pinned worker after verifying the mounted weights (issue #15). +# +# Fails closed before the server is allowed to answer: a missing pin, an unreadable model, or a +# hash mismatch stops the container instead of serving suggestions from an unknown model. +set -euo pipefail + +MODEL="${TIMMY_MODEL_FILE:-/models/SmolVLM2-2.2B-Instruct-Q4_K_M.gguf}" +MMPROJ="${TIMMY_MMPROJ_FILE:-/models/mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf}" +MODEL_SHA256="${TIMMY_MODEL_SHA256:-}" +MMPROJ_SHA256="${TIMMY_MMPROJ_SHA256:-}" +ALIAS="${TIMMY_MODEL_ALIAS:-SmolVLM2-2.2B-Instruct}" +HOST="${TIMMY_MODEL_HOST:-0.0.0.0}" +PORT="${TIMMY_MODEL_PORT:-8080}" +THREADS="${TIMMY_MODEL_THREADS:-4}" + +for pin in "$MODEL_SHA256" "$MMPROJ_SHA256"; do + if [[ ! "$pin" =~ ^[0-9a-f]{64}$ ]]; then + printf 'Refusing to start: the pinned model hashes are missing or malformed.\n' >&2 + exit 78 + fi +done + +for file in "$MODEL" "$MMPROJ"; do + if [[ ! -f "$file" ]]; then + printf 'Refusing to start: %s is not mounted. Mount the verified model directory read-only at /models.\n' "$file" >&2 + exit 78 + fi +done + +printf '%s %s\n%s %s\n' "$MODEL_SHA256" "$MODEL" "$MMPROJ_SHA256" "$MMPROJ" | sha256sum --check --status || { + printf 'Refusing to start: mounted weights do not match the pinned receipts.\n' >&2 + exit 78 +} + +printf 'Starting %s from pinned weights on %s:%s (private namespace)\n' "$ALIAS" "$HOST" "$PORT" +exec llama-server \ + --model "$MODEL" \ + --mmproj "$MMPROJ" \ + --alias "$ALIAS" \ + --host "$HOST" \ + --port "$PORT" \ + --ctx-size 4096 \ + --parallel 1 \ + --threads "$THREADS" \ + --no-webui diff --git a/deploy/worker/receipts.json b/deploy/worker/receipts.json new file mode 100644 index 0000000..fa6c461 --- /dev/null +++ b/deploy/worker/receipts.json @@ -0,0 +1,40 @@ +{ + "schemaVersion": 1, + "issue": 15, + "llamaCpp": { + "repository": "https://github.com/ggml-org/llama.cpp", + "commit": "6d05498314db1b57f81c271080018aa2d0b89be9" + }, + "baseImages": { + "build": { + "reference": "nvidia/cuda:12.4.1-devel-ubuntu22.04", + "digest": "sha256:da6791294b0b04d7e65d87b7451d6f2390b4d36225ab0701ee7dfec5769829f5" + }, + "runtime": { + "reference": "nvidia/cuda:12.4.1-runtime-ubuntu22.04", + "digest": "sha256:517da2300c184c9999ec203c2665244bdebd3578d12fcc7065e83667932643d9" + } + }, + "models": [ + { + "file": "SmolVLM2-2.2B-Instruct-Q4_K_M.gguf", + "sha256": "0cf76814555b8665149075b74ab6b5c1d428ea1d3d01c1918c12012e8d7c9f58", + "mount": "/models/SmolVLM2-2.2B-Instruct-Q4_K_M.gguf" + }, + { + "file": "mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf", + "sha256": "ae07ea1facd07dd3230c4483b63e8cda96c6944ad2481f33d531f79e892dd024", + "mount": "/models/mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf" + } + ], + "alias": "SmolVLM2-2.2B-Instruct", + "bind": { + "publishHost": "127.0.0.1", + "containerPort": 8080 + }, + "notes": [ + "Model weights are never baked into the image; they are mounted read-only and hash-verified at start-up.", + "The pins are asserted equal to scripts/bootstrap_selfhost_smolvlm.sh by tests/worker-container.test.js so host and image cannot drift.", + "The worker needs no provider credential: it is a private loopback-only service." + ] +} diff --git a/deploy/worker/run_worker_container.sh b/deploy/worker/run_worker_container.sh new file mode 100755 index 0000000..51f3dba --- /dev/null +++ b/deploy/worker/run_worker_container.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Run the pinned worker with the least privilege it can work under (issue #15). +# +# The worker is a private service. It is published on loopback only, mounts the verified weights +# read-only, keeps an immutable root filesystem, and holds no credential of any kind. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +RECEIPTS="$REPO_ROOT/deploy/worker/receipts.json" +IMAGE="${TIMMY_WORKER_IMAGE:-timmy-vision-worker}" +TAG="${TIMMY_WORKER_TAG:-$(date -u +%Y%m%d)}" +PORT="${TIMMY_MODEL_PORT:-8080}" +PUBLISH_HOST="${TIMMY_WORKER_PUBLISH_HOST:-127.0.0.1}" +MODEL_DIR="${TIMMY_WORKER_MODEL_DIR:-${TIMMY_SELFHOST_ROOT:-$HOME/.local/share/timmy-selfhost}/models}" +NAME="${TIMMY_WORKER_CONTAINER:-timmy-vision-worker}" +ALIAS="${TIMMY_MODEL_ALIAS:-SmolVLM2-2.2B-Instruct}" + +# Exposure is the one decision this recipe must never get wrong, so a non-loopback publish host is +# refused rather than warned about. +if [[ ! "$PUBLISH_HOST" =~ ^(127\.0\.0\.1|::1|localhost)$ ]]; then + printf 'Refusing to publish the worker outside loopback (127.0.0.1): got %s.\n' "$PUBLISH_HOST" >&2 + exit 64 +fi + +if [[ ! "$PORT" =~ ^[0-9]{1,5}$ ]] || (( PORT < 1 || PORT > 65535 )); then + printf 'Refusing to start: the port must be a number between 1 and 65535, got %s.\n' "$PORT" >&2 + exit 64 +fi + +RUN_ARGS=( + run + --detach + --name "$NAME" + --read-only + --cap-drop=ALL + --security-opt no-new-privileges + --tmpfs /tmp:rw,noexec,nosuid,size=64m + --volume "$MODEL_DIR:/models:ro" + --publish "127.0.0.1:${PORT}:8080" + --env "TIMMY_MODEL_PORT=8080" + --env "TIMMY_MODEL_THREADS=${TIMMY_MODEL_THREADS:-4}" + "$IMAGE:$TAG" +) + +if [[ "${1:-}" == '--print-plan' ]]; then + printf 'image: %s:%s\n' "$IMAGE" "$TAG" + printf 'model directory (read-only mount): %s\n' "$MODEL_DIR" + printf 'bind address: %s:%s\n' "$PUBLISH_HOST" "$PORT" + printf 'publish: %s:%s\n' "$PUBLISH_HOST" "$PORT" + printf 'health: %s ready (post-load gate)\n' "$ALIAS" + printf 'docker %s\n' "${RUN_ARGS[*]}" + exit 0 +fi + +if [[ ! -d "$MODEL_DIR" ]]; then + printf 'Refusing to start: %s is not a directory. Install the pinned weights with scripts/bootstrap_selfhost_smolvlm.sh install.\n' "$MODEL_DIR" >&2 + exit 78 +fi + +docker "${RUN_ARGS[@]}" +printf 'Worker container %s started on %s:%s. Readiness: docker exec %s timmy-worker-ready\n' "$NAME" "$PUBLISH_HOST" "$PORT" "$NAME" diff --git a/package.json b/package.json index faaa16a..ea13c19 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "private": true, "type": "module", "scripts": { - "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js", + "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js", "test:ui": "node tests/ui.acceptance.mjs", "test:photo": "node tests/photo-first.acceptance.mjs", "test:mobile-capture": "node tests/mobile-capture.acceptance.mjs", @@ -14,9 +14,10 @@ "test:followup": "node tests/followup.acceptance.mjs", "test:vision-refusal": "node tests/vision-refusal.acceptance.mjs", "test:staging-smoke": "node tests/staging.acceptance.mjs", - "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py", + "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py && node --check scripts/record_worker_benchmark.mjs && node --check tests/worker-readiness.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh", "check:diff": "bash scripts/check_diff.sh", - "start": "node server.mjs" + "start": "node server.mjs", + "test:worker-readiness": "node tests/worker-readiness.acceptance.mjs" }, "devDependencies": { "playwright": "^1.62.1" diff --git a/scripts/bootstrap_selfhost_smolvlm.sh b/scripts/bootstrap_selfhost_smolvlm.sh index a6a33e1..fa9b444 100755 --- a/scripts/bootstrap_selfhost_smolvlm.sh +++ b/scripts/bootstrap_selfhost_smolvlm.sh @@ -137,6 +137,10 @@ case "${1:-}" in start) start_worker ;; + ready) + # Post-load readiness (issue #15): delegate so the host and the container gate cannot drift. + exec "$(dirname "$0")/worker_ready.sh" --host "$HOST" --port "$PORT" --alias SmolVLM2-2.2B-Instruct "${@:2}" + ;; health) health_worker ;; @@ -144,7 +148,7 @@ case "${1:-}" in stop_worker ;; *) - printf 'Usage: %s {install|start|health|stop|receipt|verify}\n' "$0" >&2 + printf 'Usage: %s {install|start|health|ready|stop|receipt|verify}\n' "$0" >&2 exit 64 ;; esac diff --git a/scripts/build_release.py b/scripts/build_release.py index ffbb97d..9d5379c 100755 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -39,6 +39,7 @@ DEMO_REQUIRED_CAPTIONS = ( "Urgent language is intercepted deterministically before Hermes", "An out-of-range answer fails closed: nothing is applied and no provider detail reaches you", "A filled-in field that is out of range still fails closed, even when the other field came back blank", + "Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident", ) @@ -136,6 +137,7 @@ def main() -> int: run(["npm", "run", "test:provisional"], tree) run(["npm", "run", "test:followup"], tree) run(["npm", "run", "test:vision-refusal"], tree) + run(["npm", "run", "test:worker-readiness"], tree) demo_raw = release_dir / f"timmy-talking-turd-{version}-demo.raw.webm" caption_receipt = release_dir / f"timmy-talking-turd-{version}-demo-captions.json" demo_env = dict(server_env) @@ -177,10 +179,14 @@ def main() -> int: if not contact_sheet.exists() or contact_sheet.stat().st_size == 0: raise SystemExit("Release demo contact sheet was not produced") run(["npm", "audit", "--audit-level=high"], tree) - for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs"): + for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "scripts/record_worker_benchmark.mjs", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs", "tests/worker-readiness.acceptance.mjs"): run(["node", "--check", file], tree) run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree) run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree) + run(["bash", "-n", "scripts/worker_ready.sh"], tree) + run(["bash", "-n", "deploy/worker/entrypoint.sh"], tree) + run(["bash", "-n", "deploy/worker/build_image.sh"], tree) + run(["bash", "-n", "deploy/worker/run_worker_container.sh"], tree) run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py", "scripts/build_vision_corpus.py"], tree) run(["git", "diff", "--check", commit], tree) @@ -209,6 +215,9 @@ def main() -> int: "- AI suggestions stay provisional: suggested, confirmed, and corrected states are distinct and screen-reader described; correcting one clears the AI-confirmed state without losing user context\n" "- Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions (difficulty passing, soreness, visible blood, truly black/tarry versus dark brown, hydration); mild reports get non-diagnostic self-care, and a confirmed red-flag answer lands on the same deterministic escalation path as manual logging\n" "- Vision corpus: a committed CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) proves every reply is schema-valid or fails closed; a schema-valid reply with unreadable fields abstains instead of erroring, and unparsable or out-of-range replies return plain copy with nothing applied — including a reply that fills in one field and leaves the other blank, where an out-of-range value still fails closed instead of being reported as an unreadable photo (issue #73)\n" + "- Self-hosted worker container: a digest-pinned image that builds the pinned llama.cpp revision, mounts hash-verified weights read-only, runs unprivileged with no credential, and is published on loopback only (issue #15)\n" + "- Post-load readiness: the app offers AI only once the pinned model is actually loaded, so a reachable-but-unloaded worker falls back to manual logging in one tap instead of failing after the upload (issue #15)\n" + "- Target-GPU benchmark: scripts/record_worker_benchmark.mjs records cold start, warm latency, and VRAM on the target GPU and refuses to write a receipt it did not measure; no GPU measurement is bundled with this release\n" "- Safety: suggestions are observable-field assistance, never diagnosis; user confirmation is required\n" "- Known limitation: open-weight VLM acceptance is proven, Bristol accuracy is not clinically validated\n" "- Package excludes model weights, medical-image corpora, raw research pages, generated screenshots, and video production assets\n" @@ -265,6 +274,8 @@ def main() -> int: "provisional_suggestion_state": "passed", "hard_stool_followup": "passed", "vision_corpus_refusal": "passed", + "worker_container_receipts": "passed", + "worker_post_load_readiness": "passed", "dependency_audit_high": "passed", "syntax": "passed", "secret_scan": "passed", diff --git a/scripts/record_release_demo.mjs b/scripts/record_release_demo.mjs index 2838613..c90ff05 100644 --- a/scripts/record_release_demo.mjs +++ b/scripts/record_release_demo.mjs @@ -49,6 +49,9 @@ const errors = []; page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); page.on('pageerror', error => errors.push(error.message)); let visionOffline = false; +// Reachable is not loaded: the worker can answer /v1/models before its pinned model is resident, +// and the recording shows the app withholding AI in exactly that state (issue #15). +let modelLoaded = true; await page.route('**/api/vision-status', route => route.fulfill({ status: 200, contentType: 'application/json', @@ -59,7 +62,7 @@ await page.route('**/api/vision-status', route => route.fulfill({ model: 'SmolVLM2-2.2B-Instruct', // The second pass flips the app's own readiness contract so no AI control is offered at all. providerReady: !visionOffline, - modelSeen: !visionOffline, + modelSeen: !visionOffline && modelLoaded, }), })); // The happy path keeps one deterministic suggestion. The refusal scene swaps this for a real @@ -231,6 +234,38 @@ await tap('#discard-draft', 500); await caption('Start over clears the kept draft — nothing lingers silently', 1500); await tap('#close-sheet', 450); +// Post-load readiness pass (issue #15): the worker answers, but its pinned model is not resident +// yet. The app must withhold AI instead of offering a control that fails after the upload. Every +// claim below is asserted in the browser, so the recording cannot show a state the app does not have. +modelLoaded = false; +visionOffline = false; +await caption('Now the same worker while its pinned model is still loading', 1700); +await tap('[data-scan]', 550); +await page.getByText(/model is not loaded yet/i).first().waitFor(); +if (await page.getByText(/Self-hosted model ready/i).count() !== 0) { + throw new Error('A reachable worker without its loaded model must never be reported ready'); +} +const loadingAmber = await page.locator('.model-status.loading').first().evaluate(el => getComputedStyle(el).backgroundColor); +if (loadingAmber !== 'rgb(253, 243, 216)') throw new Error('The not-loaded state must be visually distinct from ready and offline'); +await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg'); +await page.getByText(/Ready for your review/i).waitFor(); +if (await page.locator('#analyze-photo').count() !== 0) throw new Error('No AI control may exist before the model is loaded'); +if (await page.locator('.consent-card').count() !== 0) throw new Error('No consent may be requested before the model is loaded'); +await caption('Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident', 2900); +await tap('#manual-from-scan', 500); +await page.getByText('Pick the closest form').waitFor(); +await caption('One tap still reaches manual Bristol entry while the model finishes loading', 2000); +await tap('#close-sheet', 450); +modelLoaded = true; +await caption('Once the pinned model is loaded, the same screen offers the AI analysis again', 2200); +await tap('[data-scan]', 550); +await page.getByText(/Self-hosted model ready/i).waitFor(); +await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg'); +await page.getByText(/Ready for your review/i).waitFor(); +if (await page.locator('.consent-card').isVisible() !== true) throw new Error('The AI path must return once the model is loaded'); +if (await page.locator('#analyze-photo').count() !== 1) throw new Error('The analyze control must return once the model is loaded'); +await tap('#close-sheet', 450); + // Corpus refusal pass (issue #18): the same photo flow, fed the answers Timmy has to refuse. visionOffline = false; await caption('The corpus also replays the answers Timmy has to refuse', 1700); diff --git a/scripts/record_worker_benchmark.mjs b/scripts/record_worker_benchmark.mjs new file mode 100644 index 0000000..5bfc8b4 --- /dev/null +++ b/scripts/record_worker_benchmark.mjs @@ -0,0 +1,161 @@ +#!/usr/bin/env node +// Record the target-GPU cost of the pinned worker (issue #15). +// +// This script measures; it never invents. If no GPU can be probed, if the pinned model is not +// loaded, or if the cold start cannot be observed, it exits non-zero and writes no receipt at all, +// because a receipt with zeroed or assumed numbers is worse than no receipt. +import { spawn } from 'node:child_process'; +import { readFile, writeFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; + +const LOOPBACK = /^(127\.0\.0\.1|localhost|::1)$/; + +function parseArgs(argv) { + const options = { + out: '', + host: '127.0.0.1', + port: '8080', + alias: 'SmolVLM2-2.2B-Instruct', + gpuProbe: 'nvidia-smi', + warmSamples: 20, + startCommand: '', + readyTimeoutSeconds: 900, + }; + for (let index = 0; index < argv.length; index += 1) { + const [flag, inline] = argv[index].split('='); + const value = inline ?? argv[index + 1]; + const consume = inline === undefined ? 2 : 1; + if (flag === '--out') options.out = value; + else if (flag === '--host') options.host = value; + else if (flag === '--port') options.port = value; + else if (flag === '--alias') options.alias = value; + else if (flag === '--gpu-probe') options.gpuProbe = value; + else if (flag === '--warm-samples') options.warmSamples = Number(value); + else if (flag === '--start-command') options.startCommand = value; + else if (flag === '--ready-timeout') options.readyTimeoutSeconds = Number(value); + else fail(`Unknown argument: ${flag}`); + index += consume - 1; + } + return options; +} + +function fail(message, code = 64) { + process.stderr.write(`${message}\n`); + process.exit(code); +} + +const reject = message => { + process.stderr.write(`${message}\n`); + process.exit(1); +}; + +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); + +async function modelsServed({ host, port, alias }) { + try { + const response = await fetch(`http://${host}:${port}/v1/models`, { signal: AbortSignal.timeout(3000) }); + if (!response.ok) return false; + const body = await response.json(); + const ids = Array.isArray(body?.data) ? body.data.map(item => item?.id).filter(Boolean) : []; + return ids.some(id => id === alias || id.includes(alias)); + } catch { + return false; + } +} + +function runCommand(command, args) { + return new Promise(resolve => { + const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', chunk => { stdout += chunk; }); + child.stderr.on('data', chunk => { stderr += chunk; }); + child.on('error', error => resolve({ code: 127, stdout, stderr: String(error.message) })); + child.on('close', code => resolve({ code, stdout, stderr })); + }); +} + +async function probeGpu(probeCommand) { + const result = await runCommand(probeCommand, [ + '--query-gpu=name,memory.total,memory.used', + '--format=csv,noheader,nounits', + ]); + if (result.code !== 0) { + reject(`No GPU could be probed with ${probeCommand}: ${result.stderr.trim() || `exit ${result.code}`}. A benchmark receipt needs a real device, so nothing was written.`); + } + const [name, total, used] = result.stdout.trim().split('\n')[0].split(',').map(part => part.trim()); + if (!name || !/^\d+$/.test(total) || !/^\d+$/.test(used)) { + reject(`The GPU probe returned unreadable output (${JSON.stringify(result.stdout.trim())}); refusing to record VRAM it cannot read.`); + } + return { name, vramTotalMiB: Number(total), vramUsedMiB: Number(used) }; +} + +function percentile(sorted, fraction) { + const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(fraction * sorted.length) - 1)); + return sorted[index]; +} + +const options = parseArgs(process.argv.slice(2)); +if (!options.out) fail('--out is required so the receipt has a destination'); +if (!LOOPBACK.test(options.host)) fail(`The worker is a loopback service; refusing to benchmark ${options.host}.`); +if (!Number.isInteger(options.warmSamples) || options.warmSamples < 1) fail('--warm-samples must be a positive integer'); + +const receipts = JSON.parse(await readFile(fileURLToPath(new URL('../deploy/worker/receipts.json', import.meta.url)), 'utf8')); + +// The device is probed first: nothing is timed until we know the numbers have somewhere to be real. +const gpu = await probeGpu(options.gpuProbe); + +if (!(await modelsServed(options))) { + reject(`${options.alias} is not loaded on ${options.host}:${options.port} yet, so there is no ready worker to time. Start the worker and wait for the post-load gate.`); +} + +let coldStartMs = null; +if (options.startCommand) { + const started = Date.now(); + const started_process = await runCommand('bash', ['-c', options.startCommand]); + if (started_process.code !== 0) { + reject(`The cold-start command failed (exit ${started_process.code}): ${started_process.stderr.trim()}`); + } + const deadline = Date.now() + options.readyTimeoutSeconds * 1000; + let ready = false; + while (Date.now() < deadline) { + if (await modelsServed(options)) { ready = true; break; } + await sleep(250); + } + if (!ready) reject(`The worker did not become ready within ${options.readyTimeoutSeconds}s, so the cold start could not be measured.`); + coldStartMs = Date.now() - started; +} + +const samples = []; +for (let index = 0; index < options.warmSamples; index += 1) { + const started = process.hrtime.bigint(); + const ok = await modelsServed(options); + const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6; + if (!ok) reject('The worker stopped answering during the warm pass; the receipt would be partial, so nothing was written.'); + samples.push(Math.round(elapsedMs * 1000) / 1000); +} +const sorted = [...samples].sort((left, right) => left - right); + +const receipt = { + schemaVersion: 1, + status: 'measured_on_target_gpu', + measuredAt: new Date().toISOString(), + gpu, + coldStartMs, + warmLatencyMs: { + samples: sorted.length, + min: sorted[0], + p50: percentile(sorted, 0.5), + p95: percentile(sorted, 0.95), + max: sorted[sorted.length - 1], + }, + worker: { baseUrl: `http://${options.host}:${options.port}/v1`, alias: options.alias }, + pins: { + llamaCppCommit: receipts.llamaCpp.commit, + modelSha256: receipts.models.map(model => model.sha256), + }, +}; + +await writeFile(path.resolve(options.out), `${JSON.stringify(receipt, null, 2)}\n`, 'utf8'); +process.stdout.write(`${path.resolve(options.out)}\n`); diff --git a/scripts/worker_ready.sh b/scripts/worker_ready.sh new file mode 100755 index 0000000..0051899 --- /dev/null +++ b/scripts/worker_ready.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Post-load readiness gate for Timmy's self-hosted vision worker (issue #15). +# +# A worker that answers on its port is not necessarily a worker that can answer: llama.cpp opens +# its HTTP listener before the model finishes loading and returns an error until the weights are +# resident. Readiness therefore passes only when the pinned model alias is actually served. +# +# Exit codes: +# 0 ready — the pinned model is loaded and being served +# 2 the worker is not reachable +# 3 the worker is reachable but the pinned model is not loaded yet +# 64 usage error (including a non-loopback probe host) +set -uo pipefail + +HOST="${TIMMY_MODEL_HOST:-127.0.0.1}" +PORT="${TIMMY_MODEL_PORT:-8080}" +ALIAS="${TIMMY_MODEL_ALIAS:-SmolVLM2-2.2B-Instruct}" +WAIT=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --host) HOST="${2:-}"; shift 2 ;; + --port) PORT="${2:-}"; shift 2 ;; + --alias) ALIAS="${2:-}"; shift 2 ;; + --wait) WAIT="${2:-}"; shift 2 ;; + *) + printf 'Usage: %s [--host H] [--port P] [--alias A] [--wait SECONDS]\n' "$0" >&2 + exit 64 + ;; + esac +done + +# The worker is a private, loopback-only service. Probing anything else would be a mistake about +# exposure, so refuse rather than answer about a host this gate is not responsible for. +if [[ ! "$HOST" =~ ^(127\.0\.0\.1|localhost|::1)$ ]]; then + printf 'Readiness must be probed on loopback, got %s\n' "$HOST" >&2 + exit 64 +fi + +deadline=$(( $(date +%s) + WAIT )) +outcome='unreachable' +while :; do + body="$(curl --silent --max-time 3 "http://$HOST:$PORT/v1/models" 2>/dev/null)" + if [[ $? -eq 0 && -n "$body" ]]; then + if printf '%s' "$body" | grep -q -- "$ALIAS"; then + printf 'Worker ready: %s is loaded on http://%s:%s/v1\n' "$ALIAS" "$HOST" "$PORT" + exit 0 + fi + outcome='not-loaded' + printf 'Worker reachable on http://%s:%s/v1 but %s is not loaded yet.\n' "$HOST" "$PORT" "$ALIAS" >&2 + else + outcome='unreachable' + printf 'Worker not reachable on http://%s:%s/v1.\n' "$HOST" "$PORT" >&2 + fi + [[ "$(date +%s)" -ge "$deadline" ]] && break + sleep 1 +done + +[[ "$outcome" == 'not-loaded' ]] && exit 3 +exit 2 diff --git a/src/vision-readiness.js b/src/vision-readiness.js index f969aec..d2b588e 100644 --- a/src/vision-readiness.js +++ b/src/vision-readiness.js @@ -31,5 +31,17 @@ export function visionReadiness(status) { message: 'Vision worker offline. Manual logging is still available.', }; } + // A worker that answers /v1/models has not necessarily loaded the model yet (#15), so readiness + // passes only once the pinned model is confirmed loaded. An unconfirmed model fails closed to the + // manual path instead of offering an AI control that fails after the upload. + if (status.modelSeen !== true) { + return { + phase: 'loading', + ready: false, + manualOnly: true, + manualLabel: MANUAL_LABEL, + message: 'The vision worker is up but the model is not loaded yet. Manual logging is available.', + }; + } return { phase: 'ready', ready: true, manualOnly: false, manualLabel: '', message: '' }; } diff --git a/styles.css b/styles.css index 12c7a28..6146c38 100644 --- a/styles.css +++ b/styles.css @@ -18,7 +18,7 @@ main{display:block}.eyebrow{font-size:11px;font-weight:800;letter-spacing:.1em;t .chat-page{display:flex;flex-direction:column;min-height:calc(100vh - 175px)}.chat-title{padding-bottom:8px}.agent-status{display:flex;align-items:center;gap:11px;padding:11px 13px;background:rgba(255,253,250,.7);border:1px solid var(--line);border-radius:17px;margin-bottom:12px}.agent-status>i{width:10px;height:10px;border-radius:50%;background:#a9a39d;box-shadow:0 0 0 5px rgba(169,163,157,.13)}.agent-status.connected>i{background:#209479;box-shadow:0 0 0 5px rgba(32,148,121,.13)}.agent-status.locked>i{background:#d19b27}.agent-status strong,.agent-status small{display:block}.agent-status strong{font-size:13px}.agent-status small{font-size:11px;color:var(--muted);margin-top:2px}.conversation{background:var(--surface);border:1px solid var(--line);border-radius:23px;padding:13px;box-shadow:0 10px 30px rgba(55,40,31,.05)}.chat{display:flex;flex-direction:column;gap:9px;min-height:195px;max-height:42vh;overflow:auto;padding:4px 1px 14px}.bubble{max-width:86%;padding:11px 13px;border-radius:17px;line-height:1.42;font-size:14px;white-space:pre-wrap}.bubble.timmy{align-self:flex-start;background:#efebe4;border-bottom-left-radius:5px}.bubble.user{align-self:flex-end;background:var(--teal);color:white;border-bottom-right-radius:5px}.thinking{display:flex;gap:4px}.thinking span{width:6px;height:6px;border-radius:50%;background:#8b847d;animation:blink 1s infinite}.thinking span:nth-child(2){animation-delay:.15s}.thinking span:nth-child(3){animation-delay:.3s}@keyframes blink{50%{opacity:.25;transform:translateY(-2px)}}.composer{display:grid;grid-template-columns:1fr 45px;gap:8px;align-items:end;background:#f0ece5;border-radius:18px;padding:6px}.composer textarea{border:0;background:transparent;resize:none;min-height:42px;max-height:110px;padding:10px 9px;outline:0;color:var(--ink)}.composer button{width:44px;height:44px;border:0;border-radius:14px;background:var(--ink);color:white;font-size:22px;cursor:pointer}.composer button:disabled{opacity:.4}.composer-note{font-size:10px;color:var(--muted);margin:7px 5px 0;line-height:1.35}.chat-error{font-size:12px;color:var(--red);margin:0 4px 8px}.safety-line{margin:13px 4px 0;color:var(--muted);font-size:11px;line-height:1.45}.safety-line strong{color:var(--ink)}.unlock-card{background:#fff8e6;border:1px solid #ead9a9;border-radius:20px;padding:14px;margin-bottom:12px}.unlock-card>label{display:block;font-size:12px;font-weight:800;margin-bottom:7px}.unlock-row{display:grid;grid-template-columns:1fr auto;gap:8px}.unlock-card .fine{margin:8px 2px 0} .sheet-backdrop{position:fixed;inset:0;background:rgba(28,22,18,.42);display:flex;align-items:flex-end;justify-content:center;z-index:50;padding-top:28px;backdrop-filter:blur(5px)}.sheet{width:min(100%,680px);max-height:94vh;overflow:auto;background:var(--surface);border-radius:28px 28px 0 0;padding:9px 18px calc(24px + env(safe-area-inset-bottom));box-shadow:0 -15px 50px rgba(30,22,18,.2)}.sheet-handle{width:38px;height:4px;background:#d8d1c8;border-radius:999px;margin:2px auto 15px}.sheet-header{display:flex;justify-content:space-between;align-items:flex-start;gap:12px}.sheet-header h2{margin-top:4px}.icon-btn{width:44px;height:44px;border:0;border-radius:50%;background:#efebe5;font-size:24px;cursor:pointer}.progress{height:4px;background:#eee8df;border-radius:99px;margin:7px 0 18px;overflow:hidden}.progress i{height:100%;display:block;background:var(--teal)}.progress-step-1{width:33.34%}.progress-step-2{width:66.68%}.progress-step-3{width:100%} .draft-resume{display:flex;align-items:center;justify-content:space-between;gap:10px;background:#fdf3d8;border:1px solid #ecd9a3;border-radius:14px;padding:8px 11px;margin:0 0 14px;font-size:12px;line-height:1.35;color:#5d4a12}.draft-resume strong{font-weight:800} -.scan-hero{text-align:center;padding:3px 15px 12px}.scan-hero img{width:76px}.scan-hero h3{font-size:19px;margin:4px 0 7px}.scan-hero p{font-size:13px;line-height:1.45;color:var(--muted)}.model-status{border-radius:14px;padding:9px 11px;font-size:11px;font-weight:700;margin-bottom:12px}.model-status.ready{background:var(--teal-soft);color:var(--teal)}.model-status.offline{background:var(--red-soft);color:var(--red)}.model-status.checking{background:#f0ece5;color:var(--muted)}.capture-choice-grid{display:grid;grid-template-columns:1fr 1fr;gap:10px}.capture-status{background:var(--red-soft);color:var(--red);border-radius:14px;padding:10px 12px;font-size:12px;line-height:1.4}.photo-capture{display:grid;place-items:center;text-align:center;border:1.5px dashed #b9aea1;border-radius:21px;padding:16px 10px;background:#faf7f1;cursor:pointer}.photo-capture b{font-size:30px}.photo-capture strong{margin:6px 0 2px}.photo-capture span{font-size:11px;color:var(--muted)}.photo-capture input{display:none}.photo-preview{display:block;width:100%;max-height:250px;object-fit:cover;border-radius:18px;margin:10px 0}.quality-note{font-size:12px;color:var(--muted)}.consent-card{background:#f5f1e9;border-radius:18px;padding:12px;margin:12px 0}.check{display:flex;gap:11px;align-items:flex-start;padding:9px 0}.check input{width:20px;height:20px;accent-color:var(--teal);flex:0 0 auto}.check span{font-size:13px;line-height:1.4}.analyzing{text-align:center;padding:33px 10px}.analyzing img{width:86px}.spinner{width:32px;height:32px;border:3px solid var(--soft);border-top-color:var(--teal);border-radius:50%;animation:spin .8s linear infinite;margin:10px auto}@keyframes spin{to{transform:rotate(360deg)}}.analyzing p{font-size:13px;color:var(--muted)}.scan-result{background:var(--teal-soft);border-radius:20px;padding:17px;margin-bottom:13px}.scan-result.needs-input{background:#f2eee7;text-align:center}.scan-result.needs-input>b{font-size:28px}.scan-result p{font-size:13px;line-height:1.45;margin:9px 0 0}.ai-badge{font-size:10px;font-weight:850;letter-spacing:.08em;color:var(--teal)}.suggestion-pair{display:grid;grid-template-columns:1fr 1fr;gap:9px;margin-top:12px}.suggestion-pair>div{background:var(--surface);padding:12px;border-radius:14px}.suggestion-pair small,.suggestion-pair strong{display:block}.suggestion-pair small{font-size:9px;color:var(--muted)}.suggestion-pair strong{font-size:18px;margin-top:2px}.ai-prefill{background:var(--teal-soft);border-radius:16px;padding:11px 13px;margin-bottom:12px}.ai-prefill strong,.ai-prefill small{display:block}.ai-prefill small{color:var(--muted);margin-top:3px}.choice-grid{display:grid;grid-template-columns:1fr 1fr;gap:8px}.bristol{min-height:66px;border:1px solid var(--line);background:#f7f3ed;border-radius:16px;padding:10px;text-align:left;cursor:pointer}.bristol strong,.bristol span{display:block}.bristol span{font-size:11px;color:var(--muted);margin-top:2px}.bristol.selected{background:var(--ink);color:white}.bristol.selected span{color:#ddd}.field{display:block;margin-bottom:15px}.field-label{display:block;font-size:12px;font-weight:800;margin-bottom:6px}.input{width:100%;min-height:48px;border:1px solid var(--line);border-radius:14px;background:#f8f5ef;padding:10px 12px;color:var(--ink)}textarea.input{min-height:85px;resize:vertical}.range-row{display:grid;grid-template-columns:1fr 37px;gap:10px;align-items:center}.range-row input{accent-color:var(--teal)}.range-val{width:37px;height:37px;border-radius:12px;background:var(--soft);display:grid;place-items:center;font-weight:800}.photo-drop{width:100%;background:#f3eee7}.symptoms{display:flex;flex-direction:column}.alert{background:var(--red-soft);border:1px solid #eec1ba;border-radius:16px;padding:13px;color:#75261f}.alert p{margin:5px 0 0;font-size:13px;line-height:1.4} +.scan-hero{text-align:center;padding:3px 15px 12px}.scan-hero img{width:76px}.scan-hero h3{font-size:19px;margin:4px 0 7px}.scan-hero p{font-size:13px;line-height:1.45;color:var(--muted)}.model-status{border-radius:14px;padding:9px 11px;font-size:11px;font-weight:700;margin-bottom:12px}.model-status.ready{background:var(--teal-soft);color:var(--teal)}.model-status.offline{background:var(--red-soft);color:var(--red)}.model-status.loading{background:#fdf3d8;color:#5d4a12}.capture-choice-grid{display:grid;grid-template-columns:1fr 1fr;gap:10px}.capture-status{background:var(--red-soft);color:var(--red);border-radius:14px;padding:10px 12px;font-size:12px;line-height:1.4}.photo-capture{display:grid;place-items:center;text-align:center;border:1.5px dashed #b9aea1;border-radius:21px;padding:16px 10px;background:#faf7f1;cursor:pointer}.photo-capture b{font-size:30px}.photo-capture strong{margin:6px 0 2px}.photo-capture span{font-size:11px;color:var(--muted)}.photo-capture input{display:none}.photo-preview{display:block;width:100%;max-height:250px;object-fit:cover;border-radius:18px;margin:10px 0}.quality-note{font-size:12px;color:var(--muted)}.consent-card{background:#f5f1e9;border-radius:18px;padding:12px;margin:12px 0}.check{display:flex;gap:11px;align-items:flex-start;padding:9px 0}.check input{width:20px;height:20px;accent-color:var(--teal);flex:0 0 auto}.check span{font-size:13px;line-height:1.4}.analyzing{text-align:center;padding:33px 10px}.analyzing img{width:86px}.spinner{width:32px;height:32px;border:3px solid var(--soft);border-top-color:var(--teal);border-radius:50%;animation:spin .8s linear infinite;margin:10px auto}@keyframes spin{to{transform:rotate(360deg)}}.analyzing p{font-size:13px;color:var(--muted)}.scan-result{background:var(--teal-soft);border-radius:20px;padding:17px;margin-bottom:13px}.scan-result.needs-input{background:#f2eee7;text-align:center}.scan-result.needs-input>b{font-size:28px}.scan-result p{font-size:13px;line-height:1.45;margin:9px 0 0}.ai-badge{font-size:10px;font-weight:850;letter-spacing:.08em;color:var(--teal)}.suggestion-pair{display:grid;grid-template-columns:1fr 1fr;gap:9px;margin-top:12px}.suggestion-pair>div{background:var(--surface);padding:12px;border-radius:14px}.suggestion-pair small,.suggestion-pair strong{display:block}.suggestion-pair small{font-size:9px;color:var(--muted)}.suggestion-pair strong{font-size:18px;margin-top:2px}.ai-prefill{background:var(--teal-soft);border-radius:16px;padding:11px 13px;margin-bottom:12px}.ai-prefill strong,.ai-prefill small{display:block}.ai-prefill small{color:var(--muted);margin-top:3px}.choice-grid{display:grid;grid-template-columns:1fr 1fr;gap:8px}.bristol{min-height:66px;border:1px solid var(--line);background:#f7f3ed;border-radius:16px;padding:10px;text-align:left;cursor:pointer}.bristol strong,.bristol span{display:block}.bristol span{font-size:11px;color:var(--muted);margin-top:2px}.bristol.selected{background:var(--ink);color:white}.bristol.selected span{color:#ddd}.field{display:block;margin-bottom:15px}.field-label{display:block;font-size:12px;font-weight:800;margin-bottom:6px}.input{width:100%;min-height:48px;border:1px solid var(--line);border-radius:14px;background:#f8f5ef;padding:10px 12px;color:var(--ink)}textarea.input{min-height:85px;resize:vertical}.range-row{display:grid;grid-template-columns:1fr 37px;gap:10px;align-items:center}.range-row input{accent-color:var(--teal)}.range-val{width:37px;height:37px;border-radius:12px;background:var(--soft);display:grid;place-items:center;font-weight:800}.photo-drop{width:100%;background:#f3eee7}.symptoms{display:flex;flex-direction:column}.alert{background:var(--red-soft);border:1px solid #eec1ba;border-radius:16px;padding:13px;color:#75261f}.alert p{margin:5px 0 0;font-size:13px;line-height:1.4} .privacy-list{display:flex;flex-direction:column;gap:4px}.privacy-item{display:grid;grid-template-columns:40px 1fr;gap:11px;padding:11px 0;border-bottom:1px solid var(--line)}.privacy-item:last-child{border-bottom:0}.privacy-item>b{width:38px;height:38px;border-radius:13px;background:var(--teal-soft);display:grid;place-items:center;color:var(--teal)}.privacy-item h3{margin-bottom:4px}.privacy-item p,.source-list p{font-size:13px;line-height:1.45;color:var(--muted);margin-bottom:5px}.source-list a{color:var(--teal)}.danger-zone{border-color:#e9c4bf}.toast{position:fixed;left:50%;bottom:96px;transform:translateX(-50%);background:var(--ink);color:white;border-radius:999px;padding:11px 16px;font-size:13px;font-weight:700;z-index:100;box-shadow:var(--shadow)} .staging-label{margin:8px auto 86px;text-align:center;color:var(--muted);font-size:10px;letter-spacing:.03em;opacity:.72} @media(min-width:560px){.app-shell{padding-inline:24px}.sleek-hero{padding-inline:10px}.choice-grid{grid-template-columns:repeat(3,1fr)}} diff --git a/tests/ci-workflow.test.js b/tests/ci-workflow.test.js index 9852962..b06ef17 100644 --- a/tests/ci-workflow.test.js +++ b/tests/ci-workflow.test.js @@ -119,3 +119,27 @@ test('the vision corpus and its refusal boundary are unit-, browser-, and releas assert.match(builder, /"vision_corpus_refusal": "passed"/); assert.match(builder, /build_vision_corpus\.py", "--verify"/); }); + +test('the pinned worker container, post-load gate, and GPU benchmark are gated end to end', async () => { + const [packageJsonText, workflow, builder] = await Promise.all([ + readFile(packagePath, 'utf8'), + readFile(workflowPath, 'utf8'), + readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'), + ]); + const packageJson = JSON.parse(packageJsonText); + + assert.match(packageJson.scripts.test, /tests\/worker-container\.test\.js/); + assert.match(packageJson.scripts.test, /tests\/worker-ready\.test\.js/); + assert.match(packageJson.scripts.test, /tests\/worker-benchmark\.test\.js/); + assert.equal(packageJson.scripts['test:worker-readiness'], 'node tests/worker-readiness.acceptance.mjs'); + assert.match(packageJson.scripts['check:syntax'], /node --check scripts\/record_worker_benchmark\.mjs/); + assert.match(packageJson.scripts['check:syntax'], /node --check tests\/worker-readiness\.acceptance\.mjs/); + for (const script of ['scripts/worker_ready.sh', 'deploy/worker/entrypoint.sh', 'deploy/worker/build_image.sh', 'deploy/worker/run_worker_container.sh']) { + assert.ok(packageJson.scripts['check:syntax'].includes(`bash -n ${script}`), `${script} must be syntax-gated`); + } + assert.match(workflow, /npm run test:worker-readiness/); + assert.match(builder, /"test:worker-readiness"/); + assert.match(builder, /"worker_container_receipts": "passed"/); + assert.match(builder, /"worker_post_load_readiness": "passed"/); + assert.match(builder, /"scripts\/worker_ready\.sh"/); +}); diff --git a/tests/mobile-capture.acceptance.mjs b/tests/mobile-capture.acceptance.mjs index 6bf4fee..d704f29 100644 --- a/tests/mobile-capture.acceptance.mjs +++ b/tests/mobile-capture.acceptance.mjs @@ -18,7 +18,9 @@ for (const viewport of viewports) { await page.route('**/api/vision-status', route => route.fulfill({ status: 200, contentType: 'application/json', - body: JSON.stringify({ enabled: true, profile: 'selfhost', providerReady: true, model: 'synthetic-test-model' }), + // The fixture mirrors the real /api/vision-status contract, which reports whether the pinned + // model was actually seen loaded (#15) alongside provider reachability. + body: JSON.stringify({ enabled: true, profile: 'selfhost', providerReady: true, modelSeen: true, model: 'synthetic-test-model' }), })); await page.route('**/api/analyze', route => { analysisRequests += 1; diff --git a/tests/release-demo.test.js b/tests/release-demo.test.js index c190d10..27112fd 100644 --- a/tests/release-demo.test.js +++ b/tests/release-demo.test.js @@ -185,3 +185,20 @@ test('release demo shows a filled-in out-of-range field failing closed even besi // The caption is a required gate, so a later release cannot silently drop the scene. assert.match(builder, /A filled-in field that is out of range still fails closed/); }); + +test('release demo shows readiness waiting for the loaded pinned model before AI is offered', async () => { + const demo = await readFile(demoPath, 'utf8'); + const builder = await readFile(builderPath, 'utf8'); + + // Issue #15: reachable is not loaded, and the recording must prove the app withholds AI. + assert.match(demo, /modelLoaded = false/); + assert.match(demo, /getByText\(\/model is not loaded yet\/i\)/); + assert.match(demo, /model-status\.loading/); + assert.match(demo, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/); + assert.match(demo, /No AI control may exist before the model is loaded/); + assert.match(demo, /Once the pinned model is loaded, the same screen offers the AI analysis again/); + // The scene is asserted in the browser, not narrated: the demo throws if the state is wrong. + assert.match(demo, /must never be reported ready/); + // The caption is a required release gate, so the scene cannot be dropped silently. + assert.match(builder, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/); +}); diff --git a/tests/sleek-chat.acceptance.mjs b/tests/sleek-chat.acceptance.mjs index fcf07d5..f273113 100644 --- a/tests/sleek-chat.acceptance.mjs +++ b/tests/sleek-chat.acceptance.mjs @@ -14,7 +14,7 @@ page.on('console', message => { if (message.type() === 'error') errors.push(mess page.on('pageerror', error => errors.push(error.message)); page.on('request', request => { if (new URL(request.url()).pathname.includes('/api/')) apiRequests.push(new URL(request.url()).pathname); }); -await page.route('**/api/vision-status', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ enabled: true, profile: 'selfhost', providerReady: true, model: 'SmolVLM2-2.2B-Instruct' }) })); +await page.route('**/api/vision-status', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ enabled: true, profile: 'selfhost', providerReady: true, modelSeen: true, model: 'SmolVLM2-2.2B-Instruct' }) })); await page.route('**/api/agent/status', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ enabled: true, configured: true, authenticated: true, mode: 'hermes-agent' }) })); let chatRequest; await page.route('**/api/agent/chat', async route => { diff --git a/tests/vision-readiness.test.js b/tests/vision-readiness.test.js index 122d0f4..57017b4 100644 --- a/tests/vision-readiness.test.js +++ b/tests/vision-readiness.test.js @@ -16,18 +16,48 @@ test('reports distinct phases so the UI never claims a worker is ready when it i assert.equal(visionReadiness(null).phase, 'checking'); assert.equal(visionReadiness({ enabled: false }).phase, 'disabled'); assert.equal(visionReadiness({ enabled: true, providerReady: false }).phase, 'offline'); - assert.equal(visionReadiness({ enabled: true, providerReady: true }).phase, 'ready'); + // The readiness contract widened in #15: answering the provider probe is no longer enough. + assert.equal(visionReadiness({ enabled: true, providerReady: true }).phase, 'loading'); + assert.equal(visionReadiness({ enabled: true, providerReady: true, modelSeen: false }).phase, 'loading'); + assert.equal(visionReadiness({ enabled: true, providerReady: true, modelSeen: true }).phase, 'ready'); }); -test('a ready provider keeps AI analysis available and does not force the manual path', () => { - const readiness = visionReadiness({ enabled: true, providerReady: true, model: 'SmolVLM2-2.2B-Instruct' }); +test('readiness waits for the pinned model to load before promising AI', () => { + // Issue #15: a worker that answers /v1/models has not necessarily loaded the model yet. + // Readiness must pass only after the pinned model is loaded, or the user is offered an + // AI control that fails after the upload. + const notLoaded = visionReadiness({ + enabled: true, + profile: 'selfhost', + providerReady: true, + modelSeen: false, + model: 'SmolVLM2-2.2B-Instruct', + }); + assert.equal(notLoaded.phase, 'loading'); + assert.equal(notLoaded.ready, false); + assert.equal(notLoaded.manualOnly, true); + assert.match(notLoaded.message, /model/i); + assert.match(notLoaded.manualLabel, /manual|without AI/i); + + const unverified = visionReadiness({ enabled: true, profile: 'selfhost', providerReady: true }); + assert.equal(unverified.ready, false, 'an unconfirmed model must never be treated as loaded'); +}); + +test('a loaded pinned model keeps AI analysis available and does not force the manual path', () => { + const readiness = visionReadiness({ enabled: true, providerReady: true, modelSeen: true, model: 'SmolVLM2-2.2B-Instruct' }); assert.equal(readiness.ready, true); assert.equal(readiness.manualOnly, false); assert.equal(readiness.manualLabel, ''); }); test('readiness never promises a symptom or diagnosis capability', () => { - for (const status of [null, { enabled: false }, { enabled: true, providerReady: false }, { enabled: true, providerReady: true }]) { + for (const status of [ + null, + { enabled: false }, + { enabled: true, providerReady: false }, + { enabled: true, providerReady: true, modelSeen: true }, + { enabled: true, providerReady: true, modelSeen: false }, + ]) { const readiness = visionReadiness(status); const copy = `${readiness.manualLabel} ${readiness.message}`; assert.doesNotMatch(copy, /diagnos|bleeding|cancer|symptom/i); diff --git a/tests/worker-benchmark.test.js b/tests/worker-benchmark.test.js new file mode 100644 index 0000000..2294592 --- /dev/null +++ b/tests/worker-benchmark.test.js @@ -0,0 +1,132 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync, spawn } from 'node:child_process'; +import { chmod, mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +const benchmark = new URL('../scripts/record_worker_benchmark.mjs', import.meta.url).pathname; +const receiptsPath = new URL('../deploy/worker/receipts.json', import.meta.url); +const ALIAS = 'SmolVLM2-2.2B-Instruct'; + +const FAKE_WORKER = `#!/usr/bin/env python3 +import json, os +from http.server import BaseHTTPRequestHandler, HTTPServer +loaded = os.environ['WORKER_MODE'] == 'loaded' +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = json.dumps({'data': [{'id': os.environ['WORKER_ALIAS']}] if loaded else []}).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(body) + def log_message(self, *_): + pass +HTTPServer(('127.0.0.1', int(os.environ['WORKER_PORT'])), Handler).serve_forever() +`; + +async function sandbox() { + const dir = await mkdtemp(join(tmpdir(), 'timmy-bench-')); + const gpuProbe = join(dir, 'nvidia-smi-stub'); + await writeFile(gpuProbe, '#!/bin/sh\nprintf "NVIDIA A40, 46068, 1234\\n"\n'); + await chmod(gpuProbe, 0o755); + const worker = join(dir, 'worker.py'); + await writeFile(worker, FAKE_WORKER); + return { dir, gpuProbe, worker, out: join(dir, 'worker-benchmark.json') }; +} + +async function startWorker(box, mode) { + const port = String(30000 + (process.pid % 2000)); + const proc = spawn('python3', [box.worker], { + env: { ...process.env, WORKER_MODE: mode, WORKER_PORT: port, WORKER_ALIAS: ALIAS }, + stdio: 'ignore', + }); + for (let attempt = 0; attempt < 50; attempt += 1) { + const probe = spawnSync('bash', ['-c', `curl --silent --output /dev/null --max-time 1 http://127.0.0.1:${port}/v1/models`], { encoding: 'utf8' }); + if (probe.status === 0) break; + await new Promise(resolve => setTimeout(resolve, 100)); + } + return { proc, port }; +} + +function run(box, port, extra = []) { + return spawnSync('node', [benchmark, '--out', box.out, '--port', port, '--alias', ALIAS, '--gpu-probe', box.gpuProbe, ...extra], { encoding: 'utf8' }); +} + +test('a benchmark refuses to record numbers it did not measure on a GPU', async () => { + const box = await sandbox(); + const { proc, port } = await startWorker(box, 'loaded'); + try { + const result = spawnSync('node', [benchmark, '--out', box.out, '--port', port, '--alias', ALIAS, '--gpu-probe', join(box.dir, 'missing-smi')], { encoding: 'utf8' }); + assert.notEqual(result.status, 0, 'measuring without a GPU probe must fail, not record zeros'); + assert.match(result.stderr, /GPU|probe|VRAM/i); + assert.equal(existsSync(box.out), false, 'no receipt may be written from an unmeasured run'); + } finally { + proc.kill('SIGKILL'); + } +}); + +test('a benchmark refuses to time a model that is not loaded', async () => { + const box = await sandbox(); + const { proc, port } = await startWorker(box, 'unloaded'); + try { + const result = run(box, port); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /not loaded|ready/i); + assert.equal(existsSync(box.out), false, 'an unloaded model must not produce a latency receipt'); + } finally { + proc.kill('SIGKILL'); + } +}); + +test('a benchmark records cold start, warm latency, VRAM, and the pins it measured', async () => { + const box = await sandbox(); + const { proc, port } = await startWorker(box, 'loaded'); + try { + const result = run(box, port, ['--warm-samples', '5', '--start-command', 'bash -c "sleep 0.3"']); + assert.equal(result.status, 0, result.stderr); + const receipt = JSON.parse(await readFile(box.out, 'utf8')); + assert.equal(receipt.status, 'measured_on_target_gpu'); + assert.equal(receipt.worker.alias, ALIAS); + assert.match(receipt.worker.baseUrl, new RegExp(`^http://127\\.0\\.0\\.1:${port}/v1$`)); + assert.match(receipt.gpu.name, /NVIDIA A40/); + assert.equal(receipt.gpu.vramUsedMiB, 1234); + assert.equal(receipt.gpu.vramTotalMiB, 46068); + assert.ok(receipt.coldStartMs >= 300, 'cold start is measured, not asserted'); + assert.equal(receipt.warmLatencyMs.samples, 5); + assert.ok(receipt.warmLatencyMs.p50 >= 0 && receipt.warmLatencyMs.p95 >= receipt.warmLatencyMs.p50); + + const receipts = JSON.parse(await readFile(receiptsPath, 'utf8')); + assert.equal(receipt.pins.llamaCppCommit, receipts.llamaCpp.commit); + assert.deepEqual(receipt.pins.modelSha256, receipts.models.map(model => model.sha256)); + } finally { + proc.kill('SIGKILL'); + } +}); + +test('a benchmark receipt carries no notes, image bytes, or user data', async () => { + const box = await sandbox(); + const { proc, port } = await startWorker(box, 'loaded'); + try { + const result = run(box, port, ['--warm-samples', '3']); + assert.equal(result.status, 0, result.stderr); + const raw = await readFile(box.out, 'utf8'); + const receipt = JSON.parse(raw); + assert.deepEqual(Object.keys(receipt).sort(), [ + 'coldStartMs', + 'gpu', + 'measuredAt', + 'pins', + 'schemaVersion', + 'status', + 'warmLatencyMs', + 'worker', + ]); + assert.doesNotMatch(raw, /data:image|base64|note|observations|symptom/i); + // Cold start that was not measured is recorded as unmeasured, never as zero. + assert.equal(receipt.coldStartMs, null); + } finally { + proc.kill('SIGKILL'); + } +}); diff --git a/tests/worker-container.test.js b/tests/worker-container.test.js new file mode 100644 index 0000000..37bfe8e --- /dev/null +++ b/tests/worker-container.test.js @@ -0,0 +1,151 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { spawnSync } from 'node:child_process'; + +const dockerfilePath = new URL('../deploy/worker/Dockerfile', import.meta.url); +const receiptsPath = new URL('../deploy/worker/receipts.json', import.meta.url); +const buildScriptPath = new URL('../deploy/worker/build_image.sh', import.meta.url); +const runScriptPath = new URL('../deploy/worker/run_worker_container.sh', import.meta.url); +const dockerignorePath = new URL('../.dockerignore', import.meta.url); +const bootstrapPath = new URL('../scripts/bootstrap_selfhost_smolvlm.sh', import.meta.url); + +async function recipe() { + const [dockerfile, receipts, buildScript, runScript, dockerignore, bootstrap] = await Promise.all([ + readFile(dockerfilePath, 'utf8'), + readFile(receiptsPath, 'utf8').then(JSON.parse), + readFile(buildScriptPath, 'utf8'), + readFile(runScriptPath, 'utf8'), + readFile(dockerignorePath, 'utf8'), + readFile(bootstrapPath, 'utf8'), + ]); + return { dockerfile, receipts, buildScript, runScript, dockerignore, bootstrap }; +} + +function bootstrapDefault(script, name) { + const match = new RegExp(`^${name}="\\$\\{[A-Z0-9_]*:-([^}]+)\\}"$`, 'm').exec(script); + assert.ok(match, `bootstrap script must declare a default for ${name}`); + return match[1]; +} + +test('the worker image pins llama.cpp and both model hashes to the bootstrap receipts', async () => { + const { dockerfile, receipts, bootstrap } = await recipe(); + const commit = bootstrapDefault(bootstrap, 'LLAMA_CPP_COMMIT'); + const modelSha = bootstrapDefault(bootstrap, 'MODEL_SHA256'); + const mmprojSha = bootstrapDefault(bootstrap, 'MMPROJ_SHA256'); + + // Receipts are the runtime/CI source of truth and must not drift from the host bootstrap. + assert.equal(receipts.llamaCpp.commit, commit); + assert.equal(receipts.llamaCpp.repository, 'https://github.com/ggml-org/llama.cpp'); + assert.deepEqual(receipts.models.map(model => model.sha256), [modelSha, mmprojSha]); + assert.deepEqual(receipts.models.map(model => model.file), [ + 'SmolVLM2-2.2B-Instruct-Q4_K_M.gguf', + 'mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf', + ]); + assert.match(receipts.llamaCpp.commit, /^[0-9a-f]{40}$/); + for (const model of receipts.models) assert.match(model.sha256, /^[0-9a-f]{64}$/); + + // The image builds the same pinned revision: the build args default to the receipts. + assert.match(dockerfile, new RegExp(`^ARG LLAMA_CPP_COMMIT=${commit}$`, 'm')); + assert.match(dockerfile, new RegExp(`^ARG MODEL_SHA256=${modelSha}$`, 'm')); + assert.match(dockerfile, new RegExp(`^ARG MMPROJ_SHA256=${mmprojSha}$`, 'm')); + assert.match(dockerfile, /git fetch --depth 1 origin "\$\{LLAMA_CPP_COMMIT\}"/); + assert.match(dockerfile, /git checkout --detach FETCH_HEAD/); +}); + +test('the worker image uses digest-pinned bases and refuses floating model sources', async () => { + const { dockerfile, receipts } = await recipe(); + + const froms = dockerfile.split('\n').filter(line => line.startsWith('FROM ')); + assert.equal(froms.length, 2, 'the recipe is a build stage and a runtime stage'); + for (const line of froms) assert.match(line, /@sha256:[0-9a-f]{64}/, `base must be digest-pinned: ${line}`); + // The recorded receipts and the recipe must name the same immutable bases. + const digests = froms.map(line => /@(sha256:[0-9a-f]{64})/.exec(line)[1]); + assert.deepEqual(digests, [receipts.baseImages.build.digest, receipts.baseImages.runtime.digest]); + assert.equal(receipts.baseImages.build.reference, 'nvidia/cuda:12.4.1-devel-ubuntu22.04'); + assert.equal(receipts.baseImages.runtime.reference, 'nvidia/cuda:12.4.1-runtime-ubuntu22.04'); + assert.doesNotMatch(dockerfile, /:latest|ggml-org\/llama\.cpp@?main|refs\/heads\/(?:main|master)/); + // Weights are never baked in; the model arrives as a read-only mount. + assert.doesNotMatch(dockerfile, /^COPY .*\.gguf/m); + assert.doesNotMatch(dockerfile, /^ADD /m); + assert.match(dockerfile, /\/models/); +}); + +test('the worker image runs unprivileged with no default credentials and a post-load healthcheck', async () => { + const { dockerfile } = await recipe(); + // Directives may be wrapped with a line continuation; assert on the joined directive text. + const directives = dockerfile.replace(/\\\n\s*/g, ' '); + + const user = /^USER (?!root)(\S+)$/m.exec(dockerfile); + assert.ok(user, 'the runtime stage must drop to an unprivileged user'); + assert.match(user[1], /^\d+:\d+$/, 'use a numeric uid:gid so the image needs no passwd entry'); + assert.doesNotMatch(dockerfile, /^ENV .*(?:API_KEY|TOKEN|PASSWORD|SECRET)[ =]/im); + // Health must gate on the loaded model, so the container is not "healthy" while it cannot answer. + assert.match(directives, /^HEALTHCHECK .*timmy-worker-ready/m); + assert.match(directives, /--start-period=\d+s/); + assert.match(directives, /--host", "127\.0\.0\.1"/, 'the health probe runs on the container loopback'); + // The container listens inside its own network namespace; the run recipe, not the image, decides + // exposure and publishes on loopback only (asserted separately). + assert.match(dockerfile, /--host 0\.0\.0\.0|TIMMY_MODEL_HOST=0\.0\.0\.0/); +}); + +test('the build refuses a floating or missing pin before it reaches docker', async () => { + const floating = spawnSync('bash', [buildScriptPath.pathname, '--print-plan'], { + encoding: 'utf8', + env: { ...process.env, TIMMY_LLAMA_CPP_COMMIT: 'latest' }, + }); + assert.notEqual(floating.status, 0, 'a floating llama.cpp ref must be rejected'); + assert.match(floating.stderr, /pin|40-hex|floating/i); + + const missingSha = spawnSync('bash', [buildScriptPath.pathname, '--print-plan'], { + encoding: 'utf8', + env: { ...process.env, TIMMY_MODEL_SHA256: '' }, + }); + assert.notEqual(missingSha.status, 0, 'a missing model hash must be rejected'); + + const plan = spawnSync('bash', [buildScriptPath.pathname, '--print-plan'], { encoding: 'utf8' }); + assert.equal(plan.status, 0, plan.stderr); + assert.match(plan.stdout, /llama\.cpp commit: [0-9a-f]{40}/); + assert.match(plan.stdout, /SmolVLM2-2\.2B-Instruct-Q4_K_M\.gguf sha256: [0-9a-f]{64}/); + assert.match(plan.stdout, /bind address: 127\.0\.0\.1:8080/); + assert.match(plan.stdout, /publish: 127\.0\.0\.1:8080/); +}); + +test('the run recipe publishes on loopback only and drops every privilege it can', async () => { + const { runScript } = await recipe(); + + assert.match(runScript, /127\.0\.0\.1:\$\{?PORT/); + assert.match(runScript, /--read-only/); + assert.match(runScript, /--cap-drop[= ]ALL/); + assert.match(runScript, /no-new-privileges/); + assert.match(runScript, /--security-opt/); + assert.doesNotMatch(runScript, /--privileged/); + assert.doesNotMatch(runScript, /-p "\$PORT:/); + assert.doesNotMatch(runScript, /\b0\.0\.0\.0\b/); + // The model mount is read-only, and no credential has a default value. + assert.match(runScript, /:ro\b/); + assert.doesNotMatch(runScript, /(?:API_KEY|TOKEN|PASSWORD)=[^\s"']/); +}); + +test('the run recipe refuses to publish the worker anywhere but loopback', async () => { + const hostile = spawnSync('bash', [runScriptPath.pathname, '--print-plan'], { + encoding: 'utf8', + env: { ...process.env, TIMMY_WORKER_PUBLISH_HOST: '0.0.0.0' }, + }); + assert.notEqual(hostile.status, 0, 'a non-loopback publish must be refused'); + assert.match(hostile.stderr, /loopback|127\.0\.0\.1/i); + + const plan = spawnSync('bash', [runScriptPath.pathname, '--print-plan'], { encoding: 'utf8' }); + assert.equal(plan.status, 0, plan.stderr); + assert.match(plan.stdout, /publish: 127\.0\.0\.1:\d+/); +}); + +test('the build context excludes weights, user images, releases, and secrets', async () => { + const { dockerignore } = await recipe(); + + for (const pattern of ['*.gguf', 'artifacts/', 'video/', 'models/', '.git/', 'node_modules/', '.env', '.env.*']) { + assert.ok(dockerignore.split('\n').includes(pattern), `dockerignore must exclude ${pattern}`); + } + // Committed synthetic fixtures are fine, but a real medical corpus directory is never buildable. + assert.doesNotMatch(dockerignore, /^!.*\.gguf$/m); +}); diff --git a/tests/worker-readiness.acceptance.mjs b/tests/worker-readiness.acceptance.mjs new file mode 100644 index 0000000..2326489 --- /dev/null +++ b/tests/worker-readiness.acceptance.mjs @@ -0,0 +1,91 @@ +import { chromium } from 'playwright'; +import assert from 'node:assert/strict'; +import { mkdir } from 'node:fs/promises'; + +// Issue #15: readiness must pass only after the pinned model is loaded. A worker that answers +// /v1/models is reachable, but reachable is not loaded — the app must not offer an AI control +// that fails after the upload. +await mkdir('artifacts', { recursive: true }); +const browser = await chromium.launch({ headless: true }); +const context = await browser.newContext({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 2, serviceWorkers: 'block' }); +const page = await context.newPage(); +const errors = []; +page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); +page.on('pageerror', error => errors.push(error.message)); +let modelLoaded = false; +let analysisRequests = 0; +await page.route('**/api/vision-status', route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + enabled: true, + profile: 'selfhost', + processor: 'self-hosted', + model: 'SmolVLM2-2.2B-Instruct', + providerReady: true, + modelSeen: modelLoaded, + }), +})); +await page.route('**/api/analyze', route => { + analysisRequests += 1; + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + status: 'suggestion', + isStool: true, + bristolType: 4, + color: 'brown', + confidence: 0.83, + imageQuality: 'good', + observations: 'Smooth, formed appearance.', + warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.', + }), + }); +}); + +await page.goto('http://127.0.0.1:4173', { waitUntil: 'networkidle' }); +await page.evaluate(() => localStorage.clear()); +await page.reload({ waitUntil: 'networkidle' }); + +// 1. Worker reachable, pinned model not loaded: no AI promise, no dead control, one action to manual. +await page.locator('[data-scan]').click(); +await page.getByText(/model is not loaded/i).waitFor(); +// Visual acceptance: the not-loaded state must be visually distinct from the ready and offline +// states, not an unstyled div that reads like either one of them. +const loadingStatus = page.locator('.model-status.loading'); +assert.equal(await loadingStatus.count(), 1, 'the not-loaded state needs its own status treatment'); +assert.equal(await loadingStatus.evaluate(el => getComputedStyle(el).backgroundColor), 'rgb(253, 243, 216)'); +assert.equal(await page.getByText(/Self-hosted model ready/i).count(), 0, 'an unloaded model must never be reported ready'); +assert.equal(await page.locator('#analyze-photo').count(), 0, 'no analyze control before the model is loaded'); +await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg'); +await page.getByText(/Ready for your review/i).waitFor(); +assert.equal(await page.getByText(/model is not loaded/i).count() > 0, true, 'the ready step still states why AI is unavailable'); +assert.equal(await page.locator('.consent-card').count(), 0, 'no consent is requested while the model is not loaded'); +assert.equal(await page.locator('#analyze-photo').count(), 0, 'no dead analyze control on the ready step'); +const fallback = page.locator('#manual-from-scan'); +assert.equal(await fallback.isVisible(), true); +assert.match(await fallback.innerText(), /without AI|manual/i); +await fallback.click(); +assert.equal(await page.getByText('Pick the closest form').isVisible(), true, 'one action reaches manual Bristol entry'); +assert.equal(analysisRequests, 0, 'nothing is uploaded before the pinned model is loaded'); +await page.screenshot({ path: 'artifacts/worker-model-loading-mobile.png' }); +await page.locator('#close-sheet').click(); + +// 2. The pinned model loads: the same flow now offers consent and analysis, and analysis works. +modelLoaded = true; +await page.locator('[data-scan]').click(); +await page.getByText(/Self-hosted model ready/i).waitFor(); +await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg'); +await page.getByText(/Ready for your review/i).waitFor(); +assert.equal(await page.locator('.consent-card').isVisible(), true, 'consent returns once the model is loaded'); +await page.locator('#ai-consent').check(); +await page.locator('#analyze-photo').click(); +await page.getByText(/83% confidence/i).waitFor(); +assert.equal(analysisRequests, 1, 'analysis is offered exactly when the model is loaded'); +await page.screenshot({ path: 'artifacts/worker-model-loaded-mobile.png' }); + +assert.deepEqual(errors, []); +await context.close(); +await browser.close(); +console.log('PASS readiness waits for the loaded pinned model: no AI control before load, full AI path after load'); diff --git a/tests/worker-ready.test.js b/tests/worker-ready.test.js new file mode 100644 index 0000000..1a67292 --- /dev/null +++ b/tests/worker-ready.test.js @@ -0,0 +1,107 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +const gate = new URL('../scripts/worker_ready.sh', import.meta.url).pathname; +const bootstrap = new URL('../scripts/bootstrap_selfhost_smolvlm.sh', import.meta.url); +const ALIAS = 'SmolVLM2-2.2B-Instruct'; + +const FAKE_SERVER = `#!/usr/bin/env python3 +import json, os, sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +mode = os.environ['WORKER_MODE'] +counter_path = os.environ['WORKER_COUNTER'] +count = int(open(counter_path).read() or '0') if os.path.exists(counter_path) else 0 + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + global count + count += 1 + open(counter_path, 'w').write(str(count)) + loaded = mode == 'loaded' or (mode == 'flip' and count > 2) + body = json.dumps({'data': [{'id': os.environ['WORKER_ALIAS']}] if loaded else []}).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(body) + def log_message(self, *_): + pass + +HTTPServer(('127.0.0.1', int(os.environ['WORKER_PORT'])), Handler).serve_forever() +`; + +async function withWorker(mode, run) { + const sandbox = await mkdtemp(join(tmpdir(), 'timmy-worker-ready-')); + const counter = join(sandbox, 'count.txt'); + await writeFile(counter, '0'); + const server = join(sandbox, 'server.py'); + await writeFile(server, FAKE_SERVER); + const port = String(20000 + (process.pid % 4000)); + const { spawn } = await import('node:child_process'); + const proc = spawn('python3', [server], { + env: { ...process.env, WORKER_MODE: mode, WORKER_COUNTER: counter, WORKER_PORT: port, WORKER_ALIAS: ALIAS }, + stdio: 'ignore', + }); + // Wait for the socket to accept connections before probing. + for (let attempt = 0; attempt < 50; attempt += 1) { + const probe = spawnSync('bash', ['-c', `curl --silent --output /dev/null --max-time 1 http://127.0.0.1:${port}/v1/models`], { encoding: 'utf8' }); + if (probe.status === 0) break; + await new Promise(resolve => setTimeout(resolve, 100)); + } + try { + return await run(port); + } finally { + proc.kill('SIGKILL'); + } +} + +function gateRun(port, extra = []) { + return spawnSync('bash', [gate, '--port', port, '--alias', ALIAS, ...extra], { encoding: 'utf8' }); +} + +test('readiness passes once the pinned model is actually served', async () => { + await withWorker('loaded', async port => { + const result = gateRun(port); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, new RegExp(`${ALIAS} is loaded`)); + }); +}); + +test('a reachable worker with the model still loading is not ready', async () => { + await withWorker('unloaded', async port => { + const result = gateRun(port); + assert.equal(result.status, 3, 'reachable-but-unloaded must be its own failure, not a network error'); + assert.match(result.stderr, /not loaded yet/i); + }); +}); + +test('an unreachable worker is reported as unreachable', async () => { + const result = spawnSync('bash', [gate, '--port', '9', '--alias', ALIAS], { encoding: 'utf8' }); + assert.equal(result.status, 2); + assert.match(result.stderr, /not reachable/i); +}); + +test('readiness never probes anything but loopback', () => { + const result = spawnSync('bash', [gate, '--host', '10.0.0.5', '--port', '8080'], { encoding: 'utf8' }); + assert.equal(result.status, 64); + assert.match(result.stderr, /loopback/i); +}); + +test('readiness can wait for the model load it is gating', async () => { + await withWorker('flip', async port => { + const started = Date.now(); + const result = gateRun(port, ['--wait', '15']); + assert.equal(result.status, 0, result.stderr); + assert.ok(Date.now() - started >= 300, 'the gate must actually wait rather than pass on the first probe'); + }); +}); + +test('the bootstrap exposes the same post-load gate instead of duplicating it', async () => { + const script = await readFile(bootstrap, 'utf8'); + assert.match(script, /worker_ready\.sh/); + assert.match(script, /^ ready\)$/m); +});