Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d670360560 | ||
|
|
5744969faf | ||
|
|
46f961668f | ||
|
|
9cf23df448 | ||
|
|
5c759078c4 | ||
|
|
a1777a72e8 | ||
|
|
d9f82299d0 | ||
|
|
58b7feddc8 | ||
|
|
d027a1e791 | ||
|
|
503efaf959 | ||
|
|
29ea045671 |
No files matched your search
@@ -0,0 +1,12 @@
|
||||
# The worker image is built from source; weights, releases, and private data never enter it.
|
||||
.git/
|
||||
node_modules/
|
||||
models/
|
||||
*.gguf
|
||||
artifacts/
|
||||
video/
|
||||
research/source-pages/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.env
|
||||
.env.*
|
||||
@@ -55,6 +55,9 @@ jobs:
|
||||
npm run test:manual-fallback
|
||||
npm run test:provisional
|
||||
npm run test:followup
|
||||
npm run test:vision-refusal
|
||||
npm run test:worker-readiness
|
||||
npm run test:pattern-summary
|
||||
- name: Dependency audit
|
||||
run: npm audit --audit-level=high
|
||||
- name: Syntax checks
|
||||
|
||||
@@ -97,6 +97,7 @@ npm run test:ui # server required on port 4173
|
||||
npm run test:photo # mocked positive suggestion through the real browser UX
|
||||
npm run test:sleek # simplified 390x844 shell + free-text Hermes chat
|
||||
npm run test:manual-fallback # vision-offline manual path + in-session draft recovery
|
||||
npm run test:vision-refusal # corpus refusals through a real server and a real browser
|
||||
npm audit --audit-level=high
|
||||
```
|
||||
|
||||
@@ -115,8 +116,10 @@ npm audit --audit-level=high
|
||||
- Red-flag symptom escalation
|
||||
- A short user-reported follow-up after a hard/lumpy (Bristol 1–2) log — difficulty passing, soreness, visible blood, truly black/tarry versus dark brown, and hydration. Mild reports get concise non-diagnostic self-care that never claims a photo ruled out bleeding; a confirmed red flag sets the same deterministic escalation the manual checkboxes use
|
||||
- Manual fallback that never depends on AI: when the vision worker is offline or disabled, no AI or consent control is offered and manual Bristol entry stays one action away
|
||||
- A committed CC0 synthetic vision corpus that continuously proves every reply is schema-valid or fails closed. A schema-valid reply with no readable form or colour now **abstains** ("No confident match") instead of surfacing as a worker failure, and unparsable or out-of-range replies return plain copy — "No suggestion was applied. Continue manually, or try a clearer photo." — with no provider text, image bytes, or credentials echoed. See [tests/fixtures/vision-corpus](tests/fixtures/vision-corpus/README.md)
|
||||
- Unsaved logging drafts are retained in the session with an explicit start over, and the service worker never caches analysis requests or image bodies
|
||||
- Local browser ledger, calendar, pattern summary, JSON portability, and delete-all
|
||||
- Explainable pattern summaries: the home insight card reports the confirmed-log count and form groupings for the last 28 days, every line names the exact confirmed records behind it, following a citation opens those records, and a window below the minimum says it is too sparse instead of describing a trend. No line explains a cause, rules on food, or offers a health verdict
|
||||
- Installable/offline PWA shell for the manual and saved-ledger paths
|
||||
|
||||
## What is deliberately not faked
|
||||
|
||||
@@ -4,6 +4,7 @@ import { createDraft, draftNotice, isDraftDirty } from './src/draft.js';
|
||||
import { changeVisualField, confirmSuggestion, createSuggestionState, provenanceCopy } from './src/suggestion.js';
|
||||
import { HARD_STOOL_QUESTIONS, createFollowUpAnswers, evaluateHardStoolFollowUp, needsHardStoolFollowUp } from './src/followup.js';
|
||||
import { visionReadiness } from './src/vision-readiness.js';
|
||||
import { buildPatternSummary } from './src/summary.js';
|
||||
|
||||
const runtimeConfig = {
|
||||
basePath: document.querySelector('meta[name="timmy-base-path"]')?.content || '/',
|
||||
@@ -96,6 +97,7 @@ function bindGlobal(){
|
||||
document.querySelectorAll('[data-view]').forEach(btn=>btn.onclick=()=>{view=btn.dataset.view;render()});
|
||||
document.querySelectorAll('[data-log]').forEach(btn=>btn.onclick=openLogger);
|
||||
document.querySelectorAll('[data-scan]').forEach(btn=>btn.onclick=openPhotoFirst);
|
||||
document.querySelectorAll('[data-statement-link]').forEach(btn=>btn.onclick=()=>showPatternRecords(btn.dataset.statementLink));
|
||||
}
|
||||
|
||||
function recentList(limit=5){
|
||||
@@ -105,9 +107,35 @@ function recentList(limit=5){
|
||||
function thisWeek(){const now=Date.now(),week=7*864e5;return entries.filter(e=>now-new Date(e.occurredAt).getTime()<week).length}
|
||||
function currentStreak(){const dates=new Set(entries.map(e=>e.occurredAt.slice(0,10)));let n=0,d=new Date();while(dates.has(d.toISOString().slice(0,10))){n++;d.setDate(d.getDate()-1)}return n}
|
||||
|
||||
// Issue #34: the home insight card is explainable. Every line says which confirmed records it was
|
||||
// built from, following a citation shows exactly those records, and a sparse window says so
|
||||
// instead of describing a trend the data cannot support.
|
||||
function patternSummaryCard(){
|
||||
const summary=buildPatternSummary(entries);
|
||||
const logs=`${summary.records.length} confirmed log${summary.records.length===1?'':'s'}`;
|
||||
const lines=summary.insufficient
|
||||
? `<p class="fine" data-summary-insufficient>${esc(summary.copy)}</p>`
|
||||
: `<ul class="pattern-statements">${summary.statements.map(statement=>`<li data-statement="${statement.id}" data-records="${statement.entryIds.length}">${esc(statement.text)} <button class="pattern-link" data-statement-link="${statement.id}">${statement.entryIds.length} record${statement.entryIds.length===1?'':'s'} ›</button></li>`).join('')}</ul>`;
|
||||
return `<section class="section"><div class="insight-card pattern-card" data-pattern-summary role="group" aria-label="Explainable pattern summary"><div class="timmy-orb"><img src="${appPath('assets/timmy.svg')}" alt=""></div><div><span class="eyebrow">Timmy noticed</span><p class="pattern-window" data-summary-window>${esc(summary.window.label)} · ${logs}</p>${lines}<p class="fine pattern-boundary">${esc(summary.boundary)}</p></div></div></section>`;
|
||||
}
|
||||
function showPatternRecords(statementId){
|
||||
const summary=buildPatternSummary(entries);
|
||||
const statement=summary.statements.find(item=>item.id===statementId);
|
||||
if(!statement)return;
|
||||
const byId=new Map(summary.records.map(record=>[record.id,record]));
|
||||
const rows=statement.entryIds.map(id=>byId.get(id)).filter(Boolean)
|
||||
.map(record=>`<article class="entry" data-summary-record data-entry-id="${esc(record.id)}"><div class="type-dot">T${record.bristolType}</div><div><strong>${formatDate(record.occurredAt)}</strong><span>Type ${record.bristolType} · ${esc(record.bucket)}</span></div></article>`).join('');
|
||||
document.querySelector('.sheet-backdrop')?.remove();
|
||||
const wrap=document.createElement('div');wrap.className='sheet-backdrop';
|
||||
wrap.innerHTML=`<section class="sheet" role="dialog" aria-modal="true" aria-labelledby="pattern-records-title"><div class="sheet-handle"></div><div class="sheet-header"><div><span class="eyebrow">Pattern source</span><h2 id="pattern-records-title">${esc(statement.text)}</h2></div><button class="icon-btn" id="pattern-records-close" aria-label="Close">×</button></div><div data-summary-records><p class="fine">These are the confirmed logs this line was built from.</p><div class="entries-card">${rows}</div><p class="fine">Window: ${esc(summary.window.label)}. Change or delete any of these logs and this line changes with them.</p></div></section>`;
|
||||
document.body.append(wrap);
|
||||
document.querySelector('#pattern-records-close').onclick=()=>wrap.remove();
|
||||
wrap.onclick=e=>{if(e.target===wrap)wrap.remove()};
|
||||
}
|
||||
|
||||
function home(){
|
||||
const latest=entries.slice().sort((a,b)=>new Date(b.occurredAt)-new Date(a.occurredAt))[0];
|
||||
shell(`<main class="home-main"><section class="hero sleek-hero"><span class="eyebrow">Your intelligent pooping pal</span><h1>Log it.<br>Learn the pattern.</h1><p class="lead">Start with a photo. Timmy suggests visible form and color; you review everything before it is saved.</p><button class="capture-cta btn-primary" data-scan><span class="capture-icon" aria-hidden="true">◉</span><span><strong>Start photo log</strong><small>Private, guided, about 10 seconds</small></span><i aria-hidden="true">→</i></button><button class="text-action" data-log>Log manually instead</button></section><section class="glance" aria-label="Journal at a glance"><div><strong>${thisWeek()}</strong><span>this week</span></div><i></i><div><strong>${currentStreak()}</strong><span>day streak</span></div><i></i><div><strong>${entries.length}</strong><span>all logs</span></div></section><section class="section"><div class="insight-card"><div class="timmy-orb"><img src="${appPath('assets/timmy.svg')}" alt=""></div><div><span class="eyebrow">Timmy noticed</span><p>${esc(buildTimmySummary(entries))}</p></div></div></section><section class="section recent-section"><div class="section-head"><div><span class="eyebrow">Latest</span><h2>${latest?'Recent log':'Ready when you are'}</h2></div>${latest?'<button class="text-action compact" data-view="calendar">View journal</button>':''}</div>${latest?`<div class="latest-card"><div class="type-dot">T${latest.bristolType}</div><div><strong>${formatDate(latest.occurredAt)}</strong><span>${esc(latest.color)} · ${bucketForBristolType(latest.bristolType)}</span></div><span class="chevron">›</span></div>`:'<p class="fine">One quick, confirmed entry is enough to begin seeing your pattern.</p>'}</section></main>`);
|
||||
shell(`<main class="home-main"><section class="hero sleek-hero"><span class="eyebrow">Your intelligent pooping pal</span><h1>Log it.<br>Learn the pattern.</h1><p class="lead">Start with a photo. Timmy suggests visible form and color; you review everything before it is saved.</p><button class="capture-cta btn-primary" data-scan><span class="capture-icon" aria-hidden="true">◉</span><span><strong>Start photo log</strong><small>Private, guided, about 10 seconds</small></span><i aria-hidden="true">→</i></button><button class="text-action" data-log>Log manually instead</button></section><section class="glance" aria-label="Journal at a glance"><div><strong>${thisWeek()}</strong><span>this week</span></div><i></i><div><strong>${currentStreak()}</strong><span>day streak</span></div><i></i><div><strong>${entries.length}</strong><span>all logs</span></div></section>${patternSummaryCard()}<section class="section recent-section"><div class="section-head"><div><span class="eyebrow">Latest</span><h2>${latest?'Recent log':'Ready when you are'}</h2></div>${latest?'<button class="text-action compact" data-view="calendar">View journal</button>':''}</div>${latest?`<div class="latest-card"><div class="type-dot">T${latest.bristolType}</div><div><strong>${formatDate(latest.occurredAt)}</strong><span>${esc(latest.color)} · ${bucketForBristolType(latest.bristolType)}</span></div><span class="chevron">›</span></div>`:'<p class="fine">One quick, confirmed entry is enough to begin seeing your pattern.</p>'}</section></main>`);
|
||||
}
|
||||
|
||||
function calendar(){
|
||||
@@ -176,6 +204,9 @@ async function loadVisionStatus(){
|
||||
}
|
||||
function visionStatusHtml(){
|
||||
if(!visionStatus)return '<div class="model-status checking">◌ Checking the vision worker…</div>';
|
||||
// Reachable is not loaded (#15): a worker answering before its pinned model is loaded must not be
|
||||
// shown as ready, because the AI controls are withheld until the model can actually answer.
|
||||
if(visionStatus.providerReady&&visionStatus.modelSeen!==true)return '<div class="model-status loading">◌ Vision worker up · the model is not loaded yet · manual logging is available</div>';
|
||||
if(visionStatus.profile==='selfhost'&&visionStatus.providerReady)return `<div class="model-status ready">● Self-hosted model ready · ${esc(visionStatus.model)}</div>`;
|
||||
if(visionStatus.providerReady)return `<div class="model-status ready">● Vision provider ready · ${esc(visionStatus.model)}</div>`;
|
||||
return '<div class="model-status offline">○ Vision worker offline · manual logging is still available</div>';
|
||||
@@ -192,7 +223,7 @@ function showPhotoFirst(mode='pick',error=''){
|
||||
photoFirstMode=mode;
|
||||
document.querySelector('.sheet-backdrop')?.remove();const wrap=document.createElement('div');wrap.className='sheet-backdrop';wrap.innerHTML=`<section class="sheet scan-sheet" role="dialog" aria-modal="true" aria-labelledby="scan-title"><div class="sheet-handle"></div><div class="sheet-header"><div><span class="eyebrow">Photo-first log</span><h2 id="scan-title">${mode==='result'?'Review Timmy’s suggestion':mode==='analyzing'?'Analyzing privately':'Start with the camera'}</h2></div><button class="icon-btn" id="close-sheet" aria-label="Close">×</button></div>${photoFirstBody(mode,error)}</section>`;document.body.append(wrap);document.querySelector('#close-sheet').onclick=()=>wrap.remove();wrap.onclick=e=>{if(e.target===wrap)wrap.remove()};
|
||||
document.querySelectorAll('#camera-photo,#gallery-photo').forEach(file=>{file.onchange=handleAiPhoto;file.addEventListener('cancel',()=>showPhotoFirst('pick','Camera or photo picker closed. If permission was denied, allow camera access in browser settings, choose from the gallery, or continue without AI.'))});
|
||||
const consent=document.querySelector('#ai-consent'),analyze=document.querySelector('#analyze-photo');if(consent&&analyze)consent.onchange=()=>analyze.disabled=!consent.checked||visionStatus?.providerReady===false;if(analyze)analyze.onclick=runAiAnalysis;
|
||||
const consent=document.querySelector('#ai-consent'),analyze=document.querySelector('#analyze-photo');if(consent&&analyze)consent.onchange=()=>analyze.disabled=!consent.checked||!visionReadiness(visionStatus).ready;if(analyze)analyze.onclick=runAiAnalysis;
|
||||
document.querySelector('#retake-photo')?.addEventListener('click',()=>{form.photoDataUrl='';photoHint='';resetSuggestion();showPhotoFirst('pick')});
|
||||
document.querySelector('#manual-from-scan')?.addEventListener('click',()=>{resetSuggestion();showLogStep(1)});
|
||||
document.querySelector('#use-suggestion')?.addEventListener('click',()=>{suggestionState=confirmSuggestion(suggestionState);form=mergeVisualSuggestion(form,aiSuggestion);form.typeChosen=true;showLogStep(1)});
|
||||
|
||||
|
After Width: | Height: | Size: 394 KiB |
|
After Width: | Height: | Size: 380 KiB |
|
After Width: | Height: | Size: 256 KiB |
|
After Width: | Height: | Size: 249 KiB |
|
After Width: | Height: | Size: 235 KiB |
|
After Width: | Height: | Size: 235 KiB |
|
After Width: | Height: | Size: 239 KiB |
|
After Width: | Height: | Size: 235 KiB |
|
After Width: | Height: | Size: 249 KiB |
|
After Width: | Height: | Size: 207 KiB |
@@ -0,0 +1,67 @@
|
||||
# Timmy self-hosted vision worker (issue #15).
|
||||
#
|
||||
# Reproducible by construction: the llama.cpp revision and both model hashes are pinned here and in
|
||||
# deploy/worker/receipts.json, and tests/worker-container.test.js fails if either drifts from
|
||||
# scripts/bootstrap_selfhost_smolvlm.sh on the host.
|
||||
#
|
||||
# The weights are deliberately NOT baked into the image. They are mounted read-only at /models and
|
||||
# hash-verified against the pinned receipts by the entrypoint before the server starts.
|
||||
|
||||
ARG LLAMA_CPP_COMMIT=6d05498314db1b57f81c271080018aa2d0b89be9
|
||||
ARG MODEL_SHA256=0cf76814555b8665149075b74ab6b5c1d428ea1d3d01c1918c12012e8d7c9f58
|
||||
ARG MMPROJ_SHA256=ae07ea1facd07dd3230c4483b63e8cda96c6944ad2481f33d531f79e892dd024
|
||||
|
||||
# Digest-pinned bases, recorded in deploy/worker/receipts.json. A tag alone can be repointed, so
|
||||
# the digest is what makes this build reproducible.
|
||||
FROM nvidia/cuda:12.4.1-devel-ubuntu22.04@sha256:da6791294b0b04d7e65d87b7451d6f2390b4d36225ab0701ee7dfec5769829f5 AS build
|
||||
|
||||
ARG LLAMA_CPP_COMMIT
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends git ca-certificates cmake build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /src
|
||||
RUN git init llama.cpp \
|
||||
&& cd llama.cpp \
|
||||
&& git remote add origin https://github.com/ggml-org/llama.cpp \
|
||||
&& git fetch --depth 1 origin "${LLAMA_CPP_COMMIT}" \
|
||||
&& git checkout --detach FETCH_HEAD
|
||||
RUN cmake -S llama.cpp -B llama.cpp/build -DCMAKE_BUILD_TYPE=Release -DLLAMA_CURL=OFF -DGGML_CUDA=ON \
|
||||
&& cmake --build llama.cpp/build --parallel "$(nproc)" --target llama-server
|
||||
|
||||
FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04@sha256:517da2300c184c9999ec203c2665244bdebd3578d12fcc7065e83667932643d9 AS runtime
|
||||
|
||||
ARG MODEL_SHA256
|
||||
ARG MMPROJ_SHA256
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install --yes --no-install-recommends curl ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& mkdir -p /models
|
||||
|
||||
COPY --from=build /src/llama.cpp/build/bin/ /usr/local/bin/
|
||||
COPY deploy/worker/entrypoint.sh /usr/local/bin/timmy-worker-entrypoint
|
||||
COPY scripts/worker_ready.sh /usr/local/bin/timmy-worker-ready
|
||||
RUN chmod 0755 /usr/local/bin/timmy-worker-entrypoint /usr/local/bin/timmy-worker-ready && ldconfig
|
||||
|
||||
# No credential is configured: the worker is a private service the app reaches over host loopback.
|
||||
# 0.0.0.0 here is the address inside this container's own network namespace; the run recipe decides
|
||||
# exposure and publishes on 127.0.0.1 only.
|
||||
ENV TIMMY_MODEL_DIR=/models \
|
||||
TIMMY_MODEL_FILE=/models/SmolVLM2-2.2B-Instruct-Q4_K_M.gguf \
|
||||
TIMMY_MMPROJ_FILE=/models/mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf \
|
||||
TIMMY_MODEL_SHA256=${MODEL_SHA256} \
|
||||
TIMMY_MMPROJ_SHA256=${MMPROJ_SHA256} \
|
||||
TIMMY_MODEL_ALIAS=SmolVLM2-2.2B-Instruct \
|
||||
TIMMY_MODEL_HOST=0.0.0.0 \
|
||||
TIMMY_MODEL_PORT=8080
|
||||
|
||||
USER 10001:10001
|
||||
EXPOSE 8080
|
||||
|
||||
# Health is the post-load gate: the worker is not healthy until the pinned model is being served,
|
||||
# so an orchestrator cannot route to a worker that cannot answer. start-period covers a cold load.
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=600s --retries=3 \
|
||||
CMD ["/usr/local/bin/timmy-worker-ready", "--host", "127.0.0.1", "--port", "8080", "--alias", "SmolVLM2-2.2B-Instruct"]
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/timmy-worker-entrypoint"]
|
||||
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the pinned Timmy worker image (issue #15).
|
||||
#
|
||||
# Every pin is checked before docker is invoked, so an unpinned or floating reference fails here
|
||||
# rather than producing an image nobody can reproduce.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
RECEIPTS="$REPO_ROOT/deploy/worker/receipts.json"
|
||||
IMAGE="${TIMMY_WORKER_IMAGE:-timmy-vision-worker}"
|
||||
TAG="${TIMMY_WORKER_TAG:-$(date -u +%Y%m%d)}"
|
||||
PUBLISH_HOST="${TIMMY_WORKER_PUBLISH_HOST:-127.0.0.1}"
|
||||
PORT="${TIMMY_MODEL_PORT:-8080}"
|
||||
|
||||
receipt_field() {
|
||||
python3 - "$RECEIPTS" "$1" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
document = json.load(open(sys.argv[1]))
|
||||
value = document
|
||||
for part in sys.argv[2].split('.'):
|
||||
value = value[int(part)] if part.isdigit() else value[part]
|
||||
print(value)
|
||||
PY
|
||||
}
|
||||
|
||||
# An explicit value still wins, but an empty one is treated as an invalid pin rather than a default.
|
||||
LLAMA_CPP_COMMIT="${TIMMY_LLAMA_CPP_COMMIT-$(receipt_field llamaCpp.commit)}"
|
||||
MODEL_SHA256="${TIMMY_MODEL_SHA256-$(receipt_field models.0.sha256)}"
|
||||
MMPROJ_SHA256="${TIMMY_MMPROJ_SHA256-$(receipt_field models.1.sha256)}"
|
||||
MODEL_FILE="$(receipt_field models.0.file)"
|
||||
MMPROJ_FILE="$(receipt_field models.1.file)"
|
||||
BUILD_BASE="$(receipt_field baseImages.build.reference)@$(receipt_field baseImages.build.digest)"
|
||||
RUNTIME_BASE="$(receipt_field baseImages.runtime.reference)@$(receipt_field baseImages.runtime.digest)"
|
||||
|
||||
if [[ ! "$LLAMA_CPP_COMMIT" =~ ^[0-9a-f]{40}$ ]]; then
|
||||
printf 'Refusing to build: the llama.cpp pin must be a 40-hex commit, got %s (floating refs are not reproducible).\n' "${LLAMA_CPP_COMMIT:-<empty>}" >&2
|
||||
exit 64
|
||||
fi
|
||||
for pin in "$MODEL_SHA256" "$MMPROJ_SHA256"; do
|
||||
if [[ ! "$pin" =~ ^[0-9a-f]{64}$ ]]; then
|
||||
printf 'Refusing to build: every model hash must be a pinned sha256, got %s.\n' "${pin:-<empty>}" >&2
|
||||
exit 64
|
||||
fi
|
||||
done
|
||||
for base in "$BUILD_BASE" "$RUNTIME_BASE"; do
|
||||
if [[ ! "$base" =~ @sha256:[0-9a-f]{64}$ ]]; then
|
||||
printf 'Refusing to build: base images must be digest-pinned, got %s.\n' "$base" >&2
|
||||
exit 64
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ ! "$PUBLISH_HOST" =~ ^(127\.0\.0\.1|::1|localhost)$ ]]; then
|
||||
printf 'Refusing to publish the worker outside loopback (127.0.0.1): got %s.\n' "$PUBLISH_HOST" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
print_plan() {
|
||||
printf 'image: %s:%s\n' "$IMAGE" "$TAG"
|
||||
printf 'llama.cpp commit: %s\n' "$LLAMA_CPP_COMMIT"
|
||||
printf '%s sha256: %s\n' "$MODEL_FILE" "$MODEL_SHA256"
|
||||
printf '%s sha256: %s\n' "$MMPROJ_FILE" "$MMPROJ_SHA256"
|
||||
printf 'base image (build): %s\n' "$BUILD_BASE"
|
||||
printf 'base image (runtime): %s\n' "$RUNTIME_BASE"
|
||||
printf 'bind address: %s:%s\n' "$PUBLISH_HOST" "$PORT"
|
||||
printf 'publish: %s:%s\n' "$PUBLISH_HOST" "$PORT"
|
||||
printf 'docker build --file deploy/worker/Dockerfile --build-arg LLAMA_CPP_COMMIT=%s --build-arg MODEL_SHA256=%s --build-arg MMPROJ_SHA256=%s --tag %s:%s .\n' \
|
||||
"$LLAMA_CPP_COMMIT" "$MODEL_SHA256" "$MMPROJ_SHA256" "$IMAGE" "$TAG"
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == '--print-plan' ]]; then
|
||||
print_plan
|
||||
exit 0
|
||||
fi
|
||||
|
||||
print_plan
|
||||
cd "$REPO_ROOT"
|
||||
docker build \
|
||||
--file deploy/worker/Dockerfile \
|
||||
--build-arg "LLAMA_CPP_COMMIT=$LLAMA_CPP_COMMIT" \
|
||||
--build-arg "MODEL_SHA256=$MODEL_SHA256" \
|
||||
--build-arg "MMPROJ_SHA256=$MMPROJ_SHA256" \
|
||||
--tag "$IMAGE:$TAG" \
|
||||
.
|
||||
printf 'Built %s:%s from pinned receipts. Run it with deploy/worker/run_worker_container.sh\n' "$IMAGE" "$TAG"
|
||||
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# Start the pinned worker after verifying the mounted weights (issue #15).
|
||||
#
|
||||
# Fails closed before the server is allowed to answer: a missing pin, an unreadable model, or a
|
||||
# hash mismatch stops the container instead of serving suggestions from an unknown model.
|
||||
set -euo pipefail
|
||||
|
||||
MODEL="${TIMMY_MODEL_FILE:-/models/SmolVLM2-2.2B-Instruct-Q4_K_M.gguf}"
|
||||
MMPROJ="${TIMMY_MMPROJ_FILE:-/models/mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf}"
|
||||
MODEL_SHA256="${TIMMY_MODEL_SHA256:-}"
|
||||
MMPROJ_SHA256="${TIMMY_MMPROJ_SHA256:-}"
|
||||
ALIAS="${TIMMY_MODEL_ALIAS:-SmolVLM2-2.2B-Instruct}"
|
||||
HOST="${TIMMY_MODEL_HOST:-0.0.0.0}"
|
||||
PORT="${TIMMY_MODEL_PORT:-8080}"
|
||||
THREADS="${TIMMY_MODEL_THREADS:-4}"
|
||||
|
||||
for pin in "$MODEL_SHA256" "$MMPROJ_SHA256"; do
|
||||
if [[ ! "$pin" =~ ^[0-9a-f]{64}$ ]]; then
|
||||
printf 'Refusing to start: the pinned model hashes are missing or malformed.\n' >&2
|
||||
exit 78
|
||||
fi
|
||||
done
|
||||
|
||||
for file in "$MODEL" "$MMPROJ"; do
|
||||
if [[ ! -f "$file" ]]; then
|
||||
printf 'Refusing to start: %s is not mounted. Mount the verified model directory read-only at /models.\n' "$file" >&2
|
||||
exit 78
|
||||
fi
|
||||
done
|
||||
|
||||
printf '%s %s\n%s %s\n' "$MODEL_SHA256" "$MODEL" "$MMPROJ_SHA256" "$MMPROJ" | sha256sum --check --status || {
|
||||
printf 'Refusing to start: mounted weights do not match the pinned receipts.\n' >&2
|
||||
exit 78
|
||||
}
|
||||
|
||||
printf 'Starting %s from pinned weights on %s:%s (private namespace)\n' "$ALIAS" "$HOST" "$PORT"
|
||||
exec llama-server \
|
||||
--model "$MODEL" \
|
||||
--mmproj "$MMPROJ" \
|
||||
--alias "$ALIAS" \
|
||||
--host "$HOST" \
|
||||
--port "$PORT" \
|
||||
--ctx-size 4096 \
|
||||
--parallel 1 \
|
||||
--threads "$THREADS" \
|
||||
--no-webui
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"issue": 15,
|
||||
"llamaCpp": {
|
||||
"repository": "https://github.com/ggml-org/llama.cpp",
|
||||
"commit": "6d05498314db1b57f81c271080018aa2d0b89be9"
|
||||
},
|
||||
"baseImages": {
|
||||
"build": {
|
||||
"reference": "nvidia/cuda:12.4.1-devel-ubuntu22.04",
|
||||
"digest": "sha256:da6791294b0b04d7e65d87b7451d6f2390b4d36225ab0701ee7dfec5769829f5"
|
||||
},
|
||||
"runtime": {
|
||||
"reference": "nvidia/cuda:12.4.1-runtime-ubuntu22.04",
|
||||
"digest": "sha256:517da2300c184c9999ec203c2665244bdebd3578d12fcc7065e83667932643d9"
|
||||
}
|
||||
},
|
||||
"models": [
|
||||
{
|
||||
"file": "SmolVLM2-2.2B-Instruct-Q4_K_M.gguf",
|
||||
"sha256": "0cf76814555b8665149075b74ab6b5c1d428ea1d3d01c1918c12012e8d7c9f58",
|
||||
"mount": "/models/SmolVLM2-2.2B-Instruct-Q4_K_M.gguf"
|
||||
},
|
||||
{
|
||||
"file": "mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf",
|
||||
"sha256": "ae07ea1facd07dd3230c4483b63e8cda96c6944ad2481f33d531f79e892dd024",
|
||||
"mount": "/models/mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf"
|
||||
}
|
||||
],
|
||||
"alias": "SmolVLM2-2.2B-Instruct",
|
||||
"bind": {
|
||||
"publishHost": "127.0.0.1",
|
||||
"containerPort": 8080
|
||||
},
|
||||
"notes": [
|
||||
"Model weights are never baked into the image; they are mounted read-only and hash-verified at start-up.",
|
||||
"The pins are asserted equal to scripts/bootstrap_selfhost_smolvlm.sh by tests/worker-container.test.js so host and image cannot drift.",
|
||||
"The worker needs no provider credential: it is a private loopback-only service."
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env bash
|
||||
# Run the pinned worker with the least privilege it can work under (issue #15).
|
||||
#
|
||||
# The worker is a private service. It is published on loopback only, mounts the verified weights
|
||||
# read-only, keeps an immutable root filesystem, and holds no credential of any kind.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
RECEIPTS="$REPO_ROOT/deploy/worker/receipts.json"
|
||||
IMAGE="${TIMMY_WORKER_IMAGE:-timmy-vision-worker}"
|
||||
TAG="${TIMMY_WORKER_TAG:-$(date -u +%Y%m%d)}"
|
||||
PORT="${TIMMY_MODEL_PORT:-8080}"
|
||||
PUBLISH_HOST="${TIMMY_WORKER_PUBLISH_HOST:-127.0.0.1}"
|
||||
MODEL_DIR="${TIMMY_WORKER_MODEL_DIR:-${TIMMY_SELFHOST_ROOT:-$HOME/.local/share/timmy-selfhost}/models}"
|
||||
NAME="${TIMMY_WORKER_CONTAINER:-timmy-vision-worker}"
|
||||
ALIAS="${TIMMY_MODEL_ALIAS:-SmolVLM2-2.2B-Instruct}"
|
||||
|
||||
# Exposure is the one decision this recipe must never get wrong, so a non-loopback publish host is
|
||||
# refused rather than warned about.
|
||||
if [[ ! "$PUBLISH_HOST" =~ ^(127\.0\.0\.1|::1|localhost)$ ]]; then
|
||||
printf 'Refusing to publish the worker outside loopback (127.0.0.1): got %s.\n' "$PUBLISH_HOST" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
if [[ ! "$PORT" =~ ^[0-9]{1,5}$ ]] || (( PORT < 1 || PORT > 65535 )); then
|
||||
printf 'Refusing to start: the port must be a number between 1 and 65535, got %s.\n' "$PORT" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
RUN_ARGS=(
|
||||
run
|
||||
--detach
|
||||
--name "$NAME"
|
||||
--read-only
|
||||
--cap-drop=ALL
|
||||
--security-opt no-new-privileges
|
||||
--tmpfs /tmp:rw,noexec,nosuid,size=64m
|
||||
--volume "$MODEL_DIR:/models:ro"
|
||||
--publish "127.0.0.1:${PORT}:8080"
|
||||
--env "TIMMY_MODEL_PORT=8080"
|
||||
--env "TIMMY_MODEL_THREADS=${TIMMY_MODEL_THREADS:-4}"
|
||||
"$IMAGE:$TAG"
|
||||
)
|
||||
|
||||
if [[ "${1:-}" == '--print-plan' ]]; then
|
||||
printf 'image: %s:%s\n' "$IMAGE" "$TAG"
|
||||
printf 'model directory (read-only mount): %s\n' "$MODEL_DIR"
|
||||
printf 'bind address: %s:%s\n' "$PUBLISH_HOST" "$PORT"
|
||||
printf 'publish: %s:%s\n' "$PUBLISH_HOST" "$PORT"
|
||||
printf 'health: %s ready (post-load gate)\n' "$ALIAS"
|
||||
printf 'docker %s\n' "${RUN_ARGS[*]}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ! -d "$MODEL_DIR" ]]; then
|
||||
printf 'Refusing to start: %s is not a directory. Install the pinned weights with scripts/bootstrap_selfhost_smolvlm.sh install.\n' "$MODEL_DIR" >&2
|
||||
exit 78
|
||||
fi
|
||||
|
||||
docker "${RUN_ARGS[@]}"
|
||||
printf 'Worker container %s started on %s:%s. Readiness: docker exec %s timmy-worker-ready\n' "$NAME" "$PUBLISH_HOST" "$PORT" "$NAME"
|
||||
@@ -0,0 +1,157 @@
|
||||
# Photo Data-Flow Threat Model — End-to-End Photo Path
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Issue:** [#20 — Write the end-to-end photo data-flow threat model](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/20)
|
||||
- **Parent epic:** [#5 — Clinical safety, privacy & security](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/5)
|
||||
- **Milestone:** M1 — Sovereign Photo Intelligence
|
||||
- **Date:** 2026-10-10
|
||||
- **Scope:** Every byte of a user's photo from capture to deletion on the `/api/analyze` path, plus the logs, exports, and consent records that touch it.
|
||||
|
||||
## Why this exists
|
||||
|
||||
Photo-first logging asks a user to hand Timmy an image of the most private output their body produces. That image is sensitive medical-adjacent data by default, so the data-flow boundaries must be explicit before any private beta, and every abuse case that is not yet fully mitigated must name an owner and a tracked remediation ticket. This model covers the browser, the API, the model worker, transient buffers, the training opt-in store, logs, exports, and deletion.
|
||||
|
||||
## Product boundary (unchanged by this model)
|
||||
|
||||
Timmy is non-diagnostic. AI may suggest only visible Bristol form, broad color, and image quality; every suggestion stays provisional until the user confirms or corrects it. Timmy must never diagnose disease, identify bleeding conclusively, infer pain, urgency, fever, or vomiting, or clear a food. Symptoms stay user-entered, and journal save is a separate decision from training contribution. Deterministic red-flag escalation runs on user-reported answers and is never model-gated or model-suppressible.
|
||||
|
||||
No real medical images are ever added to this repository, its fixtures, its tests, or its Git history. Every fixture used in the photo path is synthetic.
|
||||
|
||||
## Data-flow diagram
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
Browser["Browser — camera or gallery pick, in-memory draft only"]
|
||||
APIIngress["API ingress — server.mjs /api/analyze"]
|
||||
TransientBuffers["Transient buffers — bounded body bytes, decode canvas"]
|
||||
ModelWorker["Model worker — self-hosted vision, private loopback only"]
|
||||
OptInStore["Training opt-in store — consented contributions only"]
|
||||
Logs["Sanitized logs — counts and reasons, never bytes or base64"]
|
||||
Exports["Exports — user-initiated JSON portability"]
|
||||
Deletion["Deletion — withdrawal, purge, and tombstones"]
|
||||
|
||||
Browser -->|"image bytes plus explicit consent"| APIIngress
|
||||
APIIngress -->|"length, dimension, and rate limits first"| TransientBuffers
|
||||
TransientBuffers -->|"re-encoded baseline JPEG, metadata stripped"| ModelWorker
|
||||
ModelWorker -->|"observable fields only — image dropped, never persisted"| APIIngress
|
||||
APIIngress -->|"no image bytes"| Logs
|
||||
Browser --> Exports
|
||||
OptInStore --> Deletion
|
||||
TransientBuffers --> Deletion
|
||||
Logs --> Deletion
|
||||
```
|
||||
|
||||
## Trust boundaries
|
||||
|
||||
| # | Boundary | Trust posture |
|
||||
|---|---|---|
|
||||
| B1 | Browser to API ingress | Hostile network. Treat every upload as hostile and sensitive: sniff content, cap size and dimensions, rate-limit the client, and never trust the filename or declared MIME type. |
|
||||
| B2 | API ingress to model worker | Private loopback only. The worker is never exposed publicly and never receives a user identity. |
|
||||
| B3 | Model worker to provider or hosted profile | Disabled by default. When enabled, the operator carries the decision, the image is the only payload, and no ledger, symptom, or session data travels with it. |
|
||||
| B4 | App process to logs | One-way and lossy: counts, reasons, and durations leave; bytes, base64, filenames, and symptom text do not. |
|
||||
| B5 | App process to consent records | Contribution requires its own opt-in with a versioned receipt; journal save never implies contribution. |
|
||||
| B6 | Host to operator | The operator can reach data the app holds. Least privilege, documented purge, and no image bytes in operator-facing logs. |
|
||||
|
||||
## Components
|
||||
|
||||
### C1 — Browser
|
||||
|
||||
Holds the captured image as an in-memory draft and renders the consent prompt. A photo awaiting consent is deliberately not written to storage, so it cannot survive the tab by accident, and the service worker never caches analysis requests or image bodies. Lifetime: until the tab closes, consent is refused, or the draft is cleared.
|
||||
|
||||
### C2 — API ingress
|
||||
|
||||
`server.mjs` `/api/analyze` is the only door for image bytes. It owns content sniffing, dimension and body caps, re-encode, metadata stripping, rate control, and the sanitized error vocabulary. Lifetimes are per request: nothing about an image is written to disk.
|
||||
|
||||
### C3 — Model worker
|
||||
|
||||
A self-hosted vision process bound to private loopback. It receives the re-encoded image and returns schema-constrained observable fields or an abstention. It returns no free text, and its output is provisional until the user confirms it.
|
||||
|
||||
### C4 — Transient buffers
|
||||
|
||||
Request body buffer, decode canvas, and the re-encode pipe. These are the moments the image exists in readable form; they are bounded, per request, and dropped when the request ends, on success or failure alike.
|
||||
|
||||
### C5 — Training opt-in store
|
||||
|
||||
Only images and labels a user explicitly contributed, each with consent version, receipt, and provenance. This store is not on the logging path: an ordinary journal save never reaches it.
|
||||
|
||||
### C6 — Sanitized logs
|
||||
|
||||
Operational logs for health, latency, abstention, and failure. They carry counts, reason codes, and durations; they never carry image bytes, base64, the original filename, or user symptom text.
|
||||
|
||||
### C7 — Exports
|
||||
|
||||
User-initiated JSON portability of the user's own ledger. Exports are generated on request, are not written server-side, and deliberately exclude photo bytes.
|
||||
|
||||
### C8 — Deletion
|
||||
|
||||
Withdrawal, purge, and tombstones. Deletion is addressable by contribution ID so it can reach derived artifacts, and a tombstone records that deletion happened without keeping what was deleted.
|
||||
|
||||
## Threat register
|
||||
|
||||
Each entry names the abuse case, the mitigation that is already committed or queued, the owning role, the tracked ticket that will prove the mitigation, and the artifact that will count as evidence.
|
||||
|
||||
### T1 — Replay and re-submission
|
||||
|
||||
- **Scenario:** An attacker replays a captured upload to exhaust inference capacity, correlate timing with a user session, or force repeated provider work from one image.
|
||||
- **Impact:** Denial of service for real users, cost amplification on hosted profiles, and weak linkage between a session and its image bytes.
|
||||
- **Mitigation:** Per-client fixed-window rate control on `/api/analyze` that fails with `429` and `Retry-After` before any provider work, opaque client identity held server-side rather than in the image payload, and no retention of over-limit payloads.
|
||||
- **Owner:** Timmy (ingress, area/backend)
|
||||
- **Testable follow-up:** #16
|
||||
- **Evidence:** An ingress test that drives the limiter past its window and asserts a rejected request never reaches the provider, with a separate assertion that no over-limit payload is retained.
|
||||
|
||||
### T2 — Oversized input
|
||||
|
||||
- **Scenario:** A decompression bomb, an enormous dimension count, or a multi-hundred-megabyte body arrives to consume worker memory, block the queue, or stall the request path indefinitely.
|
||||
- **Impact:** Worker exhaustion and queue starvation that degrade logging availability, up to a full outage of photo analysis.
|
||||
- **Mitigation:** Decoded-body, per-dimension, and timeout caps enforced before any provider or worker work; a bounded inference queue with cancellation and an overload fallback that returns the manual path instead of hanging.
|
||||
- **Owner:** Timmy (ingress, area/backend)
|
||||
- **Testable follow-up:** #17
|
||||
- **Evidence:** A queue-full and body-timeout test proving the request is rejected inside its budget and that the manual Bristol path remains one action away, per the acceptance criteria of #17.
|
||||
|
||||
### T3 — Metadata leakage
|
||||
|
||||
- **Scenario:** An uploaded photo carries EXIF, GPS, device identifiers, XMP, or PNG text chunks; or a failure path echoes base64 image bytes into an error message or a log line.
|
||||
- **Impact:** Location and device deanonymization, plus durable copies of the image outside the intended boundary, including in logs.
|
||||
- **Mitigation:** Content sniffing independent of filename and MIME, safe re-encode to baseline JPEG with a fixed argv, metadata dropped by construction, polyglot and malformed inputs rejected, and a fixed sanitized error vocabulary with no bytes, no base64, and no stack traces.
|
||||
- **Owner:** Timmy (ingress, area/backend)
|
||||
- **Testable follow-up:** #16
|
||||
- **Evidence:** A re-encode test asserting EXIF, GPS, XMP, and PNG text chunks are absent from the accepted bytes, paired with a log-scan test asserting no image bytes or base64 appear in request or error logs.
|
||||
|
||||
### T4 — Dataset poisoning
|
||||
|
||||
- **Scenario:** Mislabeled, duplicated, or adversarially crafted contributions enter the training corpus and shift the classifier toward a wrong or unsafe behavior, including teaching it to call a not-stool image a stool form.
|
||||
- **Impact:** Silent accuracy loss that is invisible in aggregate metrics and could undermine abstention exactly where the product must fail closed.
|
||||
- **Mitigation:** Immutable provenance per contribution, contributor-level splits so a single contributor cannot straddle train and evaluation, a reviewer and adjudication tool for form, color, quality, and exclusions, and a published dataset card with the labeling guide and split policy.
|
||||
- **Owner:** Privacy reviewer (human gate), with Timmy as release authority
|
||||
- **Testable follow-up:** #29
|
||||
- **Evidence:** The contributor-held-out evaluation harness and calibration report, with the reviewer and adjudication tool as the labeling control.
|
||||
|
||||
### T5 — Unauthorized training
|
||||
|
||||
- **Scenario:** A journal save is silently treated as consent, an image is trained on without a receipt, or a withdrawal fails to reach a derived artifact so contribution survives deletion.
|
||||
- **Impact:** Use of private data without consent, which breaks the product promise and the beta consent gate, and is irreversible once a model is trained.
|
||||
- **Mitigation:** Contribution is a separate opt-in decision with a versioned consent receipt shown before the data moves; journal save defaults to no contribution; withdrawal is addressable by contribution ID and reaches every derived artifact; journal saving never requires contribution.
|
||||
- **Owner:** Privacy reviewer (human gate)
|
||||
- **Testable follow-up:** #24
|
||||
- **Evidence:** An opt-in and withdrawal test proving journal save contributes nothing by default, that the consent version and receipt are shown before contribution, and that a withdrawal removes every derived artifact by contribution ID, with tombstone handling from #25.
|
||||
|
||||
### T6 — Operator access
|
||||
|
||||
- **Scenario:** The host operator — or anyone who reaches the host with the operator's privilege — reads image bytes, symptom text, or consent records from process memory, on-disk state, backups, or logs.
|
||||
- **Impact:** A single privileged reader can deanonymize a user's most private log, and an undocumented backup can outlive a deletion request.
|
||||
- **Mitigation:** Least-privilege host access, worker bound to private loopback with no public exposure, no image bytes or base64 in operator-facing logs, documented backup and rollback boundaries, and a purge procedure that names every place an image can exist.
|
||||
- **Owner:** Operator (host owner), with Timmy as release authority
|
||||
- **Testable follow-up:** #39
|
||||
- **Evidence:** The production topology, secrets, backup, and rollback runbook, exercised against the deletion path so that purge reaches memory, temporary files, logs, and backups, with observability from #41 confirming no bytes reach the metrics surface.
|
||||
|
||||
## Accepted residual risks
|
||||
|
||||
- **Host compromise.** Data in a running process can be read by a sufficiently privileged attacker. This is accepted for the private beta and bounded by least privilege plus the purge procedure in #39; it is not claimed as eliminated.
|
||||
- **Model accuracy.** The photo prefill remains blocked behind abstention and specialist-model evidence. A wrong or confident suggestion is treated as a user-correction problem, and the user remains the author of every saved entry, per #32 and the calibration work in #4.
|
||||
- **Provider profiles.** A hosted OpenAI-compatible profile moves the image off-host by definition. It stays opt-in, operator-controlled, and off by default, with readiness reporting before it is offered, per #19.
|
||||
|
||||
## Verification
|
||||
|
||||
- This document is enforced by `tests/threat-model.test.js`, which asserts that the diagram covers every byte-bearing component, that each required abuse case is a numbered threat with an owner and a tracked follow-up ticket, and that the model keeps the non-diagnostic boundary and ships no real medical images.
|
||||
- The test runs in `npm test` and therefore inside the CI quality gate and the release build, so the threat model cannot silently drift from the code it describes.
|
||||
- Boundary and boundary-adjacent claims are reviewed by the clinical and privacy gate (#22, #23) before beta; this model does not move that gate, it feeds it.
|
||||
@@ -59,3 +59,4 @@ Routine provisioning, deployment, label sweeps, and release-candidate assembly a
|
||||
- [Product brief and safety boundary](../PRODUCT.md)
|
||||
- [AI execution evidence](../AI-EVIDENCE.md)
|
||||
- [Sovereign vision decision and measured limitations](../research/SELF-HOSTED-STOOL-VISION.md)
|
||||
- [Photo data-flow threat model](./PHOTO-DATAFLOW-THREAT-MODEL.md)
|
||||
@@ -4,7 +4,7 @@
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js",
|
||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js",
|
||||
"test:ui": "node tests/ui.acceptance.mjs",
|
||||
"test:photo": "node tests/photo-first.acceptance.mjs",
|
||||
"test:mobile-capture": "node tests/mobile-capture.acceptance.mjs",
|
||||
@@ -12,10 +12,13 @@
|
||||
"test:manual-fallback": "node tests/manual-fallback.acceptance.mjs",
|
||||
"test:provisional": "node tests/provisional-suggestion.acceptance.mjs",
|
||||
"test:followup": "node tests/followup.acceptance.mjs",
|
||||
"test:vision-refusal": "node tests/vision-refusal.acceptance.mjs",
|
||||
"test:staging-smoke": "node tests/staging.acceptance.mjs",
|
||||
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py",
|
||||
"test:pattern-summary": "node tests/pattern-summary.acceptance.mjs",
|
||||
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py && node --check scripts/record_worker_benchmark.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh",
|
||||
"check:diff": "bash scripts/check_diff.sh",
|
||||
"start": "node server.mjs"
|
||||
"start": "node server.mjs",
|
||||
"test:worker-readiness": "node tests/worker-readiness.acceptance.mjs"
|
||||
},
|
||||
"devDependencies": {
|
||||
"playwright": "^1.62.1"
|
||||
|
||||
@@ -137,6 +137,10 @@ case "${1:-}" in
|
||||
start)
|
||||
start_worker
|
||||
;;
|
||||
ready)
|
||||
# Post-load readiness (issue #15): delegate so the host and the container gate cannot drift.
|
||||
exec "$(dirname "$0")/worker_ready.sh" --host "$HOST" --port "$PORT" --alias SmolVLM2-2.2B-Instruct "${@:2}"
|
||||
;;
|
||||
health)
|
||||
health_worker
|
||||
;;
|
||||
@@ -144,7 +148,7 @@ case "${1:-}" in
|
||||
stop_worker
|
||||
;;
|
||||
*)
|
||||
printf 'Usage: %s {install|start|health|stop|receipt|verify}\n' "$0" >&2
|
||||
printf 'Usage: %s {install|start|health|ready|stop|receipt|verify}\n' "$0" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
@@ -26,6 +26,24 @@ SECRET_PATTERNS = (
|
||||
re.compile(r"(?i)(?:api[_-]?key|access[_-]?token|authorization)\s*[:=]\s*['\"](?!\.\.\.|top-secret|test-|example|placeholder)[A-Za-z0-9_./+:-]{16,}"),
|
||||
)
|
||||
|
||||
# Captions that must be proven on camera (via the demo's own receipt) for the release to pass.
|
||||
DEMO_REQUIRED_CAPTIONS = (
|
||||
"One clear photo action. Manual logging stays one tap away.",
|
||||
"Nothing uploads until explicit consent",
|
||||
"The suggestion stays provisional — nothing is applied or saved until you confirm it",
|
||||
"Type 2 is hard or lumpy, so Timmy asks about soreness, hydration, blood, and black-or-tarry — never guessed from the photo",
|
||||
"Vision worker offline — the AI and consent steps disappear instead of failing after upload",
|
||||
"Every byte-bearing control the model names, in the running app: local ledger, consent-gated upload, portable export, full deletion",
|
||||
"Delete all local data clears the ledger in this browser; there is no server copy of it to keep",
|
||||
"Hermes Agent connected — free-text, contextual, and server-side",
|
||||
"Urgent language is intercepted deterministically before Hermes",
|
||||
"An out-of-range answer fails closed: nothing is applied and no provider detail reaches you",
|
||||
"A filled-in field that is out of range still fails closed, even when the other field came back blank",
|
||||
"Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident",
|
||||
"Every pattern line names the confirmed logs behind it, and following one shows exactly those records",
|
||||
"Two logs is not a pattern: the summary refuses to describe a trend it cannot support",
|
||||
)
|
||||
|
||||
|
||||
def run(args: list[str], cwd: Path, *, capture: bool = False, env: dict[str, str] | None = None) -> str:
|
||||
print("+", " ".join(args), flush=True)
|
||||
@@ -74,6 +92,10 @@ def main() -> int:
|
||||
status = run(["git", "status", "--porcelain"], ROOT, capture=True)
|
||||
if status:
|
||||
raise SystemExit("Release requires a clean committed tree")
|
||||
release_delta = os.environ.get("TIMMY_RELEASE_DELTA",
|
||||
"Verified rebuild of the current user-visible feature path — no new feature is claimed.").strip()
|
||||
if not release_delta:
|
||||
raise SystemExit("TIMMY_RELEASE_DELTA must not be empty")
|
||||
commit = run(["git", "rev-parse", "HEAD"], ROOT, capture=True)
|
||||
short = commit[:12]
|
||||
commit_epoch = int(run(["git", "show", "-s", "--format=%ct", commit], ROOT, capture=True))
|
||||
@@ -91,6 +113,8 @@ def main() -> int:
|
||||
run(["git", "checkout", "--quiet", commit], tree)
|
||||
run(["npm", "ci", "--ignore-scripts"], tree)
|
||||
run(["npm", "test"], tree)
|
||||
# The committed corpus hashes are an integrity receipt: verify them before anything is built.
|
||||
run(["python3", "scripts/build_vision_corpus.py", "--verify"], tree)
|
||||
server_env = dict(os.environ)
|
||||
server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false"})
|
||||
server_log = (release_dir / "acceptance-server.log").open("w", encoding="utf-8")
|
||||
@@ -114,10 +138,25 @@ def main() -> int:
|
||||
run(["npm", "run", "test:manual-fallback"], tree)
|
||||
run(["npm", "run", "test:provisional"], tree)
|
||||
run(["npm", "run", "test:followup"], tree)
|
||||
run(["npm", "run", "test:vision-refusal"], tree)
|
||||
run(["npm", "run", "test:worker-readiness"], tree)
|
||||
run(["npm", "run", "test:pattern-summary"], tree)
|
||||
demo_raw = release_dir / f"timmy-talking-turd-{version}-demo.raw.webm"
|
||||
caption_receipt = release_dir / f"timmy-talking-turd-{version}-demo-captions.json"
|
||||
demo_env = dict(server_env)
|
||||
demo_env["TIMMY_RELEASE_VERSION"] = version
|
||||
demo_env["TIMMY_RELEASE_DEMO_DELTA"] = release_delta
|
||||
demo_env["TIMMY_DEMO_CAPTIONS_OUT"] = str(caption_receipt)
|
||||
run(["node", "scripts/record_release_demo.mjs", str(demo_raw)], tree, env=demo_env)
|
||||
if not caption_receipt.exists():
|
||||
raise SystemExit("Release demo produced no on-camera caption receipt")
|
||||
recorded = json.loads(caption_receipt.read_text(encoding="utf-8"))
|
||||
onscreen = [str(line) for line in (recorded.get("captions") or [])]
|
||||
if recorded.get("releaseDelta") != release_delta or not any(release_delta in line for line in onscreen):
|
||||
raise SystemExit("Release demo did not state the declared release delta on camera")
|
||||
for required in DEMO_REQUIRED_CAPTIONS:
|
||||
if not any(required in line for line in onscreen):
|
||||
raise SystemExit(f"Release demo is missing a required on-camera caption: {required}")
|
||||
finally:
|
||||
server.terminate()
|
||||
try:
|
||||
@@ -135,13 +174,23 @@ def main() -> int:
|
||||
if len(video_streams) != 1 or video_streams[0].get("codec_name") != "h264" or video_streams[0].get("pix_fmt") != "yuv420p" or video_streams[0].get("width") != 720 or video_streams[0].get("height") != 1280:
|
||||
raise SystemExit("Release demo media probe failed")
|
||||
contact_sheet = release_dir / f"timmy-talking-turd-{version}-demo-contact-sheet.jpg"
|
||||
run(["ffmpeg", "-y", "-v", "error", "-i", str(demo), "-vf", "fps=1/2,scale=180:-1,tile=4x3:padding=4:margin=4", "-frames:v", "1", str(contact_sheet)], tree)
|
||||
# Span the whole recording: a fixed 1/2 fps sheet only covered the first 24s of a longer demo.
|
||||
demo_duration = float(run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", str(demo)], tree, capture=True))
|
||||
if demo_duration <= 0:
|
||||
raise SystemExit("Release demo has no measurable duration")
|
||||
run(["ffmpeg", "-y", "-v", "error", "-i", str(demo), "-vf", f"fps={12 / demo_duration:.6f},scale=180:-1,tile=4x3:padding=4:margin=4", "-frames:v", "1", str(contact_sheet)], tree)
|
||||
if not contact_sheet.exists() or contact_sheet.stat().st_size == 0:
|
||||
raise SystemExit("Release demo contact sheet was not produced")
|
||||
run(["npm", "audit", "--audit-level=high"], tree)
|
||||
for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "tests/staging.acceptance.mjs"):
|
||||
for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "src/summary.js", "scripts/record_worker_benchmark.mjs", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs", "tests/worker-readiness.acceptance.mjs", "tests/pattern-summary.acceptance.mjs"):
|
||||
run(["node", "--check", file], tree)
|
||||
run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree)
|
||||
run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree)
|
||||
run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py"], tree)
|
||||
run(["bash", "-n", "scripts/worker_ready.sh"], tree)
|
||||
run(["bash", "-n", "deploy/worker/entrypoint.sh"], tree)
|
||||
run(["bash", "-n", "deploy/worker/build_image.sh"], tree)
|
||||
run(["bash", "-n", "deploy/worker/run_worker_container.sh"], tree)
|
||||
run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py", "scripts/build_vision_corpus.py"], tree)
|
||||
run(["git", "diff", "--check", commit], tree)
|
||||
|
||||
names = tracked_files(tree)
|
||||
@@ -160,6 +209,7 @@ def main() -> int:
|
||||
f"# Timmy review release {version}\n\n"
|
||||
f"- Source commit: [{commit}]({PUBLIC_REPO}/commit/{commit})\n"
|
||||
f"- Build date (UTC): {release_date}\n"
|
||||
f"- Release delta: {release_delta}\n"
|
||||
"- Sleek three-destination shell with one dominant photo action and quiet manual fallback\n"
|
||||
"- Manual fallback: vision-offline logging stays one action away with an in-session draft and explicit start over\n"
|
||||
"- Smart free-text Timmy chat backed by authenticated server-side Hermes session continuity\n"
|
||||
@@ -167,6 +217,11 @@ def main() -> int:
|
||||
"- Vision profiles: hosted and self-hosted SmolVLM2 bootstrap\n"
|
||||
"- AI suggestions stay provisional: suggested, confirmed, and corrected states are distinct and screen-reader described; correcting one clears the AI-confirmed state without losing user context\n"
|
||||
"- Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions (difficulty passing, soreness, visible blood, truly black/tarry versus dark brown, hydration); mild reports get non-diagnostic self-care, and a confirmed red-flag answer lands on the same deterministic escalation path as manual logging\n"
|
||||
"- Vision corpus: a committed CC0 synthetic corpus (stool, non-stool, blur, glare, empty bowl, malformed response) proves every reply is schema-valid or fails closed; a schema-valid reply with unreadable fields abstains instead of erroring, and unparsable or out-of-range replies return plain copy with nothing applied — including a reply that fills in one field and leaves the other blank, where an out-of-range value still fails closed instead of being reported as an unreadable photo (issue #73)\n"
|
||||
"- Self-hosted worker container: a digest-pinned image that builds the pinned llama.cpp revision, mounts hash-verified weights read-only, runs unprivileged with no credential, and is published on loopback only (issue #15)\n"
|
||||
"- Post-load readiness: the app offers AI only once the pinned model is actually loaded, so a reachable-but-unloaded worker falls back to manual logging in one tap instead of failing after the upload (issue #15)\n"
|
||||
"- Target-GPU benchmark: scripts/record_worker_benchmark.mjs records cold start, warm latency, and VRAM on the target GPU and refuses to write a receipt it did not measure; no GPU measurement is bundled with this release\n"
|
||||
"- Explainable pattern summaries (issue #34): the home insight card reports confirmed-log counts and form groupings per window, every line names the exact confirmed records behind it, following a citation opens those records, and a window below the minimum says it is too sparse instead of describing a trend; no line explains a cause, rules on food, or offers a health verdict\n"
|
||||
"- Safety: suggestions are observable-field assistance, never diagnosis; user confirmation is required\n"
|
||||
"- Known limitation: open-weight VLM acceptance is proven, Bristol accuracy is not clinically validated\n"
|
||||
"- Package excludes model weights, medical-image corpora, raw research pages, generated screenshots, and video production assets\n"
|
||||
@@ -197,6 +252,7 @@ def main() -> int:
|
||||
"project": "Timmy the Talking Turd",
|
||||
"version": version,
|
||||
"commit": commit,
|
||||
"release_delta": release_delta,
|
||||
"source_date_utc": source_date.isoformat(),
|
||||
"built_at_utc": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"artifact": archive.name,
|
||||
@@ -211,7 +267,7 @@ def main() -> int:
|
||||
"codec": "h264",
|
||||
"pixel_format": "yuv420p",
|
||||
"recorded_from_working_app": True,
|
||||
"fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, offline vision status, and deterministic Hermes chat response",
|
||||
"fixture_data": "synthetic-Type-4, deterministic vision suggestion reviewed, confirmed, and corrected to Type 2, user-reported hard-stool follow-up with non-diagnostic self-care and a confirmed black/tarry red flag, the CC0 synthetic vision corpus replaying a non-stool refusal, an unreadable-fields abstention, an out-of-range fail-closed reply, and a blank-plus-out-of-range fail-closed reply, offline vision status, fictional multi-day sample logs driving the explainable pattern summary and its citation sheet, a sparse fictional window showing the insufficient-data copy, and deterministic Hermes chat response",
|
||||
},
|
||||
"gates": {
|
||||
"unit_security": "passed",
|
||||
@@ -221,12 +277,17 @@ def main() -> int:
|
||||
"manual_fallback_draft_recovery": "passed",
|
||||
"provisional_suggestion_state": "passed",
|
||||
"hard_stool_followup": "passed",
|
||||
"vision_corpus_refusal": "passed",
|
||||
"worker_container_receipts": "passed",
|
||||
"worker_post_load_readiness": "passed",
|
||||
"longitudinal_pattern_summary": "passed",
|
||||
"dependency_audit_high": "passed",
|
||||
"syntax": "passed",
|
||||
"secret_scan": "passed",
|
||||
"forbidden_artifacts": "passed",
|
||||
"feature_demo_full_decode": "passed",
|
||||
"feature_demo_media_probe": "passed",
|
||||
"release_delta_visible_on_camera": "passed",
|
||||
},
|
||||
}
|
||||
manifest_path = release_dir / "manifest.json"
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate the deterministic synthetic vision corpus for issue #18.
|
||||
|
||||
Every fixture is drawn from fixed integer math: no camera, no human subject, no
|
||||
third-party content, and no randomness. That keeps the corpus redistributable and
|
||||
keeps the recorded SHA-256 hashes meaningful as an integrity receipt.
|
||||
|
||||
Usage:
|
||||
python3 scripts/build_vision_corpus.py # (re)write fixtures and manifest.json
|
||||
python3 scripts/build_vision_corpus.py --verify # verify committed fixtures against the manifest
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from PIL import Image, ImageDraw, ImageFilter
|
||||
|
||||
LICENSE = "CC0-1.0"
|
||||
CORPUS_DIR = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "vision-corpus"
|
||||
WIDTH, HEIGHT = 640, 480
|
||||
PROVENANCE = "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content."
|
||||
|
||||
OBSERVATION_STRINGS = {
|
||||
"smooth": "Smooth, formed appearance.",
|
||||
"flatware": "Flatware on a tray.",
|
||||
"unclear": "Form is unclear.",
|
||||
"highlight": "Large specular highlight hides the surface.",
|
||||
"container": "Empty container.",
|
||||
"confident": "Confident.",
|
||||
"partial": "Form read, colour not.",
|
||||
}
|
||||
|
||||
|
||||
def background(color: tuple[int, int, int]) -> Image.Image:
|
||||
"""A flat canvas with a coarse, deterministic shade band."""
|
||||
image = Image.new("RGB", (WIDTH, HEIGHT), color)
|
||||
draw = ImageDraw.Draw(image)
|
||||
for index in range(4):
|
||||
shade = tuple(min(255, channel + 6 * index) for channel in color)
|
||||
draw.rectangle([0, index * 120, WIDTH, index * 120 + 119], fill=shade)
|
||||
return image
|
||||
|
||||
|
||||
def blob(image: Image.Image, box: tuple[int, int, int, int], tone: tuple[int, int, int]) -> None:
|
||||
draw = ImageDraw.Draw(image)
|
||||
draw.ellipse(box, fill=tone)
|
||||
left, top, right, bottom = box
|
||||
inset = max(4, (right - left) // 6)
|
||||
draw.ellipse([left + inset, top + inset, right - inset, bottom - inset],
|
||||
fill=tuple(min(255, channel + 10) for channel in tone))
|
||||
|
||||
|
||||
def stool_like() -> Image.Image:
|
||||
image = background((226, 222, 212))
|
||||
blob(image, (170, 150, 470, 330), (108, 76, 48))
|
||||
return image
|
||||
|
||||
|
||||
def non_stool_scene() -> Image.Image:
|
||||
image = background((198, 202, 208))
|
||||
draw = ImageDraw.Draw(image)
|
||||
draw.ellipse([120, 90, 520, 390], fill=(244, 244, 240))
|
||||
draw.rectangle([60, 200, 300, 224], fill=(120, 126, 134))
|
||||
draw.rectangle([340, 200, 580, 224], fill=(120, 126, 134))
|
||||
return image
|
||||
|
||||
|
||||
def blur_low_detail() -> Image.Image:
|
||||
return stool_like().filter(ImageFilter.GaussianBlur(5))
|
||||
|
||||
|
||||
def glare_overexposed() -> Image.Image:
|
||||
image = background((222, 218, 208))
|
||||
blob(image, (150, 140, 430, 300), (104, 74, 46))
|
||||
draw = ImageDraw.Draw(image)
|
||||
draw.polygon([(380, 200), (640, 120), (640, 360), (420, 330)], fill=(252, 252, 250))
|
||||
draw.ellipse([470, 210, 620, 320], fill=(255, 255, 255))
|
||||
return image
|
||||
|
||||
|
||||
def empty_bowl() -> Image.Image:
|
||||
image = background((214, 210, 202))
|
||||
draw = ImageDraw.Draw(image)
|
||||
draw.ellipse([130, 90, 510, 400], fill=(238, 236, 230))
|
||||
draw.ellipse([175, 130, 465, 360], fill=(206, 204, 198))
|
||||
draw.ellipse([220, 170, 420, 320], fill=(232, 230, 224))
|
||||
return image
|
||||
|
||||
|
||||
def unusable_tone() -> Image.Image:
|
||||
image = background((202, 206, 198))
|
||||
blob(image, (190, 160, 450, 320), (128, 118, 92))
|
||||
return image
|
||||
|
||||
|
||||
def shaded_tone() -> Image.Image:
|
||||
image = background((216, 214, 206))
|
||||
blob(image, (180, 155, 460, 325), (96, 84, 52))
|
||||
draw = ImageDraw.Draw(image)
|
||||
draw.rectangle([0, 300, WIDTH, 306], fill=(180, 178, 170))
|
||||
return image
|
||||
|
||||
|
||||
def half_read_tone() -> Image.Image:
|
||||
"""A fixture for the reply shape that fills one field and blanks the other (issue #73)."""
|
||||
image = background((208, 212, 204))
|
||||
blob(image, (175, 150, 455, 322), (112, 96, 66))
|
||||
draw = ImageDraw.Draw(image)
|
||||
draw.rectangle([0, 200, WIDTH // 2, 206], fill=(196, 194, 186))
|
||||
draw.rectangle([WIDTH // 2, 260, WIDTH, 266], fill=(170, 168, 160))
|
||||
return image
|
||||
|
||||
|
||||
def content(payload: dict[str, object] | str) -> str:
|
||||
return payload if isinstance(payload, str) else json.dumps(payload, separators=(",", ":"))
|
||||
|
||||
|
||||
def cases() -> list[dict[str, Any]]:
|
||||
return [
|
||||
{
|
||||
"id": "stool-like-formed",
|
||||
"caseClass": "stool",
|
||||
"image": stool_like,
|
||||
"expectedOutcome": "suggestion",
|
||||
"expectedReason": "schema_valid_suggestion",
|
||||
"providerContent": content({
|
||||
"isStool": True, "bristolType": 4, "color": "brown", "confidence": 0.83,
|
||||
"imageQuality": "good", "observations": OBSERVATION_STRINGS["smooth"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
"id": "non-stool-scene",
|
||||
"caseClass": "non-stool",
|
||||
"image": non_stool_scene,
|
||||
"expectedOutcome": "abstained",
|
||||
"expectedReason": "not_stool",
|
||||
"providerContent": content({
|
||||
"isStool": False, "bristolType": None, "color": None, "confidence": 0.97,
|
||||
"imageQuality": "good", "observations": OBSERVATION_STRINGS["flatware"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
"id": "blur-low-detail",
|
||||
"caseClass": "blur",
|
||||
"image": blur_low_detail,
|
||||
"expectedOutcome": "abstained",
|
||||
"expectedReason": "low_confidence",
|
||||
"providerContent": content({
|
||||
"isStool": True, "bristolType": 4, "color": "brown", "confidence": 0.41,
|
||||
"imageQuality": "poor", "observations": OBSERVATION_STRINGS["unclear"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
"id": "glare-overexposed",
|
||||
"caseClass": "glare",
|
||||
"image": glare_overexposed,
|
||||
"expectedOutcome": "abstained",
|
||||
"expectedReason": "unusable_visual_fields",
|
||||
"providerContent": content({
|
||||
"isStool": True, "bristolType": None, "color": None, "confidence": 0.66,
|
||||
"imageQuality": "poor", "observations": OBSERVATION_STRINGS["highlight"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
"id": "empty-bowl",
|
||||
"caseClass": "empty-bowl",
|
||||
"image": empty_bowl,
|
||||
"expectedOutcome": "abstained",
|
||||
"expectedReason": "not_stool",
|
||||
"providerContent": content({
|
||||
"isStool": False, "bristolType": None, "color": None, "confidence": 0.93,
|
||||
"imageQuality": "good", "observations": OBSERVATION_STRINGS["container"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
"id": "malformed-response",
|
||||
"caseClass": "malformed-response",
|
||||
"image": unusable_tone,
|
||||
"expectedOutcome": "fail_closed",
|
||||
"expectedReason": "unparsable_content",
|
||||
"providerContent": "Sure! It looks like a Type 4 to me. Hope that helps.",
|
||||
},
|
||||
{
|
||||
"id": "out-of-range-fields",
|
||||
"caseClass": "malformed-response",
|
||||
"image": shaded_tone,
|
||||
"expectedOutcome": "fail_closed",
|
||||
"expectedReason": "out_of_range_or_missing_fields",
|
||||
# Confidence stays inside the contract so this reply reaches the field-range check. At 1.4 it
|
||||
# died at the confidence gate instead, which credited the `out_of_range_or_missing_fields`
|
||||
# label to a guard the case never exercised (issue #73).
|
||||
"providerContent": content({
|
||||
"isStool": True, "bristolType": 9, "color": "purple", "confidence": 0.9,
|
||||
"imageQuality": "excellent", "observations": OBSERVATION_STRINGS["confident"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
"id": "blank-plus-out-of-range",
|
||||
"caseClass": "malformed-response",
|
||||
"image": half_read_tone,
|
||||
"expectedOutcome": "fail_closed",
|
||||
"expectedReason": "out_of_range_or_missing_fields",
|
||||
# The class the blank-first ordering used to let through (issue #73): one field is blank and
|
||||
# the other is a filled-in, impossible value. A filled-in field is a claim, so this must fail
|
||||
# closed rather than abstain as `unusable_visual_fields`.
|
||||
"providerContent": content({
|
||||
"isStool": True, "bristolType": 9, "color": None, "confidence": 0.9,
|
||||
"imageQuality": "good", "observations": OBSERVATION_STRINGS["partial"],
|
||||
}),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def write_fixture(definition: dict[str, Any], directory: Path) -> dict[str, Any]:
|
||||
name = f"{definition['id']}.png"
|
||||
target = directory / name
|
||||
factory: Callable[[], Image.Image] = definition["image"]
|
||||
factory().save(target, "PNG", optimize=True)
|
||||
data = target.read_bytes()
|
||||
return {
|
||||
"id": definition["id"],
|
||||
"caseClass": definition["caseClass"],
|
||||
"file": name,
|
||||
"bytes": len(data),
|
||||
"sha256": hashlib.sha256(data).hexdigest(),
|
||||
"width": WIDTH,
|
||||
"height": HEIGHT,
|
||||
"format": "png",
|
||||
"synthetic": True,
|
||||
"license": LICENSE,
|
||||
"provenance": PROVENANCE,
|
||||
"expectedOutcome": definition["expectedOutcome"],
|
||||
"expectedReason": definition["expectedReason"],
|
||||
"providerContent": definition["providerContent"],
|
||||
}
|
||||
|
||||
|
||||
def manifest(entries: list[dict[str, object]]) -> dict[str, object]:
|
||||
return {
|
||||
"schemaVersion": 1,
|
||||
"generator": "scripts/build_vision_corpus.py",
|
||||
"license": LICENSE,
|
||||
"note": "Synthetic fixtures only. No real medical imagery, no personal data, and no third-party content.",
|
||||
"cases": entries,
|
||||
}
|
||||
|
||||
|
||||
def write(directory: Path) -> int:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
entries = [write_fixture(definition, directory) for definition in cases()]
|
||||
(directory / "manifest.json").write_text(json.dumps(manifest(entries), indent=2) + "\n", encoding="utf-8")
|
||||
print(f"wrote {len(entries)} fixtures to {directory}")
|
||||
return 0
|
||||
|
||||
|
||||
def verify(directory: Path) -> int:
|
||||
manifest_path = directory / "manifest.json"
|
||||
recorded = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
problems: list[str] = []
|
||||
for entry in recorded.get("cases", []):
|
||||
path = directory / str(entry.get("file"))
|
||||
if not path.is_file():
|
||||
problems.append(f"{entry.get('id')}: missing fixture {entry.get('file')}")
|
||||
continue
|
||||
data = path.read_bytes()
|
||||
if hashlib.sha256(data).hexdigest() != entry.get("sha256"):
|
||||
problems.append(f"{entry.get('id')}: bytes do not match the recorded sha256")
|
||||
if len(data) != entry.get("bytes"):
|
||||
problems.append(f"{entry.get('id')}: byte length does not match the manifest")
|
||||
if problems:
|
||||
for problem in problems:
|
||||
print(problem, file=sys.stderr)
|
||||
return 1
|
||||
print(f"verified {len(recorded.get('cases', []))} corpus fixtures against manifest.json")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--verify", action="store_true", help="check committed fixtures against the manifest instead of rewriting")
|
||||
parser.add_argument("--directory", default=str(CORPUS_DIR))
|
||||
args = parser.parse_args()
|
||||
directory = Path(args.directory)
|
||||
return verify(directory) if args.verify else write(directory)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,12 +1,41 @@
|
||||
import { chromium } from 'playwright';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { analyzePhoto, visionFailureResponse } from '../src/vision-service.js';
|
||||
import { loadVisionCorpus } from '../src/vision-corpus.js';
|
||||
|
||||
const output = path.resolve(process.argv[2] || 'release-demo.webm');
|
||||
const version = process.env.TIMMY_RELEASE_VERSION || 'review build';
|
||||
// The release delta is supplied by the builder so the recording states, on camera, what actually
|
||||
// changed in this release instead of implying a feature that did not ship.
|
||||
const releaseDelta = process.env.TIMMY_RELEASE_DEMO_DELTA
|
||||
|| 'Verified rebuild of the current user-visible feature path — no new feature is claimed.';
|
||||
const captionsOut = process.env.TIMMY_DEMO_CAPTIONS_OUT || '';
|
||||
const captionsShown = [];
|
||||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
await mkdir(path.dirname(output), { recursive: true });
|
||||
|
||||
// The refusal scene replays committed synthetic corpus cases through the real analysis code path,
|
||||
// so the recording shows the status and copy the server would actually send rather than copy
|
||||
// written for the video. No real photo is used anywhere in the corpus.
|
||||
const corpus = await loadVisionCorpus(fileURLToPath(new URL('../tests/fixtures/vision-corpus/', import.meta.url)));
|
||||
const corpusCase = id => corpus.cases.find(entry => entry.id === id);
|
||||
async function corpusAnalyzeResult(entry) {
|
||||
const fetchImpl = async () => ({ ok: true, json: async () => ({ choices: [{ message: { content: entry.providerContent } }] }) });
|
||||
try {
|
||||
const result = await analyzePhoto({
|
||||
payload: { imageDataUrl: 'data:image/png;base64,YQ==', consent: true },
|
||||
fetchImpl,
|
||||
config: { baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'corpus-replay', model: 'corpus-model' },
|
||||
});
|
||||
return { status: 200, body: result };
|
||||
} catch (error) {
|
||||
const failure = visionFailureResponse(error);
|
||||
return { status: failure.status, body: failure.body };
|
||||
}
|
||||
}
|
||||
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 405, height: 720 },
|
||||
@@ -20,6 +49,9 @@ const errors = [];
|
||||
page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); });
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
let visionOffline = false;
|
||||
// Reachable is not loaded: the worker can answer /v1/models before its pinned model is resident,
|
||||
// and the recording shows the app withholding AI in exactly that state (issue #15).
|
||||
let modelLoaded = true;
|
||||
await page.route('**/api/vision-status', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
@@ -30,23 +62,26 @@ await page.route('**/api/vision-status', route => route.fulfill({
|
||||
model: 'SmolVLM2-2.2B-Instruct',
|
||||
// The second pass flips the app's own readiness contract so no AI control is offered at all.
|
||||
providerReady: !visionOffline,
|
||||
modelSeen: !visionOffline,
|
||||
}),
|
||||
}));
|
||||
await page.route('**/api/analyze', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
status: 'suggestion',
|
||||
isStool: true,
|
||||
bristolType: 4,
|
||||
color: 'brown',
|
||||
confidence: 0.83,
|
||||
imageQuality: 'good',
|
||||
observations: 'Smooth, formed appearance.',
|
||||
warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
|
||||
modelSeen: !visionOffline && modelLoaded,
|
||||
}),
|
||||
}));
|
||||
// The happy path keeps one deterministic suggestion. The refusal scene swaps this for a real
|
||||
// corpus replay, so both the usable and the refused answers come from the same code path.
|
||||
const suggestionBody = {
|
||||
status: 'suggestion',
|
||||
isStool: true,
|
||||
bristolType: 4,
|
||||
color: 'brown',
|
||||
confidence: 0.83,
|
||||
imageQuality: 'good',
|
||||
observations: 'Smooth, formed appearance.',
|
||||
warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
|
||||
};
|
||||
let analyzeOverride = null;
|
||||
await page.route('**/api/analyze', route => {
|
||||
const response = analyzeOverride || { status: 200, body: suggestionBody };
|
||||
return route.fulfill({ status: response.status, contentType: 'application/json', body: JSON.stringify(response.body) });
|
||||
});
|
||||
await page.route('**/api/agent/status', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
@@ -71,6 +106,7 @@ await page.addStyleTag({ content: `
|
||||
` });
|
||||
|
||||
async function caption(text, duration = 1400) {
|
||||
captionsShown.push(text);
|
||||
await page.evaluate(text => {
|
||||
document.querySelector('#demo-caption')?.remove();
|
||||
const node = document.createElement('div');
|
||||
@@ -80,6 +116,16 @@ async function caption(text, duration = 1400) {
|
||||
node.animate([{ opacity: 0, transform: 'translateY(-8px)' }, { opacity: 1, transform: 'translateY(0)' }], { duration: 260, fill: 'forwards' });
|
||||
}, text);
|
||||
await sleep(duration);
|
||||
// A caption must not outlive its stated duration. It used to stay on screen until the next one
|
||||
// replaced it, so a view change left the previous screen's claim hanging over a different screen —
|
||||
// e.g. a delete-data caption sitting on top of the home view.
|
||||
await page.evaluate(() => {
|
||||
const node = document.querySelector('#demo-caption');
|
||||
if (!node) return;
|
||||
node.animate([{ opacity: 1 }, { opacity: 0 }], { duration: 180, fill: 'forwards' });
|
||||
setTimeout(() => node.remove(), 220);
|
||||
});
|
||||
await sleep(220);
|
||||
}
|
||||
|
||||
async function tap(selector, after = 650) {
|
||||
@@ -120,7 +166,8 @@ async function indicate(selector) {
|
||||
await sleep(650);
|
||||
}
|
||||
|
||||
await caption(`TIMMY ${version} • FEATURE DEMO`, 1200);
|
||||
await caption(`TIMMY ${version} • RELEASE DEMO`, 1200);
|
||||
await caption(`This release: ${releaseDelta}`, 2600);
|
||||
await caption('Automated checks replay this synthetic path before review', 1200);
|
||||
await caption('One clear photo action. Manual logging stays one tap away.', 1500);
|
||||
await tap('[data-scan]', 450);
|
||||
@@ -186,6 +233,146 @@ await caption('One tap still reaches manual Bristol entry, and the draft keeps y
|
||||
await tap('#discard-draft', 500);
|
||||
await caption('Start over clears the kept draft — nothing lingers silently', 1500);
|
||||
await tap('#close-sheet', 450);
|
||||
|
||||
// Post-load readiness pass (issue #15): the worker answers, but its pinned model is not resident
|
||||
// yet. The app must withhold AI instead of offering a control that fails after the upload. Every
|
||||
// claim below is asserted in the browser, so the recording cannot show a state the app does not have.
|
||||
modelLoaded = false;
|
||||
visionOffline = false;
|
||||
await caption('Now the same worker while its pinned model is still loading', 1700);
|
||||
await tap('[data-scan]', 550);
|
||||
await page.getByText(/model is not loaded yet/i).first().waitFor();
|
||||
if (await page.getByText(/Self-hosted model ready/i).count() !== 0) {
|
||||
throw new Error('A reachable worker without its loaded model must never be reported ready');
|
||||
}
|
||||
const loadingAmber = await page.locator('.model-status.loading').first().evaluate(el => getComputedStyle(el).backgroundColor);
|
||||
if (loadingAmber !== 'rgb(253, 243, 216)') throw new Error('The not-loaded state must be visually distinct from ready and offline');
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||||
await page.getByText(/Ready for your review/i).waitFor();
|
||||
if (await page.locator('#analyze-photo').count() !== 0) throw new Error('No AI control may exist before the model is loaded');
|
||||
if (await page.locator('.consent-card').count() !== 0) throw new Error('No consent may be requested before the model is loaded');
|
||||
await caption('Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident', 2900);
|
||||
await tap('#manual-from-scan', 500);
|
||||
await page.getByText('Pick the closest form').waitFor();
|
||||
await caption('One tap still reaches manual Bristol entry while the model finishes loading', 2000);
|
||||
await tap('#close-sheet', 450);
|
||||
modelLoaded = true;
|
||||
await caption('Once the pinned model is loaded, the same screen offers the AI analysis again', 2200);
|
||||
await tap('[data-scan]', 550);
|
||||
await page.getByText(/Self-hosted model ready/i).waitFor();
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||||
await page.getByText(/Ready for your review/i).waitFor();
|
||||
if (await page.locator('.consent-card').isVisible() !== true) throw new Error('The AI path must return once the model is loaded');
|
||||
if (await page.locator('#analyze-photo').count() !== 1) throw new Error('The analyze control must return once the model is loaded');
|
||||
await tap('#close-sheet', 450);
|
||||
|
||||
// Corpus refusal pass (issue #18): the same photo flow, fed the answers Timmy has to refuse.
|
||||
visionOffline = false;
|
||||
await caption('The corpus also replays the answers Timmy has to refuse', 1700);
|
||||
analyzeOverride = await corpusAnalyzeResult(corpusCase('non-stool-scene'));
|
||||
await tap('[data-scan]', 500);
|
||||
await page.getByText(/Self-hosted model ready/i).waitFor();
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/non-stool-scene.png');
|
||||
await page.locator('#ai-consent').check();
|
||||
await tap('#analyze-photo', 600);
|
||||
await page.getByText(/No confident match/i).waitFor();
|
||||
await caption('A non-stool photo gets a refusal, never a guess — manual entry stays one tap away', 2600);
|
||||
await indicate('.scan-result');
|
||||
analyzeOverride = await corpusAnalyzeResult(corpusCase('glare-overexposed'));
|
||||
await tap('#retake-photo', 400);
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/glare-overexposed.png');
|
||||
await page.locator('#ai-consent').check();
|
||||
await tap('#analyze-photo', 600);
|
||||
await page.getByText(/does not show a form Timmy can read safely/i).waitFor();
|
||||
await caption('Unreadable visual fields abstain as well: a valid reply with nothing to prefill is not an error', 2700);
|
||||
await indicate('.scan-result');
|
||||
analyzeOverride = await corpusAnalyzeResult(corpusCase('out-of-range-fields'));
|
||||
await tap('#retake-photo', 400);
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/out-of-range-fields.png');
|
||||
await page.locator('#ai-consent').check();
|
||||
await tap('#analyze-photo', 600);
|
||||
await page.getByText(/couldn’t analyze that safely/i).waitFor();
|
||||
await caption('An out-of-range answer fails closed: nothing is applied and no provider detail reaches you', 2700);
|
||||
await indicate('.scan-result');
|
||||
// Issue #73's escape class: one field blank, the other filled in and impossible. The old blank-first
|
||||
// ordering abstained here, which told the user the photo was unreadable while the model had returned a
|
||||
// value Timmy must never apply. The scene is asserted in the browser, not just captioned.
|
||||
analyzeOverride = await corpusAnalyzeResult(corpusCase('blank-plus-out-of-range'));
|
||||
await tap('#retake-photo', 400);
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/vision-corpus/blank-plus-out-of-range.png');
|
||||
await page.locator('#ai-consent').check();
|
||||
await tap('#analyze-photo', 600);
|
||||
await page.getByText(/couldn’t analyze that safely/i).waitFor();
|
||||
if (await page.getByText(/No confident match/i).count() !== 0) {
|
||||
throw new Error('A filled-in out-of-range field must not be shown as an unreadable-photo abstention');
|
||||
}
|
||||
await caption('A filled-in field that is out of range still fails closed, even when the other field came back blank', 2900);
|
||||
await indicate('.scan-result');
|
||||
await caption('You are no longer told the photo was unreadable when the model returned an impossible value', 2400);
|
||||
await indicate('.scan-result');
|
||||
await caption('Every refusal replays a committed synthetic fixture — there are no real photos in the corpus', 2500);
|
||||
analyzeOverride = null;
|
||||
await tap('#close-sheet', 450);
|
||||
|
||||
// Explainable pattern summary (issue #34). The recording seeds fictional multi-day sample logs into
|
||||
// the app's own local ledger contract, then follows one summary line to the exact records behind it.
|
||||
// Every claim below is checked in the browser, and no real health data is involved anywhere.
|
||||
const seedSyntheticLedger = async entries => {
|
||||
const key = await page.evaluate(() => `timmy:${document.querySelector('meta[name="timmy-base-path"]')?.content || '/'}:ledger-v1`);
|
||||
await page.evaluate(({ storeKey, list }) => localStorage.setItem(storeKey, JSON.stringify(list)), { storeKey: key, list: entries });
|
||||
await page.reload({ waitUntil: 'networkidle' });
|
||||
};
|
||||
const sampleDay = (id, daysAgo, bristolType) => ({
|
||||
id,
|
||||
occurredAt: new Date(Date.now() - daysAgo * 86400000).toISOString(),
|
||||
bristolType,
|
||||
color: 'brown',
|
||||
urgency: 1,
|
||||
discomfort: 1,
|
||||
note: '',
|
||||
photoDataUrl: '',
|
||||
symptoms: {},
|
||||
});
|
||||
await caption('The journal explains its own pattern lines instead of just asserting a trend', 2200);
|
||||
await seedSyntheticLedger([
|
||||
sampleDay('demo-firm', 2, 2),
|
||||
sampleDay('demo-typical-1', 5, 3),
|
||||
sampleDay('demo-typical-2', 9, 4),
|
||||
sampleDay('demo-typical-3', 12, 4),
|
||||
sampleDay('demo-old', 40, 6),
|
||||
]);
|
||||
await caption("Fictional sample logs are seeded into this browser's own storage — no real health data appears here", 2600);
|
||||
await page.locator('[data-pattern-summary]').waitFor();
|
||||
await indicate('[data-pattern-summary]');
|
||||
await caption('The summary says which confirmed logs it read, and a citation opens exactly those records', 2400);
|
||||
await indicate('[data-statement="typical"]');
|
||||
await tap('[data-statement-link="typical"]', 650);
|
||||
const citedRecords = await page.locator('[data-summary-record]').evaluateAll(nodes => nodes.map(node => node.getAttribute('data-entry-id')).sort());
|
||||
if (JSON.stringify(citedRecords) !== JSON.stringify(['demo-typical-1', 'demo-typical-2', 'demo-typical-3'])) {
|
||||
throw new Error('A summary citation must list exactly the records it cites');
|
||||
}
|
||||
await caption('Every pattern line names the confirmed logs behind it, and following one shows exactly those records', 2900);
|
||||
await indicate('[data-summary-records]');
|
||||
await tap('#pattern-records-close', 450);
|
||||
await seedSyntheticLedger([sampleDay('demo-sparse-1', 1, 4), sampleDay('demo-sparse-2', 4, 6)]);
|
||||
await page.locator('[data-summary-insufficient]').waitFor();
|
||||
await indicate('[data-summary-insufficient]');
|
||||
await caption('Two logs is not a pattern: the summary refuses to describe a trend it cannot support', 2900);
|
||||
// Restore the empty ledger the earlier scenes started from, so the rest of the recording is unchanged.
|
||||
await page.evaluate(() => localStorage.clear());
|
||||
await page.reload({ waitUntil: 'networkidle' });
|
||||
|
||||
await caption('The threat model documents this surface, so the recording shows it live rather than describing it', 2200);
|
||||
await tap('[data-view="calendar"]', 550);
|
||||
await page.getByText(/Your private journal/i).waitFor();
|
||||
await tap('[data-view="privacy"]', 550);
|
||||
await page.getByText(/Your poop\. Your phone\./i).waitFor();
|
||||
await caption('Every byte-bearing control the model names, in the running app: local ledger, consent-gated upload, portable export, full deletion', 2800);
|
||||
await indicate('#export');
|
||||
await caption('Export writes a plain JSON file you hold — the model records it as a data-lifetime boundary, not a server sync', 2600);
|
||||
await indicate('#delete-all');
|
||||
await caption('Delete all local data clears the ledger in this browser; there is no server copy of it to keep', 2600);
|
||||
await tap('[data-view="home"]', 500);
|
||||
await tap('[data-view="timmy"]', 550);
|
||||
await page.getByText(/Hermes Agent connected/i).waitFor();
|
||||
await caption('Hermes Agent connected — free-text, contextual, and server-side', 1400);
|
||||
@@ -213,7 +400,19 @@ await page.evaluate(() => {
|
||||
});
|
||||
await sleep(2200);
|
||||
|
||||
if (errors.length) throw new Error(`Browser errors: ${errors.join(' | ')}`);
|
||||
// The refusal pass deliberately produces two fail-closed 503s (an out-of-range reply, and one blank field
|
||||
// beside an out-of-range one), which the browser logs as failed resources. Requiring exactly those two
|
||||
// keeps the scene honest: if the boundary stops refusing, this fails instead of quietly recording a happy
|
||||
// path, and a dropped scene is caught rather than silently shrinking the evidence.
|
||||
const expectedFailClosed = errors.filter(message => /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/.test(message));
|
||||
if (expectedFailClosed.length !== 2 || errors.length !== expectedFailClosed.length) {
|
||||
throw new Error(`Browser errors: ${errors.join(' | ') || 'unexpected error count'}`);
|
||||
}
|
||||
// Machine-readable receipt of exactly what was rendered on camera, so the release gate can prove
|
||||
// the recorded delta statement reached the screen instead of trusting the script source.
|
||||
if (captionsOut) {
|
||||
await writeFile(captionsOut, JSON.stringify({ version, releaseDelta, captions: captionsShown }, null, 2) + '\n', 'utf8');
|
||||
}
|
||||
const recording = page.video();
|
||||
await page.close();
|
||||
await recording.saveAs(output);
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
#!/usr/bin/env node
|
||||
// Record the target-GPU cost of the pinned worker (issue #15).
|
||||
//
|
||||
// This script measures; it never invents. If no GPU can be probed, if the pinned model is not
|
||||
// loaded, or if the cold start cannot be observed, it exits non-zero and writes no receipt at all,
|
||||
// because a receipt with zeroed or assumed numbers is worse than no receipt.
|
||||
import { spawn } from 'node:child_process';
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import path from 'node:path';
|
||||
|
||||
const LOOPBACK = /^(127\.0\.0\.1|localhost|::1)$/;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
out: '',
|
||||
host: '127.0.0.1',
|
||||
port: '8080',
|
||||
alias: 'SmolVLM2-2.2B-Instruct',
|
||||
gpuProbe: 'nvidia-smi',
|
||||
warmSamples: 20,
|
||||
startCommand: '',
|
||||
readyTimeoutSeconds: 900,
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const [flag, inline] = argv[index].split('=');
|
||||
const value = inline ?? argv[index + 1];
|
||||
const consume = inline === undefined ? 2 : 1;
|
||||
if (flag === '--out') options.out = value;
|
||||
else if (flag === '--host') options.host = value;
|
||||
else if (flag === '--port') options.port = value;
|
||||
else if (flag === '--alias') options.alias = value;
|
||||
else if (flag === '--gpu-probe') options.gpuProbe = value;
|
||||
else if (flag === '--warm-samples') options.warmSamples = Number(value);
|
||||
else if (flag === '--start-command') options.startCommand = value;
|
||||
else if (flag === '--ready-timeout') options.readyTimeoutSeconds = Number(value);
|
||||
else fail(`Unknown argument: ${flag}`);
|
||||
index += consume - 1;
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function fail(message, code = 64) {
|
||||
process.stderr.write(`${message}\n`);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
const reject = message => {
|
||||
process.stderr.write(`${message}\n`);
|
||||
process.exit(1);
|
||||
};
|
||||
|
||||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
async function modelsServed({ host, port, alias }) {
|
||||
try {
|
||||
const response = await fetch(`http://${host}:${port}/v1/models`, { signal: AbortSignal.timeout(3000) });
|
||||
if (!response.ok) return false;
|
||||
const body = await response.json();
|
||||
const ids = Array.isArray(body?.data) ? body.data.map(item => item?.id).filter(Boolean) : [];
|
||||
return ids.some(id => id === alias || id.includes(alias));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function runCommand(command, args) {
|
||||
return new Promise(resolve => {
|
||||
const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
child.stdout.on('data', chunk => { stdout += chunk; });
|
||||
child.stderr.on('data', chunk => { stderr += chunk; });
|
||||
child.on('error', error => resolve({ code: 127, stdout, stderr: String(error.message) }));
|
||||
child.on('close', code => resolve({ code, stdout, stderr }));
|
||||
});
|
||||
}
|
||||
|
||||
async function probeGpu(probeCommand) {
|
||||
const result = await runCommand(probeCommand, [
|
||||
'--query-gpu=name,memory.total,memory.used',
|
||||
'--format=csv,noheader,nounits',
|
||||
]);
|
||||
if (result.code !== 0) {
|
||||
reject(`No GPU could be probed with ${probeCommand}: ${result.stderr.trim() || `exit ${result.code}`}. A benchmark receipt needs a real device, so nothing was written.`);
|
||||
}
|
||||
const [name, total, used] = result.stdout.trim().split('\n')[0].split(',').map(part => part.trim());
|
||||
if (!name || !/^\d+$/.test(total) || !/^\d+$/.test(used)) {
|
||||
reject(`The GPU probe returned unreadable output (${JSON.stringify(result.stdout.trim())}); refusing to record VRAM it cannot read.`);
|
||||
}
|
||||
return { name, vramTotalMiB: Number(total), vramUsedMiB: Number(used) };
|
||||
}
|
||||
|
||||
function percentile(sorted, fraction) {
|
||||
const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(fraction * sorted.length) - 1));
|
||||
return sorted[index];
|
||||
}
|
||||
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (!options.out) fail('--out is required so the receipt has a destination');
|
||||
if (!LOOPBACK.test(options.host)) fail(`The worker is a loopback service; refusing to benchmark ${options.host}.`);
|
||||
if (!Number.isInteger(options.warmSamples) || options.warmSamples < 1) fail('--warm-samples must be a positive integer');
|
||||
|
||||
const receipts = JSON.parse(await readFile(fileURLToPath(new URL('../deploy/worker/receipts.json', import.meta.url)), 'utf8'));
|
||||
|
||||
// The device is probed first: nothing is timed until we know the numbers have somewhere to be real.
|
||||
const gpu = await probeGpu(options.gpuProbe);
|
||||
|
||||
if (!(await modelsServed(options))) {
|
||||
reject(`${options.alias} is not loaded on ${options.host}:${options.port} yet, so there is no ready worker to time. Start the worker and wait for the post-load gate.`);
|
||||
}
|
||||
|
||||
let coldStartMs = null;
|
||||
if (options.startCommand) {
|
||||
const started = Date.now();
|
||||
const started_process = await runCommand('bash', ['-c', options.startCommand]);
|
||||
if (started_process.code !== 0) {
|
||||
reject(`The cold-start command failed (exit ${started_process.code}): ${started_process.stderr.trim()}`);
|
||||
}
|
||||
const deadline = Date.now() + options.readyTimeoutSeconds * 1000;
|
||||
let ready = false;
|
||||
while (Date.now() < deadline) {
|
||||
if (await modelsServed(options)) { ready = true; break; }
|
||||
await sleep(250);
|
||||
}
|
||||
if (!ready) reject(`The worker did not become ready within ${options.readyTimeoutSeconds}s, so the cold start could not be measured.`);
|
||||
coldStartMs = Date.now() - started;
|
||||
}
|
||||
|
||||
const samples = [];
|
||||
for (let index = 0; index < options.warmSamples; index += 1) {
|
||||
const started = process.hrtime.bigint();
|
||||
const ok = await modelsServed(options);
|
||||
const elapsedMs = Number(process.hrtime.bigint() - started) / 1e6;
|
||||
if (!ok) reject('The worker stopped answering during the warm pass; the receipt would be partial, so nothing was written.');
|
||||
samples.push(Math.round(elapsedMs * 1000) / 1000);
|
||||
}
|
||||
const sorted = [...samples].sort((left, right) => left - right);
|
||||
|
||||
const receipt = {
|
||||
schemaVersion: 1,
|
||||
status: 'measured_on_target_gpu',
|
||||
measuredAt: new Date().toISOString(),
|
||||
gpu,
|
||||
coldStartMs,
|
||||
warmLatencyMs: {
|
||||
samples: sorted.length,
|
||||
min: sorted[0],
|
||||
p50: percentile(sorted, 0.5),
|
||||
p95: percentile(sorted, 0.95),
|
||||
max: sorted[sorted.length - 1],
|
||||
},
|
||||
worker: { baseUrl: `http://${options.host}:${options.port}/v1`, alias: options.alias },
|
||||
pins: {
|
||||
llamaCppCommit: receipts.llamaCpp.commit,
|
||||
modelSha256: receipts.models.map(model => model.sha256),
|
||||
},
|
||||
};
|
||||
|
||||
await writeFile(path.resolve(options.out), `${JSON.stringify(receipt, null, 2)}\n`, 'utf8');
|
||||
process.stdout.write(`${path.resolve(options.out)}\n`);
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
# Post-load readiness gate for Timmy's self-hosted vision worker (issue #15).
|
||||
#
|
||||
# A worker that answers on its port is not necessarily a worker that can answer: llama.cpp opens
|
||||
# its HTTP listener before the model finishes loading and returns an error until the weights are
|
||||
# resident. Readiness therefore passes only when the pinned model alias is actually served.
|
||||
#
|
||||
# Exit codes:
|
||||
# 0 ready — the pinned model is loaded and being served
|
||||
# 2 the worker is not reachable
|
||||
# 3 the worker is reachable but the pinned model is not loaded yet
|
||||
# 64 usage error (including a non-loopback probe host)
|
||||
set -uo pipefail
|
||||
|
||||
HOST="${TIMMY_MODEL_HOST:-127.0.0.1}"
|
||||
PORT="${TIMMY_MODEL_PORT:-8080}"
|
||||
ALIAS="${TIMMY_MODEL_ALIAS:-SmolVLM2-2.2B-Instruct}"
|
||||
WAIT=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--host) HOST="${2:-}"; shift 2 ;;
|
||||
--port) PORT="${2:-}"; shift 2 ;;
|
||||
--alias) ALIAS="${2:-}"; shift 2 ;;
|
||||
--wait) WAIT="${2:-}"; shift 2 ;;
|
||||
*)
|
||||
printf 'Usage: %s [--host H] [--port P] [--alias A] [--wait SECONDS]\n' "$0" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# The worker is a private, loopback-only service. Probing anything else would be a mistake about
|
||||
# exposure, so refuse rather than answer about a host this gate is not responsible for.
|
||||
if [[ ! "$HOST" =~ ^(127\.0\.0\.1|localhost|::1)$ ]]; then
|
||||
printf 'Readiness must be probed on loopback, got %s\n' "$HOST" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
deadline=$(( $(date +%s) + WAIT ))
|
||||
outcome='unreachable'
|
||||
while :; do
|
||||
body="$(curl --silent --max-time 3 "http://$HOST:$PORT/v1/models" 2>/dev/null)"
|
||||
if [[ $? -eq 0 && -n "$body" ]]; then
|
||||
if printf '%s' "$body" | grep -q -- "$ALIAS"; then
|
||||
printf 'Worker ready: %s is loaded on http://%s:%s/v1\n' "$ALIAS" "$HOST" "$PORT"
|
||||
exit 0
|
||||
fi
|
||||
outcome='not-loaded'
|
||||
printf 'Worker reachable on http://%s:%s/v1 but %s is not loaded yet.\n' "$HOST" "$PORT" "$ALIAS" >&2
|
||||
else
|
||||
outcome='unreachable'
|
||||
printf 'Worker not reachable on http://%s:%s/v1.\n' "$HOST" "$PORT" >&2
|
||||
fi
|
||||
[[ "$(date +%s)" -ge "$deadline" ]] && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
[[ "$outcome" == 'not-loaded' ]] && exit 3
|
||||
exit 2
|
||||
@@ -3,7 +3,7 @@ import { isIP } from 'node:net';
|
||||
import { readFile, stat } from 'node:fs/promises';
|
||||
import { extname, join, normalize } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { analyzePhoto } from './src/vision-service.js';
|
||||
import { analyzePhoto, visionFailureResponse } from './src/vision-service.js';
|
||||
import { probeVisionProvider, resolveVisionConfig } from './src/vision-config.js';
|
||||
import { AgentGatewayError, createHermesAgentService, resolveHermesAgentConfig } from './src/hermes-agent-service.js';
|
||||
|
||||
@@ -55,7 +55,7 @@ http.createServer(async(req,res)=>{
|
||||
if(!visionConfig.enabled)return sendJson(res,503,{error:'AI analysis is disabled. Continue manually.'});
|
||||
const payload=await readJson(req);
|
||||
try{return sendJson(res,200,await analyzePhoto({payload,config:visionConfig}))}
|
||||
catch(error){const safe=/consent|JPEG|PNG|WebP|empty|too large/i.test(error.message);return sendJson(res,safe?400:503,{error:error.message})}
|
||||
catch(error){const failure=visionFailureResponse(error);return sendJson(res,failure.status,failure.body)}
|
||||
}
|
||||
if(appPath==='/api/agent/status'&&req.method==='GET')return sendJson(res,200,agentService.status(cookie(req,'timmy_agent')));
|
||||
if(appPath==='/api/agent/unlock'&&req.method==='POST'){
|
||||
|
||||
@@ -2,32 +2,60 @@ const COLORS = new Set(['brown', 'green', 'yellow', 'pale', 'red', 'black']);
|
||||
const QUALITIES = new Set(['good', 'fair', 'poor']);
|
||||
const MAX_IMAGE_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
// Fail-closed responses carry a code so the server boundary and the corpus can name the failure
|
||||
// class without matching on prose.
|
||||
export const VISION_FAILURE_CODES = Object.freeze({ invalidJson: 'invalid_json', invalidFields: 'invalid_fields' });
|
||||
|
||||
function visionResponseError(code, message) {
|
||||
const error = new Error(message);
|
||||
error.name = 'VisionResponseError';
|
||||
error.code = code;
|
||||
return error;
|
||||
}
|
||||
|
||||
function asObject(raw) {
|
||||
if (raw && typeof raw === 'object' && !Array.isArray(raw)) return raw;
|
||||
if (typeof raw !== 'string') throw new Error('Invalid AI response.');
|
||||
if (typeof raw !== 'string') throw visionResponseError(VISION_FAILURE_CODES.invalidJson, 'Invalid AI response.');
|
||||
const cleaned = raw.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
|
||||
try {
|
||||
const parsed = JSON.parse(cleaned);
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error();
|
||||
return parsed;
|
||||
} catch {
|
||||
throw new Error('Invalid AI response.');
|
||||
throw visionResponseError(VISION_FAILURE_CODES.invalidJson, 'Invalid AI response.');
|
||||
}
|
||||
}
|
||||
|
||||
export function parseVisionResponse(raw) {
|
||||
const value = asObject(raw);
|
||||
if (typeof value.isStool !== 'boolean') throw new Error('Invalid AI response: isStool is required.');
|
||||
if (typeof value.isStool !== 'boolean') throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: isStool is required.');
|
||||
const confidence = Number(value.confidence);
|
||||
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) throw new Error('Invalid AI response: confidence is required.');
|
||||
if (!value.isStool) return { status: 'needs_user_input', isStool: false, reason: 'The image does not clearly show stool.' };
|
||||
if (confidence < 0.55) return { status: 'needs_user_input', isStool: true, reason: 'The image is too uncertain to prefill safely.' };
|
||||
const bristolType = Number(value.bristolType);
|
||||
const color = String(value.color || '').toLowerCase();
|
||||
const imageQuality = String(value.imageQuality || '').toLowerCase();
|
||||
if (!Number.isInteger(bristolType) || bristolType < 1 || bristolType > 7 || !COLORS.has(color) || !QUALITIES.has(imageQuality)) {
|
||||
throw new Error('Invalid AI response: visual fields are out of range.');
|
||||
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: confidence is required.');
|
||||
if (!value.isStool) return { status: 'needs_user_input', isStool: false, abstainedFor: 'not_stool', reason: 'The image does not clearly show stool.' };
|
||||
if (confidence < 0.55) return { status: 'needs_user_input', isStool: true, abstainedFor: 'low_confidence', reason: 'The image is too uncertain to prefill safely.' };
|
||||
// The request schema lets the model return a null form and color when it cannot read them.
|
||||
// A blank field means the model could not read it, so a schema-valid answer with nothing usable
|
||||
// abstains and nothing is prefilled. A filled-in field is a claim, though: if it sits outside the
|
||||
// contract the response fails closed, even when the other field is blank (#73), because a caller
|
||||
// reading `unusable_visual_fields` would be told the photo was unreadable when the model had in
|
||||
// fact returned an impossible value.
|
||||
const blank = input => input === null || input === undefined || (typeof input === 'string' && !input.trim());
|
||||
const bristolType = blank(value.bristolType) ? null : Number(value.bristolType);
|
||||
const color = blank(value.color) ? null : String(value.color).toLowerCase();
|
||||
if ((bristolType !== null && (!Number.isInteger(bristolType) || bristolType < 1 || bristolType > 7))
|
||||
|| (color !== null && !COLORS.has(color))) {
|
||||
throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.');
|
||||
}
|
||||
if (bristolType === null || color === null) {
|
||||
return {
|
||||
status: 'needs_user_input',
|
||||
isStool: true,
|
||||
abstainedFor: 'unusable_visual_fields',
|
||||
reason: 'The photo does not show a form Timmy can read safely.',
|
||||
};
|
||||
}
|
||||
const imageQuality = String(value.imageQuality || '').toLowerCase();
|
||||
if (!QUALITIES.has(imageQuality)) throw visionResponseError(VISION_FAILURE_CODES.invalidFields, 'Invalid AI response: visual fields are out of range.');
|
||||
return {
|
||||
status: 'suggestion',
|
||||
isStool: true,
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { bucketForBristolType } from './domain.js';
|
||||
|
||||
export const PATTERN_WINDOW_DAYS = 28;
|
||||
// Two logs are an anecdote. Below this the summary is required to say so instead of describing a trend.
|
||||
export const MIN_WINDOW_LOGS = 3;
|
||||
|
||||
const DAY_MS = 86400000;
|
||||
|
||||
// The summary may only describe what the user recorded. No cause, no health verdict, no food ruling.
|
||||
const BOUNDARY = 'Built only from your own confirmed records in this window. Timmy reports what you logged and when — not why, and never a verdict on your health or your food.';
|
||||
|
||||
function inWindowRecords(entries, now, windowDays) {
|
||||
const startedAt = now - windowDays * DAY_MS;
|
||||
return (Array.isArray(entries) ? entries : [])
|
||||
.map(entry => ({ entry, at: new Date(entry?.occurredAt).getTime() }))
|
||||
.filter(({ at }) => Number.isFinite(at) && at <= now && at >= startedAt)
|
||||
.sort((left, right) => left.at - right.at)
|
||||
.map(({ entry, at }) => ({
|
||||
id: String(entry?.id ?? ''),
|
||||
occurredAt: new Date(at).toISOString(),
|
||||
bristolType: Number(entry?.bristolType),
|
||||
bucket: bucketForBristolType(entry?.bristolType),
|
||||
}));
|
||||
}
|
||||
|
||||
export function buildPatternSummary(entries = [], { now = Date.now(), windowDays = PATTERN_WINDOW_DAYS } = {}) {
|
||||
const records = inWindowRecords(entries, now, windowDays);
|
||||
const window = {
|
||||
days: windowDays,
|
||||
label: `last ${windowDays} days`,
|
||||
start: new Date(now - windowDays * DAY_MS).toISOString(),
|
||||
end: new Date(now).toISOString(),
|
||||
};
|
||||
|
||||
if (records.length < MIN_WINDOW_LOGS) {
|
||||
return {
|
||||
window,
|
||||
records,
|
||||
insufficient: true,
|
||||
statements: [],
|
||||
boundary: BOUNDARY,
|
||||
copy: `Not enough confirmed logs in this window yet — ${records.length} of ${MIN_WINDOW_LOGS} needed before Timmy describes a pattern.`,
|
||||
};
|
||||
}
|
||||
|
||||
const frequency = {
|
||||
id: 'frequency',
|
||||
text: `${records.length} confirmed log${records.length === 1 ? '' : 's'} in the ${window.label}.`,
|
||||
entryIds: records.map(record => record.id),
|
||||
};
|
||||
|
||||
const buckets = new Map();
|
||||
for (const record of records) {
|
||||
if (!buckets.has(record.bucket)) buckets.set(record.bucket, []);
|
||||
buckets.get(record.bucket).push(record.id);
|
||||
}
|
||||
const bucketLine = (bucket, label) => ({
|
||||
id: bucket === 'constipation' ? 'firm' : bucket,
|
||||
text: `${label}: ${buckets.get(bucket).length} of ${records.length} confirmed log${records.length === 1 ? '' : 's'}.`,
|
||||
entryIds: buckets.get(bucket),
|
||||
});
|
||||
const formLines = [
|
||||
['constipation', 'Firm side (Type 1–2)'],
|
||||
['typical', 'Typical range (Type 3–4)'],
|
||||
['loose', 'Loose side (Type 5–7)'],
|
||||
].filter(([bucket]) => buckets.has(bucket)).map(([bucket, label]) => bucketLine(bucket, label));
|
||||
// Both ends in one window is not a trend, and the summary has to say so out loud rather than
|
||||
// picking whichever side is marginally larger.
|
||||
const mixedLine = buckets.has('constipation') && buckets.has('loose') ? [{
|
||||
id: 'mixed',
|
||||
text: `Firm and loose logs both appear in the ${window.label} (${buckets.get('constipation').length} firm, ${buckets.get('loose').length} loose), which is a mixed pattern and not one trend.`,
|
||||
entryIds: records.filter(record => record.bucket === 'constipation' || record.bucket === 'loose').map(record => record.id),
|
||||
}] : [];
|
||||
|
||||
return { window, records, insufficient: false, statements: [frequency, ...formLines, ...mixedLine], copy: '', boundary: BOUNDARY };
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
// Contract for the deterministic vision corpus under tests/fixtures/vision-corpus.
|
||||
//
|
||||
// The corpus exists to prove one claim continuously: every provider response Timmy
|
||||
// receives is either a schema-valid visible-field suggestion or it fails closed.
|
||||
// Fixtures are synthetic images generated by scripts/build_vision_corpus.py; no real
|
||||
// medical imagery, personal data, or third-party content is redistributable here.
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { join } from 'node:path';
|
||||
import { parseVisionResponse, VISION_FAILURE_CODES } from './analysis.js';
|
||||
|
||||
export const VISION_CORPUS_LICENSE = 'CC0-1.0';
|
||||
|
||||
// Case classes the corpus must always cover, from issue #18.
|
||||
export const VISION_CORPUS_CLASSES = Object.freeze([
|
||||
'stool',
|
||||
'non-stool',
|
||||
'blur',
|
||||
'glare',
|
||||
'empty-bowl',
|
||||
'malformed-response',
|
||||
]);
|
||||
|
||||
export const VISION_CORPUS_OUTCOMES = Object.freeze(['suggestion', 'abstained', 'fail_closed']);
|
||||
|
||||
const ALLOWED_CLASSES = new Set(VISION_CORPUS_CLASSES);
|
||||
|
||||
export function validateCorpusManifest(manifest) {
|
||||
if (!manifest || typeof manifest !== 'object') throw new Error('vision corpus manifest must be an object');
|
||||
if (manifest.schemaVersion !== 1) throw new Error('vision corpus manifest must declare schemaVersion 1');
|
||||
if (manifest.license !== VISION_CORPUS_LICENSE) {
|
||||
throw new Error(`vision corpus must be redistributable under ${VISION_CORPUS_LICENSE}`);
|
||||
}
|
||||
if (!Array.isArray(manifest.cases) || manifest.cases.length === 0) throw new Error('vision corpus must list cases');
|
||||
const ids = new Set();
|
||||
for (const entry of manifest.cases) {
|
||||
if (!entry?.id) throw new Error('every corpus case needs an id');
|
||||
if (ids.has(entry.id)) throw new Error(`duplicate corpus case: ${entry.id}`);
|
||||
ids.add(entry.id);
|
||||
if (!ALLOWED_CLASSES.has(entry.caseClass)) throw new Error(`${entry.id} has an unsupported caseClass: ${entry.caseClass}`);
|
||||
if (entry.synthetic !== true) throw new Error(`${entry.id} must be a synthetic fixture`);
|
||||
if (entry.license !== VISION_CORPUS_LICENSE) throw new Error(`${entry.id} must record its ${VISION_CORPUS_LICENSE} license`);
|
||||
if (typeof entry.provenance !== 'string' || !entry.provenance.trim()) throw new Error(`${entry.id} must record provenance`);
|
||||
if (!/^[a-f0-9]{64}$/.test(String(entry.sha256 || ''))) throw new Error(`${entry.id} must record a sha256`);
|
||||
if (!VISION_CORPUS_OUTCOMES.includes(entry.expectedOutcome)) {
|
||||
throw new Error(`${entry.id} must declare an expectedOutcome from ${VISION_CORPUS_OUTCOMES.join(', ')}`);
|
||||
}
|
||||
if (!('providerContent' in entry)) throw new Error(`${entry.id} must record the provider response it replays`);
|
||||
}
|
||||
for (const required of VISION_CORPUS_CLASSES) {
|
||||
if (!manifest.cases.some(entry => entry.caseClass === required)) {
|
||||
throw new Error(`vision corpus is missing the required ${required} case`);
|
||||
}
|
||||
}
|
||||
return manifest;
|
||||
}
|
||||
|
||||
export function classifyVisionOutcome(providerContent) {
|
||||
let parsed;
|
||||
try {
|
||||
parsed = parseVisionResponse(providerContent);
|
||||
} catch (error) {
|
||||
const reason = error?.code === VISION_FAILURE_CODES.invalidJson
|
||||
? 'unparsable_content'
|
||||
: error?.code === VISION_FAILURE_CODES.invalidFields
|
||||
? 'out_of_range_or_missing_fields'
|
||||
: 'unexpected_error';
|
||||
return { outcome: 'fail_closed', reason, detail: error.message };
|
||||
}
|
||||
if (parsed.status === 'suggestion') {
|
||||
return {
|
||||
outcome: 'suggestion',
|
||||
reason: 'schema_valid_suggestion',
|
||||
bristolType: parsed.bristolType,
|
||||
color: parsed.color,
|
||||
confidence: parsed.confidence,
|
||||
};
|
||||
}
|
||||
if (parsed.status === 'needs_user_input') {
|
||||
return {
|
||||
outcome: 'abstained',
|
||||
reason: parsed.abstainedFor,
|
||||
detail: parsed.reason,
|
||||
};
|
||||
}
|
||||
return { outcome: 'fail_closed', reason: 'unparsable_content', detail: 'The response could not be used.' };
|
||||
}
|
||||
|
||||
export async function loadVisionCorpus(directory) {
|
||||
const manifestPath = join(directory, 'manifest.json');
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
||||
} catch (error) {
|
||||
throw new Error(`vision corpus manifest could not be read at ${manifestPath}: ${error.message}`);
|
||||
}
|
||||
validateCorpusManifest(manifest);
|
||||
for (const entry of manifest.cases) {
|
||||
let bytes;
|
||||
try {
|
||||
bytes = await readFile(join(directory, entry.file));
|
||||
} catch {
|
||||
throw new Error(`${entry.id} is missing its fixture ${entry.file}`);
|
||||
}
|
||||
const digest = createHash('sha256').update(bytes).digest('hex');
|
||||
if (digest !== entry.sha256 || bytes.length !== entry.bytes) {
|
||||
throw new Error(`${entry.id} fixture ${entry.file} does not match its recorded sha256; regenerate the corpus instead of editing it`);
|
||||
}
|
||||
}
|
||||
return { ...manifest, directory };
|
||||
}
|
||||
@@ -31,5 +31,17 @@ export function visionReadiness(status) {
|
||||
message: 'Vision worker offline. Manual logging is still available.',
|
||||
};
|
||||
}
|
||||
// A worker that answers /v1/models has not necessarily loaded the model yet (#15), so readiness
|
||||
// passes only once the pinned model is confirmed loaded. An unconfirmed model fails closed to the
|
||||
// manual path instead of offering an AI control that fails after the upload.
|
||||
if (status.modelSeen !== true) {
|
||||
return {
|
||||
phase: 'loading',
|
||||
ready: false,
|
||||
manualOnly: true,
|
||||
manualLabel: MANUAL_LABEL,
|
||||
message: 'The vision worker is up but the model is not loaded yet. Manual logging is available.',
|
||||
};
|
||||
}
|
||||
return { phase: 'ready', ready: true, manualOnly: false, manualLabel: '', message: '' };
|
||||
}
|
||||
@@ -1,4 +1,17 @@
|
||||
import { buildVisionRequest, parseVisionResponse, validatePhotoPayload } from './analysis.js';
|
||||
import { buildVisionRequest, parseVisionResponse, validatePhotoPayload, VISION_FAILURE_CODES } from './analysis.js';
|
||||
|
||||
// One place decides what a failed analysis request returns, so the browser copy is testable and a
|
||||
// provider's own words never reach the user.
|
||||
const FAIL_CLOSED_COPY = 'No suggestion was applied. Continue manually, or try a clearer photo.';
|
||||
const INPUT_REJECTION = /consent|JPEG|PNG|WebP|empty|too large/i;
|
||||
|
||||
export function visionFailureResponse(error) {
|
||||
const message = String(error?.message || '');
|
||||
if (INPUT_REJECTION.test(message)) return { status: 400, body: { error: message } };
|
||||
const schemaFailure = error?.code === VISION_FAILURE_CODES.invalidJson || error?.code === VISION_FAILURE_CODES.invalidFields;
|
||||
if (schemaFailure) return { status: 503, body: { error: FAIL_CLOSED_COPY } };
|
||||
return { status: 503, body: { error: message || 'AI analysis is temporarily unavailable. Continue manually.' } };
|
||||
}
|
||||
|
||||
function providerEndpoint(baseUrl) {
|
||||
let url;
|
||||
|
||||
@@ -18,7 +18,7 @@ main{display:block}.eyebrow{font-size:11px;font-weight:800;letter-spacing:.1em;t
|
||||
.chat-page{display:flex;flex-direction:column;min-height:calc(100vh - 175px)}.chat-title{padding-bottom:8px}.agent-status{display:flex;align-items:center;gap:11px;padding:11px 13px;background:rgba(255,253,250,.7);border:1px solid var(--line);border-radius:17px;margin-bottom:12px}.agent-status>i{width:10px;height:10px;border-radius:50%;background:#a9a39d;box-shadow:0 0 0 5px rgba(169,163,157,.13)}.agent-status.connected>i{background:#209479;box-shadow:0 0 0 5px rgba(32,148,121,.13)}.agent-status.locked>i{background:#d19b27}.agent-status strong,.agent-status small{display:block}.agent-status strong{font-size:13px}.agent-status small{font-size:11px;color:var(--muted);margin-top:2px}.conversation{background:var(--surface);border:1px solid var(--line);border-radius:23px;padding:13px;box-shadow:0 10px 30px rgba(55,40,31,.05)}.chat{display:flex;flex-direction:column;gap:9px;min-height:195px;max-height:42vh;overflow:auto;padding:4px 1px 14px}.bubble{max-width:86%;padding:11px 13px;border-radius:17px;line-height:1.42;font-size:14px;white-space:pre-wrap}.bubble.timmy{align-self:flex-start;background:#efebe4;border-bottom-left-radius:5px}.bubble.user{align-self:flex-end;background:var(--teal);color:white;border-bottom-right-radius:5px}.thinking{display:flex;gap:4px}.thinking span{width:6px;height:6px;border-radius:50%;background:#8b847d;animation:blink 1s infinite}.thinking span:nth-child(2){animation-delay:.15s}.thinking span:nth-child(3){animation-delay:.3s}@keyframes blink{50%{opacity:.25;transform:translateY(-2px)}}.composer{display:grid;grid-template-columns:1fr 45px;gap:8px;align-items:end;background:#f0ece5;border-radius:18px;padding:6px}.composer textarea{border:0;background:transparent;resize:none;min-height:42px;max-height:110px;padding:10px 9px;outline:0;color:var(--ink)}.composer button{width:44px;height:44px;border:0;border-radius:14px;background:var(--ink);color:white;font-size:22px;cursor:pointer}.composer button:disabled{opacity:.4}.composer-note{font-size:10px;color:var(--muted);margin:7px 5px 0;line-height:1.35}.chat-error{font-size:12px;color:var(--red);margin:0 4px 8px}.safety-line{margin:13px 4px 0;color:var(--muted);font-size:11px;line-height:1.45}.safety-line strong{color:var(--ink)}.unlock-card{background:#fff8e6;border:1px solid #ead9a9;border-radius:20px;padding:14px;margin-bottom:12px}.unlock-card>label{display:block;font-size:12px;font-weight:800;margin-bottom:7px}.unlock-row{display:grid;grid-template-columns:1fr auto;gap:8px}.unlock-card .fine{margin:8px 2px 0}
|
||||
.sheet-backdrop{position:fixed;inset:0;background:rgba(28,22,18,.42);display:flex;align-items:flex-end;justify-content:center;z-index:50;padding-top:28px;backdrop-filter:blur(5px)}.sheet{width:min(100%,680px);max-height:94vh;overflow:auto;background:var(--surface);border-radius:28px 28px 0 0;padding:9px 18px calc(24px + env(safe-area-inset-bottom));box-shadow:0 -15px 50px rgba(30,22,18,.2)}.sheet-handle{width:38px;height:4px;background:#d8d1c8;border-radius:999px;margin:2px auto 15px}.sheet-header{display:flex;justify-content:space-between;align-items:flex-start;gap:12px}.sheet-header h2{margin-top:4px}.icon-btn{width:44px;height:44px;border:0;border-radius:50%;background:#efebe5;font-size:24px;cursor:pointer}.progress{height:4px;background:#eee8df;border-radius:99px;margin:7px 0 18px;overflow:hidden}.progress i{height:100%;display:block;background:var(--teal)}.progress-step-1{width:33.34%}.progress-step-2{width:66.68%}.progress-step-3{width:100%}
|
||||
.draft-resume{display:flex;align-items:center;justify-content:space-between;gap:10px;background:#fdf3d8;border:1px solid #ecd9a3;border-radius:14px;padding:8px 11px;margin:0 0 14px;font-size:12px;line-height:1.35;color:#5d4a12}.draft-resume strong{font-weight:800}
|
||||
.scan-hero{text-align:center;padding:3px 15px 12px}.scan-hero img{width:76px}.scan-hero h3{font-size:19px;margin:4px 0 7px}.scan-hero p{font-size:13px;line-height:1.45;color:var(--muted)}.model-status{border-radius:14px;padding:9px 11px;font-size:11px;font-weight:700;margin-bottom:12px}.model-status.ready{background:var(--teal-soft);color:var(--teal)}.model-status.offline{background:var(--red-soft);color:var(--red)}.model-status.checking{background:#f0ece5;color:var(--muted)}.capture-choice-grid{display:grid;grid-template-columns:1fr 1fr;gap:10px}.capture-status{background:var(--red-soft);color:var(--red);border-radius:14px;padding:10px 12px;font-size:12px;line-height:1.4}.photo-capture{display:grid;place-items:center;text-align:center;border:1.5px dashed #b9aea1;border-radius:21px;padding:16px 10px;background:#faf7f1;cursor:pointer}.photo-capture b{font-size:30px}.photo-capture strong{margin:6px 0 2px}.photo-capture span{font-size:11px;color:var(--muted)}.photo-capture input{display:none}.photo-preview{display:block;width:100%;max-height:250px;object-fit:cover;border-radius:18px;margin:10px 0}.quality-note{font-size:12px;color:var(--muted)}.consent-card{background:#f5f1e9;border-radius:18px;padding:12px;margin:12px 0}.check{display:flex;gap:11px;align-items:flex-start;padding:9px 0}.check input{width:20px;height:20px;accent-color:var(--teal);flex:0 0 auto}.check span{font-size:13px;line-height:1.4}.analyzing{text-align:center;padding:33px 10px}.analyzing img{width:86px}.spinner{width:32px;height:32px;border:3px solid var(--soft);border-top-color:var(--teal);border-radius:50%;animation:spin .8s linear infinite;margin:10px auto}@keyframes spin{to{transform:rotate(360deg)}}.analyzing p{font-size:13px;color:var(--muted)}.scan-result{background:var(--teal-soft);border-radius:20px;padding:17px;margin-bottom:13px}.scan-result.needs-input{background:#f2eee7;text-align:center}.scan-result.needs-input>b{font-size:28px}.scan-result p{font-size:13px;line-height:1.45;margin:9px 0 0}.ai-badge{font-size:10px;font-weight:850;letter-spacing:.08em;color:var(--teal)}.suggestion-pair{display:grid;grid-template-columns:1fr 1fr;gap:9px;margin-top:12px}.suggestion-pair>div{background:var(--surface);padding:12px;border-radius:14px}.suggestion-pair small,.suggestion-pair strong{display:block}.suggestion-pair small{font-size:9px;color:var(--muted)}.suggestion-pair strong{font-size:18px;margin-top:2px}.ai-prefill{background:var(--teal-soft);border-radius:16px;padding:11px 13px;margin-bottom:12px}.ai-prefill strong,.ai-prefill small{display:block}.ai-prefill small{color:var(--muted);margin-top:3px}.choice-grid{display:grid;grid-template-columns:1fr 1fr;gap:8px}.bristol{min-height:66px;border:1px solid var(--line);background:#f7f3ed;border-radius:16px;padding:10px;text-align:left;cursor:pointer}.bristol strong,.bristol span{display:block}.bristol span{font-size:11px;color:var(--muted);margin-top:2px}.bristol.selected{background:var(--ink);color:white}.bristol.selected span{color:#ddd}.field{display:block;margin-bottom:15px}.field-label{display:block;font-size:12px;font-weight:800;margin-bottom:6px}.input{width:100%;min-height:48px;border:1px solid var(--line);border-radius:14px;background:#f8f5ef;padding:10px 12px;color:var(--ink)}textarea.input{min-height:85px;resize:vertical}.range-row{display:grid;grid-template-columns:1fr 37px;gap:10px;align-items:center}.range-row input{accent-color:var(--teal)}.range-val{width:37px;height:37px;border-radius:12px;background:var(--soft);display:grid;place-items:center;font-weight:800}.photo-drop{width:100%;background:#f3eee7}.symptoms{display:flex;flex-direction:column}.alert{background:var(--red-soft);border:1px solid #eec1ba;border-radius:16px;padding:13px;color:#75261f}.alert p{margin:5px 0 0;font-size:13px;line-height:1.4}
|
||||
.scan-hero{text-align:center;padding:3px 15px 12px}.scan-hero img{width:76px}.scan-hero h3{font-size:19px;margin:4px 0 7px}.scan-hero p{font-size:13px;line-height:1.45;color:var(--muted)}.model-status{border-radius:14px;padding:9px 11px;font-size:11px;font-weight:700;margin-bottom:12px}.model-status.ready{background:var(--teal-soft);color:var(--teal)}.model-status.offline{background:var(--red-soft);color:var(--red)}.model-status.loading{background:#fdf3d8;color:#5d4a12}.capture-choice-grid{display:grid;grid-template-columns:1fr 1fr;gap:10px}.capture-status{background:var(--red-soft);color:var(--red);border-radius:14px;padding:10px 12px;font-size:12px;line-height:1.4}.photo-capture{display:grid;place-items:center;text-align:center;border:1.5px dashed #b9aea1;border-radius:21px;padding:16px 10px;background:#faf7f1;cursor:pointer}.photo-capture b{font-size:30px}.photo-capture strong{margin:6px 0 2px}.photo-capture span{font-size:11px;color:var(--muted)}.photo-capture input{display:none}.photo-preview{display:block;width:100%;max-height:250px;object-fit:cover;border-radius:18px;margin:10px 0}.quality-note{font-size:12px;color:var(--muted)}.consent-card{background:#f5f1e9;border-radius:18px;padding:12px;margin:12px 0}.check{display:flex;gap:11px;align-items:flex-start;padding:9px 0}.check input{width:20px;height:20px;accent-color:var(--teal);flex:0 0 auto}.check span{font-size:13px;line-height:1.4}.analyzing{text-align:center;padding:33px 10px}.analyzing img{width:86px}.spinner{width:32px;height:32px;border:3px solid var(--soft);border-top-color:var(--teal);border-radius:50%;animation:spin .8s linear infinite;margin:10px auto}@keyframes spin{to{transform:rotate(360deg)}}.analyzing p{font-size:13px;color:var(--muted)}.scan-result{background:var(--teal-soft);border-radius:20px;padding:17px;margin-bottom:13px}.scan-result.needs-input{background:#f2eee7;text-align:center}.scan-result.needs-input>b{font-size:28px}.scan-result p{font-size:13px;line-height:1.45;margin:9px 0 0}.ai-badge{font-size:10px;font-weight:850;letter-spacing:.08em;color:var(--teal)}.suggestion-pair{display:grid;grid-template-columns:1fr 1fr;gap:9px;margin-top:12px}.suggestion-pair>div{background:var(--surface);padding:12px;border-radius:14px}.suggestion-pair small,.suggestion-pair strong{display:block}.suggestion-pair small{font-size:9px;color:var(--muted)}.suggestion-pair strong{font-size:18px;margin-top:2px}.ai-prefill{background:var(--teal-soft);border-radius:16px;padding:11px 13px;margin-bottom:12px}.ai-prefill strong,.ai-prefill small{display:block}.ai-prefill small{color:var(--muted);margin-top:3px}.choice-grid{display:grid;grid-template-columns:1fr 1fr;gap:8px}.bristol{min-height:66px;border:1px solid var(--line);background:#f7f3ed;border-radius:16px;padding:10px;text-align:left;cursor:pointer}.bristol strong,.bristol span{display:block}.bristol span{font-size:11px;color:var(--muted);margin-top:2px}.bristol.selected{background:var(--ink);color:white}.bristol.selected span{color:#ddd}.field{display:block;margin-bottom:15px}.field-label{display:block;font-size:12px;font-weight:800;margin-bottom:6px}.input{width:100%;min-height:48px;border:1px solid var(--line);border-radius:14px;background:#f8f5ef;padding:10px 12px;color:var(--ink)}textarea.input{min-height:85px;resize:vertical}.range-row{display:grid;grid-template-columns:1fr 37px;gap:10px;align-items:center}.range-row input{accent-color:var(--teal)}.range-val{width:37px;height:37px;border-radius:12px;background:var(--soft);display:grid;place-items:center;font-weight:800}.photo-drop{width:100%;background:#f3eee7}.symptoms{display:flex;flex-direction:column}.alert{background:var(--red-soft);border:1px solid #eec1ba;border-radius:16px;padding:13px;color:#75261f}.alert p{margin:5px 0 0;font-size:13px;line-height:1.4}
|
||||
.privacy-list{display:flex;flex-direction:column;gap:4px}.privacy-item{display:grid;grid-template-columns:40px 1fr;gap:11px;padding:11px 0;border-bottom:1px solid var(--line)}.privacy-item:last-child{border-bottom:0}.privacy-item>b{width:38px;height:38px;border-radius:13px;background:var(--teal-soft);display:grid;place-items:center;color:var(--teal)}.privacy-item h3{margin-bottom:4px}.privacy-item p,.source-list p{font-size:13px;line-height:1.45;color:var(--muted);margin-bottom:5px}.source-list a{color:var(--teal)}.danger-zone{border-color:#e9c4bf}.toast{position:fixed;left:50%;bottom:96px;transform:translateX(-50%);background:var(--ink);color:white;border-radius:999px;padding:11px 16px;font-size:13px;font-weight:700;z-index:100;box-shadow:var(--shadow)}
|
||||
.staging-label{margin:8px auto 86px;text-align:center;color:var(--muted);font-size:10px;letter-spacing:.03em;opacity:.72}
|
||||
@media(min-width:560px){.app-shell{padding-inline:24px}.sleek-hero{padding-inline:10px}.choice-grid{grid-template-columns:repeat(3,1fr)}}
|
||||
@@ -46,3 +46,9 @@ main{display:block}.eyebrow{font-size:11px;font-weight:800;letter-spacing:.1em;t
|
||||
.followup-guidance .fine{margin:8px 0 0}
|
||||
.followup-guidance.escalated{background:var(--red-soft);border-color:#eec1ba}
|
||||
.followup-guidance.escalated>strong{color:#75261f}
|
||||
.pattern-card{align-items:start}
|
||||
.pattern-window{margin:3px 0 6px;font-size:12px;font-weight:800;color:var(--teal);letter-spacing:.02em}
|
||||
.pattern-statements{margin:0;padding:0;list-style:none}
|
||||
.pattern-statements li{font-size:13.5px;line-height:1.45;margin-bottom:7px;color:#4d453e}
|
||||
.pattern-link{display:inline-block;margin-top:3px;min-height:32px;padding:5px 10px;border:1px solid #bcdfd4;border-radius:999px;background:var(--surface);color:var(--teal);font-weight:800;font-size:11px;cursor:pointer}
|
||||
.pattern-boundary{margin:8px 0 0;color:var(--muted)}
|
||||
@@ -37,14 +37,17 @@ test('validates a confident visual suggestion without inventing nonvisual fields
|
||||
});
|
||||
|
||||
test('fails closed on a non-stool or low-confidence image', () => {
|
||||
// abstainedFor names which abstention happened, so callers never have to parse prose.
|
||||
assert.deepEqual(parseVisionResponse({ isStool: false, confidence: 0.9 }), {
|
||||
status: 'needs_user_input',
|
||||
isStool: false,
|
||||
abstainedFor: 'not_stool',
|
||||
reason: 'The image does not clearly show stool.',
|
||||
});
|
||||
assert.deepEqual(parseVisionResponse({ isStool: true, bristolType: 4, color: 'brown', confidence: 0.31 }), {
|
||||
status: 'needs_user_input',
|
||||
isStool: true,
|
||||
abstainedFor: 'low_confidence',
|
||||
reason: 'The image is too uncertain to prefill safely.',
|
||||
});
|
||||
});
|
||||
@@ -78,3 +81,53 @@ test('builds a structured multimodal request that forbids diagnosis and nonvisua
|
||||
assert.match(prompt, /not a diagnosis/i);
|
||||
assert.equal(body.messages[0].content.find(part => part.type === 'image_url').image_url.url, 'data:image/jpeg;base64,YQ==');
|
||||
});
|
||||
|
||||
test('fails closed when a field the model did fill in is out of range while another field is blank', () => {
|
||||
// Issue #73: the blank-first ordering let a blank field short-circuit the whole response, so a reply
|
||||
// that carried an impossible value came back as `unusable_visual_fields` — telling the user the photo
|
||||
// was unreadable when the model had in fact returned something Timmy must never apply. A blank field
|
||||
// means "could not read it"; a filled-in field is a claim, and a claim outside the contract fails closed.
|
||||
assert.throws(
|
||||
() => parseVisionResponse({ isStool: true, bristolType: 9, color: null, confidence: 0.9, imageQuality: 'good', observations: 'Form read, colour not.' }),
|
||||
(error) => {
|
||||
assert.equal(error.code, 'invalid_fields', 'an out-of-range form next to a blank colour must fail closed');
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.throws(
|
||||
() => parseVisionResponse({ isStool: true, bristolType: null, color: 'purple', confidence: 0.9, imageQuality: 'good', observations: 'Colour read, form not.' }),
|
||||
(error) => {
|
||||
assert.equal(error.code, 'invalid_fields', 'an out-of-range colour next to a blank form must fail closed');
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('still abstains when the only filled-in visual field is inside the contract', () => {
|
||||
// The complement of the rule above: a partial read that is entirely valid is an abstention, not a
|
||||
// failure, so the glare path keeps returning 200 with nothing prefilled.
|
||||
assert.deepEqual(parseVisionResponse({ isStool: true, bristolType: 4, color: null, confidence: 0.9, imageQuality: 'good', observations: 'Form read, colour not.' }), {
|
||||
status: 'needs_user_input',
|
||||
isStool: true,
|
||||
abstainedFor: 'unusable_visual_fields',
|
||||
reason: 'The photo does not show a form Timmy can read safely.',
|
||||
});
|
||||
});
|
||||
|
||||
test('abstains instead of failing when a schema-valid response carries no readable visual fields', () => {
|
||||
// The declared JSON schema allows a null form and color whenever the model cannot read them,
|
||||
// so a schema-valid response must come back as an abstention — never as a provider failure.
|
||||
assert.deepEqual(parseVisionResponse({
|
||||
isStool: true,
|
||||
bristolType: null,
|
||||
color: null,
|
||||
confidence: 0.66,
|
||||
imageQuality: 'poor',
|
||||
observations: 'Large specular highlight hides the surface.',
|
||||
}), {
|
||||
status: 'needs_user_input',
|
||||
isStool: true,
|
||||
abstainedFor: 'unusable_visual_fields',
|
||||
reason: 'The photo does not show a form Timmy can read safely.',
|
||||
});
|
||||
});
|
||||
@@ -98,3 +98,66 @@ test('diff hygiene checks only the pull request or latest commit range', async (
|
||||
assert.match(script, /HEAD\^\.\.HEAD/);
|
||||
assert.doesNotMatch(script, /hash-object -t tree/);
|
||||
});
|
||||
|
||||
test('the vision corpus and its refusal boundary are unit-, browser-, and release-gated', async () => {
|
||||
const [packageJsonText, workflow, builder] = await Promise.all([
|
||||
readFile(packagePath, 'utf8'),
|
||||
readFile(workflowPath, 'utf8'),
|
||||
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
||||
]);
|
||||
const packageJson = JSON.parse(packageJsonText);
|
||||
|
||||
assert.match(packageJson.scripts.test, /tests\/vision-corpus\.test\.js/);
|
||||
assert.match(packageJson.scripts.test, /tests\/vision-boundary\.test\.js/);
|
||||
assert.equal(packageJson.scripts['test:vision-refusal'], 'node tests/vision-refusal.acceptance.mjs');
|
||||
assert.match(packageJson.scripts['check:syntax'], /node --check src\/vision-corpus\.js/);
|
||||
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/vision-refusal\.acceptance\.mjs/);
|
||||
assert.match(packageJson.scripts['check:syntax'], /py_compile[^&]*scripts\/build_vision_corpus\.py/);
|
||||
assert.match(workflow, /npm run test:vision-refusal/);
|
||||
// The release builder must verify the committed corpus hashes and run the refusal suite itself.
|
||||
assert.match(builder, /"test:vision-refusal"/);
|
||||
assert.match(builder, /"vision_corpus_refusal": "passed"/);
|
||||
assert.match(builder, /build_vision_corpus\.py", "--verify"/);
|
||||
});
|
||||
|
||||
test('the pinned worker container, post-load gate, and GPU benchmark are gated end to end', async () => {
|
||||
const [packageJsonText, workflow, builder] = await Promise.all([
|
||||
readFile(packagePath, 'utf8'),
|
||||
readFile(workflowPath, 'utf8'),
|
||||
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
||||
]);
|
||||
const packageJson = JSON.parse(packageJsonText);
|
||||
|
||||
assert.match(packageJson.scripts.test, /tests\/worker-container\.test\.js/);
|
||||
assert.match(packageJson.scripts.test, /tests\/worker-ready\.test\.js/);
|
||||
assert.match(packageJson.scripts.test, /tests\/worker-benchmark\.test\.js/);
|
||||
assert.equal(packageJson.scripts['test:worker-readiness'], 'node tests/worker-readiness.acceptance.mjs');
|
||||
assert.match(packageJson.scripts['check:syntax'], /node --check scripts\/record_worker_benchmark\.mjs/);
|
||||
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/worker-readiness\.acceptance\.mjs/);
|
||||
for (const script of ['scripts/worker_ready.sh', 'deploy/worker/entrypoint.sh', 'deploy/worker/build_image.sh', 'deploy/worker/run_worker_container.sh']) {
|
||||
assert.ok(packageJson.scripts['check:syntax'].includes(`bash -n ${script}`), `${script} must be syntax-gated`);
|
||||
}
|
||||
assert.match(workflow, /npm run test:worker-readiness/);
|
||||
assert.match(builder, /"test:worker-readiness"/);
|
||||
assert.match(builder, /"worker_container_receipts": "passed"/);
|
||||
assert.match(builder, /"worker_post_load_readiness": "passed"/);
|
||||
assert.match(builder, /"scripts\/worker_ready\.sh"/);
|
||||
});
|
||||
|
||||
test('explainable pattern summaries are a unit-, browser-, and release-gated contract', async () => {
|
||||
const [packageJsonText, workflow, builder] = await Promise.all([
|
||||
readFile(packagePath, 'utf8'),
|
||||
readFile(workflowPath, 'utf8'),
|
||||
readFile(new URL('../scripts/build_release.py', import.meta.url), 'utf8'),
|
||||
]);
|
||||
const packageJson = JSON.parse(packageJsonText);
|
||||
|
||||
assert.match(packageJson.scripts.test, /tests\/summary\.test\.js/);
|
||||
assert.equal(packageJson.scripts['test:pattern-summary'], 'node tests/pattern-summary.acceptance.mjs');
|
||||
assert.match(packageJson.scripts['check:syntax'], /node --check src\/summary\.js/);
|
||||
assert.match(packageJson.scripts['check:syntax'], /node --check tests\/pattern-summary\.acceptance\.mjs/);
|
||||
assert.match(workflow, /npm run test:pattern-summary/);
|
||||
assert.match(builder, /"test:pattern-summary"/);
|
||||
assert.match(builder, /"longitudinal_pattern_summary": "passed"/);
|
||||
assert.match(builder, /"src\/summary\.js"/);
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
# Vision corpus (issue #18)
|
||||
|
||||
A committed, deterministic corpus that proves one claim on every run: **every reply Timmy receives is
|
||||
either a schema-valid visible-field suggestion or it fails closed.** Nothing is guessed, and a
|
||||
refusal never carries a form or a colour.
|
||||
|
||||
## Licence and provenance
|
||||
|
||||
- All fixtures are licensed **CC0-1.0**.
|
||||
- Every fixture is a synthetic image drawn from fixed integer math by
|
||||
[`scripts/build_vision_corpus.py`](../../../scripts/build_vision_corpus.py). There is no camera, no
|
||||
human subject, no third-party content, and no real medical imagery anywhere in this directory.
|
||||
- `manifest.json` records each fixture's `sha256`, byte length, provenance, licence, case class, the
|
||||
provider reply it replays, and the outcome that reply must produce.
|
||||
- The hashes are an integrity receipt: `loadVisionCorpus()` refuses a fixture whose bytes no longer
|
||||
match the manifest, so the corpus cannot be quietly edited into a different test.
|
||||
|
||||
## Coverage
|
||||
|
||||
| Case | Class | Replayed provider reply | Required outcome |
|
||||
|---|---|---|---|
|
||||
| `stool-like-formed` | stool | valid suggestion, confidence 0.83 | `suggestion` |
|
||||
| `non-stool-scene` | non-stool | `isStool: false` | `abstained` (`not_stool`) |
|
||||
| `blur-low-detail` | blur | confidence 0.41 | `abstained` (`low_confidence`) |
|
||||
| `glare-overexposed` | glare | schema-valid with null form and colour | `abstained` (`unusable_visual_fields`) |
|
||||
| `empty-bowl` | empty-bowl | `isStool: false` | `abstained` (`not_stool`) |
|
||||
| `malformed-response` | malformed-response | plain prose, not JSON | `fail_closed` (`unparsable_content`) |
|
||||
| `out-of-range-fields` | malformed-response | Bristol 9, colour `purple`, confidence 0.9 | `fail_closed` (`out_of_range_or_missing_fields`) |
|
||||
| `blank-plus-out-of-range` | malformed-response | Bristol 9 with a `null` colour, confidence 0.9 | `fail_closed` (`out_of_range_or_missing_fields`) |
|
||||
|
||||
Both out-of-range cases keep `confidence` inside the contract on purpose: at `1.4` the reply died at the
|
||||
confidence gate, so the `out_of_range_or_missing_fields` label was credited to a guard the case never
|
||||
reached. `blank-plus-out-of-range` covers the reply shape that the blank-first ordering used to abstain on
|
||||
(issue #73) — a filled-in field is a claim, and a claim outside the contract fails closed even when the
|
||||
other field came back blank.
|
||||
|
||||
## Where the corpus is enforced
|
||||
|
||||
- `tests/vision-corpus.test.js` — corpus contract, hash integrity, and conformance of every reply.
|
||||
- `tests/vision-boundary.test.js` — the corpus replayed through the real provider call path and
|
||||
through a running `server.mjs`, asserting status codes and that no image bytes or credentials
|
||||
appear in a fail-closed body.
|
||||
- `tests/vision-refusal.acceptance.mjs` — the same refusals in a real browser.
|
||||
- `scripts/record_release_demo.mjs` — the release demo replays corpus cases through the real analysis
|
||||
code path, so the video shows the status the server would actually send.
|
||||
|
||||
## Regenerating
|
||||
|
||||
```bash
|
||||
python3 scripts/build_vision_corpus.py # rewrite fixtures and manifest.json
|
||||
python3 scripts/build_vision_corpus.py --verify # verify committed fixtures against the manifest
|
||||
```
|
||||
|
||||
The release builder runs `--verify` before anything is packaged. Regenerate deliberately and commit
|
||||
the new hashes; never hand-edit a fixture or a recorded reply.
|
||||
|
After Width: | Height: | Size: 2.3 KiB |
|
After Width: | Height: | Size: 21 KiB |
|
After Width: | Height: | Size: 3.4 KiB |
|
After Width: | Height: | Size: 3.1 KiB |
|
After Width: | Height: | Size: 2.3 KiB |
@@ -0,0 +1,136 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"generator": "scripts/build_vision_corpus.py",
|
||||
"license": "CC0-1.0",
|
||||
"note": "Synthetic fixtures only. No real medical imagery, no personal data, and no third-party content.",
|
||||
"cases": [
|
||||
{
|
||||
"id": "stool-like-formed",
|
||||
"caseClass": "stool",
|
||||
"file": "stool-like-formed.png",
|
||||
"bytes": 2414,
|
||||
"sha256": "4bb50ec6fc4ef7e9c83ed3b7ba69cdf5e73ef74b9f05e97898892bc852df970a",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"format": "png",
|
||||
"synthetic": true,
|
||||
"license": "CC0-1.0",
|
||||
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
||||
"expectedOutcome": "suggestion",
|
||||
"expectedReason": "schema_valid_suggestion",
|
||||
"providerContent": "{\"isStool\":true,\"bristolType\":4,\"color\":\"brown\",\"confidence\":0.83,\"imageQuality\":\"good\",\"observations\":\"Smooth, formed appearance.\"}"
|
||||
},
|
||||
{
|
||||
"id": "non-stool-scene",
|
||||
"caseClass": "non-stool",
|
||||
"file": "non-stool-scene.png",
|
||||
"bytes": 2406,
|
||||
"sha256": "4d12150073856c1d7dfb9332049182ee0145da6ad50f6964a7273c8c61f21f91",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"format": "png",
|
||||
"synthetic": true,
|
||||
"license": "CC0-1.0",
|
||||
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
||||
"expectedOutcome": "abstained",
|
||||
"expectedReason": "not_stool",
|
||||
"providerContent": "{\"isStool\":false,\"bristolType\":null,\"color\":null,\"confidence\":0.97,\"imageQuality\":\"good\",\"observations\":\"Flatware on a tray.\"}"
|
||||
},
|
||||
{
|
||||
"id": "blur-low-detail",
|
||||
"caseClass": "blur",
|
||||
"file": "blur-low-detail.png",
|
||||
"bytes": 21598,
|
||||
"sha256": "17000163662a42fd434bb2b2428d9e5a2156da08cbc82210e41dfc6fa7e24852",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"format": "png",
|
||||
"synthetic": true,
|
||||
"license": "CC0-1.0",
|
||||
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
||||
"expectedOutcome": "abstained",
|
||||
"expectedReason": "low_confidence",
|
||||
"providerContent": "{\"isStool\":true,\"bristolType\":4,\"color\":\"brown\",\"confidence\":0.41,\"imageQuality\":\"poor\",\"observations\":\"Form is unclear.\"}"
|
||||
},
|
||||
{
|
||||
"id": "glare-overexposed",
|
||||
"caseClass": "glare",
|
||||
"file": "glare-overexposed.png",
|
||||
"bytes": 3140,
|
||||
"sha256": "1dc87a80951dd244c9831cba8b352e24718cf5f103f4dfd0a30e8c8c1c599f95",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"format": "png",
|
||||
"synthetic": true,
|
||||
"license": "CC0-1.0",
|
||||
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
||||
"expectedOutcome": "abstained",
|
||||
"expectedReason": "unusable_visual_fields",
|
||||
"providerContent": "{\"isStool\":true,\"bristolType\":null,\"color\":null,\"confidence\":0.66,\"imageQuality\":\"poor\",\"observations\":\"Large specular highlight hides the surface.\"}"
|
||||
},
|
||||
{
|
||||
"id": "empty-bowl",
|
||||
"caseClass": "empty-bowl",
|
||||
"file": "empty-bowl.png",
|
||||
"bytes": 3479,
|
||||
"sha256": "ab443d1d9d28338e5c9f48054e7989cfce59ae25e2a9008be31cd9223be6abc4",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"format": "png",
|
||||
"synthetic": true,
|
||||
"license": "CC0-1.0",
|
||||
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
||||
"expectedOutcome": "abstained",
|
||||
"expectedReason": "not_stool",
|
||||
"providerContent": "{\"isStool\":false,\"bristolType\":null,\"color\":null,\"confidence\":0.93,\"imageQuality\":\"good\",\"observations\":\"Empty container.\"}"
|
||||
},
|
||||
{
|
||||
"id": "malformed-response",
|
||||
"caseClass": "malformed-response",
|
||||
"file": "malformed-response.png",
|
||||
"bytes": 2341,
|
||||
"sha256": "634d16337879783b307d0ecf84348cfdc07950f6893095267901383f2efc83c2",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"format": "png",
|
||||
"synthetic": true,
|
||||
"license": "CC0-1.0",
|
||||
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
||||
"expectedOutcome": "fail_closed",
|
||||
"expectedReason": "unparsable_content",
|
||||
"providerContent": "Sure! It looks like a Type 4 to me. Hope that helps."
|
||||
},
|
||||
{
|
||||
"id": "out-of-range-fields",
|
||||
"caseClass": "malformed-response",
|
||||
"file": "out-of-range-fields.png",
|
||||
"bytes": 2375,
|
||||
"sha256": "ac20ca9f3c9cbd118d3a2957110505d049a85861f97d361dcfbc4787960a5f8f",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"format": "png",
|
||||
"synthetic": true,
|
||||
"license": "CC0-1.0",
|
||||
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
||||
"expectedOutcome": "fail_closed",
|
||||
"expectedReason": "out_of_range_or_missing_fields",
|
||||
"providerContent": "{\"isStool\":true,\"bristolType\":9,\"color\":\"purple\",\"confidence\":0.9,\"imageQuality\":\"excellent\",\"observations\":\"Confident.\"}"
|
||||
},
|
||||
{
|
||||
"id": "blank-plus-out-of-range",
|
||||
"caseClass": "malformed-response",
|
||||
"file": "blank-plus-out-of-range.png",
|
||||
"bytes": 2403,
|
||||
"sha256": "d45048cd1f5bfb1be1667238c9c21fca0cd1cae8b55e12d4f325203625eeafbd",
|
||||
"width": 640,
|
||||
"height": 480,
|
||||
"format": "png",
|
||||
"synthetic": true,
|
||||
"license": "CC0-1.0",
|
||||
"provenance": "Synthetic fixture drawn deterministically by scripts/build_vision_corpus.py; no camera, no human subject, no third-party content.",
|
||||
"expectedOutcome": "fail_closed",
|
||||
"expectedReason": "out_of_range_or_missing_fields",
|
||||
"providerContent": "{\"isStool\":true,\"bristolType\":9,\"color\":null,\"confidence\":0.9,\"imageQuality\":\"good\",\"observations\":\"Form read, colour not.\"}"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
After Width: | Height: | Size: 2.3 KiB |
|
After Width: | Height: | Size: 2.3 KiB |
|
After Width: | Height: | Size: 2.4 KiB |
@@ -0,0 +1,80 @@
|
||||
// Shared harness for driving the real /api/analyze boundary with recorded corpus responses.
|
||||
//
|
||||
// Nothing here fakes the application code: the test only decides which recorded provider
|
||||
// response a stub OpenAI-compatible worker returns, and then talks to the real server.
|
||||
import http from 'node:http';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { join } from 'node:path';
|
||||
|
||||
export const repoRoot = fileURLToPath(new URL('../..', import.meta.url));
|
||||
export const corpusDir = fileURLToPath(new URL('../fixtures/vision-corpus/', import.meta.url));
|
||||
export const CORPUS_MODEL = 'corpus-model';
|
||||
|
||||
export function caseFor(corpus, id) {
|
||||
const entry = corpus.cases.find(item => item.id === id);
|
||||
if (!entry) throw new Error(`corpus case ${id} is missing`);
|
||||
return entry;
|
||||
}
|
||||
|
||||
export async function corpusPayload(entry) {
|
||||
const bytes = await readFile(join(corpusDir, entry.file));
|
||||
return { imageDataUrl: `data:image/png;base64,${bytes.toString('base64')}`, consent: true };
|
||||
}
|
||||
|
||||
/** Stub OpenAI-compatible worker whose answer the test selects per request. */
|
||||
export function startProvider() {
|
||||
const state = { content: '', requests: [] };
|
||||
const server = http.createServer((req, res) => {
|
||||
if (req.url.endsWith('/models')) {
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
return res.end(JSON.stringify({ data: [{ id: CORPUS_MODEL }] }));
|
||||
}
|
||||
let body = '';
|
||||
req.on('data', chunk => { body += chunk; });
|
||||
req.on('end', () => {
|
||||
state.requests.push(body);
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.end(JSON.stringify({ choices: [{ message: { content: state.content } }] }));
|
||||
});
|
||||
});
|
||||
return new Promise(resolve => server.listen(0, '127.0.0.1', () => resolve({
|
||||
state,
|
||||
port: server.address().port,
|
||||
close: () => new Promise(done => server.close(done)),
|
||||
})));
|
||||
}
|
||||
|
||||
/** Real server.mjs, vision enabled, pointed at the stub worker. */
|
||||
export async function startServer({ providerPort, port = 4199 }) {
|
||||
const child = spawn(process.execPath, ['server.mjs'], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
HOST: '127.0.0.1',
|
||||
PORT: String(port),
|
||||
TIMMY_VISION_ENABLED: 'true',
|
||||
TIMMY_VISION_PROFILE: 'selfhost',
|
||||
TIMMY_VISION_BASE_URL: `http://127.0.0.1:${providerPort}/v1`,
|
||||
TIMMY_VISION_MODEL: CORPUS_MODEL,
|
||||
TIMMY_VISION_TIMEOUT_MS: '5000',
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
for (let attempt = 0; attempt < 80; attempt += 1) {
|
||||
try {
|
||||
const probe = await fetch(`http://127.0.0.1:${port}/api/healthz`);
|
||||
if (probe.ok) return child;
|
||||
} catch {}
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
child.kill();
|
||||
throw new Error('acceptance server did not become ready');
|
||||
}
|
||||
|
||||
export async function stopServer(child) {
|
||||
if (!child || child.exitCode !== null) return;
|
||||
child.kill();
|
||||
await new Promise(resolve => child.once('exit', resolve));
|
||||
}
|
||||
@@ -18,7 +18,9 @@ for (const viewport of viewports) {
|
||||
await page.route('**/api/vision-status', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ enabled: true, profile: 'selfhost', providerReady: true, model: 'synthetic-test-model' }),
|
||||
// The fixture mirrors the real /api/vision-status contract, which reports whether the pinned
|
||||
// model was actually seen loaded (#15) alongside provider reachability.
|
||||
body: JSON.stringify({ enabled: true, profile: 'selfhost', providerReady: true, modelSeen: true, model: 'synthetic-test-model' }),
|
||||
}));
|
||||
await page.route('**/api/analyze', route => {
|
||||
analysisRequests += 1;
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
import { chromium } from 'playwright';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
|
||||
// Issue #34: every longitudinal summary line must point at the confirmed records behind it, and a
|
||||
// sparse window must say so instead of describing a trend. The records are fictional sample entries
|
||||
// written into the app's own local ledger contract; no real health data is involved.
|
||||
const DAY = 86400000;
|
||||
const sample = (id, daysAgo, bristolType) => ({
|
||||
id,
|
||||
occurredAt: new Date(Date.now() - daysAgo * DAY).toISOString(),
|
||||
bristolType,
|
||||
color: 'brown',
|
||||
urgency: 1,
|
||||
discomfort: 1,
|
||||
note: '',
|
||||
photoDataUrl: '',
|
||||
symptoms: {},
|
||||
});
|
||||
|
||||
await mkdir('artifacts', { recursive: true });
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 2, serviceWorkers: 'block' });
|
||||
const page = await context.newPage();
|
||||
const errors = [];
|
||||
page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); });
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
|
||||
const seed = async entries => page.evaluate(list => {
|
||||
const base = document.querySelector('meta[name="timmy-base-path"]')?.content || '/';
|
||||
localStorage.setItem(`timmy:${base}:ledger-v1`, JSON.stringify(list));
|
||||
}, entries);
|
||||
|
||||
await page.goto('http://127.0.0.1:4173', { waitUntil: 'networkidle' });
|
||||
await page.evaluate(() => localStorage.clear());
|
||||
|
||||
// 1. A sparse window refuses to describe a pattern.
|
||||
await seed([sample('sparse-a', 1, 4), sample('sparse-b', 4, 6)]);
|
||||
await page.reload({ waitUntil: 'networkidle' });
|
||||
const sparse = page.locator('[data-summary-insufficient]');
|
||||
assert.equal(await sparse.count(), 1, 'a window below the minimum must show honest insufficient-data copy');
|
||||
assert.match(await sparse.innerText(), /not enough confirmed logs/i);
|
||||
assert.match(await sparse.innerText(), /2 of 3/);
|
||||
assert.equal(await page.locator('[data-statement]').count(), 0, 'no pattern line may be claimed from two logs');
|
||||
await page.screenshot({ path: 'artifacts/pattern-summary-insufficient-mobile.png' });
|
||||
|
||||
// 2. A populated window renders explainable lines, each citing its own records.
|
||||
const seeded = [
|
||||
sample('rec-firm', 2, 1),
|
||||
sample('rec-typical-1', 5, 3),
|
||||
sample('rec-typical-2', 9, 4),
|
||||
sample('rec-typical-3', 12, 4),
|
||||
sample('rec-old', 40, 6),
|
||||
];
|
||||
await seed(seeded);
|
||||
await page.reload({ waitUntil: 'networkidle' });
|
||||
|
||||
const card = page.locator('[data-pattern-summary]');
|
||||
assert.equal(await card.count(), 1, 'the home insight card is the explainable summary');
|
||||
assert.match(await page.locator('[data-summary-window]').innerText(), /last 28 days/i);
|
||||
assert.match(await page.locator('[data-statement="frequency"]').innerText(), /4 confirmed logs/);
|
||||
assert.equal(await page.locator('[data-statement="firm"]').getAttribute('data-records'), '1');
|
||||
assert.equal(await page.locator('[data-statement="typical"]').getAttribute('data-records'), '3');
|
||||
// The out-of-window log is neither counted nor offered as a citation.
|
||||
assert.doesNotMatch(await card.innerText(), /5 confirmed logs/);
|
||||
await page.screenshot({ path: 'artifacts/pattern-summary-home-mobile.png' });
|
||||
|
||||
// 3. Following a citation shows exactly the contributing confirmed records.
|
||||
await page.locator('[data-statement-link="typical"]').click();
|
||||
const records = page.locator('[data-summary-records]');
|
||||
await records.waitFor();
|
||||
assert.equal(await page.locator('[data-summary-record]').count(), 3, 'the citation lists exactly its own records');
|
||||
const listed = await page.locator('[data-summary-record]').evaluateAll(nodes => nodes.map(node => node.getAttribute('data-entry-id')));
|
||||
assert.deepEqual(listed.sort(), ['rec-typical-1', 'rec-typical-2', 'rec-typical-3']);
|
||||
assert.match(await records.innerText(), /Type 3/);
|
||||
assert.match(await records.innerText(), /These are the confirmed logs this line was built from/);
|
||||
await page.screenshot({ path: 'artifacts/pattern-summary-records-mobile.png' });
|
||||
await page.locator('#pattern-records-close').click();
|
||||
assert.equal(await page.locator('[data-summary-records]').count(), 0, 'the citation sheet closes cleanly');
|
||||
|
||||
// 4. The summary never explains a cause or clears a food.
|
||||
const summaryText = await card.innerText();
|
||||
assert.doesNotMatch(summaryText, /because|due to|caused|safe to eat|avoid eating|should eat|diagnos/i);
|
||||
|
||||
assert.deepEqual(errors, []);
|
||||
await context.close();
|
||||
await browser.close();
|
||||
console.log('PASS longitudinal pattern summaries cite their confirmed records and refuse to describe a sparse window');
|
||||
@@ -50,6 +50,27 @@ test('release demo shows the provisional, confirmed, and corrected suggestion st
|
||||
assert.match(demo, /Type 2/, 'the retained draft in the demo must show the value the user corrected to');
|
||||
});
|
||||
|
||||
test('release demo shows the explainable pattern summary and follows a citation to its own records', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
|
||||
// Issue #34: the recording must exercise the summary, its citation sheet, and the sparse case.
|
||||
assert.match(demo, /\[data-pattern-summary\]/);
|
||||
assert.match(demo, /\[data-statement-link="typical"\]/);
|
||||
assert.match(demo, /\[data-summary-record\]/);
|
||||
assert.match(demo, /\[data-summary-insufficient\]/);
|
||||
assert.match(demo, /The summary says which confirmed logs it read, and a citation opens exactly those records/);
|
||||
assert.match(demo, /Every pattern line names the confirmed logs behind it, and following one shows exactly those records/);
|
||||
assert.match(demo, /Fictional sample logs are seeded into this browser's own storage — no real health data appears here/);
|
||||
assert.match(demo, /Two logs is not a pattern: the summary refuses to describe a trend it cannot support/);
|
||||
// The cited records are asserted in the browser, not narrated.
|
||||
assert.match(demo, /must list exactly the records it cites/);
|
||||
// Both captions are required release gates, so a later release cannot drop the scene silently.
|
||||
assert.match(builder, /Every pattern line names the confirmed logs behind it, and following one shows exactly those records/);
|
||||
assert.match(builder, /Two logs is not a pattern: the summary refuses to describe a trend it cannot support/);
|
||||
assert.match(builder, /"longitudinal_pattern_summary": "passed"/);
|
||||
});
|
||||
|
||||
test('release builder gates the sleek shell, Hermes chat, and bootstrap syntax', async () => {
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
assert.match(builder, /"test:sleek"/);
|
||||
@@ -97,3 +118,108 @@ test('release builder gates the hard-stool follow-up the same way CI does', asyn
|
||||
assert.match(builder, /"src\/followup\.js"/);
|
||||
assert.match(builder, /Hard-stool follow-up: a Bristol 1-2 entry asks user-reported questions/);
|
||||
});
|
||||
|
||||
test('release demo states the declared release delta on camera and proves it with a receipt', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
|
||||
// The delta comes from the builder, so a documentation-only release cannot imply a new feature.
|
||||
assert.match(demo, /TIMMY_RELEASE_DEMO_DELTA/);
|
||||
assert.match(demo, /This release: \$\{releaseDelta\}/);
|
||||
assert.match(demo, /no new feature is claimed/);
|
||||
// A caption alone is not evidence; the demo must emit a machine-readable receipt of what rendered.
|
||||
assert.match(demo, /TIMMY_DEMO_CAPTIONS_OUT/);
|
||||
assert.match(demo, /captionsShown\.push\(text\)/);
|
||||
assert.match(demo, /writeFile\(captionsOut/);
|
||||
});
|
||||
|
||||
test('release demo shows the byte-bearing controls the threat model governs, without inventing UI', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
|
||||
assert.match(demo, /\[data-view="privacy"\]/);
|
||||
assert.match(demo, /Every byte-bearing control the model names/);
|
||||
assert.match(demo, /indicate\('#export'\)/);
|
||||
assert.match(demo, /indicate\('#delete-all'\)/);
|
||||
assert.match(demo, /Delete all local data clears the ledger in this browser; there is no server copy of it to keep/);
|
||||
// Real controls only: the selectors must exist in the app shell, not be demo-only scaffolding.
|
||||
const app = await readFile(new URL('../app.js', import.meta.url), 'utf8');
|
||||
assert.match(app, /id="export"/);
|
||||
assert.match(app, /id="delete-all"/);
|
||||
});
|
||||
|
||||
test('release builder refuses to publish without the on-camera delta receipt', async () => {
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
|
||||
assert.match(builder, /TIMMY_RELEASE_DELTA/);
|
||||
assert.match(builder, /TIMMY_DEMO_CAPTIONS_OUT/);
|
||||
assert.match(builder, /Release demo produced no on-camera caption receipt/);
|
||||
assert.match(builder, /Release demo did not state the declared release delta on camera/);
|
||||
assert.match(builder, /Release demo is missing a required on-camera caption/);
|
||||
assert.match(builder, /"release_delta_visible_on_camera": "passed"/);
|
||||
assert.match(builder, /"release_delta": release_delta/);
|
||||
});
|
||||
|
||||
test('release demo captions expire and the contact sheet spans the whole recording', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
|
||||
// A caption that outlives its stated duration hangs the previous view's claim over the next screen.
|
||||
assert.match(demo, /creates via removal after duration|A caption must not outlive its stated duration/);
|
||||
assert.match(demo, /setTimeout\(\(\) => node\.remove\(\), 220\)/);
|
||||
// The sheet must cover the entire demo, not just its opening.
|
||||
assert.match(builder, /12 \/ demo_duration/);
|
||||
assert.match(builder, /Release demo contact sheet was not produced/);
|
||||
});
|
||||
|
||||
test('release demo visibly exercises the corpus refusal paths instead of only the happy path', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
|
||||
// The refusal responses must come from the recorded corpus and the real analysis code path,
|
||||
// never from copy pasted into the recording script.
|
||||
assert.match(demo, /loadVisionCorpus/);
|
||||
assert.match(demo, /visionFailureResponse/);
|
||||
assert.match(demo, /corpusAnalyzeResult/);
|
||||
assert.match(demo, /tests\/fixtures\/vision-corpus\/non-stool-scene\.png/);
|
||||
assert.match(demo, /tests\/fixtures\/vision-corpus\/out-of-range-fields\.png/);
|
||||
assert.match(demo, /getByText\(\/No confident match\/i\)/);
|
||||
assert.match(demo, /couldn’t analyze that safely/i);
|
||||
|
||||
assert.match(demo, /A non-stool photo gets a refusal, never a guess/);
|
||||
assert.match(demo, /Unreadable visual fields abstain as well/);
|
||||
assert.match(demo, /An out-of-range answer fails closed/);
|
||||
assert.match(demo, /no real photos in the corpus|no real photos in the recording/);
|
||||
// The refusal copy the user sees is gated on camera, so a release cannot drop it silently.
|
||||
assert.match(builder, /An out-of-range answer fails closed/);
|
||||
});
|
||||
|
||||
test('release demo shows a filled-in out-of-range field failing closed even beside a blank one', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
|
||||
// Issue #73's escape class, replayed on camera from the committed corpus rather than described.
|
||||
assert.match(demo, /corpusCase\('blank-plus-out-of-range'\)/);
|
||||
assert.match(demo, /tests\/fixtures\/vision-corpus\/blank-plus-out-of-range\.png/);
|
||||
assert.match(demo, /A filled-in field that is out of range still fails closed, even when the other field came back blank/);
|
||||
assert.match(demo, /no longer told the photo was unreadable when the model returned an impossible value/);
|
||||
// The extra fail-closed scene is a real 503, so the demo's error budget must count it.
|
||||
assert.match(demo, /expectedFailClosed\.length !== 2/);
|
||||
// The caption is a required gate, so a later release cannot silently drop the scene.
|
||||
assert.match(builder, /A filled-in field that is out of range still fails closed/);
|
||||
});
|
||||
|
||||
test('release demo shows readiness waiting for the loaded pinned model before AI is offered', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
|
||||
// Issue #15: reachable is not loaded, and the recording must prove the app withholds AI.
|
||||
assert.match(demo, /modelLoaded = false/);
|
||||
assert.match(demo, /getByText\(\/model is not loaded yet\/i\)/);
|
||||
assert.match(demo, /model-status\.loading/);
|
||||
assert.match(demo, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/);
|
||||
assert.match(demo, /No AI control may exist before the model is loaded/);
|
||||
assert.match(demo, /Once the pinned model is loaded, the same screen offers the AI analysis again/);
|
||||
// The scene is asserted in the browser, not narrated: the demo throws if the state is wrong.
|
||||
assert.match(demo, /must never be reported ready/);
|
||||
// The caption is a required release gate, so the scene cannot be dropped silently.
|
||||
assert.match(builder, /Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident/);
|
||||
});
|
||||
@@ -14,7 +14,7 @@ page.on('console', message => { if (message.type() === 'error') errors.push(mess
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
page.on('request', request => { if (new URL(request.url()).pathname.includes('/api/')) apiRequests.push(new URL(request.url()).pathname); });
|
||||
|
||||
await page.route('**/api/vision-status', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ enabled: true, profile: 'selfhost', providerReady: true, model: 'SmolVLM2-2.2B-Instruct' }) }));
|
||||
await page.route('**/api/vision-status', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ enabled: true, profile: 'selfhost', providerReady: true, modelSeen: true, model: 'SmolVLM2-2.2B-Instruct' }) }));
|
||||
await page.route('**/api/agent/status', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ enabled: true, configured: true, authenticated: true, mode: 'hermes-agent' }) }));
|
||||
let chatRequest;
|
||||
await page.route('**/api/agent/chat', async route => {
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { buildPatternSummary } from '../src/summary.js';
|
||||
|
||||
const DAY = 86400000;
|
||||
const NOW = Date.parse('2026-09-20T12:00:00.000Z');
|
||||
const entry = (id, daysAgo, bristolType) => ({
|
||||
id,
|
||||
occurredAt: new Date(NOW - daysAgo * DAY).toISOString(),
|
||||
bristolType,
|
||||
color: 'brown',
|
||||
urgency: 0,
|
||||
discomfort: 0,
|
||||
note: '',
|
||||
symptoms: {},
|
||||
});
|
||||
|
||||
test('every summary statement cites the confirmed records it was built from', () => {
|
||||
const summary = buildPatternSummary([entry('a', 1, 4), entry('b', 3, 3), entry('c', 5, 6)], { now: NOW });
|
||||
|
||||
assert.equal(summary.insufficient, false);
|
||||
assert.equal(summary.window.label, 'last 28 days');
|
||||
|
||||
const frequency = summary.statements.find(statement => statement.id === 'frequency');
|
||||
assert.ok(frequency, 'the window needs a frequency line');
|
||||
assert.match(frequency.text, /3 confirmed logs/);
|
||||
assert.match(frequency.text, /last 28 days/);
|
||||
assert.deepEqual(frequency.entryIds, ['c', 'b', 'a']);
|
||||
|
||||
// No statement may cite an id the summary cannot point at a real record for.
|
||||
const known = new Set(summary.records.map(record => record.id));
|
||||
for (const statement of summary.statements) {
|
||||
assert.ok(statement.entryIds.length > 0, `${statement.id} must cite its contributing records`);
|
||||
for (const id of statement.entryIds) assert.ok(known.has(id), `${statement.id} cited an unknown record ${id}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('each form line cites only the records in that form bucket', () => {
|
||||
const summary = buildPatternSummary([entry('a', 1, 2), entry('b', 3, 4), entry('c', 5, 4), entry('d', 6, 4)], { now: NOW });
|
||||
|
||||
const firm = summary.statements.find(statement => statement.id === 'firm');
|
||||
assert.ok(firm, 'a firm-side line is needed when firm logs exist');
|
||||
assert.match(firm.text, /Type 1–2/);
|
||||
assert.match(firm.text, /1 of 4/);
|
||||
assert.deepEqual(firm.entryIds, ['a']);
|
||||
|
||||
const typical = summary.statements.find(statement => statement.id === 'typical');
|
||||
assert.match(typical.text, /3 of 4/);
|
||||
assert.deepEqual(typical.entryIds, ['d', 'c', 'b']);
|
||||
assert.equal(summary.statements.some(statement => statement.id === 'loose'), false);
|
||||
});
|
||||
|
||||
test('a contradictory window is reported as a mixed pattern instead of one trend', () => {
|
||||
const summary = buildPatternSummary([entry('a', 1, 1), entry('b', 2, 1), entry('c', 3, 6), entry('d', 4, 7)], { now: NOW });
|
||||
|
||||
const mixed = summary.statements.find(statement => statement.id === 'mixed');
|
||||
assert.ok(mixed, 'a window with both firm and loose logs needs an explicit mixed line');
|
||||
assert.match(mixed.text, /firm and loose logs both appear/i);
|
||||
assert.match(mixed.text, /not one trend/i);
|
||||
assert.deepEqual(mixed.entryIds, ['d', 'c', 'b', 'a']);
|
||||
});
|
||||
|
||||
test('logs outside the window are neither counted nor cited', () => {
|
||||
const summary = buildPatternSummary([entry('old', 40, 1), entry('a', 1, 4), entry('b', 3, 3), entry('c', 5, 4)], { now: NOW });
|
||||
|
||||
assert.deepEqual(summary.records.map(record => record.id), ['c', 'b', 'a']);
|
||||
assert.equal(summary.statements.some(statement => statement.entryIds.includes('old')), false);
|
||||
assert.match(summary.statements.find(statement => statement.id === 'frequency').text, /3 confirmed logs/);
|
||||
|
||||
const weekly = buildPatternSummary([entry('old', 40, 1), entry('a', 1, 4), entry('b', 2, 4), entry('c', 3, 4)], { now: NOW, windowDays: 7 });
|
||||
assert.equal(weekly.window.label, 'last 7 days');
|
||||
assert.equal(weekly.window.days, 7);
|
||||
assert.deepEqual(weekly.records.map(record => record.id), ['c', 'b', 'a']);
|
||||
});
|
||||
|
||||
test('the summary states its boundary and never explains a cause or clears a food', () => {
|
||||
const summary = buildPatternSummary([entry('a', 1, 4), entry('b', 3, 3), entry('c', 5, 6)], { now: NOW });
|
||||
assert.match(summary.boundary, /your own confirmed records/i);
|
||||
|
||||
const forbidden = /because|due to|caused|triggered|safe to eat|avoid eating|should eat|clears|diagnos|treat/i;
|
||||
for (const text of [summary.boundary, summary.copy, ...summary.statements.map(statement => statement.text)]) {
|
||||
assert.doesNotMatch(text, forbidden, text);
|
||||
}
|
||||
const sparse = buildPatternSummary([entry('a', 1, 4)], { now: NOW });
|
||||
for (const text of [sparse.boundary, sparse.copy]) {
|
||||
assert.doesNotMatch(text, forbidden, text);
|
||||
}
|
||||
});
|
||||
|
||||
test('refuses to describe a pattern from too few confirmed logs', () => {
|
||||
const summary = buildPatternSummary([entry('a', 1, 4), entry('b', 3, 3)], { now: NOW });
|
||||
|
||||
assert.equal(summary.insufficient, true);
|
||||
assert.deepEqual(summary.statements, []);
|
||||
assert.match(summary.copy, /not enough confirmed logs/i);
|
||||
assert.match(summary.copy, /2 of 3/);
|
||||
// The summary still lists exactly what it did read, in the order it happened.
|
||||
assert.deepEqual(summary.records.map(record => record.id), ['b', 'a']);
|
||||
});
|
||||
@@ -0,0 +1,119 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
|
||||
const threatModelPath = new URL('../docs/PHOTO-DATAFLOW-THREAT-MODEL.md', import.meta.url);
|
||||
const productDecisionPath = new URL('../docs/PRODUCT-DECISIONS.md', import.meta.url);
|
||||
const roadmapPath = new URL('../ROADMAP.md', import.meta.url);
|
||||
|
||||
// Every hop that can hold, move, or leak a user's photo bytes has to be named
|
||||
// in the diagram, not only in prose.
|
||||
const REQUIRED_DIAGRAM_NODES = [
|
||||
['browser', /\bbrowser\b/i],
|
||||
['API ingress', /\bapi[- ]ingress\b/i],
|
||||
['model worker', /\bmodel worker\b/i],
|
||||
['transient buffers', /\btransient buffer/i],
|
||||
['training opt-in store', /\btraining opt-in\b/i],
|
||||
['logs', /\blogs?\b/i],
|
||||
['exports', /\bexports?\b/i],
|
||||
['deletion', /\bdeletion\b/i],
|
||||
];
|
||||
|
||||
// The abuse cases the ticket names explicitly, in the order they appear there.
|
||||
const REQUIRED_THREATS = [
|
||||
['replay', /\breplay\b/i],
|
||||
['oversized input', /\boversized input\b/i],
|
||||
['metadata leakage', /\bmetadata leakage\b/i],
|
||||
['poisoning', /\bpoisoning\b/i],
|
||||
['unauthorized training', /\bunauthorized training\b/i],
|
||||
['operator access', /\boperator access\b/i],
|
||||
];
|
||||
|
||||
function diagramBody(doc) {
|
||||
const match = doc.match(/```mermaid\r?\n([\s\S]*?)```/);
|
||||
assert.ok(match, 'the threat model must contain a fenced mermaid data-flow diagram');
|
||||
return match[1];
|
||||
}
|
||||
|
||||
function threatBlocks(doc) {
|
||||
const blocks = new Map();
|
||||
const pattern = /^###\s+(T\d+)\s+—\s+(.+)$/gm;
|
||||
const headings = [...doc.matchAll(pattern)];
|
||||
headings.forEach((heading, index) => {
|
||||
const start = heading.index + heading[0].length;
|
||||
const end = index + 1 < headings.length ? headings[index + 1].index : doc.length;
|
||||
blocks.set(heading[1], { name: heading[2].trim(), body: doc.slice(start, end) });
|
||||
});
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function knownIssueNumbers(roadmap) {
|
||||
return new Set([...roadmap.matchAll(/issues\/(\d+)/g)].map((match) => Number(match[1])));
|
||||
}
|
||||
|
||||
test('photo data-flow threat model diagrams every component that can hold or leak image bytes', async () => {
|
||||
const doc = await readFile(threatModelPath, 'utf8');
|
||||
const diagram = diagramBody(doc);
|
||||
|
||||
for (const [label, pattern] of REQUIRED_DIAGRAM_NODES) {
|
||||
assert.match(diagram, pattern, `the data-flow diagram must show the ${label} component`);
|
||||
}
|
||||
|
||||
// The diagram has to end somewhere: the model worker's bytes must not flow on.
|
||||
assert.match(diagram, /no retention|dropped|discarded|never persist/i,
|
||||
'the diagram must show that worker input is not retained');
|
||||
});
|
||||
|
||||
test('every required abuse case has a mitigation, an owner, and a real follow-up ticket', async () => {
|
||||
const [doc, roadmap] = await Promise.all([
|
||||
readFile(threatModelPath, 'utf8'),
|
||||
readFile(roadmapPath, 'utf8'),
|
||||
]);
|
||||
const known = knownIssueNumbers(roadmap);
|
||||
const blocks = threatBlocks(doc);
|
||||
|
||||
assert.ok(blocks.size > 0, 'threats must be numbered headings so their coverage is checkable');
|
||||
const named = [...blocks.values()].map((block) => block.name).join('\n');
|
||||
|
||||
for (const [label, pattern] of REQUIRED_THREATS) {
|
||||
assert.match(named, pattern, `the ${label} abuse case must be its own numbered threat`);
|
||||
|
||||
const block = [...blocks.values()].find((entry) => pattern.test(entry.name));
|
||||
assert.match(block.body, /^\s*-\s*\*\*Owner:\*\*\s*\S+/m,
|
||||
`${label} needs a named owner`);
|
||||
assert.match(block.body, /^\s*-\s*\*\*Testable follow-up:\*\*\s*#\d+/m,
|
||||
`${label} needs a testable follow-up ticket`);
|
||||
assert.match(block.body, /^\s*-\s*\*\*Mitigation:\*\*\s*\S+/m,
|
||||
`${label} needs a concrete mitigation, not a restatement of the risk`);
|
||||
assert.match(block.body, /^\s*-\s*\*\*Evidence:\*\*\s*\S+/m,
|
||||
`${label} needs the artifact that will prove the mitigation works`);
|
||||
|
||||
const followUp = Number(block.body.match(/^\s*-\s*\*\*Testable follow-up:\*\*\s*#(\d+)/m)[1]);
|
||||
assert.ok(known.has(followUp),
|
||||
`${label} points at #${followUp}, which is not a tracked ticket in ROADMAP.md`);
|
||||
assert.notEqual(followUp, 20, `${label} must delegate to work other than this threat model itself`);
|
||||
}
|
||||
});
|
||||
|
||||
test('threat model keeps the photo path non-diagnostic and free of real medical images', async () => {
|
||||
const doc = await readFile(threatModelPath, 'utf8');
|
||||
|
||||
assert.match(doc, /non-diagnostic/i);
|
||||
assert.match(doc, /never diagnos/i);
|
||||
assert.match(doc, /provisional/i);
|
||||
assert.match(doc, /synthetic/i, 'fixtures must be synthetic, not real medical images');
|
||||
assert.match(doc, /no real (?:medical |stool )?images? (?:are |may )?(?:ever )?(?:added|used|shipped|included)/i,
|
||||
'the model must state plainly that no real medical images enter the repo');
|
||||
assert.match(doc, /deterministic (?:red-flag )?(?:escalation|urgent-symptom)/i,
|
||||
'the model must note that escalation is deterministic and never model-gated');
|
||||
assert.doesNotMatch(doc, /\b(?:bathroom photo|patient photo|clinical image|real stool image)\b/i,
|
||||
'the model must not promise or reference real medical imagery');
|
||||
assert.doesNotMatch(doc, /\b(?:clinically validated|clinically proven|medical device|FDA-cleared|CE-marked)\b/i,
|
||||
'the model must not overstate the clinical status of the vision path');
|
||||
});
|
||||
|
||||
test('the accepted product decision record links the photo data-flow threat model', async () => {
|
||||
const decision = await readFile(productDecisionPath, 'utf8');
|
||||
assert.match(decision, /\.\/PHOTO-DATAFLOW-THREAT-MODEL\.md/,
|
||||
'the governing decision record must point reviewers at the threat model');
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { analyzePhoto, visionFailureResponse } from '../src/vision-service.js';
|
||||
import { loadVisionCorpus } from '../src/vision-corpus.js';
|
||||
import { CORPUS_MODEL, caseFor, corpusDir, corpusPayload, startProvider, startServer, stopServer } from './helpers/vision-harness.mjs';
|
||||
|
||||
const FAIL_CLOSED_COPY = /no suggestion was applied/i;
|
||||
|
||||
test('a malformed provider response fails closed with user-safe copy instead of echoing provider detail', () => {
|
||||
const failure = visionFailureResponse(Object.assign(new Error('Invalid AI response: visual fields are out of range.'), { code: 'invalid_fields' }));
|
||||
|
||||
assert.equal(failure.status, 503);
|
||||
assert.match(failure.body.error, FAIL_CLOSED_COPY);
|
||||
assert.doesNotMatch(failure.body.error, /invalid ai response|out of range/i);
|
||||
|
||||
const consent = visionFailureResponse(new Error('Explicit consent is required before AI analysis.'));
|
||||
assert.equal(consent.status, 400);
|
||||
assert.match(consent.body.error, /consent/i);
|
||||
|
||||
const offline = visionFailureResponse(new Error('AI analysis is temporarily unavailable. Continue manually.'));
|
||||
assert.equal(offline.status, 503);
|
||||
assert.match(offline.body.error, /temporarily unavailable/i);
|
||||
});
|
||||
|
||||
test('the corpus replays through the real provider call path without inventing a suggestion', async () => {
|
||||
const corpus = await loadVisionCorpus(corpusDir);
|
||||
for (const entry of corpus.cases) {
|
||||
const fetchImpl = async () => ({ ok: true, json: async () => ({ choices: [{ message: { content: entry.providerContent } }] }) });
|
||||
const options = {
|
||||
payload: { imageDataUrl: 'data:image/png;base64,YQ==', consent: true },
|
||||
fetchImpl,
|
||||
config: { baseUrl: 'http://127.0.0.1:9/v1', apiKey: 'corpus-key', model: CORPUS_MODEL },
|
||||
};
|
||||
if (entry.expectedOutcome === 'suggestion') {
|
||||
const result = await analyzePhoto(options);
|
||||
assert.equal(result.status, 'suggestion', `${entry.id} must return a suggestion`);
|
||||
assert.equal(result.imageQuality, 'good');
|
||||
} else if (entry.expectedOutcome === 'abstained') {
|
||||
const result = await analyzePhoto(options);
|
||||
assert.equal(result.status, 'needs_user_input', `${entry.id} must abstain`);
|
||||
assert.equal(result.abstainedFor, entry.expectedReason, `${entry.id} must name its abstention`);
|
||||
assert.equal('bristolType' in result, false, `${entry.id} must not prefill a form`);
|
||||
} else {
|
||||
await assert.rejects(() => analyzePhoto(options), (error) => {
|
||||
assert.equal(visionFailureResponse(error).status, 503, `${entry.id} must fail closed`);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('the running server answers every corpus case with a fail-closed status and no leaked image bytes', async () => {
|
||||
const corpus = await loadVisionCorpus(corpusDir);
|
||||
const provider = await startProvider();
|
||||
const server = await startServer({ providerPort: provider.port });
|
||||
const postAnalyze = async (payload) => {
|
||||
const response = await fetch('http://127.0.0.1:4199/api/analyze', {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
return { status: response.status, body: await response.json() };
|
||||
};
|
||||
try {
|
||||
const suggestion = caseFor(corpus, 'stool-like-formed');
|
||||
provider.state.content = suggestion.providerContent;
|
||||
const payload = await corpusPayload(suggestion);
|
||||
const ok = await postAnalyze(payload);
|
||||
assert.equal(ok.status, 200, 'a schema-valid suggestion is the only case that returns 200 with fields');
|
||||
assert.equal(ok.body.bristolType, 4);
|
||||
|
||||
const glare = caseFor(corpus, 'glare-overexposed');
|
||||
provider.state.content = glare.providerContent;
|
||||
const abstained = await postAnalyze(await corpusPayload(glare));
|
||||
assert.equal(abstained.status, 200, 'a schema-valid response with unreadable fields abstains, it is not a server failure');
|
||||
assert.equal(abstained.body.status, 'needs_user_input');
|
||||
assert.equal(abstained.body.abstainedFor, 'unusable_visual_fields');
|
||||
assert.equal('bristolType' in abstained.body, false);
|
||||
|
||||
const malformed = caseFor(corpus, 'malformed-response');
|
||||
provider.state.content = malformed.providerContent;
|
||||
const unparsable = await postAnalyze(await corpusPayload(malformed));
|
||||
assert.equal(unparsable.status, 503, 'unparsable provider content must not be served as a suggestion');
|
||||
assert.match(unparsable.body.error, FAIL_CLOSED_COPY);
|
||||
assert.doesNotMatch(JSON.stringify(unparsable.body), /base64|data:image|corpus-key|Bearer/i);
|
||||
|
||||
const outOfRangeCase = caseFor(corpus, 'out-of-range-fields');
|
||||
provider.state.content = outOfRangeCase.providerContent;
|
||||
const outOfRange = await postAnalyze(await corpusPayload(outOfRangeCase));
|
||||
assert.equal(outOfRange.status, 503, 'out-of-range fields must not be served as a suggestion');
|
||||
assert.match(outOfRange.body.error, FAIL_CLOSED_COPY);
|
||||
|
||||
// Issue #73: a filled-in field that is out of range fails closed even when the other field is blank,
|
||||
// so the user is never told the photo was unreadable when the model returned an impossible value.
|
||||
const blankAndOutOfRange = caseFor(corpus, 'blank-plus-out-of-range');
|
||||
provider.state.content = blankAndOutOfRange.providerContent;
|
||||
const blankOutOfRange = await postAnalyze(await corpusPayload(blankAndOutOfRange));
|
||||
assert.equal(blankOutOfRange.status, 503, 'a filled-in out-of-range field must not be served as an abstention');
|
||||
assert.match(blankOutOfRange.body.error, FAIL_CLOSED_COPY);
|
||||
assert.equal('abstainedFor' in blankOutOfRange.body, false, 'a contract breach must not be reported as an abstention');
|
||||
assert.doesNotMatch(JSON.stringify(blankOutOfRange.body), /base64|data:image|corpus-key|Bearer/i);
|
||||
|
||||
const noConsent = await postAnalyze({ imageDataUrl: payload.imageDataUrl, consent: false });
|
||||
assert.equal(noConsent.status, 400, 'analysis without explicit consent is a client error');
|
||||
assert.doesNotMatch(JSON.stringify(noConsent.body), /base64|data:image/i);
|
||||
} finally {
|
||||
await stopServer(server);
|
||||
await provider.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,110 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { cp, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { classifyVisionOutcome, loadVisionCorpus, VISION_CORPUS_CLASSES, VISION_CORPUS_LICENSE, VISION_CORPUS_OUTCOMES } from '../src/vision-corpus.js';
|
||||
|
||||
const corpusDir = fileURLToPath(new URL('./fixtures/vision-corpus/', import.meta.url));
|
||||
// The colour contract `src/analysis.js` enforces, restated here so the corpus can be checked against it.
|
||||
const COLORS_IN_CONTRACT = ['brown', 'green', 'yellow', 'pale', 'red', 'black'];
|
||||
|
||||
async function scratchCorpus() {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'timmy-corpus-'));
|
||||
await cp(corpusDir, directory, { recursive: true });
|
||||
return directory;
|
||||
}
|
||||
|
||||
test('vision corpus ships licensed synthetic fixtures covering every required visual case class', async () => {
|
||||
const corpus = await loadVisionCorpus(corpusDir);
|
||||
|
||||
assert.equal(corpus.license, VISION_CORPUS_LICENSE);
|
||||
const classes = new Set(corpus.cases.map(entry => entry.caseClass));
|
||||
for (const required of VISION_CORPUS_CLASSES) {
|
||||
assert.ok(classes.has(required), `corpus must cover the ${required} case`);
|
||||
}
|
||||
for (const entry of corpus.cases) {
|
||||
assert.equal(entry.synthetic, true, `${entry.id} must be synthetic; no real medical image is redistributable`);
|
||||
assert.equal(entry.license, VISION_CORPUS_LICENSE);
|
||||
assert.match(entry.provenance, /\S/, `${entry.id} must record where it came from`);
|
||||
assert.match(entry.sha256, /^[a-f0-9]{64}$/, `${entry.id} must record its content hash`);
|
||||
}
|
||||
});
|
||||
|
||||
test('vision corpus rejects fixtures whose bytes no longer match the recorded hash', async () => {
|
||||
const directory = await scratchCorpus();
|
||||
try {
|
||||
const manifest = JSON.parse(await readFile(join(directory, 'manifest.json'), 'utf8'));
|
||||
const target = join(directory, manifest.cases[0].file);
|
||||
const bytes = await readFile(target);
|
||||
await writeFile(target, Buffer.concat([bytes, Buffer.from([0])]));
|
||||
|
||||
await assert.rejects(() => loadVisionCorpus(directory), new RegExp(manifest.cases[0].id));
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('vision corpus rejects an unlicensed, non-synthetic, or incomplete manifest', async () => {
|
||||
const directory = await scratchCorpus();
|
||||
try {
|
||||
const manifestPath = join(directory, 'manifest.json');
|
||||
const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
|
||||
|
||||
manifest.cases[0].synthetic = false;
|
||||
await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
||||
await assert.rejects(() => loadVisionCorpus(directory), /synthetic/i);
|
||||
|
||||
manifest.cases[0].synthetic = true;
|
||||
manifest.cases = manifest.cases.filter(entry => entry.caseClass !== 'non-stool');
|
||||
await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
||||
await assert.rejects(() => loadVisionCorpus(directory), /non-stool/);
|
||||
|
||||
manifest.cases[0].license = 'all-rights-reserved';
|
||||
await writeFile(manifestPath, JSON.stringify(manifest, null, 2));
|
||||
await assert.rejects(() => loadVisionCorpus(directory), /CC0-1\.0/);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('the corpus earns its out-of-range label and covers the blank-plus-out-of-range class', async () => {
|
||||
const corpus = await loadVisionCorpus(corpusDir);
|
||||
const outOfRange = corpus.cases.filter(entry => entry.expectedReason === 'out_of_range_or_missing_fields');
|
||||
assert.ok(outOfRange.length > 0, 'the corpus must cover the out-of-range failure class');
|
||||
|
||||
// Issue #73: a case whose confidence breaks the gate never reaches the field-range check, so its
|
||||
// `out_of_range_or_missing_fields` label would be attributed to a guard it never exercised.
|
||||
const reachesRangeCheck = outOfRange.filter(entry => Number(JSON.parse(entry.providerContent).confidence) <= 1);
|
||||
assert.ok(reachesRangeCheck.length > 0, 'at least one out-of-range case must clear the confidence gate to exercise the range check');
|
||||
|
||||
// Issue #73's escape class: a blank field next to a non-blank, out-of-range field. This is the exact
|
||||
// reply shape that used to abstain as `unusable_visual_fields` instead of failing closed.
|
||||
const blank = value => value === null || value === undefined || (typeof value === 'string' && !value.trim());
|
||||
const colourInContract = value => COLORS_IN_CONTRACT.includes(String(value).toLowerCase());
|
||||
const formInContract = value => Number.isInteger(Number(value)) && Number(value) >= 1 && Number(value) <= 7;
|
||||
const escapeCases = outOfRange.filter(entry => {
|
||||
const value = JSON.parse(entry.providerContent);
|
||||
const fields = [value.bristolType, value.color];
|
||||
return fields.some(blank) && fields.some(field => !blank(field) && !(colourInContract(field) || formInContract(field)));
|
||||
});
|
||||
assert.ok(escapeCases.length > 0, 'the corpus must carry a blank-plus-out-of-range case so the fail-closed label is really earned');
|
||||
});
|
||||
|
||||
test('every recorded corpus response is schema-valid or fails closed, exactly as declared', async () => {
|
||||
const corpus = await loadVisionCorpus(corpusDir);
|
||||
|
||||
for (const entry of corpus.cases) {
|
||||
const result = classifyVisionOutcome(entry.providerContent);
|
||||
assert.ok(VISION_CORPUS_OUTCOMES.includes(result.outcome), `${entry.id} produced an unknown outcome ${result.outcome}`);
|
||||
assert.equal(result.outcome, entry.expectedOutcome, `${entry.id} must resolve to ${entry.expectedOutcome}`);
|
||||
assert.equal(result.reason, entry.expectedReason, `${entry.id} must report ${entry.expectedReason}`);
|
||||
// A response may never smuggle a visual claim out of a non-suggestion outcome.
|
||||
if (result.outcome !== 'suggestion') {
|
||||
assert.equal('bristolType' in result, false, `${entry.id} must not carry a suggested form`);
|
||||
assert.equal('color' in result, false, `${entry.id} must not carry a suggested color`);
|
||||
}
|
||||
assert.doesNotMatch(JSON.stringify(result), /base64|data:image|Bear\s?er|api[_-]?key/i, `${entry.id} must not echo image bytes or credentials`);
|
||||
}
|
||||
});
|
||||
@@ -16,18 +16,48 @@ test('reports distinct phases so the UI never claims a worker is ready when it i
|
||||
assert.equal(visionReadiness(null).phase, 'checking');
|
||||
assert.equal(visionReadiness({ enabled: false }).phase, 'disabled');
|
||||
assert.equal(visionReadiness({ enabled: true, providerReady: false }).phase, 'offline');
|
||||
assert.equal(visionReadiness({ enabled: true, providerReady: true }).phase, 'ready');
|
||||
// The readiness contract widened in #15: answering the provider probe is no longer enough.
|
||||
assert.equal(visionReadiness({ enabled: true, providerReady: true }).phase, 'loading');
|
||||
assert.equal(visionReadiness({ enabled: true, providerReady: true, modelSeen: false }).phase, 'loading');
|
||||
assert.equal(visionReadiness({ enabled: true, providerReady: true, modelSeen: true }).phase, 'ready');
|
||||
});
|
||||
|
||||
test('a ready provider keeps AI analysis available and does not force the manual path', () => {
|
||||
const readiness = visionReadiness({ enabled: true, providerReady: true, model: 'SmolVLM2-2.2B-Instruct' });
|
||||
test('readiness waits for the pinned model to load before promising AI', () => {
|
||||
// Issue #15: a worker that answers /v1/models has not necessarily loaded the model yet.
|
||||
// Readiness must pass only after the pinned model is loaded, or the user is offered an
|
||||
// AI control that fails after the upload.
|
||||
const notLoaded = visionReadiness({
|
||||
enabled: true,
|
||||
profile: 'selfhost',
|
||||
providerReady: true,
|
||||
modelSeen: false,
|
||||
model: 'SmolVLM2-2.2B-Instruct',
|
||||
});
|
||||
assert.equal(notLoaded.phase, 'loading');
|
||||
assert.equal(notLoaded.ready, false);
|
||||
assert.equal(notLoaded.manualOnly, true);
|
||||
assert.match(notLoaded.message, /model/i);
|
||||
assert.match(notLoaded.manualLabel, /manual|without AI/i);
|
||||
|
||||
const unverified = visionReadiness({ enabled: true, profile: 'selfhost', providerReady: true });
|
||||
assert.equal(unverified.ready, false, 'an unconfirmed model must never be treated as loaded');
|
||||
});
|
||||
|
||||
test('a loaded pinned model keeps AI analysis available and does not force the manual path', () => {
|
||||
const readiness = visionReadiness({ enabled: true, providerReady: true, modelSeen: true, model: 'SmolVLM2-2.2B-Instruct' });
|
||||
assert.equal(readiness.ready, true);
|
||||
assert.equal(readiness.manualOnly, false);
|
||||
assert.equal(readiness.manualLabel, '');
|
||||
});
|
||||
|
||||
test('readiness never promises a symptom or diagnosis capability', () => {
|
||||
for (const status of [null, { enabled: false }, { enabled: true, providerReady: false }, { enabled: true, providerReady: true }]) {
|
||||
for (const status of [
|
||||
null,
|
||||
{ enabled: false },
|
||||
{ enabled: true, providerReady: false },
|
||||
{ enabled: true, providerReady: true, modelSeen: true },
|
||||
{ enabled: true, providerReady: true, modelSeen: false },
|
||||
]) {
|
||||
const readiness = visionReadiness(status);
|
||||
const copy = `${readiness.manualLabel} ${readiness.message}`;
|
||||
assert.doesNotMatch(copy, /diagnos|bleeding|cancer|symptom/i);
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
// Browser acceptance for issue #18: the refusal paths the corpus proves must be visible in the app,
|
||||
// not just in unit tests. The app talks to a real server.mjs with vision enabled, and the only thing
|
||||
// the harness chooses is which recorded corpus response the stub worker returns.
|
||||
import { chromium } from 'playwright';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import { loadVisionCorpus } from '../src/vision-corpus.js';
|
||||
import { caseFor, corpusDir, startProvider, startServer, stopServer } from './helpers/vision-harness.mjs';
|
||||
|
||||
await mkdir('artifacts', { recursive: true });
|
||||
const corpus = await loadVisionCorpus(corpusDir);
|
||||
const provider = await startProvider();
|
||||
const server = await startServer({ providerPort: provider.port, port: 4199 });
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 2, serviceWorkers: 'block' });
|
||||
const page = await context.newPage();
|
||||
const errors = [];
|
||||
page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); });
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
// A fail-closed 503 is the intended boundary response, so the browser logs it as a failed resource.
|
||||
// It is expected exactly once; anything else is a real page error.
|
||||
const EXPECTED_FAIL_CLOSED_LOG = /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/;
|
||||
|
||||
async function closeSheet() {
|
||||
await page.locator('#close-sheet').click();
|
||||
await page.locator('.scan-sheet').waitFor({ state: 'detached' });
|
||||
}
|
||||
|
||||
async function analyzeCorpusCase(entry) {
|
||||
provider.state.content = entry.providerContent;
|
||||
await page.locator('[data-scan]').click();
|
||||
await page.getByText(/Self-hosted model ready/i).waitFor();
|
||||
await page.locator('#gallery-photo').setInputFiles(`tests/fixtures/vision-corpus/${entry.file}`);
|
||||
await page.locator('#ai-consent').check();
|
||||
await page.locator('#analyze-photo').click();
|
||||
await page.locator('.scan-result').first().waitFor();
|
||||
}
|
||||
|
||||
try {
|
||||
await page.goto('http://127.0.0.1:4199', { waitUntil: 'networkidle' });
|
||||
await page.evaluate(() => localStorage.clear());
|
||||
await page.reload({ waitUntil: 'networkidle' });
|
||||
|
||||
// 1. Non-stool photo: no prefill, a plain refusal, and manual entry still one action away.
|
||||
await analyzeCorpusCase(caseFor(corpus, 'non-stool-scene'));
|
||||
assert.equal(await page.getByText(/No confident match/i).isVisible(), true, 'a non-stool photo must refuse');
|
||||
assert.equal(await page.locator('.ai-badge').count(), 0, 'a refusal must not carry an AI suggestion badge');
|
||||
assert.equal(await page.getByText('Type 4', { exact: true }).count(), 0, 'a refusal must not prefill a form');
|
||||
assert.match(await page.locator('.scan-result').innerText(), /does not clearly show stool/i);
|
||||
const manual = page.locator('#manual-from-scan');
|
||||
assert.match(await manual.innerText(), /Choose the form yourself/i);
|
||||
await page.screenshot({ path: 'artifacts/vision-refusal-non-stool-mobile.png' });
|
||||
await manual.click();
|
||||
await page.getByText('Pick the closest form').waitFor();
|
||||
assert.equal(await page.locator('.ai-prefill').count(), 0, 'refused analysis must not leave AI provenance in the log form');
|
||||
await closeSheet();
|
||||
|
||||
// 2. Glare with unreadable visual fields: schema-valid, so it abstains calmly instead of erroring.
|
||||
await analyzeCorpusCase(caseFor(corpus, 'glare-overexposed'));
|
||||
assert.equal(await page.getByText(/No confident match/i).isVisible(), true);
|
||||
assert.match(await page.locator('.scan-result').innerText(), /does not show a form Timmy can read safely/i);
|
||||
assert.equal(await page.getByText(/couldn’t analyze that safely/i).count(), 0, 'a schema-valid abstention is not a failure');
|
||||
await page.screenshot({ path: 'artifacts/vision-refusal-unreadable-fields-mobile.png' });
|
||||
await closeSheet();
|
||||
|
||||
// 3. Out-of-range model output: fails closed, and the copy says plainly that nothing was applied.
|
||||
await analyzeCorpusCase(caseFor(corpus, 'out-of-range-fields'));
|
||||
await page.getByText(/couldn’t analyze that safely/i).waitFor();
|
||||
assert.match(await page.locator('.scan-result').innerText(), /No suggestion was applied/i);
|
||||
assert.doesNotMatch(await page.locator('.scan-result').innerText(), /out of range|Invalid AI response/i, 'provider detail must not reach the user');
|
||||
assert.equal(await page.locator('.ai-badge').count(), 0);
|
||||
await page.screenshot({ path: 'artifacts/vision-refusal-fail-closed-mobile.png' });
|
||||
await closeSheet();
|
||||
|
||||
// 4. Issue #73: a filled-in field that is out of range fails closed even when the other field is blank.
|
||||
// The old blank-first ordering abstained here, so the user was told the photo was unreadable while the
|
||||
// model had actually returned an impossible value. The abstention copy must not appear on this path.
|
||||
await analyzeCorpusCase(caseFor(corpus, 'blank-plus-out-of-range'));
|
||||
await page.getByText(/couldn’t analyze that safely/i).waitFor();
|
||||
assert.match(await page.locator('.scan-result').innerText(), /No suggestion was applied/i);
|
||||
assert.equal(await page.getByText(/No confident match/i).count(), 0, 'a contract breach must not be shown as an abstention');
|
||||
assert.equal(await page.locator('.ai-badge').count(), 0);
|
||||
assert.doesNotMatch(await page.locator('.scan-result').innerText(), /out of range|Invalid AI response/i, 'provider detail must not reach the user');
|
||||
await page.screenshot({ path: 'artifacts/vision-refusal-blank-plus-out-of-range-mobile.png' });
|
||||
await closeSheet();
|
||||
|
||||
// 5. Positive control: the same corpus still yields a real suggestion when the response is usable.
|
||||
await analyzeCorpusCase(caseFor(corpus, 'stool-like-formed'));
|
||||
await page.getByText(/83% confidence/i).waitFor();
|
||||
assert.equal(await page.getByText('Type 4', { exact: true }).isVisible(), true);
|
||||
await page.screenshot({ path: 'artifacts/vision-corpus-suggestion-mobile.png' });
|
||||
|
||||
// Two fail-closed scenes reach the browser as real 503s; the count is asserted so a scene cannot be
|
||||
// dropped silently, and any other console error still fails the run.
|
||||
const failClosedLogs = errors.filter(message => EXPECTED_FAIL_CLOSED_LOG.test(message));
|
||||
assert.equal(failClosedLogs.length, 2, 'each fail-closed response must reach the browser as a 503');
|
||||
assert.deepEqual(errors.filter(message => !EXPECTED_FAIL_CLOSED_LOG.test(message)), []);
|
||||
console.log('PASS vision corpus refusal: non-stool, unreadable fields, out-of-range, and blank-plus-out-of-range replies never prefill; the usable reply still suggests');
|
||||
} finally {
|
||||
await context.close();
|
||||
await browser.close();
|
||||
await stopServer(server);
|
||||
await provider.close();
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawnSync, spawn } from 'node:child_process';
|
||||
import { chmod, mkdtemp, readFile, writeFile } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
const benchmark = new URL('../scripts/record_worker_benchmark.mjs', import.meta.url).pathname;
|
||||
const receiptsPath = new URL('../deploy/worker/receipts.json', import.meta.url);
|
||||
const ALIAS = 'SmolVLM2-2.2B-Instruct';
|
||||
|
||||
const FAKE_WORKER = `#!/usr/bin/env python3
|
||||
import json, os
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
loaded = os.environ['WORKER_MODE'] == 'loaded'
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
body = json.dumps({'data': [{'id': os.environ['WORKER_ALIAS']}] if loaded else []}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
HTTPServer(('127.0.0.1', int(os.environ['WORKER_PORT'])), Handler).serve_forever()
|
||||
`;
|
||||
|
||||
async function sandbox() {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'timmy-bench-'));
|
||||
const gpuProbe = join(dir, 'nvidia-smi-stub');
|
||||
await writeFile(gpuProbe, '#!/bin/sh\nprintf "NVIDIA A40, 46068, 1234\\n"\n');
|
||||
await chmod(gpuProbe, 0o755);
|
||||
const worker = join(dir, 'worker.py');
|
||||
await writeFile(worker, FAKE_WORKER);
|
||||
return { dir, gpuProbe, worker, out: join(dir, 'worker-benchmark.json') };
|
||||
}
|
||||
|
||||
async function startWorker(box, mode) {
|
||||
const port = String(30000 + (process.pid % 2000));
|
||||
const proc = spawn('python3', [box.worker], {
|
||||
env: { ...process.env, WORKER_MODE: mode, WORKER_PORT: port, WORKER_ALIAS: ALIAS },
|
||||
stdio: 'ignore',
|
||||
});
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
const probe = spawnSync('bash', ['-c', `curl --silent --output /dev/null --max-time 1 http://127.0.0.1:${port}/v1/models`], { encoding: 'utf8' });
|
||||
if (probe.status === 0) break;
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
return { proc, port };
|
||||
}
|
||||
|
||||
function run(box, port, extra = []) {
|
||||
return spawnSync('node', [benchmark, '--out', box.out, '--port', port, '--alias', ALIAS, '--gpu-probe', box.gpuProbe, ...extra], { encoding: 'utf8' });
|
||||
}
|
||||
|
||||
test('a benchmark refuses to record numbers it did not measure on a GPU', async () => {
|
||||
const box = await sandbox();
|
||||
const { proc, port } = await startWorker(box, 'loaded');
|
||||
try {
|
||||
const result = spawnSync('node', [benchmark, '--out', box.out, '--port', port, '--alias', ALIAS, '--gpu-probe', join(box.dir, 'missing-smi')], { encoding: 'utf8' });
|
||||
assert.notEqual(result.status, 0, 'measuring without a GPU probe must fail, not record zeros');
|
||||
assert.match(result.stderr, /GPU|probe|VRAM/i);
|
||||
assert.equal(existsSync(box.out), false, 'no receipt may be written from an unmeasured run');
|
||||
} finally {
|
||||
proc.kill('SIGKILL');
|
||||
}
|
||||
});
|
||||
|
||||
test('a benchmark refuses to time a model that is not loaded', async () => {
|
||||
const box = await sandbox();
|
||||
const { proc, port } = await startWorker(box, 'unloaded');
|
||||
try {
|
||||
const result = run(box, port);
|
||||
assert.notEqual(result.status, 0);
|
||||
assert.match(result.stderr, /not loaded|ready/i);
|
||||
assert.equal(existsSync(box.out), false, 'an unloaded model must not produce a latency receipt');
|
||||
} finally {
|
||||
proc.kill('SIGKILL');
|
||||
}
|
||||
});
|
||||
|
||||
test('a benchmark records cold start, warm latency, VRAM, and the pins it measured', async () => {
|
||||
const box = await sandbox();
|
||||
const { proc, port } = await startWorker(box, 'loaded');
|
||||
try {
|
||||
const result = run(box, port, ['--warm-samples', '5', '--start-command', 'bash -c "sleep 0.3"']);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const receipt = JSON.parse(await readFile(box.out, 'utf8'));
|
||||
assert.equal(receipt.status, 'measured_on_target_gpu');
|
||||
assert.equal(receipt.worker.alias, ALIAS);
|
||||
assert.match(receipt.worker.baseUrl, new RegExp(`^http://127\\.0\\.0\\.1:${port}/v1$`));
|
||||
assert.match(receipt.gpu.name, /NVIDIA A40/);
|
||||
assert.equal(receipt.gpu.vramUsedMiB, 1234);
|
||||
assert.equal(receipt.gpu.vramTotalMiB, 46068);
|
||||
assert.ok(receipt.coldStartMs >= 300, 'cold start is measured, not asserted');
|
||||
assert.equal(receipt.warmLatencyMs.samples, 5);
|
||||
assert.ok(receipt.warmLatencyMs.p50 >= 0 && receipt.warmLatencyMs.p95 >= receipt.warmLatencyMs.p50);
|
||||
|
||||
const receipts = JSON.parse(await readFile(receiptsPath, 'utf8'));
|
||||
assert.equal(receipt.pins.llamaCppCommit, receipts.llamaCpp.commit);
|
||||
assert.deepEqual(receipt.pins.modelSha256, receipts.models.map(model => model.sha256));
|
||||
} finally {
|
||||
proc.kill('SIGKILL');
|
||||
}
|
||||
});
|
||||
|
||||
test('a benchmark receipt carries no notes, image bytes, or user data', async () => {
|
||||
const box = await sandbox();
|
||||
const { proc, port } = await startWorker(box, 'loaded');
|
||||
try {
|
||||
const result = run(box, port, ['--warm-samples', '3']);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
const raw = await readFile(box.out, 'utf8');
|
||||
const receipt = JSON.parse(raw);
|
||||
assert.deepEqual(Object.keys(receipt).sort(), [
|
||||
'coldStartMs',
|
||||
'gpu',
|
||||
'measuredAt',
|
||||
'pins',
|
||||
'schemaVersion',
|
||||
'status',
|
||||
'warmLatencyMs',
|
||||
'worker',
|
||||
]);
|
||||
assert.doesNotMatch(raw, /data:image|base64|note|observations|symptom/i);
|
||||
// Cold start that was not measured is recorded as unmeasured, never as zero.
|
||||
assert.equal(receipt.coldStartMs, null);
|
||||
} finally {
|
||||
proc.kill('SIGKILL');
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,151 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const dockerfilePath = new URL('../deploy/worker/Dockerfile', import.meta.url);
|
||||
const receiptsPath = new URL('../deploy/worker/receipts.json', import.meta.url);
|
||||
const buildScriptPath = new URL('../deploy/worker/build_image.sh', import.meta.url);
|
||||
const runScriptPath = new URL('../deploy/worker/run_worker_container.sh', import.meta.url);
|
||||
const dockerignorePath = new URL('../.dockerignore', import.meta.url);
|
||||
const bootstrapPath = new URL('../scripts/bootstrap_selfhost_smolvlm.sh', import.meta.url);
|
||||
|
||||
async function recipe() {
|
||||
const [dockerfile, receipts, buildScript, runScript, dockerignore, bootstrap] = await Promise.all([
|
||||
readFile(dockerfilePath, 'utf8'),
|
||||
readFile(receiptsPath, 'utf8').then(JSON.parse),
|
||||
readFile(buildScriptPath, 'utf8'),
|
||||
readFile(runScriptPath, 'utf8'),
|
||||
readFile(dockerignorePath, 'utf8'),
|
||||
readFile(bootstrapPath, 'utf8'),
|
||||
]);
|
||||
return { dockerfile, receipts, buildScript, runScript, dockerignore, bootstrap };
|
||||
}
|
||||
|
||||
function bootstrapDefault(script, name) {
|
||||
const match = new RegExp(`^${name}="\\$\\{[A-Z0-9_]*:-([^}]+)\\}"$`, 'm').exec(script);
|
||||
assert.ok(match, `bootstrap script must declare a default for ${name}`);
|
||||
return match[1];
|
||||
}
|
||||
|
||||
test('the worker image pins llama.cpp and both model hashes to the bootstrap receipts', async () => {
|
||||
const { dockerfile, receipts, bootstrap } = await recipe();
|
||||
const commit = bootstrapDefault(bootstrap, 'LLAMA_CPP_COMMIT');
|
||||
const modelSha = bootstrapDefault(bootstrap, 'MODEL_SHA256');
|
||||
const mmprojSha = bootstrapDefault(bootstrap, 'MMPROJ_SHA256');
|
||||
|
||||
// Receipts are the runtime/CI source of truth and must not drift from the host bootstrap.
|
||||
assert.equal(receipts.llamaCpp.commit, commit);
|
||||
assert.equal(receipts.llamaCpp.repository, 'https://github.com/ggml-org/llama.cpp');
|
||||
assert.deepEqual(receipts.models.map(model => model.sha256), [modelSha, mmprojSha]);
|
||||
assert.deepEqual(receipts.models.map(model => model.file), [
|
||||
'SmolVLM2-2.2B-Instruct-Q4_K_M.gguf',
|
||||
'mmproj-SmolVLM2-2.2B-Instruct-Q8_0.gguf',
|
||||
]);
|
||||
assert.match(receipts.llamaCpp.commit, /^[0-9a-f]{40}$/);
|
||||
for (const model of receipts.models) assert.match(model.sha256, /^[0-9a-f]{64}$/);
|
||||
|
||||
// The image builds the same pinned revision: the build args default to the receipts.
|
||||
assert.match(dockerfile, new RegExp(`^ARG LLAMA_CPP_COMMIT=${commit}$`, 'm'));
|
||||
assert.match(dockerfile, new RegExp(`^ARG MODEL_SHA256=${modelSha}$`, 'm'));
|
||||
assert.match(dockerfile, new RegExp(`^ARG MMPROJ_SHA256=${mmprojSha}$`, 'm'));
|
||||
assert.match(dockerfile, /git fetch --depth 1 origin "\$\{LLAMA_CPP_COMMIT\}"/);
|
||||
assert.match(dockerfile, /git checkout --detach FETCH_HEAD/);
|
||||
});
|
||||
|
||||
test('the worker image uses digest-pinned bases and refuses floating model sources', async () => {
|
||||
const { dockerfile, receipts } = await recipe();
|
||||
|
||||
const froms = dockerfile.split('\n').filter(line => line.startsWith('FROM '));
|
||||
assert.equal(froms.length, 2, 'the recipe is a build stage and a runtime stage');
|
||||
for (const line of froms) assert.match(line, /@sha256:[0-9a-f]{64}/, `base must be digest-pinned: ${line}`);
|
||||
// The recorded receipts and the recipe must name the same immutable bases.
|
||||
const digests = froms.map(line => /@(sha256:[0-9a-f]{64})/.exec(line)[1]);
|
||||
assert.deepEqual(digests, [receipts.baseImages.build.digest, receipts.baseImages.runtime.digest]);
|
||||
assert.equal(receipts.baseImages.build.reference, 'nvidia/cuda:12.4.1-devel-ubuntu22.04');
|
||||
assert.equal(receipts.baseImages.runtime.reference, 'nvidia/cuda:12.4.1-runtime-ubuntu22.04');
|
||||
assert.doesNotMatch(dockerfile, /:latest|ggml-org\/llama\.cpp@?main|refs\/heads\/(?:main|master)/);
|
||||
// Weights are never baked in; the model arrives as a read-only mount.
|
||||
assert.doesNotMatch(dockerfile, /^COPY .*\.gguf/m);
|
||||
assert.doesNotMatch(dockerfile, /^ADD /m);
|
||||
assert.match(dockerfile, /\/models/);
|
||||
});
|
||||
|
||||
test('the worker image runs unprivileged with no default credentials and a post-load healthcheck', async () => {
|
||||
const { dockerfile } = await recipe();
|
||||
// Directives may be wrapped with a line continuation; assert on the joined directive text.
|
||||
const directives = dockerfile.replace(/\\\n\s*/g, ' ');
|
||||
|
||||
const user = /^USER (?!root)(\S+)$/m.exec(dockerfile);
|
||||
assert.ok(user, 'the runtime stage must drop to an unprivileged user');
|
||||
assert.match(user[1], /^\d+:\d+$/, 'use a numeric uid:gid so the image needs no passwd entry');
|
||||
assert.doesNotMatch(dockerfile, /^ENV .*(?:API_KEY|TOKEN|PASSWORD|SECRET)[ =]/im);
|
||||
// Health must gate on the loaded model, so the container is not "healthy" while it cannot answer.
|
||||
assert.match(directives, /^HEALTHCHECK .*timmy-worker-ready/m);
|
||||
assert.match(directives, /--start-period=\d+s/);
|
||||
assert.match(directives, /--host", "127\.0\.0\.1"/, 'the health probe runs on the container loopback');
|
||||
// The container listens inside its own network namespace; the run recipe, not the image, decides
|
||||
// exposure and publishes on loopback only (asserted separately).
|
||||
assert.match(dockerfile, /--host 0\.0\.0\.0|TIMMY_MODEL_HOST=0\.0\.0\.0/);
|
||||
});
|
||||
|
||||
test('the build refuses a floating or missing pin before it reaches docker', async () => {
|
||||
const floating = spawnSync('bash', [buildScriptPath.pathname, '--print-plan'], {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, TIMMY_LLAMA_CPP_COMMIT: 'latest' },
|
||||
});
|
||||
assert.notEqual(floating.status, 0, 'a floating llama.cpp ref must be rejected');
|
||||
assert.match(floating.stderr, /pin|40-hex|floating/i);
|
||||
|
||||
const missingSha = spawnSync('bash', [buildScriptPath.pathname, '--print-plan'], {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, TIMMY_MODEL_SHA256: '' },
|
||||
});
|
||||
assert.notEqual(missingSha.status, 0, 'a missing model hash must be rejected');
|
||||
|
||||
const plan = spawnSync('bash', [buildScriptPath.pathname, '--print-plan'], { encoding: 'utf8' });
|
||||
assert.equal(plan.status, 0, plan.stderr);
|
||||
assert.match(plan.stdout, /llama\.cpp commit: [0-9a-f]{40}/);
|
||||
assert.match(plan.stdout, /SmolVLM2-2\.2B-Instruct-Q4_K_M\.gguf sha256: [0-9a-f]{64}/);
|
||||
assert.match(plan.stdout, /bind address: 127\.0\.0\.1:8080/);
|
||||
assert.match(plan.stdout, /publish: 127\.0\.0\.1:8080/);
|
||||
});
|
||||
|
||||
test('the run recipe publishes on loopback only and drops every privilege it can', async () => {
|
||||
const { runScript } = await recipe();
|
||||
|
||||
assert.match(runScript, /127\.0\.0\.1:\$\{?PORT/);
|
||||
assert.match(runScript, /--read-only/);
|
||||
assert.match(runScript, /--cap-drop[= ]ALL/);
|
||||
assert.match(runScript, /no-new-privileges/);
|
||||
assert.match(runScript, /--security-opt/);
|
||||
assert.doesNotMatch(runScript, /--privileged/);
|
||||
assert.doesNotMatch(runScript, /-p "\$PORT:/);
|
||||
assert.doesNotMatch(runScript, /\b0\.0\.0\.0\b/);
|
||||
// The model mount is read-only, and no credential has a default value.
|
||||
assert.match(runScript, /:ro\b/);
|
||||
assert.doesNotMatch(runScript, /(?:API_KEY|TOKEN|PASSWORD)=[^\s"']/);
|
||||
});
|
||||
|
||||
test('the run recipe refuses to publish the worker anywhere but loopback', async () => {
|
||||
const hostile = spawnSync('bash', [runScriptPath.pathname, '--print-plan'], {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, TIMMY_WORKER_PUBLISH_HOST: '0.0.0.0' },
|
||||
});
|
||||
assert.notEqual(hostile.status, 0, 'a non-loopback publish must be refused');
|
||||
assert.match(hostile.stderr, /loopback|127\.0\.0\.1/i);
|
||||
|
||||
const plan = spawnSync('bash', [runScriptPath.pathname, '--print-plan'], { encoding: 'utf8' });
|
||||
assert.equal(plan.status, 0, plan.stderr);
|
||||
assert.match(plan.stdout, /publish: 127\.0\.0\.1:\d+/);
|
||||
});
|
||||
|
||||
test('the build context excludes weights, user images, releases, and secrets', async () => {
|
||||
const { dockerignore } = await recipe();
|
||||
|
||||
for (const pattern of ['*.gguf', 'artifacts/', 'video/', 'models/', '.git/', 'node_modules/', '.env', '.env.*']) {
|
||||
assert.ok(dockerignore.split('\n').includes(pattern), `dockerignore must exclude ${pattern}`);
|
||||
}
|
||||
// Committed synthetic fixtures are fine, but a real medical corpus directory is never buildable.
|
||||
assert.doesNotMatch(dockerignore, /^!.*\.gguf$/m);
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
import { chromium } from 'playwright';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
|
||||
// Issue #15: readiness must pass only after the pinned model is loaded. A worker that answers
|
||||
// /v1/models is reachable, but reachable is not loaded — the app must not offer an AI control
|
||||
// that fails after the upload.
|
||||
await mkdir('artifacts', { recursive: true });
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const context = await browser.newContext({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 2, serviceWorkers: 'block' });
|
||||
const page = await context.newPage();
|
||||
const errors = [];
|
||||
page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); });
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
let modelLoaded = false;
|
||||
let analysisRequests = 0;
|
||||
await page.route('**/api/vision-status', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
enabled: true,
|
||||
profile: 'selfhost',
|
||||
processor: 'self-hosted',
|
||||
model: 'SmolVLM2-2.2B-Instruct',
|
||||
providerReady: true,
|
||||
modelSeen: modelLoaded,
|
||||
}),
|
||||
}));
|
||||
await page.route('**/api/analyze', route => {
|
||||
analysisRequests += 1;
|
||||
return route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
status: 'suggestion',
|
||||
isStool: true,
|
||||
bristolType: 4,
|
||||
color: 'brown',
|
||||
confidence: 0.83,
|
||||
imageQuality: 'good',
|
||||
observations: 'Smooth, formed appearance.',
|
||||
warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
await page.goto('http://127.0.0.1:4173', { waitUntil: 'networkidle' });
|
||||
await page.evaluate(() => localStorage.clear());
|
||||
await page.reload({ waitUntil: 'networkidle' });
|
||||
|
||||
// 1. Worker reachable, pinned model not loaded: no AI promise, no dead control, one action to manual.
|
||||
await page.locator('[data-scan]').click();
|
||||
await page.getByText(/model is not loaded/i).waitFor();
|
||||
// Visual acceptance: the not-loaded state must be visually distinct from the ready and offline
|
||||
// states, not an unstyled div that reads like either one of them.
|
||||
const loadingStatus = page.locator('.model-status.loading');
|
||||
assert.equal(await loadingStatus.count(), 1, 'the not-loaded state needs its own status treatment');
|
||||
assert.equal(await loadingStatus.evaluate(el => getComputedStyle(el).backgroundColor), 'rgb(253, 243, 216)');
|
||||
assert.equal(await page.getByText(/Self-hosted model ready/i).count(), 0, 'an unloaded model must never be reported ready');
|
||||
assert.equal(await page.locator('#analyze-photo').count(), 0, 'no analyze control before the model is loaded');
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||||
await page.getByText(/Ready for your review/i).waitFor();
|
||||
assert.equal(await page.getByText(/model is not loaded/i).count() > 0, true, 'the ready step still states why AI is unavailable');
|
||||
assert.equal(await page.locator('.consent-card').count(), 0, 'no consent is requested while the model is not loaded');
|
||||
assert.equal(await page.locator('#analyze-photo').count(), 0, 'no dead analyze control on the ready step');
|
||||
const fallback = page.locator('#manual-from-scan');
|
||||
assert.equal(await fallback.isVisible(), true);
|
||||
assert.match(await fallback.innerText(), /without AI|manual/i);
|
||||
await fallback.click();
|
||||
assert.equal(await page.getByText('Pick the closest form').isVisible(), true, 'one action reaches manual Bristol entry');
|
||||
assert.equal(analysisRequests, 0, 'nothing is uploaded before the pinned model is loaded');
|
||||
await page.screenshot({ path: 'artifacts/worker-model-loading-mobile.png' });
|
||||
await page.locator('#close-sheet').click();
|
||||
|
||||
// 2. The pinned model loads: the same flow now offers consent and analysis, and analysis works.
|
||||
modelLoaded = true;
|
||||
await page.locator('[data-scan]').click();
|
||||
await page.getByText(/Self-hosted model ready/i).waitFor();
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||||
await page.getByText(/Ready for your review/i).waitFor();
|
||||
assert.equal(await page.locator('.consent-card').isVisible(), true, 'consent returns once the model is loaded');
|
||||
await page.locator('#ai-consent').check();
|
||||
await page.locator('#analyze-photo').click();
|
||||
await page.getByText(/83% confidence/i).waitFor();
|
||||
assert.equal(analysisRequests, 1, 'analysis is offered exactly when the model is loaded');
|
||||
await page.screenshot({ path: 'artifacts/worker-model-loaded-mobile.png' });
|
||||
|
||||
assert.deepEqual(errors, []);
|
||||
await context.close();
|
||||
await browser.close();
|
||||
console.log('PASS readiness waits for the loaded pinned model: no AI control before load, full AI path after load');
|
||||
@@ -0,0 +1,107 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { mkdtemp, readFile, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
|
||||
const gate = new URL('../scripts/worker_ready.sh', import.meta.url).pathname;
|
||||
const bootstrap = new URL('../scripts/bootstrap_selfhost_smolvlm.sh', import.meta.url);
|
||||
const ALIAS = 'SmolVLM2-2.2B-Instruct';
|
||||
|
||||
const FAKE_SERVER = `#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
|
||||
mode = os.environ['WORKER_MODE']
|
||||
counter_path = os.environ['WORKER_COUNTER']
|
||||
count = int(open(counter_path).read() or '0') if os.path.exists(counter_path) else 0
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
global count
|
||||
count += 1
|
||||
open(counter_path, 'w').write(str(count))
|
||||
loaded = mode == 'loaded' or (mode == 'flip' and count > 2)
|
||||
body = json.dumps({'data': [{'id': os.environ['WORKER_ALIAS']}] if loaded else []}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
HTTPServer(('127.0.0.1', int(os.environ['WORKER_PORT'])), Handler).serve_forever()
|
||||
`;
|
||||
|
||||
async function withWorker(mode, run) {
|
||||
const sandbox = await mkdtemp(join(tmpdir(), 'timmy-worker-ready-'));
|
||||
const counter = join(sandbox, 'count.txt');
|
||||
await writeFile(counter, '0');
|
||||
const server = join(sandbox, 'server.py');
|
||||
await writeFile(server, FAKE_SERVER);
|
||||
const port = String(20000 + (process.pid % 4000));
|
||||
const { spawn } = await import('node:child_process');
|
||||
const proc = spawn('python3', [server], {
|
||||
env: { ...process.env, WORKER_MODE: mode, WORKER_COUNTER: counter, WORKER_PORT: port, WORKER_ALIAS: ALIAS },
|
||||
stdio: 'ignore',
|
||||
});
|
||||
// Wait for the socket to accept connections before probing.
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
const probe = spawnSync('bash', ['-c', `curl --silent --output /dev/null --max-time 1 http://127.0.0.1:${port}/v1/models`], { encoding: 'utf8' });
|
||||
if (probe.status === 0) break;
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
try {
|
||||
return await run(port);
|
||||
} finally {
|
||||
proc.kill('SIGKILL');
|
||||
}
|
||||
}
|
||||
|
||||
function gateRun(port, extra = []) {
|
||||
return spawnSync('bash', [gate, '--port', port, '--alias', ALIAS, ...extra], { encoding: 'utf8' });
|
||||
}
|
||||
|
||||
test('readiness passes once the pinned model is actually served', async () => {
|
||||
await withWorker('loaded', async port => {
|
||||
const result = gateRun(port);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.match(result.stdout, new RegExp(`${ALIAS} is loaded`));
|
||||
});
|
||||
});
|
||||
|
||||
test('a reachable worker with the model still loading is not ready', async () => {
|
||||
await withWorker('unloaded', async port => {
|
||||
const result = gateRun(port);
|
||||
assert.equal(result.status, 3, 'reachable-but-unloaded must be its own failure, not a network error');
|
||||
assert.match(result.stderr, /not loaded yet/i);
|
||||
});
|
||||
});
|
||||
|
||||
test('an unreachable worker is reported as unreachable', async () => {
|
||||
const result = spawnSync('bash', [gate, '--port', '9', '--alias', ALIAS], { encoding: 'utf8' });
|
||||
assert.equal(result.status, 2);
|
||||
assert.match(result.stderr, /not reachable/i);
|
||||
});
|
||||
|
||||
test('readiness never probes anything but loopback', () => {
|
||||
const result = spawnSync('bash', [gate, '--host', '10.0.0.5', '--port', '8080'], { encoding: 'utf8' });
|
||||
assert.equal(result.status, 64);
|
||||
assert.match(result.stderr, /loopback/i);
|
||||
});
|
||||
|
||||
test('readiness can wait for the model load it is gating', async () => {
|
||||
await withWorker('flip', async port => {
|
||||
const started = Date.now();
|
||||
const result = gateRun(port, ['--wait', '15']);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.ok(Date.now() - started >= 300, 'the gate must actually wait rather than pass on the first probe');
|
||||
});
|
||||
});
|
||||
|
||||
test('the bootstrap exposes the same post-load gate instead of duplicating it', async () => {
|
||||
const script = await readFile(bootstrap, 'utf8');
|
||||
assert.match(script, /worker_ready\.sh/);
|
||||
assert.match(script, /^ ready\)$/m);
|
||||
});
|
||||