Harden image ingress: sniff the container, refuse polyglots and metadata, and bound the request (#16) #82
No files matched your search
@@ -51,6 +51,7 @@ jobs:
|
||||
npm run test:ui
|
||||
npm run test:photo
|
||||
npm run test:mobile-capture
|
||||
npm run test:ingress-capture
|
||||
npm run test:sleek
|
||||
npm run test:manual-fallback
|
||||
npm run test:provisional
|
||||
|
||||
@@ -272,20 +272,23 @@ function photoFirstBody(mode,error=''){
|
||||
if(mode==='pick')return `${visionStatusHtml()}<div class="scan-hero"><img src="${appPath('assets/timmy.svg')}" alt="Timmy"><h3>One photo. Two useful suggestions.</h3><p>Timmy can suggest the visible Bristol form and color. A camera cannot know urgency, pain, symptoms, or a diagnosis.</p></div>${error?`<p class="capture-status" role="status">${esc(error)}</p>`:''}<div class="capture-choice-grid"><label class="photo-capture" for="camera-photo"><b>📷</b><strong>Take photo</strong><span>Use the rear camera</span><input id="camera-photo" type="file" accept="image/*" capture="environment"></label><label class="photo-capture" for="gallery-photo"><b>▧</b><strong>Choose from gallery</strong><span>JPEG, PNG, or WebP</span><input id="gallery-photo" type="file" accept="image/*"></label></div><button class="btn btn-ghost btn-wide section" id="manual-from-scan">Continue without AI</button>`;
|
||||
if(mode==='ready'){const readiness=visionReadiness(visionStatus);if(!readiness.ready)return `${visionStatusHtml()}<img class="photo-preview scan-preview" src="${form.photoDataUrl}" alt="Photo kept on this device"><p class="quality-note">${esc(photoHint)}</p><p class="fine">${esc(readiness.message)} Nothing is uploaded, and this photo stays on this device until you save the entry.</p><button class="btn btn-primary btn-wide" id="manual-from-scan">${esc(readiness.manualLabel)}</button><button class="btn btn-ghost btn-wide section" id="retake-photo">Use another photo</button>`;const processingCopy=visionStatus?.profile==='selfhost'?'Timmy’s server does not save it. The compressed copy stays on Timmy’s self-hosted model server.':'Timmy’s server does not save it. Your configured AI provider processes it under that provider’s terms.';return `${visionStatusHtml()}<img class="photo-preview scan-preview" src="${form.photoDataUrl}" alt="Photo awaiting AI analysis"><p class="quality-note">${esc(photoHint)}</p><div class="consent-card"><label class="check"><input id="ai-consent" type="checkbox"><span><strong>Send this compressed copy for one-time AI analysis.</strong><br>${processingCopy}</span></label></div><button class="btn btn-primary btn-wide" id="analyze-photo" disabled>Analyze visible form + color</button><button class="btn btn-ghost btn-wide section" id="retake-photo">Use another photo</button>`;}
|
||||
if(mode==='analyzing')return `<div class="analyzing"><img src="${appPath('assets/timmy.svg')}" alt="Timmy"><div class="spinner" aria-hidden="true"></div><h3>Timmy is looking at form and color…</h3><p>Not symptoms. Not disease. Not whether Taco Bell was a strategic error.</p></div>`;
|
||||
// An unreadable file is its own state, not a failed analysis: nothing was sent, nothing was
|
||||
// attempted, and saying "couldn't analyze that safely" here implied an upload that never happened.
|
||||
if(mode==='unreadable')return `<div class="scan-result needs-input" data-capture-error="unreadable"><b>⚠</b><h3>That file isn’t a photo Timmy can read.</h3><p>Nothing was uploaded. Choose a JPEG, PNG, or WebP photo, take a new one, or fill it out manually.</p></div><button class="btn btn-primary btn-wide" id="manual-from-scan">Fill it out manually</button><button class="btn btn-ghost btn-wide section" id="retake-photo">Choose another photo</button>`;
|
||||
if(mode==='error')return `<div class="scan-result needs-input"><b>↻</b><h3>Timmy couldn’t analyze that safely.</h3><p>${esc(error||'Continue manually or try a clearer photo.')}</p></div><button class="btn btn-primary btn-wide" id="manual-from-scan">Fill it out manually</button><button class="btn btn-ghost btn-wide section" id="retake-photo">Try another photo</button>`;
|
||||
if(suggestionState.status==='pending'){const copy=provenanceCopy(suggestionState,form);return `<div class="scan-result provisional" data-provenance="${copy.status}" aria-label="${esc(copy.aria)}"><span class="ai-badge">${esc(copy.label)} · ${Math.round(suggestionState.suggestion.confidence*100)}% CONFIDENCE</span><div class="suggestion-pair"><div><small>BRISTOL FORM</small><strong>Type ${suggestionState.suggestion.bristolType}</strong></div><div><small>VISIBLE COLOR</small><strong>${esc(suggestionState.suggestion.color)}</strong></div></div><p>${esc(aiSuggestion.observations||'Visual match found.')}</p><p class="provenance-note" role="status">${esc(copy.detail)}</p><p class="fine">${esc(aiSuggestion.warning)}</p></div><button class="btn btn-primary btn-wide" id="use-suggestion">Use these suggestions</button><button class="btn btn-ghost btn-wide section" id="manual-from-scan">Review everything manually</button>`;}
|
||||
return `<div class="scan-result needs-input"><b>?</b><h3>No confident match.</h3><p>${esc(aiSuggestion?.reason||'The image was too uncertain to prefill safely.')}</p></div><button class="btn btn-primary btn-wide" id="manual-from-scan">Choose the form yourself</button><button class="btn btn-ghost btn-wide section" id="retake-photo">Try another photo</button>`;
|
||||
}
|
||||
function showPhotoFirst(mode='pick',error=''){
|
||||
photoFirstMode=mode;
|
||||
document.querySelector('.sheet-backdrop')?.remove();const wrap=document.createElement('div');wrap.className='sheet-backdrop';wrap.innerHTML=`<section class="sheet scan-sheet" role="dialog" aria-modal="true" aria-labelledby="scan-title"><div class="sheet-handle"></div><div class="sheet-header"><div><span class="eyebrow">Photo-first log</span><h2 id="scan-title">${mode==='result'?'Review Timmy’s suggestion':mode==='analyzing'?'Analyzing privately':'Start with the camera'}</h2></div><button class="icon-btn" id="close-sheet" aria-label="Close">×</button></div>${photoFirstBody(mode,error)}</section>`;document.body.append(wrap);document.querySelector('#close-sheet').onclick=()=>wrap.remove();wrap.onclick=e=>{if(e.target===wrap)wrap.remove()};
|
||||
document.querySelector('.sheet-backdrop')?.remove();const wrap=document.createElement('div');wrap.className='sheet-backdrop';wrap.innerHTML=`<section class="sheet scan-sheet" role="dialog" aria-modal="true" aria-labelledby="scan-title"><div class="sheet-handle"></div><div class="sheet-header"><div><span class="eyebrow">Photo-first log</span><h2 id="scan-title">${mode==='result'?'Review Timmy’s suggestion':mode==='analyzing'?'Analyzing privately':mode==='unreadable'?'That photo could not be read':'Start with the camera'}</h2></div><button class="icon-btn" id="close-sheet" aria-label="Close">×</button></div>${photoFirstBody(mode,error)}</section>`;document.body.append(wrap);document.querySelector('#close-sheet').onclick=()=>wrap.remove();wrap.onclick=e=>{if(e.target===wrap)wrap.remove()};
|
||||
document.querySelectorAll('#camera-photo,#gallery-photo').forEach(file=>{file.onchange=handleAiPhoto;file.addEventListener('cancel',()=>showPhotoFirst('pick','Camera or photo picker closed. If permission was denied, allow camera access in browser settings, choose from the gallery, or continue without AI.'))});
|
||||
const consent=document.querySelector('#ai-consent'),analyze=document.querySelector('#analyze-photo');if(consent&&analyze)consent.onchange=()=>analyze.disabled=!consent.checked||!visionReadiness(visionStatus).ready;if(analyze)analyze.onclick=runAiAnalysis;
|
||||
document.querySelector('#retake-photo')?.addEventListener('click',()=>{form.photoDataUrl='';photoHint='';resetSuggestion();showPhotoFirst('pick')});
|
||||
document.querySelector('#manual-from-scan')?.addEventListener('click',()=>{resetSuggestion();showLogStep(1)});
|
||||
document.querySelector('#use-suggestion')?.addEventListener('click',()=>{suggestionState=confirmSuggestion(suggestionState);form=mergeVisualSuggestion(form,aiSuggestion);form.typeChosen=true;showLogStep(1)});
|
||||
}
|
||||
async function handleAiPhoto(e){const file=e.target.files[0];if(!file)return;try{const result=await compressPhoto(file);form.photoDataUrl=result.dataUrl;photoHint=photoQualityMessage(result);showPhotoFirst('ready')}catch{showPhotoFirst('error','That image could not be read. Try another photo.')}}
|
||||
async function handleAiPhoto(e){const file=e.target.files[0];if(!file)return;try{const result=await compressPhoto(file);form.photoDataUrl=result.dataUrl;photoHint=photoQualityMessage(result);showPhotoFirst('ready')}catch{form.photoDataUrl='';photoHint='';showPhotoFirst('unreadable')}}
|
||||
async function runAiAnalysis(){showPhotoFirst('analyzing');try{const response=await fetch(appPath('api/analyze'),{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({imageDataUrl:form.photoDataUrl,consent:true})});const data=await response.json();if(!response.ok)throw new Error(data.error||'AI analysis is unavailable.');aiSuggestion=data;suggestionState=createSuggestionState(data);showPhotoFirst('result')}catch(error){showPhotoFirst('error',error.message||'AI analysis is unavailable. Continue manually.')}}
|
||||
|
||||
function openLogger(){beginDraft();photoHint='';resetSuggestion();showLogStep(1)}
|
||||
|
||||
+3
-2
@@ -4,10 +4,11 @@
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js",
|
||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js tests/image-ingress.test.js tests/rate-limit.test.js tests/http-body.test.js tests/analyze-ingress.acceptance.test.js",
|
||||
"test:ui": "node tests/ui.acceptance.mjs",
|
||||
"test:photo": "node tests/photo-first.acceptance.mjs",
|
||||
"test:mobile-capture": "node tests/mobile-capture.acceptance.mjs",
|
||||
"test:ingress-capture": "node tests/ingress-capture.acceptance.mjs",
|
||||
"test:sleek": "node tests/sleek-chat.acceptance.mjs",
|
||||
"test:manual-fallback": "node tests/manual-fallback.acceptance.mjs",
|
||||
"test:provisional": "node tests/provisional-suggestion.acceptance.mjs",
|
||||
@@ -17,7 +18,7 @@
|
||||
"test:pattern-summary": "node tests/pattern-summary.acceptance.mjs",
|
||||
"test:grounded-chat": "node tests/grounded-chat.acceptance.mjs",
|
||||
"test:photo-lifecycle": "node tests/photo-lifecycle.acceptance.mjs",
|
||||
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py && node --check scripts/record_worker_benchmark.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh",
|
||||
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/image-ingress.js && node --check src/rate-limit.js && node --check src/http-body.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py && node --check scripts/record_worker_benchmark.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh && node --check tests/ingress-capture.acceptance.mjs && node --check tests/analyze-ingress.acceptance.test.js",
|
||||
"check:diff": "bash scripts/check_diff.sh",
|
||||
"start": "node server.mjs",
|
||||
"test:worker-readiness": "node tests/worker-readiness.acceptance.mjs"
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { chromium } from 'playwright';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
// Issue #16, as a user experiences it: a file that only claims to be a photo never becomes an upload,
|
||||
// never reaches the model worker, and still leaves manual logging one tap away. The second half of
|
||||
// this suite asks the real running server to refuse the same bytes, so the browser copy and the
|
||||
// server boundary are proven in the same run.
|
||||
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 390, height: 844 },
|
||||
deviceScaleFactor: 2,
|
||||
serviceWorkers: 'block',
|
||||
});
|
||||
const page = await context.newPage();
|
||||
let analysisRequests = 0;
|
||||
const analyzePayloads = [];
|
||||
await page.route('**/api/vision-status', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ enabled: true, profile: 'selfhost', providerReady: true, modelSeen: true, model: 'synthetic-test-model' }),
|
||||
}));
|
||||
await page.route('**/api/analyze', route => {
|
||||
analysisRequests += 1;
|
||||
analyzePayloads.push(route.request().postData() || '');
|
||||
return route.continue();
|
||||
});
|
||||
|
||||
await page.goto('http://127.0.0.1:4173', { waitUntil: 'networkidle' });
|
||||
await page.locator('[data-scan]').click();
|
||||
|
||||
const spoofed = Buffer.from('<html><script>alert(1)</script></html>', 'utf8');
|
||||
await page.locator('#gallery-photo').setInputFiles({ name: 'photo.jpg', mimeType: 'image/jpeg', buffer: spoofed });
|
||||
|
||||
const refusal = page.locator('[data-capture-error="unreadable"]');
|
||||
await refusal.waitFor();
|
||||
const copy = await refusal.innerText();
|
||||
assert.match(copy, /isn.t a photo Timmy can read/i, 'the refusal names what is wrong with the file');
|
||||
assert.match(copy, /Nothing was uploaded/i, 'the refusal states that nothing left the device');
|
||||
assert.equal(await page.getByText(/couldn.t analyze that safely/i).count(), 0,
|
||||
'an unreadable file must not be reported as a failed analysis');
|
||||
assert.equal(analysisRequests, 0, 'an unreadable file must never be posted to /api/analyze');
|
||||
assert.equal(await page.locator('#manual-from-scan').isVisible(), true, 'manual entry stays one tap away');
|
||||
assert.equal(await page.locator('#retake-photo').isVisible(), true, 'the picker stays reachable');
|
||||
|
||||
// The same bytes, offered to the running server directly, are refused before any provider work.
|
||||
const serverResponse = await page.evaluate(async payload => {
|
||||
const response = await fetch(`${document.querySelector('meta[name="timmy-base-path"]')?.content.replace(/\/$/, '') || ''}/api/analyze`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
return { status: response.status, body: await response.json() };
|
||||
}, { consent: true, imageDataUrl: `data:image/jpeg;base64,${spoofed.toString('base64')}` });
|
||||
assert.equal(serverResponse.status, 400, `the server must refuse non-image bytes, saw ${serverResponse.status}`);
|
||||
assert.match(serverResponse.body.error, /not a JPEG, PNG, or WebP photo/i);
|
||||
// Only this deliberate probe reached the endpoint; the app itself never posted the unreadable file.
|
||||
assert.equal(analysisRequests, 1, 'only the deliberate server probe may reach /api/analyze so far');
|
||||
assert.match(analyzePayloads[0], /"consent":true/);
|
||||
|
||||
// A real (synthetic) photo still reaches the consent step, so the gate is a boundary and not a wall.
|
||||
await page.locator('#retake-photo').click();
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||||
await page.locator('#ai-consent').waitFor();
|
||||
assert.equal(await page.locator('[data-capture-error="unreadable"]').count(), 0);
|
||||
assert.equal(analysisRequests, 1, 'a photo that is only selected is still not uploaded before consent');
|
||||
|
||||
await context.close();
|
||||
await browser.close();
|
||||
console.log('PASS unreadable photo bytes are refused on device and at the server, with manual entry intact');
|
||||
@@ -47,8 +47,11 @@ for (const viewport of viewports) {
|
||||
mimeType: 'image/jpeg',
|
||||
buffer: Buffer.from('not an image'),
|
||||
});
|
||||
assert.match(await page.locator('.scan-result').innerText(), /could not be read/i);
|
||||
await page.getByText('Try another photo', { exact: true }).click();
|
||||
assert.match(await page.locator('.scan-result').innerText(), /isn.t a photo Timmy can read/i);
|
||||
// Issue #16: the refusal is its own state and states plainly that nothing left the device, instead
|
||||
// of borrowing the "couldn't analyze that safely" copy from a failed upload.
|
||||
assert.match(await page.locator('.scan-result').innerText(), /Nothing was uploaded/i);
|
||||
await page.getByText('Choose another photo', { exact: true }).click();
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||||
assert.equal(await page.locator('#analyze-photo').isDisabled(), true);
|
||||
assert.equal(await page.locator('#retake-photo').isVisible(), true);
|
||||
|
||||
Reference in new issue
Block a user