Compare commits
13
Commits
daily-2026-10-30
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
664a909957 | ||
|
|
ef83ad93bb | ||
|
|
48979bf744 | ||
|
|
e08b537866 | ||
|
|
3e2253a052 | ||
|
|
9067ce5306 | ||
|
|
2019d515c0 | ||
|
|
c31d0a510f | ||
|
|
8ce5bc174b | ||
|
|
bc64055196 | ||
|
|
2b773ba58c | ||
|
|
9b72872466 | ||
|
|
ba9afe0a1d |
No files matched your search
@@ -56,6 +56,7 @@ jobs:
|
||||
npm run test:ui
|
||||
npm run test:photo
|
||||
npm run test:mobile-capture
|
||||
npm run test:ingress-capture
|
||||
npm run test:sleek
|
||||
npm run test:manual-fallback
|
||||
npm run test:provisional
|
||||
|
||||
@@ -7,6 +7,7 @@ A working mobile-first bowel diary with an optional **photo-first AI assist**:
|
||||
- [Product boundary and release authority decision](docs/PRODUCT-DECISIONS.md)
|
||||
- [Self-hosted stool-vision decision](research/SELF-HOSTED-STOOL-VISION.md)
|
||||
- [Production topology, secrets, backup, and rollback runbook](docs/PRODUCTION-RUNBOOK.md)
|
||||
- [Privacy-safe model health counters (operator view)](docs/MODEL-METRICS.md)
|
||||
|
||||
## Build a gated review release
|
||||
|
||||
|
||||
@@ -313,7 +313,7 @@ function withdrawTrainingContribution(contributionId){
|
||||
}
|
||||
|
||||
function privacy(){
|
||||
shell(`<main><div class="page-title"><span class="eyebrow">Private by design</span><h1>Your poop. Your phone.</h1><p>This prototype has no account, analytics, ad tracker, or server database.</p></div><section class="card privacy-list"><div class="privacy-item"><b>⌂</b><div><h3>Stored locally</h3><p>Saved entries and optional photos live in this browser’s local storage. You can remove one photo without deleting its confirmed log.</p></div></div><div class="privacy-item"><b>⇩</b><div><h3>Portable</h3><p>Export a readable JSON file. Import it in another copy of Timmy.</p></div></div><div class="privacy-item"><b>◎</b><div><h3>AI only when you ask</h3><p>Manual logging never uploads. Analyze a photo sends one compressed copy after consent. Hermes chat may receive up to 20 confirmed text-only entries after you connect; photos and backend credentials never enter chat.</p></div></div></section><section class="section card"><h2>Data controls</h2><div class="row"><button class="btn btn-primary" id="export">Export JSON</button><label class="btn btn-secondary" for="import">Import JSON</label><input class="hidden" type="file" id="import" accept="application/json"></div><p class="fine section">Exports can contain sensitive health notes and photos. Store them somewhere you trust.</p></section>${portabilityNoticeBlock()}${trainingConsentSection()}<section class="section card source-list"><h2>Health sources</h2><p class="fine">The Bristol groupings and urgent-symptom copy are grounded in public clinical guidance.</p><p><a target="_blank" rel="noreferrer" href="https://www.continence.org.au/about-incontinence/bowel-incontinence/bristol-stool-chart/">Continence Health Australia</a></p><p><a target="_blank" rel="noreferrer" href="https://www.nhs.uk/conditions/bleeding-from-the-bottom-rectal-bleeding/">NHS rectal bleeding guidance</a></p><p><a target="_blank" rel="noreferrer" href="https://www.niddk.nih.gov/health-information/digestive-diseases/constipation/symptoms-causes">NIDDK constipation guidance</a></p></section><section class="section card danger-zone"><h2>Delete everything</h2><p class="fine">Permanently removes Timmy’s local ledger and every saved photo from this browser.</p><button class="btn btn-danger" id="delete-all">Delete all local data</button></section></main>`);
|
||||
shell(`<main><div class="page-title"><span class="eyebrow">Private by design</span><h1>Your poop. Your phone.</h1><p>This prototype has no account, analytics, ad tracker, or server database.</p></div><section class="card privacy-list"><div class="privacy-item"><b>⌂</b><div><h3>Stored locally</h3><p>Saved entries and optional photos live in this browser’s local storage. You can remove one photo without deleting its confirmed log.</p></div></div><div class="privacy-item"><b>⇩</b><div><h3>Portable</h3><p>Export a readable JSON file. Import it in another copy of Timmy.</p></div></div><div class="privacy-item"><b>◎</b><div><h3>AI only when you ask</h3><p>Manual logging never uploads. Analyze a photo sends one compressed copy after consent. Hermes chat may receive up to 20 confirmed text-only entries after you connect; photos and backend credentials never enter chat.</p></div></div><div class="privacy-item" data-health-disclosure><b>♥</b><div><h3>Health counters</h3><p>Timmy’s server keeps in-memory counters of how analysis requests went — allowed, refused, failed, and how long they took — so an outage can be seen without opening anyone’s journal. They never include your notes, photos, image fingerprints, or identity, and they reset when the server restarts.</p></div></div></section><section class="section card"><h2>Data controls</h2><div class="row"><button class="btn btn-primary" id="export">Export JSON</button><label class="btn btn-secondary" for="import">Import JSON</label><input class="hidden" type="file" id="import" accept="application/json"></div><p class="fine section">Exports can contain sensitive health notes and photos. Store them somewhere you trust.</p></section>${portabilityNoticeBlock()}${trainingConsentSection()}<section class="section card source-list"><h2>Health sources</h2><p class="fine">The Bristol groupings and urgent-symptom copy are grounded in public clinical guidance.</p><p><a target="_blank" rel="noreferrer" href="https://www.continence.org.au/about-incontinence/bowel-incontinence/bristol-stool-chart/">Continence Health Australia</a></p><p><a target="_blank" rel="noreferrer" href="https://www.nhs.uk/conditions/bleeding-from-the-bottom-rectal-bleeding/">NHS rectal bleeding guidance</a></p><p><a target="_blank" rel="noreferrer" href="https://www.niddk.nih.gov/health-information/digestive-diseases/constipation/symptoms-causes">NIDDK constipation guidance</a></p></section><section class="section card danger-zone"><h2>Delete everything</h2><p class="fine">Permanently removes Timmy’s local ledger and every saved photo from this browser.</p><button class="btn btn-danger" id="delete-all">Delete all local data</button></section></main>`);
|
||||
document.querySelector('#export').onclick=exportData;document.querySelector('#import').onchange=importData;document.querySelector('#delete-all').onclick=deleteData;
|
||||
}
|
||||
function exportData(){const blob=new Blob([exportLedgerBackup({entries,contributions})],{type:'application/json'}),a=document.createElement('a');a.href=URL.createObjectURL(blob);a.download=`timmy-backup-v${LEDGER_EXPORT_VERSION}.json`;a.click();URL.revokeObjectURL(a.href);portabilityNotice={kind:'ok',version:LEDGER_EXPORT_VERSION,text:`Backup written as version ${LEDGER_EXPORT_VERSION}: ${entries.length} log${entries.length===1?'':'s'} and ${contributions.length} training receipt${contributions.length===1?'':'s'}.`};render();toast('Backup exported');}
|
||||
@@ -340,20 +340,23 @@ function photoFirstBody(mode,error=''){
|
||||
if(mode==='pick')return `${visionStatusHtml()}<div class="scan-hero"><img src="${appPath('assets/timmy.svg')}" alt="Timmy"><h3>One photo. Two useful suggestions.</h3><p>Timmy can suggest the visible Bristol form and color. A camera cannot know urgency, pain, symptoms, or a diagnosis.</p></div>${error?`<p class="capture-status" role="status">${esc(error)}</p>`:''}<div class="capture-choice-grid"><label class="photo-capture" for="camera-photo"><b>📷</b><strong>Take photo</strong><span>Use the rear camera</span><input id="camera-photo" type="file" accept="image/*" capture="environment"></label><label class="photo-capture" for="gallery-photo"><b>▧</b><strong>Choose from gallery</strong><span>JPEG, PNG, or WebP</span><input id="gallery-photo" type="file" accept="image/*"></label></div><button class="btn btn-ghost btn-wide section" id="manual-from-scan">Continue without AI</button>`;
|
||||
if(mode==='ready'){const readiness=visionReadiness(visionStatus);if(!readiness.ready)return `${visionStatusHtml()}<img class="photo-preview scan-preview" src="${form.photoDataUrl}" alt="Photo kept on this device"><p class="quality-note">${esc(photoHint)}</p><p class="fine">${esc(readiness.message)} Nothing is uploaded, and this photo stays on this device until you save the entry.</p><button class="btn btn-primary btn-wide" id="manual-from-scan">${esc(readiness.manualLabel)}</button><button class="btn btn-ghost btn-wide section" id="retake-photo">Use another photo</button>`;const processingCopy=visionStatus?.profile==='selfhost'?'Timmy’s server does not save it. The compressed copy stays on Timmy’s self-hosted model server.':'Timmy’s server does not save it. Your configured AI provider processes it under that provider’s terms.';return `${visionStatusHtml()}<img class="photo-preview scan-preview" src="${form.photoDataUrl}" alt="Photo awaiting AI analysis"><p class="quality-note">${esc(photoHint)}</p><div class="consent-card"><label class="check"><input id="ai-consent" type="checkbox"><span><strong>Send this compressed copy for one-time AI analysis.</strong><br>${processingCopy}</span></label></div><button class="btn btn-primary btn-wide" id="analyze-photo" disabled>Analyze visible form + color</button><button class="btn btn-ghost btn-wide section" id="retake-photo">Use another photo</button>`;}
|
||||
if(mode==='analyzing')return `<div class="analyzing"><img src="${appPath('assets/timmy.svg')}" alt="Timmy"><div class="spinner" aria-hidden="true"></div><h3>Timmy is looking at form and color…</h3><p>Not symptoms. Not disease. Not whether Taco Bell was a strategic error.</p></div>`;
|
||||
// An unreadable file is its own state, not a failed analysis: nothing was sent, nothing was
|
||||
// attempted, and saying "couldn't analyze that safely" here implied an upload that never happened.
|
||||
if(mode==='unreadable')return `<div class="scan-result needs-input" data-capture-error="unreadable"><b>⚠</b><h3>That file isn’t a photo Timmy can read.</h3><p>Nothing was uploaded. Choose a JPEG, PNG, or WebP photo, take a new one, or fill it out manually.</p></div><button class="btn btn-primary btn-wide" id="manual-from-scan">Fill it out manually</button><button class="btn btn-ghost btn-wide section" id="retake-photo">Choose another photo</button>`;
|
||||
if(mode==='error')return `<div class="scan-result needs-input"><b>↻</b><h3>Timmy couldn’t analyze that safely.</h3><p>${esc(error||'Continue manually or try a clearer photo.')}</p></div><button class="btn btn-primary btn-wide" id="manual-from-scan">Fill it out manually</button><button class="btn btn-ghost btn-wide section" id="retake-photo">Try another photo</button>`;
|
||||
if(suggestionState.status==='pending'){const copy=provenanceCopy(suggestionState,form);return `<div class="scan-result provisional" data-provenance="${copy.status}" aria-label="${esc(copy.aria)}"><span class="ai-badge">${esc(copy.label)} · ${Math.round(suggestionState.suggestion.confidence*100)}% CONFIDENCE</span><div class="suggestion-pair"><div><small>BRISTOL FORM</small><strong>Type ${suggestionState.suggestion.bristolType}</strong></div><div><small>VISIBLE COLOR</small><strong>${esc(suggestionState.suggestion.color)}</strong></div></div><p>${esc(aiSuggestion.observations||'Visual match found.')}</p><p class="provenance-note" role="status">${esc(copy.detail)}</p><p class="fine">${esc(aiSuggestion.warning)}</p></div><button class="btn btn-primary btn-wide" id="use-suggestion">Use these suggestions</button><button class="btn btn-ghost btn-wide section" id="manual-from-scan">Review everything manually</button>`;}
|
||||
return `<div class="scan-result needs-input"><b>?</b><h3>No confident match.</h3><p>${esc(aiSuggestion?.reason||'The image was too uncertain to prefill safely.')}</p></div><button class="btn btn-primary btn-wide" id="manual-from-scan">Choose the form yourself</button><button class="btn btn-ghost btn-wide section" id="retake-photo">Try another photo</button>`;
|
||||
}
|
||||
function showPhotoFirst(mode='pick',error=''){
|
||||
photoFirstMode=mode;
|
||||
document.querySelector('.sheet-backdrop')?.remove();const wrap=document.createElement('div');wrap.className='sheet-backdrop';wrap.innerHTML=`<section class="sheet scan-sheet" role="dialog" aria-modal="true" aria-labelledby="scan-title"><div class="sheet-handle"></div><div class="sheet-header"><div><span class="eyebrow">Photo-first log</span><h2 id="scan-title">${mode==='result'?'Review Timmy’s suggestion':mode==='analyzing'?'Analyzing privately':'Start with the camera'}</h2></div><button class="icon-btn" id="close-sheet" aria-label="Close">×</button></div>${photoFirstBody(mode,error)}</section>`;document.body.append(wrap);document.querySelector('#close-sheet').onclick=()=>wrap.remove();wrap.onclick=e=>{if(e.target===wrap)wrap.remove()};
|
||||
document.querySelector('.sheet-backdrop')?.remove();const wrap=document.createElement('div');wrap.className='sheet-backdrop';wrap.innerHTML=`<section class="sheet scan-sheet" role="dialog" aria-modal="true" aria-labelledby="scan-title"><div class="sheet-handle"></div><div class="sheet-header"><div><span class="eyebrow">Photo-first log</span><h2 id="scan-title">${mode==='result'?'Review Timmy’s suggestion':mode==='analyzing'?'Analyzing privately':mode==='unreadable'?'That photo could not be read':'Start with the camera'}</h2></div><button class="icon-btn" id="close-sheet" aria-label="Close">×</button></div>${photoFirstBody(mode,error)}</section>`;document.body.append(wrap);document.querySelector('#close-sheet').onclick=()=>wrap.remove();wrap.onclick=e=>{if(e.target===wrap)wrap.remove()};
|
||||
document.querySelectorAll('#camera-photo,#gallery-photo').forEach(file=>{file.onchange=handleAiPhoto;file.addEventListener('cancel',()=>showPhotoFirst('pick','Camera or photo picker closed. If permission was denied, allow camera access in browser settings, choose from the gallery, or continue without AI.'))});
|
||||
const consent=document.querySelector('#ai-consent'),analyze=document.querySelector('#analyze-photo');if(consent&&analyze)consent.onchange=()=>analyze.disabled=!consent.checked||!visionReadiness(visionStatus).ready;if(analyze)analyze.onclick=runAiAnalysis;
|
||||
document.querySelector('#retake-photo')?.addEventListener('click',()=>{form.photoDataUrl='';photoHint='';resetSuggestion();showPhotoFirst('pick')});
|
||||
document.querySelector('#manual-from-scan')?.addEventListener('click',()=>{resetSuggestion();showLogStep(1)});
|
||||
document.querySelector('#use-suggestion')?.addEventListener('click',()=>{suggestionState=confirmSuggestion(suggestionState);form=mergeVisualSuggestion(form,aiSuggestion);form.typeChosen=true;showLogStep(1)});
|
||||
}
|
||||
async function handleAiPhoto(e){const file=e.target.files[0];if(!file)return;try{const result=await compressPhoto(file);form.photoDataUrl=result.dataUrl;photoHint=photoQualityMessage(result);showPhotoFirst('ready')}catch{showPhotoFirst('error','That image could not be read. Try another photo.')}}
|
||||
async function handleAiPhoto(e){const file=e.target.files[0];if(!file)return;try{const result=await compressPhoto(file);form.photoDataUrl=result.dataUrl;photoHint=photoQualityMessage(result);showPhotoFirst('ready')}catch{form.photoDataUrl='';photoHint='';showPhotoFirst('unreadable')}}
|
||||
async function runAiAnalysis(){showPhotoFirst('analyzing');try{const response=await fetch(appPath('api/analyze'),{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({imageDataUrl:form.photoDataUrl,consent:true})});const data=await response.json();if(!response.ok)throw new Error(data.error||'AI analysis is unavailable.');aiSuggestion=data;suggestionState=createSuggestionState(data);showPhotoFirst('result')}catch(error){showPhotoFirst('error',error.message||'AI analysis is unavailable. Continue manually.')}}
|
||||
|
||||
function openLogger(){beginDraft();photoHint='';resetSuggestion();showLogStep(1)}
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 486 KiB |
@@ -0,0 +1,111 @@
|
||||
# Privacy-safe model health counters
|
||||
|
||||
Issue [#19](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/19).
|
||||
This is the operator surface for "is the worker healthy?", and it is deliberately the smallest one
|
||||
that answers the question. The journal stays the only place a person's bowel data lives.
|
||||
|
||||
## Endpoint
|
||||
|
||||
```
|
||||
GET /api/metrics
|
||||
Authorization: Bearer $TIMMY_METRICS_TOKEN
|
||||
```
|
||||
|
||||
- `TIMMY_METRICS_TOKEN` unset → **404**. A default deployment has no metrics route at all; nothing is
|
||||
exposed by forgetting to configure something.
|
||||
- Token set, header missing or wrong → **403**, with a body that contains no counters.
|
||||
- Token set and matched → **200** with the snapshot below.
|
||||
|
||||
The snapshot is read from process memory. It survives no restart, is written to no log, and is sent
|
||||
to no third party.
|
||||
|
||||
## What is in the snapshot
|
||||
|
||||
```json
|
||||
{
|
||||
"schema": 1,
|
||||
"generatedAt": "2026-11-01T10:00:00.000Z",
|
||||
"app": { "uptimeMs": 0, "analyzeRequests": 0, "refusedBeforeModel": 0 },
|
||||
"queue": { "inFlight": 0, "limit": 4, "atCapacity": false, "accepted": 0 },
|
||||
"model": {
|
||||
"outcomes": { "suggestion": 0, "abstained": 0, "failure": 0 },
|
||||
"failureClasses": { "ingress_refused": 0, "rate_limited": 0, "disabled": 0,
|
||||
"provider_unreachable": 0, "provider_rejected": 0,
|
||||
"schema_invalid": 0, "timeout": 0, "unexpected": 0 },
|
||||
"abstentionRate": null,
|
||||
"readiness": { "phase": "checking", "ready": false, "coldLoadMs": null },
|
||||
"latency": { "count": 0, "meanMs": null, "maxMs": 0,
|
||||
"bucketsMs": { "1000": 0, "2000": 0, "5000": 0, "15000": 0, "60000": 0 } }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
The three blocks are the three boundaries, and each counts only what it owns:
|
||||
|
||||
| Block | Owns | Does not own |
|
||||
|---|---|---|
|
||||
| `app` | Every `POST /api/analyze` the process saw, and the ones refused before any model work (bad bytes, per-device budget, analysis disabled). | Queue occupancy or model answers. |
|
||||
| `queue` | Slots handed out and released, and the configured concurrency ceiling. | Whether the request succeeded. |
|
||||
| `model` | Answers (`suggestion` / `abstained`), classified failures, readiness phase, and model-work latency. | Anything decided before a slot was taken. |
|
||||
|
||||
`abstentionRate` is `abstained / (suggestion + abstained)`; it is `null` until the model has answered
|
||||
at least once. A failed request is **not** an abstention — an outage must not look like a cautious
|
||||
model.
|
||||
|
||||
`latency.bucketsMs` is cumulative: `"5000"` counts every sample at or below 5 s. `coldLoadMs` is the
|
||||
interval from process start to the first observation in which the pinned model was resident; a worker
|
||||
that flaps back to `offline` keeps the original figure.
|
||||
|
||||
`readiness.phase` uses the same vocabulary the browser uses for the AI affordance (`checking`,
|
||||
`disabled`, `offline`, `loading`, `ready`), and the value is produced by the same
|
||||
`visionReadiness()` function the app renders, so a metric and a screen cannot disagree.
|
||||
|
||||
## What can never appear in it
|
||||
|
||||
The only strings this module stores are members of three frozen lists — outcomes, failure classes,
|
||||
and readiness phases — plus a fixed latency-bucket edge set. Anything else is bucketed, not copied:
|
||||
|
||||
- an unrecognised outcome becomes `failure`;
|
||||
- an unrecognised failure class becomes `unexpected`;
|
||||
- an unrecognised readiness phase becomes `checking`.
|
||||
|
||||
Therefore no note text, no image bytes, no image hash or fingerprint, no subject key, no caller
|
||||
address, and no session or cookie value can reach a label. `tests/model-metrics.test.js` asserts this
|
||||
by feeding hostile strings into every label position, and
|
||||
`tests/metrics-endpoint.acceptance.test.js` asserts it through the shipping server.
|
||||
|
||||
## Reading it
|
||||
|
||||
```bash
|
||||
curl -sS -H "Authorization: Bearer $TIMMY_METRICS_TOKEN" http://127.0.0.1:4173/api/metrics | jq .
|
||||
```
|
||||
|
||||
For a quick look that does not need `jq`, the report renders the same snapshot as three labelled blocks:
|
||||
|
||||
```bash
|
||||
TIMMY_METRICS_TOKEN=... node scripts/model_metrics_report.mjs --url http://127.0.0.1:4173
|
||||
```
|
||||
|
||||
Exit codes are the point of the command as much as the numbers: `0` means a snapshot was rendered,
|
||||
`1` means the server answered non-200 (404 says the route is disabled because no token is configured,
|
||||
403 says the supplied token was rejected), `2` means the command itself was called wrongly or the
|
||||
server could not be reached. The token is read from the environment or `--token` and is never echoed
|
||||
in the report or in an error line.
|
||||
|
||||
What the numbers mean to an operator:
|
||||
|
||||
| Signal | Reading |
|
||||
|---|---|
|
||||
| `readiness.phase: offline` | The worker is not answering. Manual logging is unaffected; do not restart the app for this. |
|
||||
| `readiness.phase: loading` | The worker answered but the pinned model is not resident. AI is withheld rather than offered and failed. |
|
||||
| `queue.atCapacity: true` | Every model slot is in use. Watch `latency.maxMs`; a growing figure with no completions is a stuck inference, not load. |
|
||||
| `failureClasses.provider_unreachable` climbing | Network or worker outage, not caller error. |
|
||||
| `failureClasses.schema_invalid` climbing | The worker is answering off-contract. Treat as a regression in the pinned model or prompt, not a user problem. |
|
||||
| `app.refusedBeforeModel` climbing with no model traffic | Abuse or a broken client; the model never saw it. |
|
||||
|
||||
## Known gap
|
||||
|
||||
`queue.inFlight` is the app's own admission depth for `POST /api/analyze`. The full bounded queue
|
||||
(concurrency limit enforcement, cancellation, overload fallback) is issue
|
||||
[#17](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/17); this
|
||||
snapshot reports the slot accounting that exists today and does not claim the rest.
|
||||
@@ -144,6 +144,26 @@ Each entry names the abuse case, the mitigation that is already committed or que
|
||||
- **Testable follow-up:** #39
|
||||
- **Evidence:** The production topology, secrets, backup, and rollback runbook, exercised against the deletion path so that purge reaches memory, temporary files, logs, and backups, with observability from #41 confirming no bytes reach the metrics surface.
|
||||
|
||||
## Ingress controls implemented for #16
|
||||
|
||||
`/api/analyze` is the only route that accepts image bytes, and every limit below is decided inside
|
||||
`server.mjs` before `analyzePhoto` runs, so a refusal never costs a provider call.
|
||||
|
||||
| Control | Where | Behaviour |
|
||||
|---|---|---|
|
||||
| Per-caller request budget | `src/rate-limit.js` | Sliding window, 12 requests per 60s by default (`TIMMY_ANALYZE_RATE_LIMIT`, `TIMMY_ANALYZE_RATE_WINDOW_MS`); refusal is `429` with `Retry-After`. The key is a salted SHA-256 of the socket address held in process memory only — no address, no identity, no bytes are stored. |
|
||||
| Body ceiling and deadline | `src/http-body.js` | 6 MB ceiling and a 15s deadline (`TIMMY_BODY_TIMEOUT_MS`); `413` and `408` respectively, and the half-read socket is dropped after the refusal is written. |
|
||||
| Container sniffing | `src/image-ingress.js` | The declared data-URL MIME is treated as a claim, never evidence: JPEG/PNG/WebP is decided from magic bytes, and a mismatch is refused. |
|
||||
| Polyglot / trailing payloads | `src/image-ingress.js` | The container is walked to its own end marker (PNG `IEND`, JPEG `EOI`, RIFF length); anything appended after the image is refused instead of forwarded. |
|
||||
| Embedded metadata | `src/image-ingress.js` | JPEG APP1/APP13/COM segments and PNG `eXIf`/`tEXt`/`zTXt`/`iTXt`/`tIME` chunks are refused. The app's own upload is a canvas re-encode (JPEG q0.7, longest edge 1200px) and carries none, so the accepted path is already metadata-free. |
|
||||
| Pixel limits | `src/image-ingress.js` | Dimensions are read from the header (PNG `IHDR`, JPEG SOF, WebP `VP8X`/`VP8`/`VP8L`) and refused above a 4096px edge or 16 megapixels, so a decode bomb never reaches the worker. |
|
||||
| Log hygiene | `server.mjs` | Refusal copy is fixed prose; no request body, image bytes, hash, or address is written to a log or returned in an error. |
|
||||
|
||||
Residual, and not claimed as solved here: the server refuses metadata-bearing or mismatched
|
||||
containers rather than transcoding them itself, so a client that cannot produce a clean re-encode
|
||||
must use the app's capture path. PNG chunk CRCs are not validated (structure is), and the request
|
||||
budget is per process, not per cluster.
|
||||
|
||||
## Accepted residual risks
|
||||
|
||||
- **Host compromise.** Data in a running process can be read by a sufficiently privileged attacker. This is accepted for the private beta and bounded by least privilege plus the purge procedure in #39; it is not claimed as eliminated.
|
||||
@@ -154,4 +174,5 @@ Each entry names the abuse case, the mitigation that is already committed or que
|
||||
|
||||
- This document is enforced by `tests/threat-model.test.js`, which asserts that the diagram covers every byte-bearing component, that each required abuse case is a numbered threat with an owner and a tracked follow-up ticket, and that the model keeps the non-diagnostic boundary and ships no real medical images.
|
||||
- The test runs in `npm test` and therefore inside the CI quality gate and the release build, so the threat model cannot silently drift from the code it describes.
|
||||
- The ingress controls above are enforced by `tests/image-ingress.test.js`, `tests/rate-limit.test.js`, `tests/http-body.test.js`, and `tests/analyze-ingress.acceptance.test.js`, which runs the shipping server against synthetic bytes and a counting provider stub; `tests/ingress-capture.acceptance.mjs` proves the same refusal in the running app. All of them run in `npm test` or the browser gate, so a regression fails the release instead of a promise.
|
||||
- Boundary and boundary-adjacent claims are reviewed by the clinical and privacy gate (#22, #23) before beta; this model does not move that gate, it feeds it.
|
||||
@@ -57,13 +57,14 @@ enforces that distinction and the app withholds the AI control until it passes.
|
||||
|
||||
## Secrets
|
||||
|
||||
Four secrets exist, and every one of them is injected at runtime:
|
||||
Five secrets exist, and every one of them is injected at runtime:
|
||||
|
||||
| Secret | Source | Scope |
|
||||
|---|---|---|
|
||||
| `TIMMY_DATASET_SALT` | mode-600 environment file on the host | Corpus subject keys and split assignment. Rotating it re-splits future ingests, so it is not rotated casually. |
|
||||
| Vision provider key or worker token | mode-600 environment file on the host | Only the app process reads it; the browser never receives it. |
|
||||
| Agent access code | mode-600 environment file on the host | Unlocks `/api/agent/unlock`; the cookie it mints is `HttpOnly` and same-origin. |
|
||||
| `TIMMY_METRICS_TOKEN` | mode-600 environment file on the host | Optional. Bearer token for the operator-only `/api/metrics` snapshot. When it is unset the route returns 404, so an unconfigured deployment exposes no counters at all. |
|
||||
| Gitea access token | local operator shell or secret store | Issue and PR automation only. It is never written into a release tree, an archive, or a log. |
|
||||
|
||||
Rules:
|
||||
|
||||
@@ -59,13 +59,15 @@ contacts a real model provider, never uses a real medical image, and never touch
|
||||
|
||||
## Current state (read the run, not this line)
|
||||
|
||||
The suite is expected to come back **BLOCKED** until the ingress hardening in
|
||||
[#16](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/16) lands: bytes
|
||||
are still trusted from the data-URL label, so an HTML file dressed as a JPEG reaches the worker and
|
||||
only fails afterwards on schema. That is the finding, and it is why the release gate treats a failing
|
||||
case as a block instead of a warning. Queue bounding is
|
||||
[#17](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/17) and is held
|
||||
here as P1.
|
||||
This section used to predict **BLOCKED** until the ingress hardening in
|
||||
[#16](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/16) landed. With
|
||||
that ingress in place the same command returns **GO** on this tree, because the format is decided from
|
||||
the bytes rather than the data-URL label: `ingress-polyglot-html` and `ingress-garbage-bytes` are now
|
||||
refused with `400` before the worker is called, and an oversized body is refused with a `413` the
|
||||
caller can read instead of a dropped connection. Latency for the load phase is measured against a
|
||||
second target with the request budget opened up, so those samples are analyses rather than `429`
|
||||
refusals. Queue bounding is still [#17](https://forge.alexanderwhitestone.com/git/stackchain/timmy-talking-turd/issues/17)
|
||||
and is held here as P1.
|
||||
|
||||
Deletion is governed by the contribution ledger, not by this HTTP boundary; the case stays
|
||||
`covered-elsewhere` against `tests/training-contribution.test.js` and blocks if that file disappears.
|
||||
|
||||
+5
-3
@@ -4,10 +4,11 @@
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js tests/training-contribution.test.js tests/portability.test.js tests/abstention-policy.test.js tests/release-security-gate.test.js",
|
||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/draft.test.js tests/suggestion.test.js tests/followup.test.js tests/vision-readiness.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/threat-model.test.js tests/vision-corpus.test.js tests/vision-boundary.test.js tests/worker-benchmark.test.js tests/worker-ready.test.js tests/worker-container.test.js tests/summary.test.js tests/escalation-regression.test.js tests/grounded-chat.test.js tests/photo-lifecycle.test.js tests/training-contribution.test.js tests/portability.test.js tests/abstention-policy.test.js tests/release-security-gate.test.js tests/model-metrics.test.js tests/metrics-endpoint.acceptance.test.js tests/image-ingress.test.js tests/rate-limit.test.js tests/http-body.test.js tests/analyze-ingress.acceptance.test.js tests/model-metrics-report-format.test.js tests/model-metrics-report.acceptance.test.js",
|
||||
"test:ui": "node tests/ui.acceptance.mjs",
|
||||
"test:photo": "node tests/photo-first.acceptance.mjs",
|
||||
"test:mobile-capture": "node tests/mobile-capture.acceptance.mjs",
|
||||
"test:ingress-capture": "node tests/ingress-capture.acceptance.mjs",
|
||||
"test:sleek": "node tests/sleek-chat.acceptance.mjs",
|
||||
"test:manual-fallback": "node tests/manual-fallback.acceptance.mjs",
|
||||
"test:provisional": "node tests/provisional-suggestion.acceptance.mjs",
|
||||
@@ -18,7 +19,7 @@
|
||||
"test:grounded-chat": "node tests/grounded-chat.acceptance.mjs",
|
||||
"test:photo-lifecycle": "node tests/photo-lifecycle.acceptance.mjs",
|
||||
"test:training-contribution": "node tests/training-contribution.acceptance.mjs",
|
||||
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check src/training-contribution.js && node --check src/abstention-policy.js && node --check src/release-security-gate.js && node --check scripts/security_attack_suite.mjs && node --check tests/security-attack.acceptance.mjs && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/corpus_manifest.py scripts/review_corpus.py scripts/dataset_card.py scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py scripts/calibration_report.py scripts/operator_backup.py && node --check scripts/record_worker_benchmark.mjs && node --check scripts/abstention_policy_receipt.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/training-contribution.acceptance.mjs && node --check tests/portability.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh",
|
||||
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/draft.js && node --check src/suggestion.js && node --check src/followup.js && node --check src/vision-readiness.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check src/vision-corpus.js && node --check src/summary.js && node --check src/photo-lifecycle.js && node --check src/grounded-chat.js && node --check src/training-contribution.js && node --check src/abstention-policy.js && node --check src/release-security-gate.js && node --check scripts/security_attack_suite.mjs && node --check tests/security-attack.acceptance.mjs && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/manual-fallback.acceptance.mjs && node --check tests/provisional-suggestion.acceptance.mjs && node --check tests/followup.acceptance.mjs && node --check tests/vision-refusal.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/corpus_manifest.py scripts/review_corpus.py scripts/dataset_card.py scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py scripts/calibration_report.py scripts/operator_backup.py && node --check scripts/record_worker_benchmark.mjs && node --check scripts/abstention_policy_receipt.mjs && node --check tests/worker-readiness.acceptance.mjs && node --check tests/pattern-summary.acceptance.mjs && node --check tests/photo-lifecycle.acceptance.mjs && node --check tests/training-contribution.acceptance.mjs && node --check tests/portability.acceptance.mjs && node --check tests/grounded-chat.acceptance.mjs && bash -n scripts/worker_ready.sh && bash -n deploy/worker/entrypoint.sh && bash -n deploy/worker/build_image.sh && bash -n deploy/worker/run_worker_container.sh && node --check src/model-metrics.js && node --check src/model-metrics-report.js && node --check scripts/model_metrics_report.mjs && node --check src/image-ingress.js && node --check src/rate-limit.js && node --check src/http-body.js && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/build_vision_corpus.py && node --check tests/ingress-capture.acceptance.mjs && node --check tests/analyze-ingress.acceptance.test.js",
|
||||
"check:diff": "bash scripts/check_diff.sh",
|
||||
"start": "node server.mjs",
|
||||
"test:worker-readiness": "node tests/worker-readiness.acceptance.mjs",
|
||||
@@ -30,7 +31,8 @@
|
||||
"test:dataset-card": "python3 tests/dataset-card.test.py",
|
||||
"test:backup": "python3 tests/backup-restore.test.py",
|
||||
"test:runbook": "python3 tests/production-runbook.test.py",
|
||||
"test:security-attack": "node tests/security-attack.acceptance.mjs"
|
||||
"test:security-attack": "node tests/security-attack.acceptance.mjs",
|
||||
"test:metrics-report": "node --test tests/model-metrics-report-format.test.js tests/model-metrics-report.acceptance.test.js"
|
||||
},
|
||||
"devDependencies": {
|
||||
"playwright": "^1.62.1"
|
||||
|
||||
@@ -50,6 +50,9 @@ DEMO_REQUIRED_CAPTIONS = (
|
||||
"Reachable is not loaded: no AI control, no consent, and no upload until the pinned model is resident",
|
||||
"Every pattern line names the confirmed logs behind it, and following one shows exactly those records",
|
||||
"Two logs is not a pattern: the summary refuses to describe a trend it cannot support",
|
||||
"A file that only claims to be a photo is refused on the device — nothing is uploaded",
|
||||
"The device guard is not the only one: the same bytes, sent straight to the running server",
|
||||
"Refused at the server too: 400 not_an_image, before any model work",
|
||||
"The answer states the count and cites the four confirmed logs it counted",
|
||||
"Following a citation opens exactly the confirmed log behind that answer",
|
||||
"Ask something the journal cannot answer and Timmy refuses instead of reassuring",
|
||||
@@ -68,6 +71,10 @@ DEMO_REQUIRED_CAPTIONS = (
|
||||
"An older backup is upgraded on import, and the notice says which version the file came from",
|
||||
"A file from a newer Timmy is refused with an instruction, and the ledger it would have replaced is untouched",
|
||||
"A restored log still says it began as an AI suggestion you corrected — provenance is part of the record",
|
||||
"Health counters are disclosed where you read privacy facts — they never include notes, photos, image fingerprints, or identity",
|
||||
"The operator health report is a command line, not a screen in the app — this runs it against this recording’s own server",
|
||||
"Every counter on this card came from the requests this recording just made, and a wrong token is refused instead of answered",
|
||||
"The snapshot holds counts, a readiness phase, latency, and enumerated failure classes — no note, photo, image fingerprint, or identity",
|
||||
"The review tool is a command line, not a screen in the app — this runs it against the committed synthetic sample",
|
||||
"The queue shows the proposal and hides the model answer; revealing it is refused until a reviewer commits",
|
||||
"The model said Type 6 and the reviewer said Type 5: the disagreement is recorded, not smoothed over",
|
||||
@@ -192,7 +199,10 @@ def main() -> int:
|
||||
run(["python3", "tests/backup-restore.test.py", "-v"], tree)
|
||||
run(["python3", "tests/production-runbook.test.py", "-v"], tree)
|
||||
server_env = dict(os.environ)
|
||||
server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false"})
|
||||
# TIMMY_METRICS_TOKEN gates the operator snapshot for this recording only. It is a demo value
|
||||
# against a loopback-only server, not a credential, and it never appears in an artifact.
|
||||
server_env.update({"HOST": "127.0.0.1", "PORT": "4173", "TIMMY_VISION_ENABLED": "false",
|
||||
"TIMMY_METRICS_TOKEN": "demo-metrics-token-not-a-credential"})
|
||||
# A leftover server from another checkout answers on this port and the readiness probe cannot
|
||||
# tell it apart from ours, so every gate and the recorded demo would silently describe a
|
||||
# different tree. Refuse to build over a live listener, and verify the served bytes afterwards.
|
||||
@@ -217,6 +227,7 @@ def main() -> int:
|
||||
run(["npm", "run", "test:ui"], tree)
|
||||
run(["npm", "run", "test:photo"], tree)
|
||||
run(["npm", "run", "test:mobile-capture"], tree)
|
||||
run(["npm", "run", "test:ingress-capture"], tree)
|
||||
run(["npm", "run", "test:sleek"], tree)
|
||||
run(["npm", "run", "test:manual-fallback"], tree)
|
||||
run(["npm", "run", "test:provisional"], tree)
|
||||
@@ -275,7 +286,7 @@ def main() -> int:
|
||||
if not contact_sheet.exists() or contact_sheet.stat().st_size == 0:
|
||||
raise SystemExit("Release demo contact sheet was not produced")
|
||||
run(["npm", "audit", "--audit-level=high"], tree)
|
||||
for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "src/summary.js", "src/photo-lifecycle.js", "src/training-contribution.js", "scripts/record_worker_benchmark.mjs", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs", "tests/worker-readiness.acceptance.mjs", "tests/pattern-summary.acceptance.mjs", "tests/photo-lifecycle.acceptance.mjs", "tests/training-contribution.acceptance.mjs"):
|
||||
for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/draft.js", "src/suggestion.js", "src/followup.js", "src/vision-readiness.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/vision-corpus.js", "src/summary.js", "src/photo-lifecycle.js", "src/training-contribution.js", "scripts/record_worker_benchmark.mjs", "tests/staging.acceptance.mjs", "tests/vision-refusal.acceptance.mjs", "tests/worker-readiness.acceptance.mjs", "tests/pattern-summary.acceptance.mjs", "tests/photo-lifecycle.acceptance.mjs", "tests/training-contribution.acceptance.mjs", "src/image-ingress.js", "src/rate-limit.js", "src/http-body.js", "tests/ingress-capture.acceptance.mjs"):
|
||||
run(["node", "--check", file], tree)
|
||||
run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree)
|
||||
run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree)
|
||||
@@ -392,6 +403,8 @@ def main() -> int:
|
||||
"production_backup_restore": "passed",
|
||||
"production_runbook_commands": "passed",
|
||||
"portable_backup_restore": "passed",
|
||||
"image_ingress_refusal": "passed",
|
||||
"privacy_safe_health_metrics": "passed",
|
||||
"dependency_audit_high": "passed",
|
||||
"syntax": "passed",
|
||||
"secret_scan": "passed",
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
#!/usr/bin/env node
|
||||
// Operator report for the privacy-safe model counters (issue #19).
|
||||
//
|
||||
// It reads one snapshot from a running Timmy server and prints it. The token is a credential for the
|
||||
// operator surface, so it is read from the environment or argv and never echoed — not in errors, not
|
||||
// in the report, not in a stack trace.
|
||||
import { formatOperatorReport } from '../src/model-metrics-report.js';
|
||||
|
||||
const USAGE = `Usage: node scripts/model_metrics_report.mjs [options]
|
||||
|
||||
--url <origin> base URL of the Timmy server (default http://127.0.0.1:4173)
|
||||
--token <token> operator token (default: $TIMMY_METRICS_TOKEN)
|
||||
--json print the raw snapshot instead of the text report
|
||||
--timeout-ms <ms> request deadline (default 5000)
|
||||
--help show this text
|
||||
`;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
url: process.env.TIMMY_METRICS_URL || 'http://127.0.0.1:4173',
|
||||
token: process.env.TIMMY_METRICS_TOKEN || '',
|
||||
json: false,
|
||||
timeoutMs: 5_000,
|
||||
help: false,
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--help' || arg === '-h') {
|
||||
options.help = true;
|
||||
} else if (arg === '--json') {
|
||||
options.json = true;
|
||||
} else if (arg === '--url') {
|
||||
options.url = argv[++index] ?? '';
|
||||
} else if (arg === '--token') {
|
||||
options.token = argv[++index] ?? '';
|
||||
} else if (arg === '--timeout-ms') {
|
||||
options.timeoutMs = Number(argv[++index]);
|
||||
} else {
|
||||
throw new Error(`unknown option: ${arg}`);
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const options = parseArgs(argv);
|
||||
if (options.help) {
|
||||
process.stdout.write(USAGE);
|
||||
return 0;
|
||||
}
|
||||
if (!options.token) {
|
||||
throw new Error('operator token required: pass --token or set TIMMY_METRICS_TOKEN');
|
||||
}
|
||||
const endpoint = new URL('api/metrics', options.url.endsWith('/') ? options.url : `${options.url}/`);
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(endpoint, {
|
||||
headers: { authorization: `Bearer ${options.token}`, accept: 'application/json' },
|
||||
signal: AbortSignal.timeout(Number.isFinite(options.timeoutMs) ? options.timeoutMs : 5_000),
|
||||
});
|
||||
} catch (error) {
|
||||
process.stderr.write(`Could not read ${endpoint.origin}/api/metrics: ${error?.message || 'request failed'}\n`);
|
||||
return 1;
|
||||
}
|
||||
if (!response.ok) {
|
||||
if (response.status === 404) {
|
||||
// The route is absent rather than guarded: that is a deployment fact, not a credential problem.
|
||||
process.stderr.write('Metrics are disabled on this server: the route only exists when TIMMY_METRICS_TOKEN is set.\n');
|
||||
return 1;
|
||||
}
|
||||
if (response.status === 403) {
|
||||
process.stderr.write('The server rejected the operator token.\n');
|
||||
return 1;
|
||||
}
|
||||
process.stderr.write(`The server answered ${response.status} for /api/metrics.\n`);
|
||||
return 1;
|
||||
}
|
||||
const snapshot = await response.json();
|
||||
process.stdout.write(options.json ? `${JSON.stringify(snapshot, null, 2)}\n` : formatOperatorReport(snapshot));
|
||||
return 0;
|
||||
}
|
||||
|
||||
main(process.argv.slice(2)).then(code => {
|
||||
process.exitCode = code;
|
||||
}).catch(error => {
|
||||
process.stderr.write(`${error?.message || 'report failed'}\n`);
|
||||
process.exitCode = 2;
|
||||
});
|
||||
@@ -90,7 +90,9 @@ const suggestionBody = {
|
||||
warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
|
||||
};
|
||||
let analyzeOverride = null;
|
||||
let analyzeCalls = 0;
|
||||
await page.route('**/api/analyze', route => {
|
||||
analyzeCalls += 1;
|
||||
const response = analyzeOverride || { status: 200, body: suggestionBody };
|
||||
return route.fulfill({ status: response.status, contentType: 'application/json', body: JSON.stringify(response.body) });
|
||||
});
|
||||
@@ -310,6 +312,74 @@ await indicate('label[for="camera-photo"]');
|
||||
await page.locator('#camera-photo').dispatchEvent('cancel');
|
||||
await page.getByText(/Camera or photo picker closed/i).waitFor();
|
||||
await caption('Camera closed cleanly — gallery and manual logging are still available', 1600);
|
||||
// Issue #16, on camera: the file below is committed as text and only claims to be a photo. It must
|
||||
// be refused on the device, the refusal must say nothing was uploaded, and the app must never post
|
||||
// it — asserted here, not narrated.
|
||||
await indicate('label[for="gallery-photo"]');
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/not-a-photo.jpg');
|
||||
await page.locator('[data-capture-error="unreadable"]').waitFor();
|
||||
const unreadableCopy = await page.locator('[data-capture-error="unreadable"]').innerText();
|
||||
if (!/isn’t a photo Timmy can read/.test(unreadableCopy) || !/Nothing was uploaded/.test(unreadableCopy)) {
|
||||
throw new Error(`An unreadable file must be refused with the upload-free copy, saw: ${unreadableCopy}`);
|
||||
}
|
||||
if (await page.getByText(/couldn’t analyze that safely/i).count() !== 0) {
|
||||
throw new Error('A file that never left the device must not be reported as a failed analysis');
|
||||
}
|
||||
if (analyzeCalls !== 0) throw new Error('An unreadable file was posted to /api/analyze');
|
||||
await caption('A file that only claims to be a photo is refused on the device — nothing is uploaded', 2600);
|
||||
await indicate('[data-capture-error="unreadable"]');
|
||||
await caption('Manual logging stays one tap away, and the photo picker is one tap back', 2000);
|
||||
// Issue #16 has two boundaries, and the device guard is only the first. The bytes below are offered
|
||||
// straight to this running app with the demo's own /api/analyze interceptor lifted, so the status and
|
||||
// the refusal copy on the card are read back from the real server rather than narrated.
|
||||
await caption('The device guard is not the only one: the same bytes, sent straight to the running server', 2600);
|
||||
await page.unroute('**/api/analyze');
|
||||
let serverRefusal;
|
||||
try {
|
||||
serverRefusal = await page.evaluate(async () => {
|
||||
const base = (document.querySelector('meta[name="timmy-base-path"]')?.content || '/').replace(/\/$/, '');
|
||||
const bytes = new TextEncoder().encode('<!doctype html><html><body>named .jpg, not an image</body></html>');
|
||||
let binary = '';
|
||||
for (const byte of bytes) binary += String.fromCharCode(byte);
|
||||
const response = await fetch(`${base}/api/analyze`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ consent: true, imageDataUrl: `data:image/jpeg;base64,${btoa(binary)}` }),
|
||||
});
|
||||
return { status: response.status, body: await response.json(), contentType: response.headers.get('content-type') };
|
||||
});
|
||||
} finally {
|
||||
await page.route('**/api/analyze', route => {
|
||||
analyzeCalls += 1;
|
||||
const response = analyzeOverride || { status: 200, body: suggestionBody };
|
||||
return route.fulfill({ status: response.status, contentType: 'application/json', body: JSON.stringify(response.body) });
|
||||
});
|
||||
}
|
||||
// A 200 here would mean the boundary is untested and the demo is theatre, so the scene fails instead.
|
||||
if (serverRefusal.status !== 400 || !/not a JPEG, PNG, or WebP photo/i.test(serverRefusal.body?.error || '')) {
|
||||
throw new Error(`The running server must refuse non-image bytes, saw ${serverRefusal.status} ${JSON.stringify(serverRefusal.body)}`);
|
||||
}
|
||||
await evidenceCard({
|
||||
id: 'demo-ingress-card',
|
||||
title: 'IMAGE INGRESS, REFUSED BY THE SERVER',
|
||||
subtitle: 'The same non-image bytes, posted to this running app in this recording',
|
||||
rows: [
|
||||
['status', `${serverRefusal.status} refused`],
|
||||
['code', String(serverRefusal.body?.code || 'not_an_image')],
|
||||
['copy', String(serverRefusal.body?.error || '')],
|
||||
['—', ''],
|
||||
['provider', 'never called — no model request for these bytes'],
|
||||
['budget', 'counted by the per-device limiter before the provider is asked'],
|
||||
['after', 'the journal, the draft, and manual logging are untouched'],
|
||||
['—', ''],
|
||||
['scope', 'this probe is a non-image container. Polyglot tails, embedded metadata, dimensions, body size, and the rate limit are enforced and asserted in the suite'],
|
||||
],
|
||||
});
|
||||
const ingressProof = await evidenceCardProof('demo-ingress-card');
|
||||
if (!ingressProof.ok) throw new Error(`server refusal card is not legible: ${JSON.stringify(ingressProof)}`);
|
||||
await caption('Refused at the server too: 400 not_an_image, before any model work', 2600);
|
||||
await page.evaluate(() => document.querySelector('#demo-ingress-card')?.remove());
|
||||
await tap('#retake-photo', 450);
|
||||
await indicate('label[for="gallery-photo"]');
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||||
await caption('Nothing uploads until explicit consent', 1100);
|
||||
@@ -1455,6 +1525,102 @@ await tap('#entry-sheet-close', 450);
|
||||
await tap('[data-view="privacy"]', 550);
|
||||
await page.locator('#delete-all').waitFor();
|
||||
|
||||
// Issue #19 — the operator health counters are disclosed where a user reads privacy facts, and the
|
||||
// recording shows the real sentence the app renders, checked in the browser first.
|
||||
const healthDisclosure = page.locator('[data-health-disclosure]');
|
||||
await healthDisclosure.waitFor();
|
||||
const healthCopy = (await healthDisclosure.innerText()).trim();
|
||||
if (!/how analysis requests went/i.test(healthCopy) || !/image fingerprints/i.test(healthCopy) || !/reset/i.test(healthCopy)) {
|
||||
throw new Error(`the privacy screen must disclose the privacy-safe health counters: ${healthCopy}`);
|
||||
}
|
||||
await page.evaluate(node => node.scrollIntoView({ block: 'center' }), await healthDisclosure.elementHandle());
|
||||
await caption('Health counters are disclosed where you read privacy facts — they never include notes, photos, image fingerprints, or identity', 3200);
|
||||
await indicate('[data-health-disclosure]');
|
||||
|
||||
// Issue #19, on camera: the operator report is a command line, not a screen in the app. It runs the
|
||||
// real script against this recording's own server (started with a demo-only loopback token), so every
|
||||
// number printed on the card was produced by the requests this recording just made — the ingress
|
||||
// refusals above are counted, not staged. The same script with a wrong token must be refused rather
|
||||
// than answered, and the token itself must never reach the report.
|
||||
const operatorToken = process.env.TIMMY_METRICS_TOKEN || '';
|
||||
if (!operatorToken) {
|
||||
throw new Error('the demo server must be started with a demo TIMMY_METRICS_TOKEN so the operator report can be exercised');
|
||||
}
|
||||
async function operatorReport(args) {
|
||||
return execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', ...args], { cwd: process.cwd() })
|
||||
.then(result => ({ code: 0, stdout: result.stdout, stderr: result.stderr }))
|
||||
.catch(error => ({ code: typeof error.code === 'number' ? error.code : 1, stdout: error.stdout || '', stderr: error.stderr || '' }));
|
||||
}
|
||||
const reportRun = await operatorReport(['--url', 'http://127.0.0.1:4173', '--token', operatorToken]);
|
||||
if (reportRun.code !== 0) throw new Error(`the operator report must read the snapshot: ${reportRun.stderr.trim()}`);
|
||||
const refusedRun = await operatorReport(['--url', 'http://127.0.0.1:4173', '--token', 'demo-token-that-is-wrong']);
|
||||
if (refusedRun.code !== 1 || !/rejected the operator token/i.test(refusedRun.stderr)) {
|
||||
throw new Error('a wrong operator token must be refused with a stated reason instead of answered');
|
||||
}
|
||||
if (reportRun.stdout.includes(operatorToken) || refusedRun.stderr.includes('demo-token-that-is-wrong')) {
|
||||
throw new Error('the operator token must never appear in the report or in an error line');
|
||||
}
|
||||
const parsedRows = [];
|
||||
let sections = 0;
|
||||
for (const line of reportRun.stdout.split('\n')) {
|
||||
const match = /^ {2}(\S.*?)\s{2,}(\S.*)$/.exec(line);
|
||||
if (match) { parsedRows.push([match[1].trim(), match[2].trim()]); continue; }
|
||||
const heading = line.trim();
|
||||
if (heading === 'APP' || heading === 'QUEUE' || heading === 'MODEL') {
|
||||
parsedRows.push(['—', '']);
|
||||
parsedRows.push(['section', heading]);
|
||||
sections += 1;
|
||||
}
|
||||
}
|
||||
if (sections !== 3) throw new Error(`the operator report must separate the app, queue, and model boundaries, not ${sections} of them`);
|
||||
const rowValue = pattern => {
|
||||
const found = parsedRows.find(([key]) => pattern.test(key));
|
||||
if (!found) throw new Error(`the operator report is missing the ${pattern} row: ${reportRun.stdout}`);
|
||||
return found[1];
|
||||
};
|
||||
const failureRows = parsedRows.filter(([key]) => /^failure /i.test(key));
|
||||
if (failureRows.length !== 8) throw new Error(`the report must print all eight enumerated failure classes, not ${failureRows.length}`);
|
||||
const failureList = failureRows.map(([key, value]) => `${key.replace(/^failure /i, '')} ${value}`).join(' · ');
|
||||
const analyzeCount = Number(rowValue(/^analyze requests$/i));
|
||||
if (!Number.isFinite(analyzeCount) || analyzeCount < 1) {
|
||||
throw new Error(`the report must count this recording's own requests, not a zero: ${reportRun.stdout}`);
|
||||
}
|
||||
const metricsCard = await evidenceCard({
|
||||
id: 'operator-health-report',
|
||||
title: 'OPERATOR HEALTH REPORT',
|
||||
// Long CLI labels are folded into values on purpose: a key column narrow enough to fit the phone
|
||||
// splits `analyze requests` across two lines and detaches it from its count.
|
||||
subtitle: 'command line, not app UI — loopback :4173, demo token',
|
||||
rows: [
|
||||
['section', 'APP'],
|
||||
['uptime', rowValue(/^uptime$/i)],
|
||||
['requests', `${analyzeCount} analyzed · ${rowValue(/^refused before model$/i)} refused before the model`],
|
||||
['section', 'QUEUE'],
|
||||
['in flight', rowValue(/^in flight$/i)],
|
||||
['accepted', rowValue(/^accepted$/i)],
|
||||
['at capacity', rowValue(/^at capacity$/i)],
|
||||
['section', 'MODEL'],
|
||||
['readiness', rowValue(/^readiness$/i)],
|
||||
['outcomes', rowValue(/^outcomes$/i)],
|
||||
['abstention', rowValue(/^abstention$/i)],
|
||||
['latency', rowValue(/^latency$/i)],
|
||||
['failures', failureList],
|
||||
['—', ''],
|
||||
['wrong token', 'refused · exit 1 · no counters'],
|
||||
],
|
||||
font: '600 9.5px/1.45 ui-monospace, SFMono-Regular, Menlo, monospace',
|
||||
keyWidth: 72,
|
||||
gap: '1px',
|
||||
rowGap: '6px',
|
||||
});
|
||||
await sleep(420);
|
||||
const metricsCardProof = await evidenceCardProof('operator-health-report');
|
||||
if (!metricsCardProof.ok) throw new Error(`Operator report card is not fully readable on screen: ${JSON.stringify(metricsCardProof)}`);
|
||||
await caption('The operator health report is a command line, not a screen in the app — this runs it against this recording’s own server', 3800);
|
||||
await caption('Every counter on this card came from the requests this recording just made, and a wrong token is refused instead of answered', 3600);
|
||||
await caption('The snapshot holds counts, a readiness phase, latency, and enumerated failure classes — no note, photo, image fingerprint, or identity', 4000);
|
||||
await page.evaluate(() => document.querySelector('#operator-health-report')?.remove());
|
||||
|
||||
await indicate('#delete-all');
|
||||
await tap('#delete-all', 650);
|
||||
await page.getByText(/Local ledger and photos deleted/).waitFor();
|
||||
@@ -1576,12 +1742,14 @@ await page.evaluate(() => {
|
||||
});
|
||||
await sleep(2200);
|
||||
|
||||
// The refusal pass deliberately produces two fail-closed 503s (an out-of-range reply, and one blank field
|
||||
// beside an out-of-range one), which the browser logs as failed resources. Requiring exactly those two
|
||||
// keeps the scene honest: if the boundary stops refusing, this fails instead of quietly recording a happy
|
||||
// path, and a dropped scene is caught rather than silently shrinking the evidence.
|
||||
// Two deliberate refusals show up in the browser console as failed resources, and both are counted
|
||||
// rather than tolerated: the ingress probe above posts non-image bytes to the real server (one 400),
|
||||
// and the refusal pass produces two fail-closed 503s (an out-of-range reply, and one blank field
|
||||
// beside an out-of-range one). Enumerating them keeps the scene honest: if the ingress boundary stops
|
||||
// refusing, or a dropped scene shrinks the evidence, this fails instead of recording a happy path.
|
||||
const expectedFailClosed = errors.filter(message => /Failed to load resource: the server responded with a status of 503 \(Service Unavailable\)/.test(message));
|
||||
if (expectedFailClosed.length !== 2 || errors.length !== expectedFailClosed.length) {
|
||||
const expectedIngressRefusal = errors.filter(message => /Failed to load resource: the server responded with a status of 400 \(Bad Request\)/.test(message));
|
||||
if (expectedFailClosed.length !== 2 || expectedIngressRefusal.length !== 1 || errors.length !== 3) {
|
||||
throw new Error(`Browser errors: ${errors.join(' | ') || 'unexpected error count'}`);
|
||||
}
|
||||
// Machine-readable receipt of exactly what was rendered on camera, so the release gate can prove
|
||||
|
||||
@@ -60,7 +60,7 @@ function startStubWorker() {
|
||||
|
||||
// The real server, with its own stdout/stderr captured from the first byte: the log scan is only
|
||||
// worth something if it read a log that actually exists.
|
||||
async function startTarget({ providerPort, port }) {
|
||||
async function startTarget({ providerPort, port, env = {} }) {
|
||||
const child = spawn(process.execPath, ['server.mjs'], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
@@ -72,6 +72,7 @@ async function startTarget({ providerPort, port }) {
|
||||
TIMMY_VISION_BASE_URL: `http://127.0.0.1:${providerPort}/v1`,
|
||||
TIMMY_VISION_MODEL: 'stub-vision-worker',
|
||||
TIMMY_VISION_TIMEOUT_MS: '5000',
|
||||
...env,
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
@@ -104,9 +105,9 @@ async function main() {
|
||||
const photo = await readFile(join(repoRoot, 'tests/fixtures/synthetic-type4.jpg'));
|
||||
const imageDataUrl = `data:image/jpeg;base64,${photo.toString('base64')}`;
|
||||
|
||||
const post = async (body) => {
|
||||
const post = async (body, targetBase = baseUrl) => {
|
||||
const before = worker.state.requests;
|
||||
const response = await fetch(`${baseUrl}/api/analyze`, {
|
||||
const response = await fetch(`${targetBase}/api/analyze`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: typeof body === 'string' ? body : JSON.stringify(body),
|
||||
@@ -245,13 +246,25 @@ async function main() {
|
||||
});
|
||||
|
||||
// Load: sequential valid analyses, abstaining answers, latency measured with the same boundary.
|
||||
// The abuse cases above spend the ingress request budget, so latency is measured against a second
|
||||
// target that starts with the budget wide open; otherwise these samples would be 429 refusals.
|
||||
const loadPort = explicitBaseUrl ? null : await freePort();
|
||||
const loadTarget = explicitBaseUrl
|
||||
? null
|
||||
: await startTarget({
|
||||
providerPort: worker.port,
|
||||
port: loadPort,
|
||||
env: { TIMMY_ANALYZE_RATE_LIMIT: '100000', TIMMY_ANALYZE_RATE_WINDOW_MS: '60000' },
|
||||
});
|
||||
const loadBase = explicitBaseUrl || `http://127.0.0.1:${loadPort}`;
|
||||
worker.state.content = JSON.stringify({ isStool: false, confidence: 0.9 });
|
||||
const samples = [];
|
||||
let errors = 0;
|
||||
let abstentions = 0;
|
||||
try {
|
||||
for (let index = 0; index < 8; index += 1) {
|
||||
const started = performance.now();
|
||||
const result = await post({ consent: true, imageDataUrl });
|
||||
const result = await post({ consent: true, imageDataUrl }, loadBase);
|
||||
samples.push(Math.round(performance.now() - started));
|
||||
if (result.status !== 200) errors += 1;
|
||||
else {
|
||||
@@ -260,6 +273,9 @@ async function main() {
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await stop(loadTarget?.child);
|
||||
}
|
||||
const percentiles = latencyPercentiles(samples);
|
||||
const load = {
|
||||
requests: samples.length,
|
||||
@@ -270,6 +286,12 @@ async function main() {
|
||||
max: percentiles.max,
|
||||
note: 'Latency measured against a counting stub on loopback; it measures this boundary, not a model.',
|
||||
};
|
||||
// Issue #16 adds a per-caller request budget at the ingress, so a suite that spends its budget on
|
||||
// the abuse cases above would measure 429 refusals here and publish them as analysis latency.
|
||||
// A refusal is not a sample: the load phase runs against its own permissive target instead.
|
||||
if (load.errors > 0) {
|
||||
throw new Error(`the load phase measured ${load.errors}/${load.requests} refusals instead of analyses, so it cannot publish latency`);
|
||||
}
|
||||
|
||||
const report = buildReport({
|
||||
generatedAt: new Date().toISOString(),
|
||||
|
||||
+105
-5
@@ -1,11 +1,17 @@
|
||||
import http from 'node:http';
|
||||
import { isIP } from 'node:net';
|
||||
import { createHash, randomBytes, timingSafeEqual } from 'node:crypto';
|
||||
import { readFile, stat } from 'node:fs/promises';
|
||||
import { extname, join, normalize } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { analyzePhoto, visionFailureResponse } from './src/vision-service.js';
|
||||
import { probeVisionProvider, resolveVisionConfig } from './src/vision-config.js';
|
||||
import { AgentGatewayError, createHermesAgentService, resolveHermesAgentConfig } from './src/hermes-agent-service.js';
|
||||
import { readJsonBody } from './src/http-body.js';
|
||||
import { inspectImageBytes, IngressError } from './src/image-ingress.js';
|
||||
import { createRateLimiter } from './src/rate-limit.js';
|
||||
import { createModelMetrics } from './src/model-metrics.js';
|
||||
import { visionReadiness } from './src/vision-readiness.js';
|
||||
|
||||
const root=fileURLToPath(new URL('.',import.meta.url));
|
||||
const port=Number(process.env.PORT||4173);
|
||||
@@ -32,7 +38,57 @@ const stagingLabel=process.env.TIMMY_STAGING_LABEL?`Staging · ${release} · ${c
|
||||
|
||||
function sendJson(res,status,value){res.writeHead(status,{'content-type':'application/json; charset=utf-8','cache-control':'no-store','x-content-type-options':'nosniff'});res.end(JSON.stringify(value));}
|
||||
function escapeHtmlAttribute(value){return String(value).replace(/[&<>"]/g,character=>({'&':'&','<':'<','>':'>','"':'"'}[character]));}
|
||||
function readJson(req,maxBytes=6*1024*1024){return new Promise((resolve,reject)=>{let size=0,tooLarge=false;const chunks=[];req.on('data',chunk=>{size+=chunk.length;if(size>maxBytes){tooLarge=true;return}chunks.push(chunk)});req.on('end',()=>{if(tooLarge)return reject(new AgentGatewayError(413,'Request is too large.'));try{resolve(JSON.parse(Buffer.concat(chunks).toString('utf8')))}catch{return reject(new AgentGatewayError(400,'Invalid JSON request.'))}});req.on('error',reject)})}
|
||||
// Photo ingress (issue #16). Every limit here is decided from the request itself and fails before
|
||||
// any provider call: a per-caller request budget, a deadline for a body that never arrives, a byte
|
||||
// ceiling, and a container/pixel inspection of the decoded image. The bucket key is a salted hash
|
||||
// that lives only in this process's memory, so no address and no image byte is ever logged.
|
||||
const bodyTimeoutMs=Number(process.env.TIMMY_BODY_TIMEOUT_MS||15_000);
|
||||
const analyzeLimiter=createRateLimiter({limit:Number(process.env.TIMMY_ANALYZE_RATE_LIMIT||12),windowMs:Number(process.env.TIMMY_ANALYZE_RATE_WINDOW_MS||60_000)});
|
||||
// Issue #19: operator health counters. The token is what makes this an operator surface rather than
|
||||
// a public one — with no token configured the route does not exist at all.
|
||||
const metricsToken=String(process.env.TIMMY_METRICS_TOKEN||'');
|
||||
const modelMetrics=createModelMetrics({limit:Number(process.env.TIMMY_MODEL_CONCURRENCY||4)});
|
||||
function classifyFailure(error){
|
||||
const code=String(error?.code||'');
|
||||
if(code==='invalid_json'||code==='invalid_fields')return 'schema_invalid';
|
||||
const message=String(error?.message||'');
|
||||
if(/timed out|timeout|abort/i.test(message))return 'timeout';
|
||||
// A payload the semantic validator rejects never reached the model, so it is an ingress refusal
|
||||
// rather than a model fault.
|
||||
if(/consent|JPEG|PNG|WebP|empty|too large/i.test(message))return 'ingress_refused';
|
||||
if(/invalid response/i.test(message))return 'provider_rejected';
|
||||
return 'provider_unreachable';
|
||||
}
|
||||
// Both the browser's readiness probe and the operator snapshot record the same observation, so the
|
||||
// cold-load figure reflects the moment the model actually became resident, not the first time an
|
||||
// operator happened to look.
|
||||
function observeReadiness(provider){
|
||||
const readiness=visionReadiness({enabled:visionConfig.enabled,providerReady:provider.ready,modelSeen:provider.modelSeen});
|
||||
modelMetrics.recordReadiness({phase:readiness.phase,ready:readiness.ready});
|
||||
return readiness;
|
||||
}
|
||||
// A token comparison that does not leak its prefix through response timing. The digests are equal
|
||||
// length by construction, so timingSafeEqual can be used even for a wrong-length guess.
|
||||
function operatorTokenMatches(supplied){
|
||||
if(!metricsToken||!supplied)return false;
|
||||
const digest=value=>createHash('sha256').update(String(value)).digest();
|
||||
return timingSafeEqual(digest(supplied),digest(metricsToken));
|
||||
}
|
||||
const bucketSalt=randomBytes(16).toString('hex');
|
||||
function clientBucket(req){return createHash('sha256').update(`${bucketSalt}:${req.socket.remoteAddress||'unknown'}`).digest('hex').slice(0,24);}
|
||||
function readJson(req,maxBytes=6*1024*1024){return readJsonBody(req,{maxBytes,timeoutMs:bodyTimeoutMs});}
|
||||
function ingressGuard(payload){
|
||||
const raw=String(payload?.imageDataUrl||'');
|
||||
const match=/^data:(image\/(?:jpeg|png|webp));base64,([A-Za-z0-9+/=]+)$/.exec(raw);
|
||||
// A payload that is not shaped like a data URL is left to the semantic validator, so the user still
|
||||
// gets the specific "upload a JPEG, PNG, or WebP photo" copy rather than a second opinion here.
|
||||
if(!match)return null;
|
||||
return inspectImageBytes(Buffer.from(match[2],'base64'),{declaredMime:match[1]});
|
||||
}
|
||||
function sendIngressError(res,error){
|
||||
if(error instanceof IngressError)return sendJson(res,400,{error:error.message,code:error.code});
|
||||
return sendJson(res,error.status||400,{error:error.message,code:error.code});
|
||||
}
|
||||
function cookie(req,name){for(const part of String(req.headers.cookie||'').split(';')){const [key,...value]=part.trim().split('=');if(key===name)return decodeURIComponent(value.join('='))}return ''}
|
||||
function requestOrigin(req){return String(req.headers.origin||'').replace(/\/$/,'')}
|
||||
function rejectCrossSite(req){const site=String(req.headers['sec-fetch-site']||'');if(site&&site!=='same-origin')throw new AgentGatewayError(403,'Cross-site agent requests are not allowed.')}
|
||||
@@ -48,14 +104,58 @@ http.createServer(async(req,res)=>{
|
||||
if(appPath==='/api/healthz'&&req.method==='GET')return sendJson(res,200,{ok:true,release,commit,visionEnabled:visionConfig.enabled,agentEnabled:agentConfig.enabled});
|
||||
if(appPath==='/api/vision-status'&&req.method==='GET'){
|
||||
const provider=await probeVisionProvider(visionConfig);
|
||||
observeReadiness(provider);
|
||||
const privacy=visionConfig.profile==='selfhost'?'The compressed image is processed by Timmy’s self-hosted model and is not forwarded to a third-party model provider.':'A compressed copy is sent to the configured AI provider only when you explicitly request analysis.';
|
||||
return sendJson(res,200,{...visionConfig.publicStatus(),providerReady:provider.ready,modelSeen:provider.modelSeen,privacy});
|
||||
}
|
||||
// Operator-only health snapshot (issue #19). With TIMMY_METRICS_TOKEN unset the route is absent,
|
||||
// so a default deployment exposes no counters at all.
|
||||
if(appPath==='/api/metrics'&&req.method==='GET'){
|
||||
if(!metricsToken)return sendJson(res,404,{error:'Not found'});
|
||||
if(!operatorTokenMatches(String(req.headers.authorization||'').replace(/^Bearer\s+/i,'')))return sendJson(res,403,{error:'Operator token required.'});
|
||||
observeReadiness(await probeVisionProvider(visionConfig));
|
||||
return sendJson(res,200,modelMetrics.snapshot());
|
||||
}
|
||||
if(appPath==='/api/analyze'&&req.method==='POST'){
|
||||
if(!visionConfig.enabled)return sendJson(res,503,{error:'AI analysis is disabled. Continue manually.'});
|
||||
const payload=await readJson(req);
|
||||
try{return sendJson(res,200,await analyzePhoto({payload,config:visionConfig}))}
|
||||
catch(error){const failure=visionFailureResponse(error);return sendJson(res,failure.status,failure.body)}
|
||||
// Refusals first, in cost order: the request budget, then the body, then the image bytes. None of
|
||||
// these depend on the model, so hostile bytes are refused as hostile even when analysis is off —
|
||||
// otherwise the disabled switch hides the ingress boundary behind a 503 and the refusal is untested.
|
||||
const startedAtMs=Date.now();
|
||||
const recordFailure=failureClass=>modelMetrics.recordAnalyze({outcome:'failure',failureClass,durationMs:Date.now()-startedAtMs});
|
||||
const budget=analyzeLimiter.take(clientBucket(req));
|
||||
if(!budget.allowed){
|
||||
recordFailure('rate_limited');
|
||||
res.setHeader('retry-after',String(Math.max(1,Math.ceil(budget.retryAfterMs/1000))));
|
||||
return sendJson(res,429,{error:'Too many photos from this device. Wait a moment, then try again — your journal and manual logging still work.'});
|
||||
}
|
||||
let payload;
|
||||
try{payload=await readJson(req)}
|
||||
catch(error){
|
||||
recordFailure('ingress_refused');
|
||||
const sent=sendIngressError(res,error);
|
||||
// The refusal is written first, then the half-read socket is dropped so a stalled body cannot
|
||||
// hold a connection or its partial buffer open.
|
||||
if(error?.code==='body_timeout'||error?.code==='too_large')req.destroy?.();
|
||||
return sent;
|
||||
}
|
||||
try{ingressGuard(payload)}
|
||||
catch(error){recordFailure('ingress_refused');return sendIngressError(res,error)}
|
||||
// Bytes are known to be a real photo at this point, so the disabled switch is the honest answer.
|
||||
if(!visionConfig.enabled){recordFailure('disabled');return sendJson(res,503,{error:'AI analysis is disabled. Continue manually.'})}
|
||||
// The slot is what the queue boundary reports, and it is released on every exit path so a failed
|
||||
// analysis cannot quietly consume concurrency.
|
||||
const slot=modelMetrics.enter();
|
||||
try{
|
||||
const result=await analyzePhoto({payload,config:visionConfig});
|
||||
modelMetrics.recordAnalyze({outcome:result?.status==='suggestion'?'suggestion':'abstained',durationMs:Date.now()-startedAtMs});
|
||||
return sendJson(res,200,result);
|
||||
}
|
||||
catch(error){
|
||||
const failure=visionFailureResponse(error);
|
||||
recordFailure(classifyFailure(error));
|
||||
return sendJson(res,failure.status,failure.body);
|
||||
}
|
||||
finally{modelMetrics.leave(slot)}
|
||||
}
|
||||
if(appPath==='/api/agent/status'&&req.method==='GET')return sendJson(res,200,agentService.status(cookie(req,'timmy_agent')));
|
||||
if(appPath==='/api/agent/unlock'&&req.method==='POST'){
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
// Bounded request-body reading for the API boundary (issue #16).
|
||||
//
|
||||
// Three limits have to fail before any provider work starts: the byte ceiling, a deadline for a body
|
||||
// that never arrives, and JSON validity. All three refuse with a client-visible status and a fixed
|
||||
// message; none of them logs or returns the body.
|
||||
|
||||
const COPY = Object.freeze({
|
||||
too_large: [413, 'Request is too large.'],
|
||||
invalid_json: [400, 'Invalid JSON request.'],
|
||||
body_timeout: [408, 'The request took too long to arrive. Try again.'],
|
||||
});
|
||||
|
||||
export class BodyError extends Error {
|
||||
constructor(code) {
|
||||
const [status, message] = COPY[code] || COPY.invalid_json;
|
||||
super(message);
|
||||
this.name = 'BodyError';
|
||||
this.code = code;
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export function readJsonBody(req, { maxBytes = 6 * 1024 * 1024, timeoutMs = 15_000 } = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
let tooLarge = false;
|
||||
let size = 0;
|
||||
const chunks = [];
|
||||
const finish = callback => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(deadline);
|
||||
req.removeListener('data', onData);
|
||||
req.removeListener('end', onEnd);
|
||||
req.removeListener('error', onError);
|
||||
callback();
|
||||
};
|
||||
const deadline = setTimeout(() => {
|
||||
finish(() => {
|
||||
// Stop reading and let the caller answer 408; the server destroys the socket once the
|
||||
// refusal is written, so a half-read body cannot be held open indefinitely.
|
||||
req.pause?.();
|
||||
reject(new BodyError('body_timeout'));
|
||||
});
|
||||
}, timeoutMs);
|
||||
|
||||
function onData(chunk) {
|
||||
if (settled) return;
|
||||
// Past the ceiling the body is already refused: keep draining it without buffering so the
|
||||
// caller can still read the 413. Destroying the socket here would surface as "fetch failed"
|
||||
// and leave the caller with no status to act on.
|
||||
if (tooLarge) return;
|
||||
size += chunk.length;
|
||||
if (size > maxBytes) {
|
||||
tooLarge = true;
|
||||
chunks.length = 0;
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
}
|
||||
function onEnd() {
|
||||
finish(() => {
|
||||
if (tooLarge) return reject(new BodyError('too_large'));
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(chunks).toString('utf8')));
|
||||
} catch {
|
||||
reject(new BodyError('invalid_json'));
|
||||
}
|
||||
});
|
||||
}
|
||||
function onError(error) {
|
||||
finish(() => reject(error));
|
||||
}
|
||||
|
||||
req.on('data', onData);
|
||||
req.on('end', onEnd);
|
||||
req.on('error', onError);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
// Transport-level hardening for the one endpoint that accepts image bytes (issue #16).
|
||||
//
|
||||
// The declared MIME type in a data URL is a claim made by the caller, never evidence, so every
|
||||
// decision here comes from the bytes: the container is sniffed, the container is walked to its own
|
||||
// end marker so nothing can ride along behind a valid image, embedded metadata segments are refused
|
||||
// (the browser always uploads a canvas re-encode, which carries none), and the pixel dimensions are
|
||||
// read from the header so a decode bomb never reaches the worker.
|
||||
//
|
||||
// Refusal copy is fixed and deliberate: it names what is wrong with the file and never echoes the
|
||||
// bytes, a hash, or any part of the upload back to the caller or into a log line.
|
||||
|
||||
export const MAX_EDGE = 4096;
|
||||
export const MAX_PIXELS = 16_000_000;
|
||||
|
||||
const COPY = Object.freeze({
|
||||
empty: 'The photo is empty.',
|
||||
not_an_image: 'That file is not a JPEG, PNG, or WebP photo.',
|
||||
container_mismatch: 'This photo is not the format it claims. Choose a JPEG, PNG, or WebP photo.',
|
||||
truncated: 'This photo is incomplete or damaged. Choose another photo.',
|
||||
trailing_data: 'This photo file has extra data after the image. Choose another photo.',
|
||||
metadata_present: 'This photo still carries embedded metadata. Choose it through Timmy’s camera or gallery picker so it is re-encoded without metadata.',
|
||||
dimensions_too_large: 'This photo is too large in pixels. Choose a smaller photo.',
|
||||
});
|
||||
|
||||
export class IngressError extends Error {
|
||||
constructor(code) {
|
||||
super(COPY[code] || COPY.not_an_image);
|
||||
this.name = 'IngressError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
const refuse = code => {
|
||||
throw new IngressError(code);
|
||||
};
|
||||
|
||||
const PNG_CHUNKS = { metadata: new Set(['eXIf', 'tEXt', 'zTXt', 'iTXt', 'tIME']) };
|
||||
// Segment markers that carry metadata rather than image data: APP1 (Exif/XMP), APP13 (Photoshop
|
||||
// resource block) and COM (free-text comment). The app's own upload is a canvas re-encode and
|
||||
// contains none of them, so a file that does was not produced by the capture path.
|
||||
const JPEG_METADATA_MARKERS = new Set([0xe1, 0xed, 0xfe]);
|
||||
const JPEG_SOF_MARKERS = new Set([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf]);
|
||||
|
||||
function sniff(bytes) {
|
||||
if (bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return 'image/png';
|
||||
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'image/jpeg';
|
||||
if (bytes.length >= 12 && bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP') return 'image/webp';
|
||||
return '';
|
||||
}
|
||||
|
||||
function inspectPng(bytes) {
|
||||
let offset = 8;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
let sawHeader = false;
|
||||
while (offset + 8 <= bytes.length) {
|
||||
const length = bytes.readUInt32BE(offset);
|
||||
const type = bytes.toString('ascii', offset + 4, offset + 8);
|
||||
const dataStart = offset + 8;
|
||||
const next = dataStart + length + 4;
|
||||
if (next > bytes.length) refuse('truncated');
|
||||
if (!sawHeader) {
|
||||
if (type !== 'IHDR' || length !== 13) refuse('truncated');
|
||||
width = bytes.readUInt32BE(dataStart);
|
||||
height = bytes.readUInt32BE(dataStart + 4);
|
||||
sawHeader = true;
|
||||
}
|
||||
if (PNG_CHUNKS.metadata.has(type)) refuse('metadata_present');
|
||||
if (type === 'IEND') {
|
||||
if (next !== bytes.length) refuse('trailing_data');
|
||||
return { width, height };
|
||||
}
|
||||
offset = next;
|
||||
}
|
||||
return refuse('truncated');
|
||||
}
|
||||
|
||||
function inspectJpeg(bytes) {
|
||||
let offset = 2;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
while (offset + 4 <= bytes.length) {
|
||||
if (bytes[offset] !== 0xff) refuse('truncated');
|
||||
const marker = bytes[offset + 1];
|
||||
if (marker === 0xd9) {
|
||||
if (offset + 2 !== bytes.length) refuse('trailing_data');
|
||||
break;
|
||||
}
|
||||
// Standalone markers carry no length field.
|
||||
if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7) || marker === 0xff) {
|
||||
offset += marker === 0xff ? 1 : 2;
|
||||
continue;
|
||||
}
|
||||
const length = bytes.readUInt16BE(offset + 2);
|
||||
if (length < 2 || offset + 2 + length > bytes.length) refuse('truncated');
|
||||
if (JPEG_METADATA_MARKERS.has(marker)) refuse('metadata_present');
|
||||
if (JPEG_SOF_MARKERS.has(marker) && length >= 7) {
|
||||
height = bytes.readUInt16BE(offset + 5);
|
||||
width = bytes.readUInt16BE(offset + 7);
|
||||
}
|
||||
offset += 2 + length;
|
||||
if (marker === 0xda) {
|
||||
// Entropy-coded scan data follows the start-of-scan header. The only thing left to trust is
|
||||
// the end-of-image marker, and it has to be the last two bytes of the file.
|
||||
if (bytes[bytes.length - 2] !== 0xff || bytes[bytes.length - 1] !== 0xd9) {
|
||||
return refuse(bytes.includes(Buffer.from([0xff, 0xd9])) ? 'trailing_data' : 'truncated');
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!width || !height) refuse('truncated');
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
function inspectWebp(bytes) {
|
||||
const declared = bytes.readUInt32LE(4) + 8;
|
||||
if (declared !== bytes.length) refuse(declared < bytes.length ? 'trailing_data' : 'truncated');
|
||||
let offset = 12;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
while (offset + 8 <= bytes.length) {
|
||||
const type = bytes.toString('ascii', offset, offset + 4);
|
||||
const size = bytes.readUInt32LE(offset + 4);
|
||||
const dataStart = offset + 8;
|
||||
if (dataStart + size > bytes.length) refuse('truncated');
|
||||
if (type === 'VP8X' && size >= 10) {
|
||||
width = bytes.readUIntLE(dataStart + 4, 3) + 1;
|
||||
height = bytes.readUIntLE(dataStart + 7, 3) + 1;
|
||||
} else if (type === 'VP8 ' && size >= 10) {
|
||||
width = bytes.readUInt16LE(dataStart + 6) & 0x3fff;
|
||||
height = bytes.readUInt16LE(dataStart + 8) & 0x3fff;
|
||||
} else if (type === 'VP8L' && size >= 5) {
|
||||
const bits = bytes.readUInt32LE(dataStart + 1);
|
||||
width = (bits & 0x3fff) + 1;
|
||||
height = ((bits >> 14) & 0x3fff) + 1;
|
||||
}
|
||||
offset = dataStart + size + (size % 2);
|
||||
}
|
||||
if (!width || !height) refuse('truncated');
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
// Returns the real container, its pixel dimensions, and the byte count. Throws IngressError on
|
||||
// anything the model worker must never be handed.
|
||||
export function inspectImageBytes(bytes, { declaredMime = '' } = {}) {
|
||||
const buffer = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes || []);
|
||||
if (!buffer.length) refuse('empty');
|
||||
const actual = sniff(buffer);
|
||||
if (!actual) refuse('not_an_image');
|
||||
if (declaredMime && declaredMime !== actual) refuse('container_mismatch');
|
||||
const dimensions = actual === 'image/png' ? inspectPng(buffer)
|
||||
: actual === 'image/jpeg' ? inspectJpeg(buffer)
|
||||
: inspectWebp(buffer);
|
||||
if (!dimensions.width || !dimensions.height) refuse('truncated');
|
||||
if (dimensions.width > MAX_EDGE || dimensions.height > MAX_EDGE
|
||||
|| dimensions.width * dimensions.height > MAX_PIXELS) refuse('dimensions_too_large');
|
||||
return { mime: actual, width: dimensions.width, height: dimensions.height, bytes: buffer.length };
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
// Renders the operator snapshot from `src/model-metrics.js` as a plain-text report.
|
||||
//
|
||||
// This module is a pure function of the snapshot, on purpose: the report is built from numbers and
|
||||
// from the three frozen vocabularies, so there is no path by which request content could reach the
|
||||
// terminal even if the counters were compromised. Anything outside the enumerated vocabularies is
|
||||
// not printed — an unknown failure class is visible as a missing row, never as a new label.
|
||||
import { FAILURE_CLASSES, OUTCOMES } from './model-metrics.js';
|
||||
|
||||
// Wide enough for the longest row label (`failure provider_unreachable`) plus a visible gap: a label
|
||||
// that fills the column renders run-together with its count (`failure ingress_refused0`), which reads
|
||||
// as one broken token instead of a metric.
|
||||
const LABEL_WIDTH = 30;
|
||||
|
||||
function row(label, value) {
|
||||
return ` ${String(label).padEnd(LABEL_WIDTH)}${value}`;
|
||||
}
|
||||
|
||||
function duration(value) {
|
||||
if (!Number.isFinite(value) || value < 0) return 'n/a';
|
||||
if (value < 1_000) return `${Math.round(value)}ms`;
|
||||
const seconds = value / 1_000;
|
||||
if (seconds < 60) return `${seconds.toFixed(1)}s`;
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
if (minutes < 60) return `${minutes}m ${Math.round(seconds % 60)}s`;
|
||||
return `${Math.floor(minutes / 60)}h ${minutes % 60}m`;
|
||||
}
|
||||
|
||||
function count(value) {
|
||||
return Number.isFinite(value) ? String(value) : '0';
|
||||
}
|
||||
|
||||
function readinessLine(model) {
|
||||
const readiness = model?.readiness ?? {};
|
||||
const phase = typeof readiness.phase === 'string' && readiness.phase ? readiness.phase : 'checking';
|
||||
return readiness.coldLoadMs === null || readiness.coldLoadMs === undefined
|
||||
? phase
|
||||
: `${phase} · cold load ${duration(readiness.coldLoadMs)}`;
|
||||
}
|
||||
|
||||
function abstentionLine(model) {
|
||||
const rate = model?.abstentionRate;
|
||||
const answered = (model?.outcomes?.suggestion ?? 0) + (model?.outcomes?.abstained ?? 0);
|
||||
if (rate === null || rate === undefined || answered === 0) return 'no answers yet';
|
||||
return `${(rate * 100).toFixed(1)}% of ${answered} answer${answered === 1 ? '' : 's'}`;
|
||||
}
|
||||
|
||||
export function formatOperatorReport(snapshot) {
|
||||
if (!snapshot || typeof snapshot !== 'object') throw new Error('a metrics snapshot is required');
|
||||
const app = snapshot.app ?? {};
|
||||
const queue = snapshot.queue ?? {};
|
||||
const model = snapshot.model ?? {};
|
||||
const latency = model.latency ?? {};
|
||||
const outcomes = OUTCOMES.map(name => `${name} ${count(model.outcomes?.[name])}`).join(' · ');
|
||||
|
||||
const lines = [
|
||||
`Timmy model health · ${snapshot.generatedAt ?? 'unknown time'} · schema ${snapshot.schema ?? '?'}`,
|
||||
'',
|
||||
'APP',
|
||||
row('uptime', duration(app.uptimeMs)),
|
||||
row('analyze requests', count(app.analyzeRequests)),
|
||||
row('refused before model', count(app.refusedBeforeModel)),
|
||||
'',
|
||||
'QUEUE',
|
||||
row('in flight', `${count(queue.inFlight)} / ${count(queue.limit)}`),
|
||||
row('accepted', count(queue.accepted)),
|
||||
row('at capacity', queue.atCapacity === true ? 'yes' : 'no'),
|
||||
'',
|
||||
'MODEL',
|
||||
row('readiness', readinessLine(model)),
|
||||
row('outcomes', outcomes),
|
||||
row('abstention', abstentionLine(model)),
|
||||
row('latency', `mean ${duration(latency.meanMs)} · max ${duration(latency.maxMs)} · samples ${count(latency.count)}`),
|
||||
...FAILURE_CLASSES.map(name => row(`failure ${name}`, count(model.failureClasses?.[name]))),
|
||||
];
|
||||
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
// Privacy-safe model health metrics for issue #19.
|
||||
//
|
||||
// The point of this module is the boundary it does not cross: a metric sample is assembled from
|
||||
// enumerated outcomes and numbers, never from a request. No note text, no photo hash, no image
|
||||
// bytes, and no caller identity can reach a label, because the only strings this module stores are
|
||||
// members of the frozen lists below. Anything else is coerced to a bucket instead of copied.
|
||||
|
||||
export const OUTCOMES = Object.freeze(['suggestion', 'abstained', 'failure']);
|
||||
|
||||
// Failure classes are the operator's vocabulary for "what went wrong". They are chosen here, not
|
||||
// observed from an error, so an upstream message can never become a metric label.
|
||||
export const FAILURE_CLASSES = Object.freeze([
|
||||
'ingress_refused',
|
||||
'rate_limited',
|
||||
'disabled',
|
||||
'provider_unreachable',
|
||||
'provider_rejected',
|
||||
'schema_invalid',
|
||||
'timeout',
|
||||
'unexpected',
|
||||
]);
|
||||
|
||||
// Readiness is the same vocabulary the browser already uses for the AI affordance, so a metric and
|
||||
// the screen can never disagree about why AI is withheld.
|
||||
export const READINESS_PHASES = Object.freeze(['checking', 'disabled', 'offline', 'loading', 'ready']);
|
||||
|
||||
// Failure classes that are decided before a request can occupy a queue slot or wait on the model.
|
||||
const REFUSED_BEFORE_MODEL = Object.freeze(['ingress_refused', 'rate_limited', 'disabled']);
|
||||
|
||||
export function createModelMetrics({ now = () => Date.now(), limit = 4 } = {}) {
|
||||
const outcomes = { suggestion: 0, abstained: 0, failure: 0 };
|
||||
const failureClasses = Object.fromEntries(FAILURE_CLASSES.map(name => [name, 0]));
|
||||
// The queue boundary is tracked by opaque tickets, not by request identity: the only thing that
|
||||
// survives a request here is the count of slots in use.
|
||||
const inUse = new Set();
|
||||
let accepted = 0;
|
||||
const startedAt = now();
|
||||
let readiness = { phase: 'checking', ready: false, coldLoadMs: null };
|
||||
let analyzeRequests = 0;
|
||||
let refusedBeforeModel = 0;
|
||||
// Latency is a histogram, never a list of individual timings: a bucket count says how the worker
|
||||
// behaves without carrying a per-request record of when someone photographed their stool.
|
||||
const bucketEdgesMs = Object.freeze([1_000, 2_000, 5_000, 15_000, 60_000]);
|
||||
const bucketsMs = Object.fromEntries(bucketEdgesMs.map(edge => [String(edge), 0]));
|
||||
let latencyCount = 0;
|
||||
let latencySumMs = 0;
|
||||
let latencyMaxMs = 0;
|
||||
|
||||
function abstentionRate() {
|
||||
const answered = outcomes.suggestion + outcomes.abstained;
|
||||
return answered === 0 ? null : outcomes.abstained / answered;
|
||||
}
|
||||
|
||||
function recordLatency(durationMs) {
|
||||
if (!Number.isFinite(durationMs) || durationMs < 0) return;
|
||||
latencyCount += 1;
|
||||
latencySumMs += durationMs;
|
||||
latencyMaxMs = Math.max(latencyMaxMs, durationMs);
|
||||
// Edges are cumulative on purpose: `bucketsMs['5000']` answers "how many samples were at or
|
||||
// below five seconds", so an operator can read a percentile without this module storing the
|
||||
// individual timings that produced it.
|
||||
for (const edge of bucketEdgesMs) {
|
||||
if (durationMs <= edge) bucketsMs[String(edge)] += 1;
|
||||
}
|
||||
// Anything slower than the last edge stays counted in the total but out of the histogram, so a
|
||||
// hanging request cannot invent a bucket.
|
||||
}
|
||||
|
||||
return {
|
||||
enter() {
|
||||
const ticket = Symbol('model-slot');
|
||||
inUse.add(ticket);
|
||||
accepted += 1;
|
||||
return ticket;
|
||||
},
|
||||
leave(ticket) {
|
||||
// Deleting a ticket that is already gone is a no-op, so a double release cannot free a slot
|
||||
// that a different request still holds.
|
||||
inUse.delete(ticket);
|
||||
},
|
||||
recordReadiness({ phase, ready }) {
|
||||
const next = READINESS_PHASES.includes(phase) ? phase : 'checking';
|
||||
// A worker can flap between ready and offline. Only the first time the model became resident
|
||||
// is a cold load; every later observation is just the current phase.
|
||||
const coldLoadMs = readiness.coldLoadMs === null && next === 'ready' && ready === true
|
||||
? Math.max(0, now() - startedAt)
|
||||
: readiness.coldLoadMs;
|
||||
readiness = { phase: next, ready: ready === true, coldLoadMs };
|
||||
},
|
||||
recordAnalyze({ outcome, failureClass, durationMs }) {
|
||||
// The label is looked up, never stored: an unrecognised outcome becomes `failure` so the
|
||||
// counters stay a closed set no matter what a caller sends.
|
||||
const bucket = OUTCOMES.includes(outcome) ? outcome : 'failure';
|
||||
outcomes[bucket] += 1;
|
||||
analyzeRequests += 1;
|
||||
if (bucket === 'failure') {
|
||||
failureClasses[FAILURE_CLASSES.includes(failureClass) ? failureClass : 'unexpected'] += 1;
|
||||
// A refusal decided before any model work (bad bytes, the per-device budget, or the disabled
|
||||
// switch) is an app fact: it never occupied a queue slot and never waited on the model.
|
||||
if (REFUSED_BEFORE_MODEL.includes(failureClass)) {
|
||||
refusedBeforeModel += 1;
|
||||
} else {
|
||||
recordLatency(durationMs);
|
||||
}
|
||||
} else {
|
||||
recordLatency(durationMs);
|
||||
}
|
||||
},
|
||||
snapshot() {
|
||||
return {
|
||||
schema: 1,
|
||||
generatedAt: new Date(now()).toISOString(),
|
||||
app: {
|
||||
uptimeMs: Math.max(0, now() - startedAt),
|
||||
analyzeRequests,
|
||||
refusedBeforeModel,
|
||||
},
|
||||
queue: {
|
||||
inFlight: inUse.size,
|
||||
limit,
|
||||
atCapacity: inUse.size >= limit,
|
||||
accepted,
|
||||
},
|
||||
model: {
|
||||
outcomes: { ...outcomes },
|
||||
failureClasses: { ...failureClasses },
|
||||
abstentionRate: abstentionRate(),
|
||||
readiness: { ...readiness },
|
||||
latency: {
|
||||
count: latencyCount,
|
||||
meanMs: latencyCount === 0 ? null : Math.round(latencySumMs / latencyCount),
|
||||
maxMs: latencyMaxMs,
|
||||
bucketsMs: { ...bucketsMs },
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Bounded, in-process request budget for the photo-analysis endpoint (issue #16).
|
||||
//
|
||||
// The limiter exists so a caller cannot make Timmy the transport for a flood of images to the
|
||||
// model worker. It keeps timestamps only, never the bytes, the file name, or the caller's address:
|
||||
// the server hands it a short-lived opaque bucket key. Refusals never extend a window, so a caller
|
||||
// cannot earn a longer penalty than the request it already made.
|
||||
|
||||
export function createRateLimiter({ limit = 12, windowMs = 60_000, maxKeys = 2048, now = () => Date.now() } = {}) {
|
||||
const hits = new Map();
|
||||
let lastPrune = now();
|
||||
|
||||
function prune(at) {
|
||||
const cutoff = at - windowMs;
|
||||
for (const [key, stamps] of hits) {
|
||||
const kept = stamps.filter(stamp => stamp > cutoff);
|
||||
if (kept.length) hits.set(key, kept);
|
||||
else hits.delete(key);
|
||||
}
|
||||
lastPrune = at;
|
||||
}
|
||||
|
||||
return {
|
||||
take(key) {
|
||||
const at = now();
|
||||
// Pruning is at most once per window, plus whenever the map outgrows its ceiling, so the
|
||||
// bookkeeping stays bounded no matter how many distinct callers appear.
|
||||
if (at - lastPrune >= windowMs || hits.size > maxKeys) prune(at);
|
||||
const window = (hits.get(key) || []).filter(stamp => stamp > at - windowMs);
|
||||
if (window.length >= limit) {
|
||||
hits.set(key, window);
|
||||
return { allowed: false, remaining: 0, retryAfterMs: Math.max(0, window[0] + windowMs - at) };
|
||||
}
|
||||
window.push(at);
|
||||
hits.set(key, window);
|
||||
if (hits.size > maxKeys) {
|
||||
for (const oldest of hits.keys()) {
|
||||
if (hits.size <= maxKeys) break;
|
||||
hits.delete(oldest);
|
||||
}
|
||||
}
|
||||
return { allowed: true, remaining: Math.max(0, limit - window.length), retryAfterMs: 0 };
|
||||
},
|
||||
size() {
|
||||
return hits.size;
|
||||
},
|
||||
reset() {
|
||||
hits.clear();
|
||||
lastPrune = now();
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -84,8 +84,8 @@ export function buildReport({ generatedAt, target, cases, load, suite = 'scripts
|
||||
unproven: verdict.unproven,
|
||||
coveredElsewhere: verdict.coveredElsewhere,
|
||||
proven: verdict.proven,
|
||||
cases: verdict.cases.map(({ id, category, severity, requirement, expected, observed, status, mismatch, snippet, surface, evidence }) =>
|
||||
({ id, category, severity, requirement, expected, observed, status, mismatch, surface, evidence, snippet: snippet ? redactImageBytes(snippet) : undefined })),
|
||||
cases: verdict.cases.map(({ id, category, severity, requirement, expected, observed, status, mismatch, snippet, surface, evidence, evidenceVerified }) =>
|
||||
({ id, category, severity, requirement, expected, observed, status, mismatch, surface, evidence, evidenceVerified, snippet: snippet ? redactImageBytes(snippet) : undefined })),
|
||||
load,
|
||||
disclaimer: 'Synthetic hostile bytes only. Every case drives the real server against a stub worker; no real medical image and no live user data is involved.',
|
||||
};
|
||||
|
||||
@@ -0,0 +1,245 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import http from 'node:http';
|
||||
import net from 'node:net';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { once } from 'node:events';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
// End-to-end proof for issue #16 at the real boundary: the server process that ships, not a mock of
|
||||
// it. Every image here is synthetic container bytes assembled in this file, and the provider is a
|
||||
// local stub that counts requests, so "the provider was never called" is measurable rather than
|
||||
// assumed.
|
||||
|
||||
const repoRoot = fileURLToPath(new URL('..', import.meta.url));
|
||||
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
|
||||
function pngChunk(type, data = Buffer.alloc(0)) {
|
||||
const length = Buffer.alloc(4);
|
||||
length.writeUInt32BE(data.length);
|
||||
return Buffer.concat([length, Buffer.from(type, 'ascii'), data, Buffer.alloc(4)]);
|
||||
}
|
||||
function cleanJpeg({ width = 1200, height = 900, tail = Buffer.alloc(0) } = {}) {
|
||||
const sof = Buffer.alloc(17);
|
||||
sof.writeUInt16BE(0xffc0, 0);
|
||||
sof.writeUInt16BE(15, 2);
|
||||
sof[4] = 8;
|
||||
sof.writeUInt16BE(height, 5);
|
||||
sof.writeUInt16BE(width, 7);
|
||||
sof[9] = 3;
|
||||
for (let i = 0; i < 3; i += 1) {
|
||||
sof[10 + i * 3] = i + 1;
|
||||
sof[11 + i * 3] = 0x11;
|
||||
}
|
||||
return Buffer.concat([Buffer.from([0xff, 0xd8]), sof, Buffer.from([0xff, 0xd9]), tail]);
|
||||
}
|
||||
function polyglotPng() {
|
||||
const ihdr = Buffer.alloc(13);
|
||||
ihdr.writeUInt32BE(1200, 0);
|
||||
ihdr.writeUInt32BE(900, 4);
|
||||
ihdr[8] = 8;
|
||||
ihdr[9] = 2;
|
||||
return Buffer.concat([
|
||||
PNG_SIGNATURE,
|
||||
pngChunk('IHDR', ihdr),
|
||||
pngChunk('IDAT', Buffer.from([0x78, 0x9c, 0x01])),
|
||||
pngChunk('IEND'),
|
||||
Buffer.from('<html><script>alert(1)</script></html>', 'utf8'),
|
||||
]);
|
||||
}
|
||||
|
||||
const dataUrl = (mime, bytes) => `data:${mime};base64,${bytes.toString('base64')}`;
|
||||
|
||||
async function freePort() {
|
||||
const server = net.createServer();
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const { port } = server.address();
|
||||
server.close();
|
||||
return port;
|
||||
}
|
||||
|
||||
async function startProvider() {
|
||||
const state = { calls: 0 };
|
||||
const server = http.createServer(async (req, res) => {
|
||||
if (req.url === '/v1/models') {
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.end(JSON.stringify({ data: [{ id: 'ingress-test-model' }] }));
|
||||
return;
|
||||
}
|
||||
state.calls += 1;
|
||||
for await (const _ of req) { /* drain */ }
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
choices: [{
|
||||
message: {
|
||||
content: JSON.stringify({
|
||||
isStool: true, bristolType: 4, color: 'brown', confidence: 0.8,
|
||||
imageQuality: 'good', observations: 'Synthetic test card.',
|
||||
}),
|
||||
},
|
||||
}],
|
||||
}));
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
return {
|
||||
state,
|
||||
port: server.address().port,
|
||||
// fetch keeps connections alive, so a plain close() would wait for sockets the test already
|
||||
// finished with and hang teardown.
|
||||
close: () => new Promise(resolve => {
|
||||
server.closeAllConnections?.();
|
||||
server.close(resolve);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async function withServer(env, run) {
|
||||
const provider = await startProvider();
|
||||
const port = await freePort();
|
||||
const child = spawn(process.execPath, ['server.mjs'], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
HOST: '127.0.0.1',
|
||||
PORT: String(port),
|
||||
TIMMY_VISION_ENABLED: 'true',
|
||||
TIMMY_VISION_PROFILE: 'selfhost',
|
||||
TIMMY_VISION_BASE_URL: `http://127.0.0.1:${provider.port}/v1`,
|
||||
TIMMY_VISION_MODEL: 'ingress-test-model',
|
||||
...env,
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
const logs = [];
|
||||
child.stdout.on('data', chunk => logs.push(chunk.toString()));
|
||||
child.stderr.on('data', chunk => logs.push(chunk.toString()));
|
||||
const base = `http://127.0.0.1:${port}`;
|
||||
try {
|
||||
// Readiness is the server's own health route, not a sleep.
|
||||
const deadline = Date.now() + 15_000;
|
||||
for (;;) {
|
||||
try {
|
||||
const response = await fetch(`${base}/api/healthz`);
|
||||
if (response.ok) break;
|
||||
} catch { /* not up yet */ }
|
||||
if (Date.now() > deadline) throw new Error(`server did not start: ${logs.join('')}`);
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
await run({ base, provider, logs });
|
||||
} finally {
|
||||
child.kill('SIGKILL');
|
||||
await provider.close();
|
||||
}
|
||||
}
|
||||
|
||||
const postAnalyze = (base, payload) => fetch(`${base}/api/analyze`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
|
||||
test('the shipping server refuses bytes that are not an image before any provider work', async () => {
|
||||
await withServer({}, async ({ base, provider }) => {
|
||||
const response = await postAnalyze(base, {
|
||||
consent: true,
|
||||
imageDataUrl: dataUrl('image/jpeg', Buffer.from('<html><script>alert(1)</script></html>', 'utf8')),
|
||||
});
|
||||
const body = await response.json();
|
||||
assert.equal(response.status, 400);
|
||||
assert.match(body.error, /not a JPEG, PNG, or WebP photo/i);
|
||||
assert.equal(provider.state.calls, 0, 'a non-image must never reach the model provider');
|
||||
});
|
||||
});
|
||||
|
||||
test('the shipping server refuses a valid image with a hidden payload appended', async () => {
|
||||
await withServer({}, async ({ base, provider, logs }) => {
|
||||
const response = await postAnalyze(base, { consent: true, imageDataUrl: dataUrl('image/png', polyglotPng()) });
|
||||
const body = await response.json();
|
||||
assert.equal(response.status, 400);
|
||||
assert.match(body.error, /extra data after the image/i);
|
||||
assert.equal(provider.state.calls, 0, 'a polyglot file must never reach the model provider');
|
||||
// The refusal path must not copy the upload into the process log, where it would outlive the request.
|
||||
const logged = logs.join('');
|
||||
assert.doesNotMatch(logged, /base64/i);
|
||||
assert.doesNotMatch(logged, /[A-Za-z0-9+/]{120,}/, 'no run of image bytes may appear in the log');
|
||||
});
|
||||
});
|
||||
|
||||
test('the shipping server still analyses a clean re-encoded image', async () => {
|
||||
await withServer({}, async ({ base, provider }) => {
|
||||
const response = await postAnalyze(base, { consent: true, imageDataUrl: dataUrl('image/jpeg', cleanJpeg()) });
|
||||
const body = await response.json();
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(body.status, 'suggestion');
|
||||
assert.equal(provider.state.calls, 1, 'the clean path must reach the provider exactly once');
|
||||
});
|
||||
});
|
||||
|
||||
test('the shipping server refuses an oversized body with a refusal the caller can read', async () => {
|
||||
// Issue #16 sets a byte ceiling, and issue #40 asks the release candidate to prove that ceiling
|
||||
// behaves like a refusal rather than a dropped connection. Destroying the socket while the caller
|
||||
// is still uploading leaves the caller with "fetch failed" and no status to act on, so the refusal
|
||||
// has to be written and read before the connection goes away.
|
||||
await withServer({}, async ({ base, provider }) => {
|
||||
const oversized = JSON.stringify({
|
||||
consent: true,
|
||||
imageDataUrl: `data:image/jpeg;base64,${'A'.repeat(6.6 * 1024 * 1024)}`,
|
||||
});
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(`${base}/api/analyze`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: oversized,
|
||||
});
|
||||
} catch (error) {
|
||||
assert.fail(`an oversized body must be refused with a readable status, not a dropped connection: ${error.message}`);
|
||||
}
|
||||
const body = await response.json();
|
||||
assert.equal(response.status, 413);
|
||||
assert.match(body.error, /too large/i);
|
||||
assert.equal(provider.state.calls, 0, 'an oversized body must never reach the provider');
|
||||
});
|
||||
});
|
||||
|
||||
test('the shipping server refuses a flood with 429 and stops calling the provider', async () => {
|
||||
await withServer({ TIMMY_ANALYZE_RATE_LIMIT: '2', TIMMY_ANALYZE_RATE_WINDOW_MS: '60000' }, async ({ base, provider }) => {
|
||||
const payload = { consent: true, imageDataUrl: dataUrl('image/jpeg', cleanJpeg()) };
|
||||
assert.equal((await postAnalyze(base, payload)).status, 200);
|
||||
assert.equal((await postAnalyze(base, payload)).status, 200);
|
||||
const refused = await postAnalyze(base, payload);
|
||||
const body = await refused.json();
|
||||
assert.equal(refused.status, 429);
|
||||
assert.match(body.error, /too many photos|wait a minute/i);
|
||||
assert.ok(Number(refused.headers.get('retry-after')) > 0, 'a refusal must say how long to wait');
|
||||
assert.equal(provider.state.calls, 2, 'a refused request must not reach the provider');
|
||||
});
|
||||
});
|
||||
|
||||
test('the shipping server gives up on a body that never arrives', async () => {
|
||||
await withServer({ TIMMY_BODY_TIMEOUT_MS: '250' }, async ({ base, provider }) => {
|
||||
const port = Number(new URL(base).port);
|
||||
const status = await new Promise((resolve, reject) => {
|
||||
const socket = net.connect(port, '127.0.0.1', () => {
|
||||
socket.write([
|
||||
'POST /api/analyze HTTP/1.1',
|
||||
'Host: 127.0.0.1',
|
||||
'Content-Type: application/json',
|
||||
'Content-Length: 400',
|
||||
'', '',
|
||||
].join('\r\n'));
|
||||
});
|
||||
socket.setTimeout(10_000, () => { socket.destroy(); reject(new Error('no response before the client timeout')); });
|
||||
socket.on('data', chunk => {
|
||||
const line = chunk.toString('utf8').split('\r\n')[0];
|
||||
socket.destroy();
|
||||
resolve(Number(line.split(' ')[1]));
|
||||
});
|
||||
socket.on('error', error => { if (error.code !== 'ECONNRESET') reject(error); });
|
||||
});
|
||||
assert.equal(status, 408);
|
||||
assert.equal(provider.state.calls, 0, 'a stalled body must never reach the provider');
|
||||
});
|
||||
});
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
<!doctype html><html><body>This file is named .jpg but it is not an image. Synthetic fixture for the image-ingress refusal path.</body></html>
|
||||
@@ -0,0 +1,58 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { PassThrough } from 'node:stream';
|
||||
|
||||
import { BodyError, readJsonBody } from '../src/http-body.js';
|
||||
|
||||
const bodyStream = chunks => {
|
||||
const stream = new PassThrough();
|
||||
for (const chunk of chunks) stream.write(chunk);
|
||||
stream.end();
|
||||
return stream;
|
||||
};
|
||||
|
||||
test('reads a bounded JSON request body', async () => {
|
||||
const parsed = await readJsonBody(bodyStream([Buffer.from('{"consent":'), Buffer.from('true}')]), { maxBytes: 1024 });
|
||||
assert.deepEqual(parsed, { consent: true });
|
||||
});
|
||||
|
||||
test('refuses a body above the byte limit instead of buffering it', async () => {
|
||||
await assert.rejects(
|
||||
() => readJsonBody(bodyStream([Buffer.alloc(64, 0x61)]), { maxBytes: 16 }),
|
||||
error => {
|
||||
assert.ok(error instanceof BodyError);
|
||||
assert.equal(error.code, 'too_large');
|
||||
assert.equal(error.status, 413);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('refuses malformed JSON with a client error', async () => {
|
||||
await assert.rejects(
|
||||
() => readJsonBody(bodyStream([Buffer.from('{not json')]), { maxBytes: 1024 }),
|
||||
error => {
|
||||
assert.ok(error instanceof BodyError);
|
||||
assert.equal(error.code, 'invalid_json');
|
||||
assert.equal(error.status, 400);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
test('gives up on a body that never finishes and stops reading it', async () => {
|
||||
const stalled = new PassThrough();
|
||||
stalled.write(Buffer.from('{"consent":true'));
|
||||
const started = Date.now();
|
||||
await assert.rejects(
|
||||
() => readJsonBody(stalled, { maxBytes: 1024, timeoutMs: 40 }),
|
||||
error => {
|
||||
assert.ok(error instanceof BodyError);
|
||||
assert.equal(error.code, 'body_timeout');
|
||||
assert.equal(error.status, 408);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.ok(Date.now() - started < 2_000, 'the deadline must fire promptly');
|
||||
assert.equal(stalled.isPaused(), true, 'the stalled body must stop being read after the deadline');
|
||||
});
|
||||
@@ -0,0 +1,175 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { IngressError, inspectImageBytes, MAX_EDGE, MAX_PIXELS } from '../src/image-ingress.js';
|
||||
|
||||
// Every fixture built here is synthetic container bytes assembled in code. No photograph of any
|
||||
// person or specimen is used, and nothing is fetched.
|
||||
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
|
||||
function pngChunk(type, data = Buffer.alloc(0)) {
|
||||
const length = Buffer.alloc(4);
|
||||
length.writeUInt32BE(data.length);
|
||||
const body = Buffer.concat([Buffer.from(type, 'ascii'), data]);
|
||||
// The parser walks chunk lengths to find the end of the container; the CRC is not read, so a
|
||||
// synthetic zero CRC is enough to be structurally complete.
|
||||
return Buffer.concat([length, body, Buffer.alloc(4)]);
|
||||
}
|
||||
|
||||
function png({ width = 1200, height = 900, extraChunks = [], tail = Buffer.alloc(0) } = {}) {
|
||||
const ihdr = Buffer.alloc(13);
|
||||
ihdr.writeUInt32BE(width, 0);
|
||||
ihdr.writeUInt32BE(height, 4);
|
||||
ihdr[8] = 8; // bit depth
|
||||
ihdr[9] = 2; // truecolour
|
||||
return Buffer.concat([
|
||||
PNG_SIGNATURE,
|
||||
pngChunk('IHDR', ihdr),
|
||||
...extraChunks,
|
||||
pngChunk('IDAT', Buffer.from([0x78, 0x9c, 0x01])),
|
||||
pngChunk('IEND'),
|
||||
tail,
|
||||
]);
|
||||
}
|
||||
|
||||
function jpeg({ width = 1200, height = 900, segments = [], tail = Buffer.alloc(0) } = {}) {
|
||||
const sof = Buffer.alloc(17);
|
||||
sof.writeUInt16BE(0xffc0, 0);
|
||||
sof.writeUInt16BE(15, 2); // payload length
|
||||
sof[4] = 8; // precision
|
||||
sof.writeUInt16BE(height, 5);
|
||||
sof.writeUInt16BE(width, 7);
|
||||
sof[9] = 3; // component count
|
||||
for (let i = 0; i < 3; i += 1) {
|
||||
sof[10 + i * 3] = i + 1;
|
||||
sof[11 + i * 3] = 0x11;
|
||||
sof[12 + i * 3] = 0;
|
||||
}
|
||||
return Buffer.concat([
|
||||
Buffer.from([0xff, 0xd8]),
|
||||
...segments,
|
||||
sof,
|
||||
Buffer.from([0xff, 0xd9]),
|
||||
tail,
|
||||
]);
|
||||
}
|
||||
|
||||
function exifSegment() {
|
||||
const payload = Buffer.concat([
|
||||
Buffer.from('Exif', 'ascii'),
|
||||
Buffer.from([0x00, 0x00]),
|
||||
Buffer.from('GPSLatitude=0.000', 'ascii'),
|
||||
]);
|
||||
const header = Buffer.alloc(4);
|
||||
header.writeUInt16BE(0xffe1, 0);
|
||||
header.writeUInt16BE(payload.length + 2, 2);
|
||||
return Buffer.concat([header, payload]);
|
||||
}
|
||||
|
||||
function webp({ width = 1200, height = 900 } = {}) {
|
||||
// VP8X canvas payload: flags(1) + reserved(3) + width-1(3, LE) + height-1(3, LE).
|
||||
const payload = Buffer.alloc(10);
|
||||
payload[0] = 0;
|
||||
payload.writeUIntLE(width - 1, 4, 3);
|
||||
payload.writeUIntLE(height - 1, 7, 3);
|
||||
const chunk = Buffer.alloc(8);
|
||||
chunk.write('VP8X', 0, 'ascii');
|
||||
chunk.writeUInt32LE(payload.length, 4);
|
||||
const body = Buffer.concat([Buffer.from('WEBP', 'ascii'), chunk, payload]);
|
||||
const riff = Buffer.alloc(8);
|
||||
riff.write('RIFF', 0, 'ascii');
|
||||
riff.writeUInt32LE(body.length, 4);
|
||||
return Buffer.concat([riff, body]);
|
||||
}
|
||||
|
||||
const expectRefusal = (bytes, declaredMime, pattern) => {
|
||||
assert.throws(
|
||||
() => inspectImageBytes(bytes, { declaredMime }),
|
||||
error => {
|
||||
assert.ok(error instanceof IngressError, `expected an IngressError, got ${error?.name}`);
|
||||
assert.match(error.message, pattern);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
test('refuses a file whose bytes are not a JPEG, PNG, or WebP', () => {
|
||||
expectRefusal(Buffer.from('<html><script>alert(1)</script></html>', 'utf8'), 'image/jpeg', /not a JPEG, PNG, or WebP photo/i);
|
||||
});
|
||||
|
||||
test('refuses a PNG that claims to be a JPEG', () => {
|
||||
expectRefusal(png(), 'image/jpeg', /not the format it claims/i);
|
||||
});
|
||||
|
||||
test('refuses a JPEG that claims to be a PNG', () => {
|
||||
expectRefusal(jpeg(), 'image/png', /not the format it claims/i);
|
||||
});
|
||||
|
||||
test('refuses a truncated container instead of forwarding a partial image', () => {
|
||||
expectRefusal(PNG_SIGNATURE.subarray(0, 8), 'image/png', /incomplete or damaged/i);
|
||||
expectRefusal(Buffer.from([0xff, 0xd8, 0xff, 0xe0]), 'image/jpeg', /incomplete or damaged/i);
|
||||
});
|
||||
|
||||
test('refuses a PNG with bytes appended after its end marker', () => {
|
||||
expectRefusal(png({ tail: Buffer.from('PK\u0003\u0004zip-appended', 'binary') }), 'image/png', /extra data after the image/i);
|
||||
});
|
||||
|
||||
test('refuses a JPEG with bytes appended after its end marker', () => {
|
||||
expectRefusal(jpeg({ tail: Buffer.from('<script>alert(1)</script>', 'utf8') }), 'image/jpeg', /extra data after the image/i);
|
||||
});
|
||||
|
||||
test('refuses a JPEG that still carries an EXIF segment', () => {
|
||||
expectRefusal(jpeg({ segments: [exifSegment()] }), 'image/jpeg', /embedded metadata/i);
|
||||
});
|
||||
|
||||
test('refuses a PNG that carries an eXIf chunk', () => {
|
||||
const exifChunk = pngChunk('eXIf', Buffer.from('MM\u0000*metadata', 'binary'));
|
||||
expectRefusal(png({ extraChunks: [exifChunk] }), 'image/png', /embedded metadata/i);
|
||||
});
|
||||
|
||||
test('accepts a clean JPEG and reports its real dimensions', () => {
|
||||
const intake = inspectImageBytes(jpeg({ width: 1200, height: 900 }), { declaredMime: 'image/jpeg' });
|
||||
assert.equal(intake.mime, 'image/jpeg');
|
||||
assert.equal(intake.width, 1200);
|
||||
assert.equal(intake.height, 900);
|
||||
assert.equal(intake.bytes, jpeg({ width: 1200, height: 900 }).length);
|
||||
});
|
||||
|
||||
test('accepts a clean PNG and reports its real dimensions', () => {
|
||||
const intake = inspectImageBytes(png({ width: 800, height: 600 }), { declaredMime: 'image/png' });
|
||||
assert.equal(intake.mime, 'image/png');
|
||||
assert.equal(intake.width, 800);
|
||||
assert.equal(intake.height, 600);
|
||||
});
|
||||
|
||||
test('accepts a clean WebP and reports its canvas dimensions', () => {
|
||||
const intake = inspectImageBytes(webp({ width: 640, height: 480 }), { declaredMime: 'image/webp' });
|
||||
assert.equal(intake.mime, 'image/webp');
|
||||
assert.equal(intake.width, 640);
|
||||
assert.equal(intake.height, 480);
|
||||
});
|
||||
|
||||
test(`refuses an edge longer than ${MAX_EDGE} pixels even when the file is small`, () => {
|
||||
expectRefusal(png({ width: MAX_EDGE + 1, height: 200 }), 'image/png', /too large in pixels/i);
|
||||
});
|
||||
|
||||
test(`refuses an image above ${MAX_PIXELS} pixels when each edge fits`, () => {
|
||||
const edge = 4096;
|
||||
expectRefusal(jpeg({ width: edge, height: Math.ceil(MAX_PIXELS / edge) + 1 }), 'image/jpeg', /too large in pixels/i);
|
||||
});
|
||||
|
||||
test('never repeats image bytes back to the caller in a refusal message', () => {
|
||||
const marker = 'SECRETPIXELMARKER';
|
||||
try {
|
||||
inspectImageBytes(Buffer.from(`not an image ${marker}`, 'utf8'), { declaredMime: 'image/jpeg' });
|
||||
assert.fail('expected a refusal');
|
||||
} catch (error) {
|
||||
assert.ok(error instanceof IngressError);
|
||||
assert.doesNotMatch(error.message, new RegExp(marker));
|
||||
assert.doesNotMatch(error.message, /[A-Za-z0-9+/=]{40,}/);
|
||||
}
|
||||
});
|
||||
|
||||
test('refuses an empty body with a bounded message', () => {
|
||||
expectRefusal(Buffer.alloc(0), 'image/jpeg', /empty/i);
|
||||
});
|
||||
@@ -0,0 +1,70 @@
|
||||
import { chromium } from 'playwright';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
// Issue #16, as a user experiences it: a file that only claims to be a photo never becomes an upload,
|
||||
// never reaches the model worker, and still leaves manual logging one tap away. The second half of
|
||||
// this suite asks the real running server to refuse the same bytes, so the browser copy and the
|
||||
// server boundary are proven in the same run.
|
||||
|
||||
const browser = await chromium.launch({ headless: true });
|
||||
const context = await browser.newContext({
|
||||
viewport: { width: 390, height: 844 },
|
||||
deviceScaleFactor: 2,
|
||||
serviceWorkers: 'block',
|
||||
});
|
||||
const page = await context.newPage();
|
||||
let analysisRequests = 0;
|
||||
const analyzePayloads = [];
|
||||
await page.route('**/api/vision-status', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ enabled: true, profile: 'selfhost', providerReady: true, modelSeen: true, model: 'synthetic-test-model' }),
|
||||
}));
|
||||
await page.route('**/api/analyze', route => {
|
||||
analysisRequests += 1;
|
||||
analyzePayloads.push(route.request().postData() || '');
|
||||
return route.continue();
|
||||
});
|
||||
|
||||
await page.goto('http://127.0.0.1:4173', { waitUntil: 'networkidle' });
|
||||
await page.locator('[data-scan]').click();
|
||||
|
||||
const spoofed = Buffer.from('<html><script>alert(1)</script></html>', 'utf8');
|
||||
await page.locator('#gallery-photo').setInputFiles({ name: 'photo.jpg', mimeType: 'image/jpeg', buffer: spoofed });
|
||||
|
||||
const refusal = page.locator('[data-capture-error="unreadable"]');
|
||||
await refusal.waitFor();
|
||||
const copy = await refusal.innerText();
|
||||
assert.match(copy, /isn.t a photo Timmy can read/i, 'the refusal names what is wrong with the file');
|
||||
assert.match(copy, /Nothing was uploaded/i, 'the refusal states that nothing left the device');
|
||||
assert.equal(await page.getByText(/couldn.t analyze that safely/i).count(), 0,
|
||||
'an unreadable file must not be reported as a failed analysis');
|
||||
assert.equal(analysisRequests, 0, 'an unreadable file must never be posted to /api/analyze');
|
||||
assert.equal(await page.locator('#manual-from-scan').isVisible(), true, 'manual entry stays one tap away');
|
||||
assert.equal(await page.locator('#retake-photo').isVisible(), true, 'the picker stays reachable');
|
||||
|
||||
// The same bytes, offered to the running server directly, are refused before any provider work.
|
||||
const serverResponse = await page.evaluate(async payload => {
|
||||
const response = await fetch(`${document.querySelector('meta[name="timmy-base-path"]')?.content.replace(/\/$/, '') || ''}/api/analyze`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
return { status: response.status, body: await response.json() };
|
||||
}, { consent: true, imageDataUrl: `data:image/jpeg;base64,${spoofed.toString('base64')}` });
|
||||
assert.equal(serverResponse.status, 400, `the server must refuse non-image bytes, saw ${serverResponse.status}`);
|
||||
assert.match(serverResponse.body.error, /not a JPEG, PNG, or WebP photo/i);
|
||||
// Only this deliberate probe reached the endpoint; the app itself never posted the unreadable file.
|
||||
assert.equal(analysisRequests, 1, 'only the deliberate server probe may reach /api/analyze so far');
|
||||
assert.match(analyzePayloads[0], /"consent":true/);
|
||||
|
||||
// A real (synthetic) photo still reaches the consent step, so the gate is a boundary and not a wall.
|
||||
await page.locator('#retake-photo').click();
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||||
await page.locator('#ai-consent').waitFor();
|
||||
assert.equal(await page.locator('[data-capture-error="unreadable"]').count(), 0);
|
||||
assert.equal(analysisRequests, 1, 'a photo that is only selected is still not uploaded before consent');
|
||||
|
||||
await context.close();
|
||||
await browser.close();
|
||||
console.log('PASS unreadable photo bytes are refused on device and at the server, with manual entry intact');
|
||||
@@ -0,0 +1,181 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import http from 'node:http';
|
||||
import net from 'node:net';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { once } from 'node:events';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
// Issue #19 at the real boundary: the server that ships, behind a provider stub this test owns, so
|
||||
// "the metric reflects a real request" and "the metric never contains the request" are both
|
||||
// measurable instead of asserted from a mock.
|
||||
|
||||
const repoRoot = fileURLToPath(new URL('..', import.meta.url));
|
||||
const OPERATOR_TOKEN = 'test-operator-token-9f2a';
|
||||
|
||||
function cleanJpeg({ width = 1200, height = 900 } = {}) {
|
||||
const sof = Buffer.alloc(17);
|
||||
sof.writeUInt16BE(0xffc0, 0);
|
||||
sof.writeUInt16BE(15, 2);
|
||||
sof[4] = 8;
|
||||
sof.writeUInt16BE(height, 5);
|
||||
sof.writeUInt16BE(width, 7);
|
||||
sof[9] = 3;
|
||||
for (let i = 0; i < 3; i += 1) {
|
||||
sof[10 + i * 3] = i + 1;
|
||||
sof[11 + i * 3] = 0x11;
|
||||
}
|
||||
return Buffer.concat([Buffer.from([0xff, 0xd8]), sof, Buffer.from([0xff, 0xd9])]);
|
||||
}
|
||||
|
||||
const dataUrl = (mime, bytes) => `data:${mime};base64,${bytes.toString('base64')}`;
|
||||
|
||||
async function freePort() {
|
||||
const server = net.createServer();
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const { port } = server.address();
|
||||
server.close();
|
||||
return port;
|
||||
}
|
||||
|
||||
async function startProvider() {
|
||||
const state = { calls: 0 };
|
||||
const server = http.createServer(async (req, res) => {
|
||||
if (req.url === '/v1/models') {
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.end(JSON.stringify({ data: [{ id: 'metrics-test-model' }] }));
|
||||
return;
|
||||
}
|
||||
state.calls += 1;
|
||||
for await (const _ of req) { /* drain */ }
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
choices: [{
|
||||
message: {
|
||||
content: JSON.stringify({
|
||||
isStool: true, bristolType: 4, color: 'brown', confidence: 0.8,
|
||||
imageQuality: 'good', observations: 'Synthetic test card.',
|
||||
}),
|
||||
},
|
||||
}],
|
||||
}));
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
return {
|
||||
state,
|
||||
port: server.address().port,
|
||||
close: () => new Promise(resolve => {
|
||||
server.closeAllConnections?.();
|
||||
server.close(resolve);
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
async function withServer(env, run) {
|
||||
const provider = await startProvider();
|
||||
const port = await freePort();
|
||||
const child = spawn(process.execPath, ['server.mjs'], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
HOST: '127.0.0.1',
|
||||
PORT: String(port),
|
||||
TIMMY_VISION_ENABLED: 'true',
|
||||
TIMMY_VISION_PROFILE: 'selfhost',
|
||||
TIMMY_VISION_BASE_URL: `http://127.0.0.1:${provider.port}/v1`,
|
||||
TIMMY_VISION_MODEL: 'metrics-test-model',
|
||||
...env,
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
const logs = [];
|
||||
child.stdout.on('data', chunk => logs.push(chunk.toString()));
|
||||
child.stderr.on('data', chunk => logs.push(chunk.toString()));
|
||||
const base = `http://127.0.0.1:${port}`;
|
||||
try {
|
||||
const deadline = Date.now() + 15_000;
|
||||
for (;;) {
|
||||
try {
|
||||
const response = await fetch(`${base}/api/healthz`);
|
||||
if (response.ok) break;
|
||||
} catch { /* not up yet */ }
|
||||
if (Date.now() > deadline) throw new Error(`server did not start: ${logs.join('')}`);
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
await run({ base, provider, logs });
|
||||
} finally {
|
||||
child.kill('SIGKILL');
|
||||
await provider.close();
|
||||
}
|
||||
}
|
||||
|
||||
const postAnalyze = (base, payload) => fetch(`${base}/api/analyze`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const getMetrics = (base, token = OPERATOR_TOKEN) => fetch(`${base}/api/metrics`, {
|
||||
headers: token ? { authorization: `Bearer ${token}` } : {},
|
||||
});
|
||||
|
||||
test('the metrics snapshot reports the request the server actually analysed', async () => {
|
||||
await withServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN }, async ({ base, provider }) => {
|
||||
const analyzed = await postAnalyze(base, { consent: true, imageDataUrl: dataUrl('image/jpeg', cleanJpeg()) });
|
||||
assert.equal(analyzed.status, 200);
|
||||
assert.equal(provider.state.calls, 1);
|
||||
|
||||
const response = await getMetrics(base);
|
||||
const snapshot = await response.json();
|
||||
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(snapshot.app.analyzeRequests, 1);
|
||||
assert.equal(snapshot.queue.accepted, 1);
|
||||
assert.equal(snapshot.queue.inFlight, 0, 'a finished request must release its slot');
|
||||
assert.equal(snapshot.model.outcomes.suggestion, 1);
|
||||
assert.equal(snapshot.model.latency.count, 1);
|
||||
assert.equal(snapshot.model.abstentionRate, 0);
|
||||
assert.equal(snapshot.model.readiness.ready, true, 'readiness is probed from the configured worker');
|
||||
});
|
||||
});
|
||||
|
||||
test('the metrics route is operator-only, and absent when no token is configured', async () => {
|
||||
await withServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN }, async ({ base }) => {
|
||||
const wrong = await getMetrics(base, 'not-the-operator-token');
|
||||
assert.equal(wrong.status, 403);
|
||||
assert.doesNotMatch(await wrong.text(), /analyzeRequests/, 'a rejected caller learns nothing about traffic');
|
||||
|
||||
const anonymous = await getMetrics(base, '');
|
||||
assert.equal(anonymous.status, 403);
|
||||
});
|
||||
|
||||
await withServer({}, async ({ base }) => {
|
||||
const response = await getMetrics(base, '');
|
||||
assert.equal(response.status, 404, 'an unconfigured deployment exposes no metrics route at all');
|
||||
});
|
||||
});
|
||||
|
||||
test('the snapshot proves the privacy boundary from the outside', async () => {
|
||||
await withServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN }, async ({ base }) => {
|
||||
// A real synthetic photo goes through the real path, so any leak would have real bytes to leak.
|
||||
const bytes = cleanJpeg();
|
||||
const analyzed = await postAnalyze(base, {
|
||||
consent: true,
|
||||
note: 'fictional note about a very private symptom',
|
||||
imageDataUrl: dataUrl('image/jpeg', bytes),
|
||||
});
|
||||
assert.equal(analyzed.status, 200);
|
||||
|
||||
const snapshot = await getMetrics(base).then(response => response.json());
|
||||
const serialized = JSON.stringify(snapshot);
|
||||
|
||||
assert.doesNotMatch(serialized, /very private symptom/i, 'note text must never be a metric');
|
||||
const base64Run = bytes.toString('base64').slice(0, 24);
|
||||
assert.ok(base64Run.length === 24, 'sanity: the synthetic image has bytes to leak');
|
||||
assert.doesNotMatch(serialized, new RegExp(base64Run.replace(/[+/=]/g, '\\$&')), 'image bytes must never be a metric');
|
||||
assert.doesNotMatch(serialized, /data:image/, 'no data URL may appear in the snapshot');
|
||||
assert.doesNotMatch(serialized, /127\.0\.0\.1|::1/, 'no caller address may appear in the snapshot');
|
||||
assert.doesNotMatch(serialized, /image|note|photo/i, 'no request-derived key may appear in the snapshot');
|
||||
});
|
||||
});
|
||||
@@ -47,8 +47,11 @@ for (const viewport of viewports) {
|
||||
mimeType: 'image/jpeg',
|
||||
buffer: Buffer.from('not an image'),
|
||||
});
|
||||
assert.match(await page.locator('.scan-result').innerText(), /could not be read/i);
|
||||
await page.getByText('Try another photo', { exact: true }).click();
|
||||
assert.match(await page.locator('.scan-result').innerText(), /isn.t a photo Timmy can read/i);
|
||||
// Issue #16: the refusal is its own state and states plainly that nothing left the device, instead
|
||||
// of borrowing the "couldn't analyze that safely" copy from a failed upload.
|
||||
assert.match(await page.locator('.scan-result').innerText(), /Nothing was uploaded/i);
|
||||
await page.getByText('Choose another photo', { exact: true }).click();
|
||||
await page.locator('#gallery-photo').setInputFiles('tests/fixtures/synthetic-type4.jpg');
|
||||
assert.equal(await page.locator('#analyze-photo').isDisabled(), true);
|
||||
assert.equal(await page.locator('#retake-photo').isVisible(), true);
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { formatOperatorReport } from '../src/model-metrics-report.js';
|
||||
import { createModelMetrics } from '../src/model-metrics.js';
|
||||
|
||||
function snapshot(overrides = {}) {
|
||||
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') });
|
||||
metrics.recordReadiness({ phase: 'ready', ready: true });
|
||||
return { ...metrics.snapshot(), ...overrides };
|
||||
}
|
||||
|
||||
test('the report names the app, queue, and model boundaries in that order', () => {
|
||||
const report = formatOperatorReport(snapshot());
|
||||
|
||||
assert.match(report, /^APP$/m);
|
||||
assert.match(report, /^QUEUE$/m);
|
||||
assert.match(report, /^MODEL$/m);
|
||||
assert.ok(report.indexOf('APP') < report.indexOf('QUEUE'), 'app boundary comes before queue');
|
||||
assert.ok(report.indexOf('QUEUE') < report.indexOf('MODEL'), 'queue boundary comes before model');
|
||||
});
|
||||
|
||||
test('the report prints the readiness phase the worker actually reported', () => {
|
||||
const report = formatOperatorReport(snapshot({
|
||||
model: { ...snapshot().model, readiness: { phase: 'loading', ready: false, coldLoadMs: null } },
|
||||
}));
|
||||
|
||||
assert.match(report, /readiness\s+loading/);
|
||||
// A loading worker is not ready, and the report must not imply AI is available.
|
||||
assert.doesNotMatch(report, /readiness\s+ready/);
|
||||
});
|
||||
|
||||
test('an abstention rate with no answers is absence, never zero percent', () => {
|
||||
const report = formatOperatorReport(snapshot());
|
||||
|
||||
assert.match(report, /abstention\s+no answers yet/i);
|
||||
assert.doesNotMatch(report, /abstention\s+0(\.0)?%/);
|
||||
});
|
||||
|
||||
test('an abstention rate is rendered from answers only, so failures cannot dilute it', () => {
|
||||
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') });
|
||||
metrics.recordReadiness({ phase: 'ready', ready: true });
|
||||
metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 });
|
||||
metrics.recordAnalyze({ outcome: 'abstained', durationMs: 1200 });
|
||||
metrics.recordAnalyze({ outcome: 'failure', failureClass: 'timeout', durationMs: 30_000 });
|
||||
|
||||
const report = formatOperatorReport(metrics.snapshot());
|
||||
|
||||
assert.match(report, /abstention\s+50\.0%/);
|
||||
assert.match(report, /failure\s+timeout\s+1|timeout 1/i);
|
||||
});
|
||||
|
||||
test('the report renders only the enumerated failure vocabulary, never a caller-supplied label', () => {
|
||||
const base = snapshot();
|
||||
const report = formatOperatorReport({
|
||||
...base,
|
||||
model: {
|
||||
...base.model,
|
||||
failureClasses: {
|
||||
...base.model.failureClasses,
|
||||
'note: watery stool at 06:40': 3,
|
||||
'user@example.com': 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.doesNotMatch(report, /watery stool/);
|
||||
assert.doesNotMatch(report, /user@example\.com/);
|
||||
// The closed set is still fully accounted for, so the extra keys are visibly absent rather than
|
||||
// silently folded into a bucket the operator would read as something else.
|
||||
assert.match(report, /unexpected\s+0/);
|
||||
});
|
||||
|
||||
test('the report shows queue occupancy against the configured ceiling', () => {
|
||||
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z'), limit: 3 });
|
||||
metrics.enter();
|
||||
metrics.enter();
|
||||
|
||||
const report = formatOperatorReport(metrics.snapshot());
|
||||
|
||||
assert.match(report, /in flight\s+2 \/ 3/);
|
||||
assert.match(report, /at capacity\s+no/);
|
||||
});
|
||||
|
||||
test('every row keeps a visible gap between its label and its count', () => {
|
||||
const metrics = createModelMetrics({ now: () => Date.parse('2026-11-02T10:00:00.000Z') });
|
||||
// The longest label is the one that used to run together with its value: a column that exactly fits
|
||||
// the text reads as `failure ingress_refused0`, one broken token instead of a metric and its count.
|
||||
metrics.recordAnalyze({ outcome: 'failure', failureClass: 'ingress_refused', durationMs: 5 });
|
||||
const report = formatOperatorReport(metrics.snapshot());
|
||||
|
||||
for (const line of report.split('\n')) {
|
||||
if (!/^ {2}\S/.test(line)) continue;
|
||||
assert.match(line, /^ {2}\S.*?\s{2,}\S/, `row has no gap between label and value: ${JSON.stringify(line)}`);
|
||||
}
|
||||
assert.match(report, /failure ingress_refused\s+1/);
|
||||
assert.match(report, /failure provider_unreachable\s+0/);
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import http from 'node:http';
|
||||
import net from 'node:net';
|
||||
import { spawn, execFile } from 'node:child_process';
|
||||
import { once } from 'node:events';
|
||||
import { promisify } from 'node:util';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const repoRoot = fileURLToPath(new URL('..', import.meta.url));
|
||||
const OPERATOR_TOKEN = 'report-operator-token-4b1c';
|
||||
|
||||
async function freePort() {
|
||||
const server = net.createServer();
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
const { port } = server.address();
|
||||
server.close();
|
||||
return port;
|
||||
}
|
||||
|
||||
async function startProvider() {
|
||||
const server = http.createServer(async (req, res) => {
|
||||
if (req.url === '/v1/models') {
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.end(JSON.stringify({ data: [{ id: 'report-model' }] }));
|
||||
return;
|
||||
}
|
||||
for await (const _ of req) { /* drain */ }
|
||||
res.writeHead(200, { 'content-type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
choices: [{ message: { content: JSON.stringify({
|
||||
isStool: true, bristolType: 4, color: 'brown', confidence: 0.8,
|
||||
imageQuality: 'good', observations: 'Synthetic test card.',
|
||||
}) } }],
|
||||
}));
|
||||
});
|
||||
server.listen(0, '127.0.0.1');
|
||||
await once(server, 'listening');
|
||||
return { port: server.address().port, close: () => new Promise(resolve => { server.closeAllConnections?.(); server.close(resolve); }) };
|
||||
}
|
||||
|
||||
test('the operator report renders the app, queue, and model boundaries from a live snapshot', async () => {
|
||||
const provider = await startProvider();
|
||||
const port = await freePort();
|
||||
const child = spawn(process.execPath, ['server.mjs'], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
HOST: '127.0.0.1',
|
||||
PORT: String(port),
|
||||
TIMMY_VISION_ENABLED: 'true',
|
||||
TIMMY_VISION_PROFILE: 'selfhost',
|
||||
TIMMY_VISION_BASE_URL: `http://127.0.0.1:${provider.port}/v1`,
|
||||
TIMMY_VISION_MODEL: 'report-model',
|
||||
TIMMY_METRICS_TOKEN: OPERATOR_TOKEN,
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
const logs = [];
|
||||
child.stdout.on('data', c => logs.push(c.toString()));
|
||||
child.stderr.on('data', c => logs.push(c.toString()));
|
||||
const base = `http://127.0.0.1:${port}`;
|
||||
try {
|
||||
const deadline = Date.now() + 15_000;
|
||||
for (;;) {
|
||||
try {
|
||||
const response = await fetch(`${base}/api/healthz`);
|
||||
if (response.ok) break;
|
||||
} catch { /* not up yet */ }
|
||||
if (Date.now() > deadline) throw new Error(`server did not start: ${logs.join('')}`);
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
|
||||
const { stdout } = await execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', '--url', base, '--token', OPERATOR_TOKEN], { cwd: repoRoot });
|
||||
|
||||
assert.match(stdout, /APP/);
|
||||
assert.match(stdout, /QUEUE/);
|
||||
assert.match(stdout, /MODEL/);
|
||||
assert.match(stdout, /readiness\s+ready/, 'the report must name the readiness phase it observed');
|
||||
assert.match(stdout, /abstention/i);
|
||||
assert.match(stdout, /failure/i);
|
||||
assert.doesNotMatch(stdout, /OPERATOR_TOKEN|report-operator-token/, 'the report must never print the token');
|
||||
} finally {
|
||||
child.kill('SIGKILL');
|
||||
await provider.close();
|
||||
}
|
||||
});
|
||||
|
||||
async function startServer(env) {
|
||||
const port = await freePort();
|
||||
const child = spawn(process.execPath, ['server.mjs'], {
|
||||
cwd: repoRoot,
|
||||
env: { ...process.env, HOST: '127.0.0.1', PORT: String(port), ...env },
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
const base = `http://127.0.0.1:${port}`;
|
||||
const deadline = Date.now() + 15_000;
|
||||
for (;;) {
|
||||
try {
|
||||
const response = await fetch(`${base}/api/healthz`);
|
||||
if (response.ok) break;
|
||||
} catch { /* not up yet */ }
|
||||
if (Date.now() > deadline) throw new Error('server did not start');
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
return { base, stop: () => child.kill('SIGKILL') };
|
||||
}
|
||||
|
||||
test('the report refuses a missing or wrong operator token without echoing it', async () => {
|
||||
const tokenless = await startServer({ TIMMY_METRICS_ENABLED: undefined, TIMMY_METRICS_TOKEN: '' });
|
||||
try {
|
||||
await assert.rejects(
|
||||
execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', '--url', tokenless.base, '--token', 'x'], { cwd: repoRoot }),
|
||||
error => {
|
||||
assert.notEqual(error.code, 0);
|
||||
assert.match(error.stderr, /disabled/i, 'a server with no metrics route must say so, not "unauthorized"');
|
||||
return true;
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
tokenless.stop();
|
||||
}
|
||||
|
||||
const guarded = await startServer({ TIMMY_METRICS_TOKEN: OPERATOR_TOKEN });
|
||||
try {
|
||||
await assert.rejects(
|
||||
execFileAsync(process.execPath, ['scripts/model_metrics_report.mjs', '--url', guarded.base, '--token', 'not-the-token'], { cwd: repoRoot }),
|
||||
error => {
|
||||
assert.notEqual(error.code, 0);
|
||||
assert.match(error.stderr, /rejected/i);
|
||||
assert.doesNotMatch(`${error.stderr}${error.stdout}`, /not-the-token/, 'the rejected token must never be echoed');
|
||||
return true;
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
guarded.stop();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,151 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { createModelMetrics } from '../src/model-metrics.js';
|
||||
|
||||
// Issue #19: the worker has to be operable without the metrics becoming a second copy of the
|
||||
// journal. These tests are the boundary: what a metric sample may say, and what it may never say.
|
||||
|
||||
test('abstention rate is computed from the model answers that came back at all', async () => {
|
||||
const metrics = createModelMetrics({ now: () => 1_000 });
|
||||
|
||||
metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 });
|
||||
metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 1_100 });
|
||||
metrics.recordAnalyze({ outcome: 'abstained', durationMs: 700 });
|
||||
metrics.recordAnalyze({ outcome: 'abstained', durationMs: 800 });
|
||||
|
||||
const snapshot = metrics.snapshot();
|
||||
|
||||
assert.equal(snapshot.model.outcomes.suggestion, 2);
|
||||
assert.equal(snapshot.model.outcomes.abstained, 2);
|
||||
assert.equal(snapshot.model.abstentionRate, 0.5);
|
||||
});
|
||||
|
||||
test('a label that is not an enumerated outcome is counted, never copied into the snapshot', async () => {
|
||||
const metrics = createModelMetrics({ now: () => 1_000 });
|
||||
|
||||
// The worst case for a metrics boundary: a caller-supplied string arrives where a label belongs.
|
||||
metrics.recordAnalyze({ outcome: 'Type 4, urgent, 2026-11-01, milkY4B2cQ9w==', durationMs: 900 });
|
||||
|
||||
const snapshot = metrics.snapshot();
|
||||
const serialized = JSON.stringify(snapshot);
|
||||
|
||||
assert.deepEqual(Object.keys(snapshot.model.outcomes).sort(), ['abstained', 'failure', 'suggestion']);
|
||||
assert.doesNotMatch(serialized, /urgent/i);
|
||||
assert.doesNotMatch(serialized, /milkY4B2cQ9w/);
|
||||
assert.equal(snapshot.model.outcomes.failure, 1, 'an unrecognised outcome is one failed answer');
|
||||
});
|
||||
|
||||
test('a failed request is classified and kept out of the abstention denominator', async () => {
|
||||
const metrics = createModelMetrics({ now: () => 1_000 });
|
||||
|
||||
metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 });
|
||||
metrics.recordAnalyze({ outcome: 'failure', failureClass: 'provider_unreachable', durationMs: 2_000 });
|
||||
metrics.recordAnalyze({ outcome: 'failure', failureClass: 'provider_unreachable', durationMs: 2_100 });
|
||||
metrics.recordAnalyze({ outcome: 'abstained', durationMs: 800 });
|
||||
|
||||
const snapshot = metrics.snapshot();
|
||||
|
||||
assert.equal(snapshot.model.failureClasses.provider_unreachable, 2);
|
||||
assert.equal(snapshot.model.abstentionRate, 0.5, 'a failed answer is not an abstention');
|
||||
});
|
||||
|
||||
test('a failure class outside the enumerated set collapses instead of being echoed', async () => {
|
||||
const metrics = createModelMetrics({ now: () => 1_000 });
|
||||
|
||||
metrics.recordAnalyze({ outcome: 'failure', failureClass: 'note:blood in stool, image 9f2a', durationMs: 2_000 });
|
||||
|
||||
const snapshot = metrics.snapshot();
|
||||
|
||||
assert.equal(snapshot.model.failureClasses.unexpected, 1);
|
||||
assert.doesNotMatch(JSON.stringify(snapshot), /blood in stool/);
|
||||
});
|
||||
|
||||
test('analysis latency is reported as warm work, with counts and a bounded histogram', async () => {
|
||||
const metrics = createModelMetrics({ now: () => 1_000 });
|
||||
|
||||
metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 900 });
|
||||
metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 1_500 });
|
||||
metrics.recordAnalyze({ outcome: 'abstained', durationMs: 6_000 });
|
||||
// A refused request never waited on the model, so it must not be sampled as latency.
|
||||
metrics.recordAnalyze({ outcome: 'failure', failureClass: 'ingress_refused', durationMs: 5 });
|
||||
|
||||
const latency = metrics.snapshot().model.latency;
|
||||
|
||||
assert.equal(latency.count, 3, 'latency counts model work only');
|
||||
assert.equal(latency.maxMs, 6_000);
|
||||
assert.equal(latency.meanMs, 2_800);
|
||||
// Fixed bucket edges keep the snapshot a histogram of numbers, not a list of timings.
|
||||
assert.deepEqual(latency.bucketsMs, { '1000': 1, '2000': 2, '5000': 2, '15000': 3, '60000': 3 });
|
||||
});
|
||||
|
||||
test('queue depth is the in-flight model work, and a released slot is never counted twice', async () => {
|
||||
const metrics = createModelMetrics({ now: () => 1_000, limit: 2 });
|
||||
|
||||
const first = metrics.enter();
|
||||
const second = metrics.enter();
|
||||
assert.equal(metrics.snapshot().queue.inFlight, 2);
|
||||
assert.equal(metrics.snapshot().queue.atCapacity, true);
|
||||
|
||||
metrics.leave(first);
|
||||
metrics.leave(first); // a double release must not free a slot that is still in use
|
||||
const queue = metrics.snapshot().queue;
|
||||
|
||||
assert.equal(queue.inFlight, 1, 'the second request still holds its slot');
|
||||
assert.equal(queue.atCapacity, false);
|
||||
assert.equal(queue.accepted, 2);
|
||||
|
||||
metrics.leave(second);
|
||||
assert.equal(metrics.snapshot().queue.inFlight, 0);
|
||||
assert.equal(metrics.snapshot().queue.accepted, 2, 'a completed request is still an accepted request');
|
||||
});
|
||||
|
||||
test('readiness is reported as a phase, and the cold model load is measured once', async () => {
|
||||
let clock = 10_000;
|
||||
const metrics = createModelMetrics({ now: () => clock });
|
||||
|
||||
metrics.recordReadiness({ phase: 'loading', ready: false });
|
||||
clock = 74_000; // 64s of weights loading
|
||||
metrics.recordReadiness({ phase: 'ready', ready: true });
|
||||
clock = 90_000;
|
||||
metrics.recordReadiness({ phase: 'offline', ready: false });
|
||||
|
||||
const readiness = metrics.snapshot().model.readiness;
|
||||
|
||||
assert.equal(readiness.phase, 'offline');
|
||||
assert.equal(readiness.ready, false);
|
||||
assert.equal(readiness.coldLoadMs, 64_000, 'the cold load is the first time the model became resident');
|
||||
});
|
||||
|
||||
test('a readiness phase outside the enumerated set is reported as checking', async () => {
|
||||
const metrics = createModelMetrics({ now: () => 1_000 });
|
||||
|
||||
metrics.recordReadiness({ phase: 'user says: bloody stool', ready: true });
|
||||
|
||||
const snapshot = metrics.snapshot();
|
||||
assert.equal(snapshot.model.readiness.phase, 'checking');
|
||||
assert.doesNotMatch(JSON.stringify(snapshot), /bloody stool/);
|
||||
});
|
||||
|
||||
test('the snapshot keeps the app, queue, and model boundaries separate', async () => {
|
||||
let clock = 1_000;
|
||||
const metrics = createModelMetrics({ now: () => clock, limit: 2 });
|
||||
|
||||
// One request is refused at the app boundary and never takes a model slot; the other is admitted,
|
||||
// answered, and released. Each of the three boundaries should account for exactly that.
|
||||
const slot = metrics.enter();
|
||||
metrics.recordAnalyze({ outcome: 'suggestion', durationMs: 800 });
|
||||
metrics.leave(slot);
|
||||
metrics.recordAnalyze({ outcome: 'failure', failureClass: 'rate_limited', durationMs: 0 });
|
||||
clock = 41_000;
|
||||
|
||||
const snapshot = metrics.snapshot();
|
||||
|
||||
assert.deepEqual(Object.keys(snapshot).sort(), ['app', 'generatedAt', 'model', 'queue', 'schema']);
|
||||
assert.equal(snapshot.app.uptimeMs, 40_000);
|
||||
assert.equal(snapshot.app.analyzeRequests, 2, 'the app boundary counts every request it saw');
|
||||
assert.equal(snapshot.app.refusedBeforeModel, 1, 'a refusal that never reached the queue is an app fact');
|
||||
assert.equal(snapshot.queue.accepted, 1, 'the queue boundary counts what it admitted, not what it answered');
|
||||
assert.equal(snapshot.queue.inFlight, 0, 'a released slot is not in flight');
|
||||
assert.equal(snapshot.model.outcomes.suggestion, 1, 'the model boundary counts answers only');
|
||||
});
|
||||
@@ -0,0 +1,91 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { createRateLimiter } from '../src/rate-limit.js';
|
||||
|
||||
// A clock the test owns, so no test sleeps and every window boundary is exact.
|
||||
function manualClock(start = 1_700_000_000_000) {
|
||||
let current = start;
|
||||
return { now: () => current, advance: ms => { current += ms; } };
|
||||
}
|
||||
|
||||
test('allows requests up to the limit and refuses the next one', () => {
|
||||
const clock = manualClock();
|
||||
const limiter = createRateLimiter({ limit: 3, windowMs: 60_000, now: clock.now });
|
||||
|
||||
for (let attempt = 1; attempt <= 3; attempt += 1) {
|
||||
const verdict = limiter.take('device-a');
|
||||
assert.equal(verdict.allowed, true, `attempt ${attempt} must be allowed`);
|
||||
assert.equal(verdict.remaining, 3 - attempt);
|
||||
}
|
||||
|
||||
const refused = limiter.take('device-a');
|
||||
assert.equal(refused.allowed, false);
|
||||
assert.equal(refused.remaining, 0);
|
||||
assert.equal(refused.retryAfterMs, 60_000);
|
||||
});
|
||||
|
||||
test('counts each caller on its own budget', () => {
|
||||
const clock = manualClock();
|
||||
const limiter = createRateLimiter({ limit: 1, windowMs: 60_000, now: clock.now });
|
||||
|
||||
assert.equal(limiter.take('device-a').allowed, true);
|
||||
assert.equal(limiter.take('device-a').allowed, false);
|
||||
assert.equal(limiter.take('device-b').allowed, true);
|
||||
});
|
||||
|
||||
test('lets a caller back in once its window has passed', () => {
|
||||
const clock = manualClock();
|
||||
const limiter = createRateLimiter({ limit: 1, windowMs: 60_000, now: clock.now });
|
||||
|
||||
assert.equal(limiter.take('device-a').allowed, true);
|
||||
assert.equal(limiter.take('device-a').allowed, false);
|
||||
clock.advance(60_001);
|
||||
const allowedAgain = limiter.take('device-a');
|
||||
assert.equal(allowedAgain.allowed, true);
|
||||
assert.equal(allowedAgain.remaining, 0);
|
||||
});
|
||||
|
||||
test('a refused request does not extend the block', () => {
|
||||
const clock = manualClock();
|
||||
const limiter = createRateLimiter({ limit: 1, windowMs: 1_000, now: clock.now });
|
||||
|
||||
assert.equal(limiter.take('device-a').allowed, true);
|
||||
clock.advance(400);
|
||||
assert.equal(limiter.take('device-a').allowed, false);
|
||||
clock.advance(601);
|
||||
// The original hit is now 1001ms old, so the window has moved on even though the caller hammered
|
||||
// the endpoint in between. A refusal must never buy a longer penalty than the caller earned.
|
||||
assert.equal(limiter.take('device-a').allowed, true);
|
||||
});
|
||||
|
||||
test('reports the wait from the oldest hit still inside the window', () => {
|
||||
const clock = manualClock();
|
||||
const limiter = createRateLimiter({ limit: 2, windowMs: 10_000, now: clock.now });
|
||||
|
||||
limiter.take('device-a');
|
||||
clock.advance(4_000);
|
||||
limiter.take('device-a');
|
||||
clock.advance(1_000);
|
||||
const refused = limiter.take('device-a');
|
||||
assert.equal(refused.allowed, false);
|
||||
assert.equal(refused.retryAfterMs, 5_000);
|
||||
});
|
||||
|
||||
test('keeps its caller map bounded when many callers appear', () => {
|
||||
const clock = manualClock();
|
||||
const limiter = createRateLimiter({ limit: 5, windowMs: 60_000, maxKeys: 8, now: clock.now });
|
||||
|
||||
for (let index = 0; index < 50; index += 1) limiter.take(`device-${index}`);
|
||||
assert.ok(limiter.size() <= 8, `expected at most 8 tracked callers, saw ${limiter.size()}`);
|
||||
});
|
||||
|
||||
test('prunes expired callers instead of growing for the life of the process', () => {
|
||||
const clock = manualClock();
|
||||
const limiter = createRateLimiter({ limit: 5, windowMs: 1_000, now: clock.now });
|
||||
|
||||
for (let index = 0; index < 20; index += 1) limiter.take(`device-${index}`);
|
||||
clock.advance(1_001);
|
||||
limiter.take('device-fresh');
|
||||
assert.equal(limiter.size(), 1);
|
||||
});
|
||||
@@ -113,6 +113,23 @@ test('release demo visibly exercises the red-flag language coverage instead of o
|
||||
assert.match(builder, /A tarry note already saved in the journal escalates the same way, before any provider call/);
|
||||
});
|
||||
|
||||
test('release demo shows a file that only claims to be a photo being refused before upload', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
|
||||
// Issue #16: the recording must show the device-side refusal of a non-image, prove nothing was
|
||||
// posted, and keep manual entry reachable — all asserted in the browser, not narrated.
|
||||
assert.match(demo, /tests\/fixtures\/not-a-photo\.jpg/);
|
||||
assert.match(demo, /\[data-capture-error="unreadable"\]/);
|
||||
assert.match(demo, /Nothing was uploaded/);
|
||||
assert.match(demo, /An unreadable file was posted to \/api\/analyze/);
|
||||
assert.match(demo, /A file that only claims to be a photo is refused on the device — nothing is uploaded/);
|
||||
assert.match(demo, /Manual logging stays one tap away, and the photo picker is one tap back/);
|
||||
// The caption is a required release gate, so a later release cannot drop the scene silently.
|
||||
assert.match(builder, /A file that only claims to be a photo is refused on the device — nothing is uploaded/);
|
||||
assert.match(builder, /"image_ingress_refusal": "passed"/);
|
||||
});
|
||||
|
||||
test('release builder gates the sleek shell, Hermes chat, and bootstrap syntax', async () => {
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
assert.match(builder, /"test:sleek"/);
|
||||
@@ -294,8 +311,11 @@ test('release demo shows a filled-in out-of-range field failing closed even besi
|
||||
assert.match(demo, /tests\/fixtures\/vision-corpus\/blank-plus-out-of-range\.png/);
|
||||
assert.match(demo, /A filled-in field that is out of range still fails closed, even when the other field came back blank/);
|
||||
assert.match(demo, /no longer told the photo was unreadable when the model returned an impossible value/);
|
||||
// The extra fail-closed scene is a real 503, so the demo's error budget must count it.
|
||||
// The extra fail-closed scene is a real 503, so the demo's error budget must count it, and the
|
||||
// deliberate ingress probe is the only 400 the recording is allowed to produce.
|
||||
assert.match(demo, /expectedFailClosed\.length !== 2/);
|
||||
assert.match(demo, /expectedIngressRefusal\.length !== 1/);
|
||||
assert.match(demo, /errors\.length !== 3/);
|
||||
// The caption is a required gate, so a later release cannot silently drop the scene.
|
||||
assert.match(builder, /A filled-in field that is out of range still fails closed/);
|
||||
});
|
||||
@@ -541,3 +561,35 @@ test('release demo runs the RC attack suite on camera and prints the verdict it
|
||||
assert.match(demo, /'providerTextLeaked' in observed/);
|
||||
assert.doesNotMatch(demo, /http \$\{observed\.status\} · worker \$\{observed\.providerCalls\}`/);
|
||||
});
|
||||
|
||||
test('release demo shows the privacy-safe health counters where the app discloses them', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
|
||||
// Issue #19 added an operator surface. The recording has to show the user-facing half of it — the
|
||||
// disclosure on the privacy screen — and the claim has to be checked in the browser first.
|
||||
assert.match(demo, /\[data-health-disclosure\]/);
|
||||
assert.match(demo, /the privacy screen must disclose the privacy-safe health counters/);
|
||||
assert.match(demo, /Health counters are disclosed where you read privacy facts — they never include notes, photos, image fingerprints, or identity/);
|
||||
// The caption is a release gate, so a later release cannot drop the scene silently.
|
||||
assert.match(builder, /Health counters are disclosed where you read privacy facts/);
|
||||
assert.match(builder, /"privacy_safe_health_metrics": "passed"/);
|
||||
});
|
||||
|
||||
test('release demo runs the operator health report against its own server and shows a refused token', async () => {
|
||||
const demo = await readFile(demoPath, 'utf8');
|
||||
const builder = await readFile(builderPath, 'utf8');
|
||||
|
||||
// The disclosure alone does not demonstrate the feature. The recording must run the real CLI, show
|
||||
// counters this recording produced, and prove the wrong-token path is refused rather than answered.
|
||||
assert.match(demo, /scripts\/model_metrics_report\.mjs/);
|
||||
assert.match(demo, /the operator report must read the snapshot/);
|
||||
assert.match(demo, /a wrong operator token must be refused with a stated reason instead of answered/);
|
||||
assert.match(demo, /the report must count this recording's own requests, not a zero/);
|
||||
assert.match(demo, /const metricsCardProof = await evidenceCardProof\('operator-health-report'\)/);
|
||||
// The bearer must never be printed on camera or into an error line.
|
||||
assert.match(demo, /the operator token must never appear in the report or in an error line/);
|
||||
// The demo server has to carry a token, or the report cannot be exercised at all.
|
||||
assert.match(builder, /"TIMMY_METRICS_TOKEN": "demo-metrics-token-not-a-credential"/);
|
||||
assert.match(builder, /The operator health report is a command line, not a screen in the app/);
|
||||
});
|
||||
@@ -43,6 +43,19 @@ await page.waitForTimeout(500);
|
||||
assert.equal(await page.evaluate(() => window.scrollY), 0, 'reply should not push the page header/navigation out of frame');
|
||||
await page.screenshot({ path: 'artifacts/timmy-chat-mobile.png', fullPage: false });
|
||||
|
||||
// Issue #19: the operator counters have to be disclosed where the user already looks for privacy
|
||||
// facts, and the disclosure has to name what is not collected. A settings screen that stays silent
|
||||
// about a new counter is the failure this asserts against.
|
||||
await page.locator('[data-view="calendar"]').first().click();
|
||||
await page.locator('[data-view="privacy"]').first().click();
|
||||
const disclosure = page.locator('[data-health-disclosure]');
|
||||
assert.equal(await disclosure.isVisible(), true, 'the privacy screen must disclose the health counters');
|
||||
const disclosureText = await disclosure.innerText();
|
||||
assert.match(disclosureText, /how analysis requests went/i);
|
||||
assert.match(disclosureText, /never (contain|hold|include) your notes, photos, image fingerprints, or identity/i);
|
||||
assert.match(disclosureText, /reset/i);
|
||||
await page.screenshot({ path: 'artifacts/privacy-health-disclosure.png', fullPage: true });
|
||||
|
||||
assert.deepEqual(errors, []);
|
||||
await browser.close();
|
||||
console.log('PASS mobile home → log → red-flag → save → Timmy boundary');
|
||||
Reference in new issue
Block a user