feat: ship photo-first Timmy prototype and sovereign vision spike
This commit is contained in:
commit
58906f11d0
57 files changed
+3065
No files matched your search
+102
@@ -0,0 +1,102 @@
|
||||
const COLORS = new Set(['brown', 'green', 'yellow', 'pale', 'red', 'black']);
|
||||
const QUALITIES = new Set(['good', 'fair', 'poor']);
|
||||
const MAX_IMAGE_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
function asObject(raw) {
|
||||
if (raw && typeof raw === 'object' && !Array.isArray(raw)) return raw;
|
||||
if (typeof raw !== 'string') throw new Error('Invalid AI response.');
|
||||
const cleaned = raw.trim().replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, '');
|
||||
try {
|
||||
const parsed = JSON.parse(cleaned);
|
||||
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error();
|
||||
return parsed;
|
||||
} catch {
|
||||
throw new Error('Invalid AI response.');
|
||||
}
|
||||
}
|
||||
|
||||
export function parseVisionResponse(raw) {
|
||||
const value = asObject(raw);
|
||||
if (typeof value.isStool !== 'boolean') throw new Error('Invalid AI response: isStool is required.');
|
||||
const confidence = Number(value.confidence);
|
||||
if (!Number.isFinite(confidence) || confidence < 0 || confidence > 1) throw new Error('Invalid AI response: confidence is required.');
|
||||
if (!value.isStool) return { status: 'needs_user_input', isStool: false, reason: 'The image does not clearly show stool.' };
|
||||
if (confidence < 0.55) return { status: 'needs_user_input', isStool: true, reason: 'The image is too uncertain to prefill safely.' };
|
||||
const bristolType = Number(value.bristolType);
|
||||
const color = String(value.color || '').toLowerCase();
|
||||
const imageQuality = String(value.imageQuality || '').toLowerCase();
|
||||
if (!Number.isInteger(bristolType) || bristolType < 1 || bristolType > 7 || !COLORS.has(color) || !QUALITIES.has(imageQuality)) {
|
||||
throw new Error('Invalid AI response: visual fields are out of range.');
|
||||
}
|
||||
return {
|
||||
status: 'suggestion',
|
||||
isStool: true,
|
||||
bristolType,
|
||||
color,
|
||||
confidence: Math.round(confidence * 100) / 100,
|
||||
imageQuality,
|
||||
observations: String(value.observations || '').trim().slice(0, 240),
|
||||
warning: 'Visual suggestion only. Confirm it yourself; this is not a diagnosis.',
|
||||
};
|
||||
}
|
||||
|
||||
export function mergeVisualSuggestion(form, suggestion) {
|
||||
if (suggestion?.status !== 'suggestion') return { ...form };
|
||||
return { ...form, bristolType: suggestion.bristolType, color: suggestion.color };
|
||||
}
|
||||
|
||||
export function validatePhotoPayload(payload = {}) {
|
||||
if (payload.consent !== true) throw new Error('Explicit consent is required before AI analysis.');
|
||||
const match = /^data:(image\/(?:jpeg|png|webp));base64,([A-Za-z0-9+/=]+)$/.exec(String(payload.imageDataUrl || ''));
|
||||
if (!match) throw new Error('Upload a JPEG, PNG, or WebP photo.');
|
||||
const padding = (match[2].match(/=*$/) || [''])[0].length;
|
||||
const bytes = Math.floor(match[2].length * 3 / 4) - padding;
|
||||
if (bytes <= 0) throw new Error('The photo is empty.');
|
||||
if (bytes > MAX_IMAGE_BYTES) throw new Error('The photo is too large. Use an image under 4 MB.');
|
||||
return { imageDataUrl: payload.imageDataUrl, mime: match[1], bytes };
|
||||
}
|
||||
|
||||
export function buildVisionRequest({ imageDataUrl, model }) {
|
||||
return {
|
||||
model,
|
||||
temperature: 0,
|
||||
max_tokens: 500,
|
||||
messages: [{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: [
|
||||
'You are a conservative visual form-suggestion tool for a bowel diary, not a clinician.',
|
||||
'First decide whether the image clearly shows human stool. If it does not, set isStool=false.',
|
||||
'If it does, suggest only the closest Bristol Stool Form Scale type (1-7), visible color (brown, green, yellow, pale, red, or black), image quality, confidence, and a short neutral visual observation.',
|
||||
'Do not infer urgency, discomfort, pain, symptoms, bleeding, disease, diet safety, cause, or treatment from the image.',
|
||||
'Red or black is a visible color description only and is not a diagnosis. Be uncertain when lighting or visibility is poor.',
|
||||
'The user must confirm every suggestion. Return only the requested JSON.',
|
||||
].join(' '),
|
||||
},
|
||||
{ type: 'image_url', image_url: { url: imageDataUrl } },
|
||||
],
|
||||
}],
|
||||
response_format: {
|
||||
type: 'json_schema',
|
||||
json_schema: {
|
||||
name: 'timmy_visual_suggestion',
|
||||
strict: true,
|
||||
schema: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['isStool', 'bristolType', 'color', 'confidence', 'imageQuality', 'observations'],
|
||||
properties: {
|
||||
isStool: { type: 'boolean' },
|
||||
bristolType: { anyOf: [{ type: 'integer', minimum: 1, maximum: 7 }, { type: 'null' }] },
|
||||
color: { anyOf: [{ type: 'string', enum: [...COLORS] }, { type: 'null' }] },
|
||||
confidence: { type: 'number', minimum: 0, maximum: 1 },
|
||||
imageQuality: { type: 'string', enum: [...QUALITIES] },
|
||||
observations: { type: 'string', maxLength: 240 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
const URGENT_KEYS = ['blood', 'blackOrDarkRed', 'severePain', 'vomiting', 'fever', 'cannotPassGas'];
|
||||
|
||||
export function bucketForBristolType(type) {
|
||||
const value = Number(type);
|
||||
if (value === 1 || value === 2) return 'constipation';
|
||||
if (value === 3 || value === 4) return 'typical';
|
||||
if (value >= 5 && value <= 7) return 'loose';
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
export function detectUrgentFlags(symptoms = {}) {
|
||||
const flags = URGENT_KEYS.filter((key) => symptoms[key] === true);
|
||||
return {
|
||||
urgent: flags.length > 0,
|
||||
flags,
|
||||
message: flags.length
|
||||
? 'These reported symptoms can need prompt medical care. Contact a clinician or urgent service now; call emergency services for heavy or nonstop bleeding, fainting, or severe worsening symptoms.'
|
||||
: 'No urgent symptom was selected. This tracker is not a diagnosis; seek care whenever you are worried or symptoms persist.',
|
||||
};
|
||||
}
|
||||
|
||||
export function buildTimmySummary(entries = []) {
|
||||
if (!entries.length) return 'No logs yet. Add one when you are ready and I’ll summarize the pattern—not diagnose it.';
|
||||
const counts = entries.reduce((acc, entry) => {
|
||||
const bucket = bucketForBristolType(entry.bristolType);
|
||||
acc[bucket] = (acc[bucket] || 0) + 1;
|
||||
return acc;
|
||||
}, {});
|
||||
const pieces = [`${entries.length} ${entries.length === 1 ? 'log' : 'logs'}`];
|
||||
if (counts.typical) pieces.push(`${counts.typical} typical`);
|
||||
if (counts.constipation) pieces.push(`${counts.constipation} on the firm side`);
|
||||
if (counts.loose) pieces.push(`${counts.loose} on the loose side`);
|
||||
return `${pieces.join(' · ')}. Patterns matter more than one entry. You choose what to eat; I only help you notice changes.`;
|
||||
}
|
||||
|
||||
export function sanitizeEntry(input = {}) {
|
||||
const symptoms = {};
|
||||
for (const key of URGENT_KEYS) symptoms[key] = input.symptoms?.[key] === true;
|
||||
const bristolType = Math.min(7, Math.max(1, Number(input.bristolType) || 4));
|
||||
return {
|
||||
id: String(input.id || globalThis.crypto?.randomUUID?.() || `${Date.now()}-${Math.random()}`),
|
||||
occurredAt: new Date(input.occurredAt || Date.now()).toISOString(),
|
||||
bristolType,
|
||||
color: ['brown', 'green', 'yellow', 'pale', 'red', 'black'].includes(input.color) ? input.color : 'brown',
|
||||
urgency: Math.min(4, Math.max(0, Number(input.urgency) || 0)),
|
||||
discomfort: Math.min(4, Math.max(0, Number(input.discomfort) || 0)),
|
||||
note: String(input.note || '').trim().slice(0, 500),
|
||||
photoDataUrl: typeof input.photoDataUrl === 'string' && input.photoDataUrl.startsWith('data:image/') ? input.photoDataUrl : '',
|
||||
symptoms,
|
||||
};
|
||||
}
|
||||
|
||||
export function photoQualityMessage({ width = 0, height = 0, brightness = 0.5 } = {}) {
|
||||
if (width < 640 || height < 480) return 'Move a little closer or use a higher-resolution photo. The image stays on this device.';
|
||||
if (brightness < 0.12) return 'Add more light before saving. Timmy only checks whether the photo is usable.';
|
||||
if (brightness > 0.95) return 'Reduce glare before saving. Timmy only checks whether the photo is usable.';
|
||||
return 'Ready for your review. Choose the matching Bristol form yourself; Timmy does not interpret the picture.';
|
||||
}
|
||||
|
||||
export function exportLedger(entries, exportedAt = new Date().toISOString()) {
|
||||
return JSON.stringify({
|
||||
product: 'Timmy the Talking Turd',
|
||||
schemaVersion: 1,
|
||||
exportedAt,
|
||||
entries: Array.isArray(entries) ? entries : [],
|
||||
}, null, 2);
|
||||
}
|
||||
|
||||
export function importLedger(text) {
|
||||
const parsed = JSON.parse(text);
|
||||
if (parsed?.schemaVersion !== 1 || !Array.isArray(parsed.entries)) throw new Error('This is not a supported Timmy export.');
|
||||
return parsed.entries.map(sanitizeEntry);
|
||||
}
|
||||
|
||||
export const urgentSymptomKeys = Object.freeze([...URGENT_KEYS]);
|
||||
@@ -0,0 +1,58 @@
|
||||
const PROFILES = {
|
||||
hosted: {
|
||||
baseUrl: 'http://127.0.0.1:8645/v1',
|
||||
apiKey: 'local-proxy',
|
||||
model: 'stepfun/step-3.7-flash:free',
|
||||
processor: 'third-party',
|
||||
},
|
||||
selfhost: {
|
||||
baseUrl: 'http://127.0.0.1:8080/v1',
|
||||
apiKey: 'local-selfhost',
|
||||
model: 'SmolVLM2-2.2B-Instruct',
|
||||
processor: 'self-hosted',
|
||||
},
|
||||
};
|
||||
|
||||
export function resolveVisionConfig(env = process.env) {
|
||||
const profile = env.TIMMY_VISION_PROFILE || 'hosted';
|
||||
if (!PROFILES[profile]) throw new Error('TIMMY_VISION_PROFILE must be hosted or selfhost.');
|
||||
const defaults = PROFILES[profile];
|
||||
const config = {
|
||||
enabled: env.TIMMY_VISION_ENABLED !== '0',
|
||||
profile,
|
||||
processor: defaults.processor,
|
||||
baseUrl: env.TIMMY_VISION_BASE_URL || defaults.baseUrl,
|
||||
apiKey: env.TIMMY_VISION_API_KEY || defaults.apiKey,
|
||||
model: env.TIMMY_VISION_MODEL || defaults.model,
|
||||
requestTimeoutMs: Number(env.TIMMY_VISION_TIMEOUT_MS || (profile === 'selfhost' ? 120_000 : 60_000)),
|
||||
};
|
||||
config.publicStatus = () => ({
|
||||
enabled: config.enabled,
|
||||
profile: config.profile,
|
||||
processor: config.processor,
|
||||
model: config.enabled ? config.model : null,
|
||||
});
|
||||
return config;
|
||||
}
|
||||
|
||||
function modelsEndpoint(baseUrl) {
|
||||
const url = new URL(baseUrl);
|
||||
if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid AI provider URL.');
|
||||
return `${url.toString().replace(/\/$/, '')}/models`;
|
||||
}
|
||||
|
||||
export async function probeVisionProvider(config, fetchImpl = fetch) {
|
||||
if (!config.enabled) return { ready: false, modelSeen: false };
|
||||
try {
|
||||
const response = await fetchImpl(modelsEndpoint(config.baseUrl), {
|
||||
headers: { authorization: `Bearer ${config.apiKey}` },
|
||||
signal: AbortSignal.timeout(2_000),
|
||||
});
|
||||
if (!response.ok) return { ready: false, modelSeen: false };
|
||||
const body = await response.json();
|
||||
const ids = Array.isArray(body?.data) ? body.data.map(item => item?.id).filter(Boolean) : [];
|
||||
return { ready: ids.length > 0, modelSeen: ids.some(id => id === config.model || id.includes(config.model)) };
|
||||
} catch {
|
||||
return { ready: false, modelSeen: false };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { buildVisionRequest, parseVisionResponse, validatePhotoPayload } from './analysis.js';
|
||||
|
||||
function providerEndpoint(baseUrl) {
|
||||
let url;
|
||||
try { url = new URL(baseUrl); } catch { throw new Error('Invalid AI provider URL.'); }
|
||||
if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Invalid AI provider URL.');
|
||||
return `${url.toString().replace(/\/$/, '')}/chat/completions`;
|
||||
}
|
||||
|
||||
export async function analyzePhoto({ payload, fetchImpl = fetch, config }) {
|
||||
const photo = validatePhotoPayload(payload);
|
||||
if (!config?.model) throw new Error('AI analysis is not configured.');
|
||||
const endpoint = providerEndpoint(config.baseUrl);
|
||||
const response = await fetchImpl(endpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
authorization: `Bearer ${config.apiKey || 'local-proxy'}`,
|
||||
},
|
||||
body: JSON.stringify(buildVisionRequest({ imageDataUrl: photo.imageDataUrl, model: config.model })),
|
||||
signal: AbortSignal.timeout(config.requestTimeoutMs || 60_000),
|
||||
}).catch(() => { throw new Error('AI analysis is temporarily unavailable. Continue manually.'); });
|
||||
if (!response.ok) throw new Error('AI analysis is temporarily unavailable. Continue manually.');
|
||||
let data;
|
||||
try { data = await response.json(); } catch { throw new Error('Invalid response from the AI provider.'); }
|
||||
const content = data?.choices?.[0]?.message?.content;
|
||||
if (typeof content !== 'string' && (typeof content !== 'object' || content === null)) throw new Error('Invalid response from the AI provider.');
|
||||
return parseVisionResponse(content);
|
||||
}
|
||||
Reference in new issue
Block a user