feat(ingress): decide photo validity from bytes, budget, and header before any provider work (#16)
The declared data-URL MIME was a claim the boundary believed. Every image decision now comes from the bytes: the container is sniffed, walked to its own end marker, refused when metadata segments remain, and refused above a 4096px edge or 16MP. The request path adds a per-caller sliding-window budget (429 + Retry-After) and a body deadline (408), so a flood, a stalled body, or a decode bomb fails before a provider call. Log hygiene is part of the contract: refusal copy is fixed prose, the bucket key is a salted hash kept in memory only, and the acceptance test asserts no image bytes reach the process log.
This commit is contained in:
timmy committed
1 parent
d0b3c10aea
commit
ba9afe0a1d
9 files changed
+869
-2
No files matched your search
@@ -0,0 +1,75 @@
|
||||
// Bounded request-body reading for the API boundary (issue #16).
|
||||
//
|
||||
// Three limits have to fail before any provider work starts: the byte ceiling, a deadline for a body
|
||||
// that never arrives, and JSON validity. All three refuse with a client-visible status and a fixed
|
||||
// message; none of them logs or returns the body.
|
||||
|
||||
const COPY = Object.freeze({
|
||||
too_large: [413, 'Request is too large.'],
|
||||
invalid_json: [400, 'Invalid JSON request.'],
|
||||
body_timeout: [408, 'The request took too long to arrive. Try again.'],
|
||||
});
|
||||
|
||||
export class BodyError extends Error {
|
||||
constructor(code) {
|
||||
const [status, message] = COPY[code] || COPY.invalid_json;
|
||||
super(message);
|
||||
this.name = 'BodyError';
|
||||
this.code = code;
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export function readJsonBody(req, { maxBytes = 6 * 1024 * 1024, timeoutMs = 15_000 } = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
let size = 0;
|
||||
const chunks = [];
|
||||
const finish = callback => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(deadline);
|
||||
req.removeListener('data', onData);
|
||||
req.removeListener('end', onEnd);
|
||||
req.removeListener('error', onError);
|
||||
callback();
|
||||
};
|
||||
const deadline = setTimeout(() => {
|
||||
finish(() => {
|
||||
// Stop reading and let the caller answer 408; the server destroys the socket once the
|
||||
// refusal is written, so a half-read body cannot be held open indefinitely.
|
||||
req.pause?.();
|
||||
reject(new BodyError('body_timeout'));
|
||||
});
|
||||
}, timeoutMs);
|
||||
|
||||
function onData(chunk) {
|
||||
if (settled) return;
|
||||
size += chunk.length;
|
||||
if (size > maxBytes) {
|
||||
finish(() => {
|
||||
req.destroy?.();
|
||||
reject(new BodyError('too_large'));
|
||||
});
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
}
|
||||
function onEnd() {
|
||||
finish(() => {
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(chunks).toString('utf8')));
|
||||
} catch {
|
||||
reject(new BodyError('invalid_json'));
|
||||
}
|
||||
});
|
||||
}
|
||||
function onError(error) {
|
||||
finish(() => reject(error));
|
||||
}
|
||||
|
||||
req.on('data', onData);
|
||||
req.on('end', onEnd);
|
||||
req.on('error', onError);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
// Transport-level hardening for the one endpoint that accepts image bytes (issue #16).
|
||||
//
|
||||
// The declared MIME type in a data URL is a claim made by the caller, never evidence, so every
|
||||
// decision here comes from the bytes: the container is sniffed, the container is walked to its own
|
||||
// end marker so nothing can ride along behind a valid image, embedded metadata segments are refused
|
||||
// (the browser always uploads a canvas re-encode, which carries none), and the pixel dimensions are
|
||||
// read from the header so a decode bomb never reaches the worker.
|
||||
//
|
||||
// Refusal copy is fixed and deliberate: it names what is wrong with the file and never echoes the
|
||||
// bytes, a hash, or any part of the upload back to the caller or into a log line.
|
||||
|
||||
export const MAX_EDGE = 4096;
|
||||
export const MAX_PIXELS = 16_000_000;
|
||||
|
||||
const COPY = Object.freeze({
|
||||
empty: 'The photo is empty.',
|
||||
not_an_image: 'That file is not a JPEG, PNG, or WebP photo.',
|
||||
container_mismatch: 'This photo is not the format it claims. Choose a JPEG, PNG, or WebP photo.',
|
||||
truncated: 'This photo is incomplete or damaged. Choose another photo.',
|
||||
trailing_data: 'This photo file has extra data after the image. Choose another photo.',
|
||||
metadata_present: 'This photo still carries embedded metadata. Choose it through Timmy’s camera or gallery picker so it is re-encoded without metadata.',
|
||||
dimensions_too_large: 'This photo is too large in pixels. Choose a smaller photo.',
|
||||
});
|
||||
|
||||
export class IngressError extends Error {
|
||||
constructor(code) {
|
||||
super(COPY[code] || COPY.not_an_image);
|
||||
this.name = 'IngressError';
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
const refuse = code => {
|
||||
throw new IngressError(code);
|
||||
};
|
||||
|
||||
const PNG_CHUNKS = { metadata: new Set(['eXIf', 'tEXt', 'zTXt', 'iTXt', 'tIME']) };
|
||||
// Segment markers that carry metadata rather than image data: APP1 (Exif/XMP), APP13 (Photoshop
|
||||
// resource block) and COM (free-text comment). The app's own upload is a canvas re-encode and
|
||||
// contains none of them, so a file that does was not produced by the capture path.
|
||||
const JPEG_METADATA_MARKERS = new Set([0xe1, 0xed, 0xfe]);
|
||||
const JPEG_SOF_MARKERS = new Set([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf]);
|
||||
|
||||
function sniff(bytes) {
|
||||
if (bytes.length >= 8 && bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) return 'image/png';
|
||||
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) return 'image/jpeg';
|
||||
if (bytes.length >= 12 && bytes.toString('ascii', 0, 4) === 'RIFF' && bytes.toString('ascii', 8, 12) === 'WEBP') return 'image/webp';
|
||||
return '';
|
||||
}
|
||||
|
||||
function inspectPng(bytes) {
|
||||
let offset = 8;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
let sawHeader = false;
|
||||
while (offset + 8 <= bytes.length) {
|
||||
const length = bytes.readUInt32BE(offset);
|
||||
const type = bytes.toString('ascii', offset + 4, offset + 8);
|
||||
const dataStart = offset + 8;
|
||||
const next = dataStart + length + 4;
|
||||
if (next > bytes.length) refuse('truncated');
|
||||
if (!sawHeader) {
|
||||
if (type !== 'IHDR' || length !== 13) refuse('truncated');
|
||||
width = bytes.readUInt32BE(dataStart);
|
||||
height = bytes.readUInt32BE(dataStart + 4);
|
||||
sawHeader = true;
|
||||
}
|
||||
if (PNG_CHUNKS.metadata.has(type)) refuse('metadata_present');
|
||||
if (type === 'IEND') {
|
||||
if (next !== bytes.length) refuse('trailing_data');
|
||||
return { width, height };
|
||||
}
|
||||
offset = next;
|
||||
}
|
||||
return refuse('truncated');
|
||||
}
|
||||
|
||||
function inspectJpeg(bytes) {
|
||||
let offset = 2;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
while (offset + 4 <= bytes.length) {
|
||||
if (bytes[offset] !== 0xff) refuse('truncated');
|
||||
const marker = bytes[offset + 1];
|
||||
if (marker === 0xd9) {
|
||||
if (offset + 2 !== bytes.length) refuse('trailing_data');
|
||||
break;
|
||||
}
|
||||
// Standalone markers carry no length field.
|
||||
if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7) || marker === 0xff) {
|
||||
offset += marker === 0xff ? 1 : 2;
|
||||
continue;
|
||||
}
|
||||
const length = bytes.readUInt16BE(offset + 2);
|
||||
if (length < 2 || offset + 2 + length > bytes.length) refuse('truncated');
|
||||
if (JPEG_METADATA_MARKERS.has(marker)) refuse('metadata_present');
|
||||
if (JPEG_SOF_MARKERS.has(marker) && length >= 7) {
|
||||
height = bytes.readUInt16BE(offset + 5);
|
||||
width = bytes.readUInt16BE(offset + 7);
|
||||
}
|
||||
offset += 2 + length;
|
||||
if (marker === 0xda) {
|
||||
// Entropy-coded scan data follows the start-of-scan header. The only thing left to trust is
|
||||
// the end-of-image marker, and it has to be the last two bytes of the file.
|
||||
if (bytes[bytes.length - 2] !== 0xff || bytes[bytes.length - 1] !== 0xd9) {
|
||||
return refuse(bytes.includes(Buffer.from([0xff, 0xd9])) ? 'trailing_data' : 'truncated');
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!width || !height) refuse('truncated');
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
function inspectWebp(bytes) {
|
||||
const declared = bytes.readUInt32LE(4) + 8;
|
||||
if (declared !== bytes.length) refuse(declared < bytes.length ? 'trailing_data' : 'truncated');
|
||||
let offset = 12;
|
||||
let width = 0;
|
||||
let height = 0;
|
||||
while (offset + 8 <= bytes.length) {
|
||||
const type = bytes.toString('ascii', offset, offset + 4);
|
||||
const size = bytes.readUInt32LE(offset + 4);
|
||||
const dataStart = offset + 8;
|
||||
if (dataStart + size > bytes.length) refuse('truncated');
|
||||
if (type === 'VP8X' && size >= 10) {
|
||||
width = bytes.readUIntLE(dataStart + 4, 3) + 1;
|
||||
height = bytes.readUIntLE(dataStart + 7, 3) + 1;
|
||||
} else if (type === 'VP8 ' && size >= 10) {
|
||||
width = bytes.readUInt16LE(dataStart + 6) & 0x3fff;
|
||||
height = bytes.readUInt16LE(dataStart + 8) & 0x3fff;
|
||||
} else if (type === 'VP8L' && size >= 5) {
|
||||
const bits = bytes.readUInt32LE(dataStart + 1);
|
||||
width = (bits & 0x3fff) + 1;
|
||||
height = ((bits >> 14) & 0x3fff) + 1;
|
||||
}
|
||||
offset = dataStart + size + (size % 2);
|
||||
}
|
||||
if (!width || !height) refuse('truncated');
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
// Returns the real container, its pixel dimensions, and the byte count. Throws IngressError on
|
||||
// anything the model worker must never be handed.
|
||||
export function inspectImageBytes(bytes, { declaredMime = '' } = {}) {
|
||||
const buffer = Buffer.isBuffer(bytes) ? bytes : Buffer.from(bytes || []);
|
||||
if (!buffer.length) refuse('empty');
|
||||
const actual = sniff(buffer);
|
||||
if (!actual) refuse('not_an_image');
|
||||
if (declaredMime && declaredMime !== actual) refuse('container_mismatch');
|
||||
const dimensions = actual === 'image/png' ? inspectPng(buffer)
|
||||
: actual === 'image/jpeg' ? inspectJpeg(buffer)
|
||||
: inspectWebp(buffer);
|
||||
if (!dimensions.width || !dimensions.height) refuse('truncated');
|
||||
if (dimensions.width > MAX_EDGE || dimensions.height > MAX_EDGE
|
||||
|| dimensions.width * dimensions.height > MAX_PIXELS) refuse('dimensions_too_large');
|
||||
return { mime: actual, width: dimensions.width, height: dimensions.height, bytes: buffer.length };
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
// Bounded, in-process request budget for the photo-analysis endpoint (issue #16).
|
||||
//
|
||||
// The limiter exists so a caller cannot make Timmy the transport for a flood of images to the
|
||||
// model worker. It keeps timestamps only, never the bytes, the file name, or the caller's address:
|
||||
// the server hands it a short-lived opaque bucket key. Refusals never extend a window, so a caller
|
||||
// cannot earn a longer penalty than the request it already made.
|
||||
|
||||
export function createRateLimiter({ limit = 12, windowMs = 60_000, maxKeys = 2048, now = () => Date.now() } = {}) {
|
||||
const hits = new Map();
|
||||
let lastPrune = now();
|
||||
|
||||
function prune(at) {
|
||||
const cutoff = at - windowMs;
|
||||
for (const [key, stamps] of hits) {
|
||||
const kept = stamps.filter(stamp => stamp > cutoff);
|
||||
if (kept.length) hits.set(key, kept);
|
||||
else hits.delete(key);
|
||||
}
|
||||
lastPrune = at;
|
||||
}
|
||||
|
||||
return {
|
||||
take(key) {
|
||||
const at = now();
|
||||
// Pruning is at most once per window, plus whenever the map outgrows its ceiling, so the
|
||||
// bookkeeping stays bounded no matter how many distinct callers appear.
|
||||
if (at - lastPrune >= windowMs || hits.size > maxKeys) prune(at);
|
||||
const window = (hits.get(key) || []).filter(stamp => stamp > at - windowMs);
|
||||
if (window.length >= limit) {
|
||||
hits.set(key, window);
|
||||
return { allowed: false, remaining: 0, retryAfterMs: Math.max(0, window[0] + windowMs - at) };
|
||||
}
|
||||
window.push(at);
|
||||
hits.set(key, window);
|
||||
if (hits.size > maxKeys) {
|
||||
for (const oldest of hits.keys()) {
|
||||
if (hits.size <= maxKeys) break;
|
||||
hits.delete(oldest);
|
||||
}
|
||||
}
|
||||
return { allowed: true, remaining: Math.max(0, limit - window.length), retryAfterMs: 0 };
|
||||
},
|
||||
size() {
|
||||
return hits.size;
|
||||
},
|
||||
reset() {
|
||||
hits.clear();
|
||||
lastPrune = now();
|
||||
},
|
||||
};
|
||||
}
|
||||
Reference in new issue
Block a user