diff --git a/.gitea/workflows/quality.yml b/.gitea/workflows/quality.yml index c77cc35..35f7dd4 100644 --- a/.gitea/workflows/quality.yml +++ b/.gitea/workflows/quality.yml @@ -23,16 +23,22 @@ jobs: with: node-version: 22 cache: npm + - name: Set up pinned Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' - name: Install reproducibly run: | npm ci python3 -m pip install --break-system-packages -r requirements-test.txt + printf 'TIMMY_PYTHON=%s\n' "$(python3 -c 'import sys; print(sys.executable)')" >> "$GITHUB_ENV" - name: Install browser run: npx playwright install --with-deps chromium - name: Unit and security tests run: | npm test python3 tests/staging-deploy.test.py -v + python3 tests/reencode-image.test.py -v - name: Mobile browser acceptance run: | npm start > /tmp/timmy-server.log 2>&1 & @@ -57,5 +63,7 @@ jobs: run: | npm run check:syntax node --check tests/staging.acceptance.mjs + - name: Image runtime pin and re-encode smoke + run: python3 -c "import importlib.util,json,pathlib,sys; s=importlib.util.spec_from_file_location('d','scripts/deploy_staging.py'); m=importlib.util.module_from_spec(s); sys.modules[s.name]=m; s.loader.exec_module(m); print(json.dumps(m.verify_image_runtime(pathlib.Path('.'))))" - name: Diff hygiene run: npm run check:diff diff --git a/package.json b/package.json index 31839e0..2d32133 100644 --- a/package.json +++ b/package.json @@ -4,12 +4,12 @@ "private": true, "type": "module", "scripts": { - "test": "node --test tests/domain.test.js tests/analysis.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js", + "test": "node --test tests/domain.test.js tests/analysis.test.js tests/image-ingress.test.js tests/rate-limiter.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/image-polyglot.test.js tests/image-dataurl.test.js tests/image-concurrency.test.js tests/image-unavailable.test.js tests/rate-identity.test.js tests/body-timeout.test.js", "test:ui": "node tests/ui.acceptance.mjs", "test:photo": "node tests/photo-first.acceptance.mjs", "test:sleek": "node tests/sleek-chat.acceptance.mjs", "test:staging-smoke": "node tests/staging.acceptance.mjs", - "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py", + "check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/image-ingress.js && node --check src/rate-limiter.js && node --check src/client-identity.js && node --check src/image-container.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/image-polyglot.test.js && node --check tests/image-dataurl.test.js && node --check tests/image-concurrency.test.js && node --check tests/image-unavailable.test.js && node --check tests/rate-identity.test.js && node --check tests/body-timeout.test.js && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/reencode_image.py", "check:diff": "bash scripts/check_diff.sh", "start": "node server.mjs" }, diff --git a/scripts/build_release.py b/scripts/build_release.py index 5d2b391..e7ddbcb 100755 --- a/scripts/build_release.py +++ b/scripts/build_release.py @@ -133,11 +133,16 @@ def main() -> int: contact_sheet = release_dir / f"timmy-talking-turd-{version}-demo-contact-sheet.jpg" run(["ffmpeg", "-y", "-v", "error", "-i", str(demo), "-vf", "fps=1/2,scale=180:-1,tile=4x3:padding=4:margin=4", "-frames:v", "1", str(contact_sheet)], tree) run(["npm", "audit", "--audit-level=high"], tree) - for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "tests/staging.acceptance.mjs"): + for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/image-ingress.js", "src/rate-limiter.js", "src/client-identity.js", "src/image-container.js", "tests/staging.acceptance.mjs", "tests/image-polyglot.test.js", "tests/image-dataurl.test.js", "tests/image-concurrency.test.js", "tests/image-unavailable.test.js", "tests/rate-identity.test.js", "tests/body-timeout.test.js"): run(["node", "--check", file], tree) run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree) run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree) - run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py"], tree) + run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py", "scripts/deploy_staging.py", "scripts/reencode_image.py"], tree) + # Gated re-encode runtime smoke: the production image re-encoder must be + # the pinned immutable toolchain and must actually re-encode a synthetic + # 1x1 under the hard 512 MiB service budget. This is the deployment + # smoke gate, run here as part of the release build itself. + deploy_mod = run([sys.executable, "-c", "import importlib.util,sys; s=importlib.util.spec_from_file_location('d','scripts/deploy_staging.py'); m=importlib.util.module_from_spec(s); s.loader.exec_module(m); print(__import__('json').dumps(m.verify_image_runtime(__import__('pathlib').Path('.'))))"], tree, capture=True) run(["git", "diff", "--check", commit], tree) names = tracked_files(tree) diff --git a/scripts/deploy_staging.py b/scripts/deploy_staging.py index 5d98212..69258d8 100755 --- a/scripts/deploy_staging.py +++ b/scripts/deploy_staging.py @@ -238,6 +238,59 @@ def _run_argv(argv: Sequence[str], **options) -> subprocess.CompletedProcess: return subprocess.run(list(argv), check=True, text=True, capture_output=True, shell=False, timeout=timeout) +# Pinned, immutable production image runtime. The deployment smoke gate refuses +# to promote unless the re-encoder provisioned in the release matches this exact +# toolchain. This is the same record scripts/reencode_image.py publishes, so a +# runtime drift is caught before a single request is served. +PINNED_PYTHON_VERSION = "3.11" +PINNED_PILLOW_VERSION = "12.3.0" +REENCODE_SCRIPT = "scripts/reencode_image.py" + + +def verify_image_runtime(root: Path, *, timeout: float = 30.0) -> dict: + """Smoke-test the release's pinned, immutable image re-encode runtime. + + Runs the release's re-encoder with --verify-pin and, if it matches, performs + a synthetic re-encode of a 1x1 pixel image to prove the runtime can actually + load, decode, and strip metadata under the hard 512 MiB service budget. No + live host or network change: this only exercises the release on disk. + """ + script = root / REENCODE_SCRIPT + if not script.is_file(): + raise DeploymentError("release is missing the image re-encoder script") + probe = json.loads(subprocess.run( + [sys.executable, str(script), "--verify-pin"], + check=True, text=True, capture_output=True, shell=False, timeout=timeout, + ).stdout.strip()) + if probe.get("ok") is not True: + raise DeploymentError( + f"image runtime mismatch: provisioned {probe.get('python')}/{probe.get('pillow')} " + f"does not match pinned {PINNED_PYTHON_VERSION}/{PINNED_PILLOW_VERSION}" + ) + import base64 + # Minimal valid 1x1 baseline JPEG (no metadata) used only for the smoke. + # Generated with Pillow to avoid an embed errors in a hand-written blob. + from PIL import Image as _PILImage + import io as _io + _buf = _io.BytesIO() + _PILImage.new("RGB", (1, 1), (128, 128, 128)).save(_buf, "JPEG", quality=85) + pixel = _buf.getvalue() + with tempfile.TemporaryDirectory(prefix="timmy-runtime-smoke-") as work: + src = Path(work) / "pixel.jpg" + dst = Path(work) / "reencoded.jpg" + src.write_bytes(pixel) + result = subprocess.run( + [sys.executable, str(script), "--in", str(src), "--out", str(dst)], + check=True, text=True, capture_output=True, shell=False, timeout=timeout, + ) + verdict = json.loads(result.stdout.strip().splitlines()[-1]) + if not verdict.get("ok"): + raise DeploymentError(f"image runtime re-encode smoke failed: {verdict}") + if not dst.is_file() or dst.stat().st_size == 0 or dst.read_bytes()[:3] != b"\xff\xd8\xff": + raise DeploymentError("image runtime re-encode smoke produced no valid JPEG") + return {"ok": True, "python": probe["python"], "pillow": probe["pillow"]} + + def poll_health(url: str, expected_commit: str, timeout: float) -> dict: deadline = time.monotonic() + timeout while True: @@ -265,6 +318,9 @@ def _verify(config: DeploymentConfig, commit: str, run_command: RunCommand, heal health_check(config.health_url, commit, config.health_timeout) if smoke: run_command(config.smoke_command, check=True, text=True, capture_output=True, shell=False, timeout=config.command_timeout) + # Pinned, immutable production image runtime: refuse the promotion if the + # release's re-encoder is not the exact toolchain we test against. + verify_image_runtime(config.releases / commit, timeout=config.command_timeout) def promote(*, config: DeploymentConfig, tag: str, archive: Path, expected_sha256: str, commit: str, diff --git a/scripts/gen_ingress_fixtures.py b/scripts/gen_ingress_fixtures.py new file mode 100644 index 0000000..2147977 --- /dev/null +++ b/scripts/gen_ingress_fixtures.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Generate synthetic hostile fixtures for image-ingress security tests. + +All content is synthetic test data. No real medical images, no external uploads. +""" +import os + +from PIL import Image, PngImagePlugin + +D = os.path.join(os.path.dirname(__file__), "..", "tests", "fixtures") +D = os.path.abspath(D) +os.makedirs(D, exist_ok=True) + +img = Image.new("RGB", (64, 64)) +for y in range(64): + for x in range(64): + img.putpixel((x, y), (x * 4 % 256, y * 4 % 256, 128)) + +# 1. Clean small JPEG (valid control) +img.save(os.path.join(D, "ingress-clean.jpg"), "JPEG", quality=90) + +# 2. JPEG with EXIF metadata (Make/Model + GPS IFD pointer) +ex = img.getexif() +ex[0x010F] = "HostileCam" +ex[0x0110] = "Model-X" +gps_ifd = ex.get_ifd(0x8825) +gps_ifd[1] = "N" # GPSLatitudeRef +gps_ifd[2] = (44, 30, 0) # GPSLatitude (degrees, minutes, seconds) +gps_ifd[4] = (68, 15, 0) # GPSLongitude +img.save(os.path.join(D, "ingress-exif.jpg"), "JPEG", quality=90, exif=ex) + +# 3. PNG with tEXt metadata chunks +png_meta = PngImagePlugin.PngInfo() +png_meta.add_text("Comment", "sensitive-metadata") +png_meta.add_text("GPS", "lat:44.0 lon:-68.0") +img.save(os.path.join(D, "ingress-metadata.png"), "PNG", pnginfo=png_meta) + +# 4. Spoofed content: HTML masquerading as an image +with open(os.path.join(D, "ingress-spoofed.html"), "wb") as f: + f.write(b"not an image") + +# 5. GIF-header polyglot with embedded script payload +poly = (b"GIF89a\x01\x00\x01\x00\x00\xff\x00,\x00\x00\x00\x00\x01\x00\x01\x00" + b"\x00\x02\x00;" + b"" * 4) +with open(os.path.join(D, "ingress-polyglot.gif"), "wb") as f: + f.write(poly) + +# 6. ZIP-in-JPEG polyglot (GIFAR-style) +jpg_bytes = open(os.path.join(D, "ingress-clean.jpg"), "rb").read() +zip_poly = jpg_bytes[:2] + b"PK\x03\x04" + jpg_bytes[2:10] + b"PK\x05\x06" + b"\x00" * 18 +with open(os.path.join(D, "ingress-zip-polyglot.jpg"), "wb") as f: + f.write(zip_poly) + +# 7. Truncated JPEG (SOI present, cut before EOI) +with open(os.path.join(D, "ingress-truncated.jpg"), "wb") as f: + f.write(jpg_bytes[: len(jpg_bytes) // 2]) + +# 8. Decompression bomb: 12000x12000 sparse PNG, tiny on disk +bomb = Image.new("L", (12000, 12000), 7) +bomb.save(os.path.join(D, "ingress-bomb.png"), "PNG", optimize=True) +print("bomb size:", os.path.getsize(os.path.join(D, "ingress-bomb.png"))) + +# 9. Oversized-dimension JPEG (6000x6000, small on disk) +big = Image.new("RGB", (6000, 6000), (90, 90, 90)) +big.save(os.path.join(D, "ingress-oversized.jpg"), "JPEG", quality=40) +print("oversized size:", os.path.getsize(os.path.join(D, "ingress-oversized.jpg"))) + +# 10. Random garbage with jpeg extension +with open(os.path.join(D, "ingress-garbage.jpg"), "wb") as f: + f.write(os.urandom(2048)) + +# 11. Empty file +open(os.path.join(D, "ingress-empty.jpg"), "wb").close() + +# 12. SVG with embedded script +with open(os.path.join(D, "ingress-script.svg"), "wb") as f: + f.write(b'' + b"") + +# --- Trailing-data / container polyglots that bypass naive substring scans --- +# Every one of these is a structurally valid image followed by appended bytes. +# A canonical parser must reject them on the trailing data itself, not on a +# signature keyword, so casing and container choice cannot evade the check. +TRAILERS = { + "ingress-tail-upper-script.jpg": b"", + "ingress-tail-mixed-script.jpg": b"", + "ingress-tail-html.jpg": b"", + "ingress-tail-zip-eocd.jpg": b"PK\x05\x06" + b"\x00" * 18, + "ingress-tail-zip-local.jpg": b"PK\x03\x04" + b"\x00" * 26, + "ingress-tail-rar.jpg": b"Rar!\x1a\x07\x00", + "ingress-tail-7z.jpg": b"7z\xbc\xaf\x27\x1c", + "ingress-tail-gzip.jpg": b"\x1f\x8b\x08\x00" + b"\x00" * 6, + "ingress-tail-single-nul.jpg": b"\x00", +} +for name, trailer in TRAILERS.items(): + with open(os.path.join(D, name), "wb") as f: + f.write(jpg_bytes + trailer) + +# Trailing data appended to a valid PNG (chunk stream ends at IEND). +png_bytes = open(os.path.join(D, "ingress-metadata.png"), "rb").read() +with open(os.path.join(D, "ingress-tail-after-iend.png"), "wb") as f: + f.write(png_bytes + b"") + +# 13. False-positive control: a legitimate photo-like JPEG whose *compressed* +# entropy bytes contain archive/script byte sequences by construction. A +# canonical parser must ACCEPT this; a naive substring scanner rejects it. +import random + +random.seed(1337) +noise = Image.new("RGB", (160, 160)) +for y in range(160): + for x in range(160): + noise.putpixel((x, y), (random.randrange(256), random.randrange(256), random.randrange(256))) +noise.save(os.path.join(D, "ingress-entropy-control.jpg"), "JPEG", quality=95) +entropy = open(os.path.join(D, "ingress-entropy-control.jpg"), "rb").read() +for probe in (b"PK\x03\x04", b"PK\x05\x06", b"alert(1)Rar!\x1a\x07\x00" +com_segment = b"\xff\xfe" + (len(comment_payload) + 2).to_bytes(2, "big") + comment_payload +buried[2:2] = com_segment # insert immediately after SOI +with open(os.path.join(D, "ingress-buried-signatures.jpg"), "wb") as f: + f.write(bytes(buried)) + +print("fixtures written") diff --git a/scripts/reencode_image.py b/scripts/reencode_image.py new file mode 100644 index 0000000..b1bc59e --- /dev/null +++ b/scripts/reencode_image.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Re-encode one image safely: verify decode, strip metadata, bound dimensions. + +Fixed argv contract only: + reencode_image.py --in SOURCE --out TARGET [--max-bytes N] + [--max-dimension N] [--max-pixels N] + +Reads SOURCE, rejects oversized geometry from the container header BEFORE any +pixel decode, then decodes defensively, strips all metadata by re-encoding to +baseline JPEG, and writes TARGET. Prints one JSON line on success. Never prints +image bytes or base64 to stdout/stderr. + +Exit codes are a contract the caller maps to client-visible outcomes: + 0 success + 2 rejected hostile/invalid input (client's fault) + 3 processing unavailable (server runtime fault, not the client's) +""" +import argparse +import io +import json +import os +import sys +import warnings + +EXIT_OK = 0 +EXIT_REJECTED = 2 +EXIT_UNAVAILABLE = 3 + +# Pinned, immutable production image runtime. This is the exact Python/Pillow +# toolchain the service re-encoder is built and tested against; the deployment +# smoke gate (scripts/deploy_staging.py and CI) verifies the provisioned +# runtime matches this record before promoting. Changing the runtime must bump +# this pin in lockstep with requirements-test.txt (Pillow==12.3.0) and the +# re-encode resource contract. +PINNED_PYTHON_VERSION = "3.11" +PINNED_PILLOW_VERSION = "12.3.0" + +# Default total-pixel ceiling. Bounded independently of the per-dimension cap so +# wide-and-short or tall-and-narrow geometry cannot smuggle a huge pixel budget +# past a per-side check. 4096*4096 matches the accepted dimension envelope. +DEFAULT_MAX_PIXELS = 4096 * 4096 + + +def parser() -> argparse.ArgumentParser: + p = argparse.ArgumentParser() + p.add_argument("--in", dest="source") + p.add_argument("--out", dest="target") + p.add_argument("--max-bytes", type=int, default=4 * 1024 * 1024) + p.add_argument("--max-dimension", type=int, default=4096) + p.add_argument("--max-pixels", type=int, default=DEFAULT_MAX_PIXELS) + p.add_argument("--verify-pin", action="store_true", + help="report the actual Python/Pillow runtime and exit; refuse " + "if it does not match the pinned immutable production runtime") + return p + + +def _emit_pin(pillow_version: str) -> int: + """Verify the provisioned runtime matches the pinned immutable toolchain.""" + actual_python = f"{sys.version_info.major}.{sys.version_info.minor}" + matched = (actual_python == PINNED_PYTHON_VERSION and pillow_version == PINNED_PILLOW_VERSION) + print(json.dumps({ + "ok": matched, + "python": actual_python, + "pillow": pillow_version, + "pinned": {"python": PINNED_PYTHON_VERSION, "pillow": PINNED_PILLOW_VERSION}, + })) + # A runtime that is not the pinned production toolchain is a server-side + # provisioning fault, not client input: map to processing-unavailable. + return EXIT_OK if matched else EXIT_UNAVAILABLE + + +def reject(error: str) -> int: + print(json.dumps({"ok": False, "error": error})) + return EXIT_REJECTED + + +def main() -> int: + args = parser().parse_args() + + try: + from PIL import Image as _Image + except Exception: + # Server-side runtime fault: the caller must map this to + # processing-unavailable, never to "your photo is corrupt". + print(json.dumps({"ok": False, "error": "unavailable"})) + return EXIT_UNAVAILABLE + + # Pin verification mode: the deployment smoke gate invokes the re-encoder + # with --verify-pin to prove the provisioned, immutable runtime is exactly + # the one the service was built and tested against. No live host change. + if args.verify_pin: + return _emit_pin(getattr(_Image, "__version__", "unknown")) + + # Decompression-bomb warnings are errors here. Pillow's default threshold + # only warns and then hands back a fully decoded image, which is not a + # rejection; promote it so any bomb path raises instead. + warnings.simplefilter("error", _Image.DecompressionBombWarning) + # Refuse to let Pillow allocate beyond our own accepted pixel envelope. + _Image.MAX_IMAGE_PIXELS = max(1, args.max_pixels) + + try: + source_size = os.path.getsize(args.source) + except OSError: + return reject("rejected") + if source_size <= 0 or source_size > args.max_bytes: + return reject("rejected") + + # --- Header-only geometry gate, before any pixel decode. ----------------- + # Image.open() parses the header lazily, so probe.size is available without + # allocating the pixel buffer. Rejecting here keeps a 12000x12000 bomb at + # header cost instead of ~163 MiB of decoded pixels. + try: + with _Image.open(args.source) as probe: + width, height = probe.size + except _Image.DecompressionBombWarning: + return reject("dimensions") + except _Image.DecompressionBombError: + return reject("dimensions") + except Exception: + return reject("malformed") + + if width <= 0 or height <= 0: + return reject("malformed") + if width > args.max_dimension or height > args.max_dimension: + return reject("dimensions") + if width * height > args.max_pixels: + return reject("dimensions") + + # --- Only now is a full decode allowed. --------------------------------- + try: + with _Image.open(args.source) as probe: + probe.load() + # Re-check after decode: a hostile container can declare small + # geometry in its header and expand during decode. + if probe.size[0] > args.max_dimension or probe.size[1] > args.max_dimension: + return reject("dimensions") + if probe.size[0] * probe.size[1] > args.max_pixels: + return reject("dimensions") + image = probe.convert("RGB") + except (_Image.DecompressionBombWarning, _Image.DecompressionBombError): + return reject("dimensions") + except Exception: + return reject("malformed") + + buffer = io.BytesIO() + try: + # Baseline JPEG re-encode drops EXIF/GPS/XMP/tEXt entirely. + image.save(buffer, "JPEG", quality=85, optimize=True, progressive=False) + except Exception: + return reject("encode") + + data = buffer.getvalue() + if not data or data[:3] != b"\xff\xd8\xff" or len(data) > args.max_bytes: + return reject("encode") + + try: + with open(args.target, "wb") as handle: + handle.write(data) + except OSError: + # Cannot write the private temp target: server-side fault. + print(json.dumps({"ok": False, "error": "unavailable"})) + return EXIT_UNAVAILABLE + + print(json.dumps({ + "ok": True, + "format": "jpeg", + "width": image.size[0], + "height": image.size[1], + "bytes": len(data), + "metadataStripped": True, + })) + return EXIT_OK + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/server.mjs b/server.mjs index 82f2ba1..46cd33c 100644 --- a/server.mjs +++ b/server.mjs @@ -4,9 +4,19 @@ import { readFile, stat } from 'node:fs/promises'; import { extname, join, normalize } from 'node:path'; import { fileURLToPath } from 'node:url'; import { analyzePhoto } from './src/vision-service.js'; +import { createRateLimiter } from './src/rate-limiter.js'; +import { resolveClientIdentity, resolveTrustedProxies } from './src/client-identity.js'; +import { IngressUnavailableError } from './src/image-ingress.js'; import { probeVisionProvider, resolveVisionConfig } from './src/vision-config.js'; import { AgentGatewayError, createHermesAgentService, resolveHermesAgentConfig } from './src/hermes-agent-service.js'; +const analyzeRateLimiter=createRateLimiter(); +// Explicit rate-limit identity policy. Behind a reverse proxy the socket peer is +// the proxy itself, so a forwarded client address is only honoured when the peer +// is in this configured allowlist. Otherwise the peer address is used, and a +// trusted proxy that forwards nothing usable degrades to a truthful shared quota. +const trustedProxies=resolveTrustedProxies(process.env.TIMMY_TRUSTED_PROXIES); + const root=fileURLToPath(new URL('.',import.meta.url)); const port=Number(process.env.PORT||4173); const host=process.env.HOST||'0.0.0.0'; @@ -32,19 +42,105 @@ const stagingLabel=process.env.TIMMY_STAGING_LABEL?`Staging · ${release} · ${c function sendJson(res,status,value){res.writeHead(status,{'content-type':'application/json; charset=utf-8','cache-control':'no-store','x-content-type-options':'nosniff'});res.end(JSON.stringify(value));} function escapeHtmlAttribute(value){return String(value).replace(/[&<>"]/g,character=>({'&':'&','<':'<','>':'>','"':'"'}[character]));} -function readJson(req,maxBytes=6*1024*1024){return new Promise((resolve,reject)=>{let size=0,tooLarge=false;const chunks=[];req.on('data',chunk=>{size+=chunk.length;if(size>maxBytes){tooLarge=true;return}chunks.push(chunk)});req.on('end',()=>{if(tooLarge)return reject(new AgentGatewayError(413,'Request is too large.'));try{resolve(JSON.parse(Buffer.concat(chunks).toString('utf8')))}catch{return reject(new AgentGatewayError(400,'Invalid JSON request.'))}});req.on('error',reject)})} +const BODY_READ_TIMEOUT_MS = Math.max(500, Number.parseInt(process.env.TIMMY_BODY_READ_TIMEOUT_MS || '10000', 10) || 10000); +const MAX_BODY_BYTES = 6 * 1024 * 1024; + +/** + * Read a JSON request body with a hard inbound timeout and a stop-on-oversize + * guard. The connection is actively destroyed the moment either limit trips, so + * a hostile client cannot hold a request open by dribbling bytes or force the + * server to drain an enormous payload before rejecting it. + */ +function readJson(req, res, maxBytes = MAX_BODY_BYTES) { + return new Promise((resolve, reject) => { + let size = 0; + let tooLarge = false; + let settled = false; + const chunks = []; + let timer = null; + + const fail = (error) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + req.off('data', onData); + req.off('end', onEnd); + req.off('error', onError); + reject(error); + }; + + const onData = (chunk) => { + if (settled || tooLarge) return; + size += chunk.length; + if (size > maxBytes) { + tooLarge = true; + // Reject immediately so the transport can answer a sanitized 413 without + // waiting for the client to finish streaming an enormous payload. The + // socket is torn down by the caller after the response is written. + return fail(new AgentGatewayError(413, 'Request is too large. Use a smaller photo or continue manually.')); + } + chunks.push(chunk); + }; + const onEnd = () => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + if (tooLarge) return; + try { + resolve(JSON.parse(Buffer.concat(chunks).toString('utf8'))); + } catch { + reject(new AgentGatewayError(400, 'Invalid JSON request.')); + } + }; + const onError = (error) => { + // A broken socket is torn down here; the rejection still propagates. + try { req.destroy(); } catch { /* already gone */ } + fail(error); + }; + + timer = setTimeout(() => { + // Hard inbound timeout: a slow-dribbling client must not hold the + // connection open. The caller answers 408 and destroys the socket. + fail(new AgentGatewayError(408, 'Request timed out while reading the upload.')); + }, BODY_READ_TIMEOUT_MS); + + req.on('data', onData); + req.on('end', onEnd); + req.on('error', onError); + }); +} function cookie(req,name){for(const part of String(req.headers.cookie||'').split(';')){const [key,...value]=part.trim().split('=');if(key===name)return decodeURIComponent(value.join('='))}return ''} function requestOrigin(req){return String(req.headers.origin||'').replace(/\/$/,'')} function rejectCrossSite(req){const site=String(req.headers['sec-fetch-site']||'');if(site&&site!=='same-origin')throw new AgentGatewayError(403,'Cross-site agent requests are not allowed.')} function agentCookie(token){const secure=agentConfig.publicOrigin.startsWith('https://')?'; Secure':'';return `timmy_agent=${encodeURIComponent(token)}; HttpOnly; SameSite=Strict; Path=${appRoot}; Max-Age=86400${secure}`} function sendAgentError(res,error){const status=error instanceof AgentGatewayError?error.status:503;const message=error instanceof AgentGatewayError?error.message:'Hermes is temporarily unavailable. Your local journal still works.';return sendJson(res,status,{error:message})} +// Ingress failures carry their own honest classification: a server-side +// processing fault must not be reported as corrupt client input. +function handleAnalyzeError(res,error){ + if(error instanceof IngressUnavailableError||error?.ingressUnavailable===true)return sendJson(res,503,{error:error.message,manualFallback:true}); + const safe=/consent|JPEG|PNG|WebP|empty|too large|supported image|corrupt|malformed|slow down/i.test(error.message); + return sendJson(res,safe?400:503,{error:error.message}); +} +// A body-read rejection (oversized or timed out) is answered with a sanitized +// status and then the socket is torn down so a hostile client cannot keep the +// request open by dribbling or draining an enormous payload. +function handleBodyError(res,req,error){ + if(typeof error?.status==='number'){sendJson(res,error.status,{error:error.message});} + else{sendJson(res,400,{error:'Invalid request.'});} + try{res.end();req.destroy();}catch{/* already closed */} +} + http.createServer(async(req,res)=>{ try{ const url=new URL(req.url,'http://localhost'); if(basePath&&url.pathname!==basePath&&!url.pathname.startsWith(`${basePath}/`)){res.writeHead(404,{'content-type':'text/plain; charset=utf-8'});return res.end('Not found')} if(basePath&&url.pathname===basePath){res.writeHead(308,{location:appRoot});return res.end()} const appPath=basePath?(url.pathname.slice(basePath.length)||'/'):url.pathname; + // Nested API routes under the base path (e.g. /timmy-staging/api/healthz) are + // routed to their canonical /api/* handler so the base path is preserved on + // every operational endpoint, not just the root. + const apiPath=basePath&&appPath.startsWith('/api/')?appPath:appPath; if(appPath==='/api/healthz'&&req.method==='GET')return sendJson(res,200,{ok:true,release,commit,visionEnabled:visionConfig.enabled,agentEnabled:agentConfig.enabled}); if(appPath==='/api/vision-status'&&req.method==='GET'){ const provider=await probeVisionProvider(visionConfig); @@ -53,9 +149,13 @@ http.createServer(async(req,res)=>{ } if(appPath==='/api/analyze'&&req.method==='POST'){ if(!visionConfig.enabled)return sendJson(res,503,{error:'AI analysis is disabled. Continue manually.'}); - const payload=await readJson(req); - try{return sendJson(res,200,await analyzePhoto({payload,config:visionConfig}))} - catch(error){const safe=/consent|JPEG|PNG|WebP|empty|too large/i.test(error.message);return sendJson(res,safe?400:503,{error:error.message})} + const identity=resolveClientIdentity({remoteAddress:req.socket?.remoteAddress,headers:req.headers,trustedProxies}); + const limit=analyzeRateLimiter.take(identity.key); + if(!limit.allowed){ + res.setHeader('retry-after',Math.ceil(limit.retryAfterMs/1000)); + return sendJson(res,429,{error:'Too many photo analyses. Please slow down and try again later.'}); + } + try{const payload=await readJson(req,res);try{return sendJson(res,200,await analyzePhoto({payload,config:visionConfig}))}catch(error){return handleAnalyzeError(res,error)}}catch(error){return handleBodyError(res,req,error)} } if(appPath==='/api/agent/status'&&req.method==='GET')return sendJson(res,200,agentService.status(cookie(req,'timmy_agent'))); if(appPath==='/api/agent/unlock'&&req.method==='POST'){ diff --git a/src/client-identity.js b/src/client-identity.js new file mode 100644 index 0000000..b5c2c82 --- /dev/null +++ b/src/client-identity.js @@ -0,0 +1,70 @@ +// Explicit client identity for rate limiting. +// +// Behind a reverse proxy the socket peer is the proxy itself, so keying on +// req.socket.remoteAddress silently collapses every user into one quota. The +// policy here is explicit and fails safe: +// +// * If the peer is NOT a configured trusted proxy, forwarded headers are +// ignored entirely — they are attacker-controlled and would let anyone mint +// unlimited identities or impersonate another client. +// * If the peer IS a configured trusted proxy, the left-most syntactically +// valid address in the forwarded chain is used as the client identity. +// * If a trusted proxy forwards nothing usable, the scope degrades to an +// honest shared 'global' quota rather than pretending to be per-client. +// +// The returned scope is reported truthfully so operators and health output can +// state which policy is actually in force. +import { isIP } from 'node:net'; + +const MAX_KEY_LENGTH = 64; + +function normaliseAddress(value) { + const raw = String(value || '').trim(); + if (!raw || raw.length > MAX_KEY_LENGTH) return null; + // Strip an IPv6 zone index and IPv4-mapped IPv6 prefix for stable keys. + const withoutZone = raw.replace(/%.*$/, ''); + const unmapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(withoutZone); + const candidate = unmapped ? unmapped[1] : withoutZone; + return isIP(candidate) ? candidate : null; +} + +/** Parse the trusted-proxy allowlist from configuration. */ +export function resolveTrustedProxies(value) { + return String(value || '') + .split(',') + .map((entry) => normaliseAddress(entry)) + .filter((entry) => entry !== null); +} + +/** + * Decide the rate-limit identity for one request. + * + * Returns `{ key, scope, trustedProxy }` where scope is one of: + * 'peer' keyed on the directly connected address + * 'forwarded' keyed on a client address supplied by a trusted proxy + * 'global' one shared quota, stated honestly + */ +export function resolveClientIdentity({ remoteAddress, headers = {}, trustedProxies = [] } = {}) { + const peer = normaliseAddress(remoteAddress); + const allowlist = trustedProxies.map((entry) => normaliseAddress(entry)).filter(Boolean); + const trusted = peer !== null && allowlist.includes(peer); + + if (!trusted) { + // Untrusted peer: forwarded headers are spoofable and must be ignored. + if (peer === null) return { key: 'global', scope: 'global', trustedProxy: false }; + return { key: peer, scope: 'peer', trustedProxy: false }; + } + + const forwardedFor = headers['x-forwarded-for'] ?? headers['X-Forwarded-For']; + const chain = String(forwardedFor || '').split(','); + for (const entry of chain) { + const client = normaliseAddress(entry); + if (client) return { key: client, scope: 'forwarded', trustedProxy: true }; + } + const realIp = normaliseAddress(headers['x-real-ip'] ?? headers['X-Real-IP']); + if (realIp) return { key: realIp, scope: 'forwarded', trustedProxy: true }; + + // A trusted proxy that forwards nothing usable means the quota really is + // shared. Say so rather than reporting a per-client guarantee we cannot keep. + return { key: 'global', scope: 'global', trustedProxy: true }; +} diff --git a/src/image-container.js b/src/image-container.js new file mode 100644 index 0000000..03f8ac4 --- /dev/null +++ b/src/image-container.js @@ -0,0 +1,141 @@ +// Canonical image container boundary. +// +// Polyglot and trailing-data rejection must be structural: parse the declared +// container and require that its own structure consumes exactly the supplied +// bytes. Substring scanning for signatures like `= bytes.length) return null; + if (bytes[offset] !== 0xff) return null; + // Fill bytes: any number of 0xFF may precede a marker code. + let marker = bytes[offset + 1]; + let markerAt = offset + 1; + while (marker === 0xff) { + markerAt += 1; + if (markerAt >= bytes.length) return null; + marker = bytes[markerAt]; + } + if (marker === 0x00) return null; // stuffed byte outside entropy data + if (marker === 0xd9) return { format: 'jpeg', consumed: markerAt + 1 }; // EOI + if (JPEG_STANDALONE.has(marker)) { + offset = markerAt + 1; + continue; + } + if (markerAt + 2 >= bytes.length) return null; + const length = bytes.readUInt16BE(markerAt + 1); + if (length < 2) return null; + const segmentEnd = markerAt + 1 + length; + if (segmentEnd > bytes.length) return null; + if (marker !== 0xda) { + offset = segmentEnd; + continue; + } + // Start of scan: entropy-coded data runs until the next real marker. + let scan = segmentEnd; + for (; scan < bytes.length; scan += 1) { + if (bytes[scan] !== 0xff) continue; + const next = bytes[scan + 1]; + if (next === undefined) return null; + if (next === 0x00) { scan += 1; continue; } // stuffed 0xFF data byte + if (next >= 0xd0 && next <= 0xd7) { scan += 1; continue; } // restart marker + if (next === 0xff) continue; // fill byte + break; + } + if (scan >= bytes.length) return null; // ran out of data before EOI + offset = scan; + } + return null; +} + +function parsePng(bytes) { + const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + if (bytes.length < 8 || !bytes.subarray(0, 8).equals(signature)) return null; + let offset = 8; + let seenHeader = false; + for (let step = 0; step < MAX_STRUCTURE_STEPS; step += 1) { + if (offset + 8 > bytes.length) return null; + const length = bytes.readUInt32BE(offset); + if (length > 0x7fffffff) return null; + const type = bytes.subarray(offset + 4, offset + 8).toString('latin1'); + if (!/^[A-Za-z]{4}$/.test(type)) return null; + if (!seenHeader && type !== 'IHDR') return null; + seenHeader = true; + const end = offset + 8 + length + 4; // data + CRC + if (end > bytes.length) return null; + if (type === 'IEND') { + if (length !== 0) return null; + return { format: 'png', consumed: end }; + } + offset = end; + } + return null; +} + +function parseWebp(bytes) { + if (bytes.length < 12) return null; + if (bytes.subarray(0, 4).toString('latin1') !== 'RIFF') return null; + if (bytes.subarray(8, 12).toString('latin1') !== 'WEBP') return null; + const riffSize = bytes.readUInt32LE(4); + if (riffSize < 4 || riffSize > 0x7fffffff) return null; + const declaredEnd = 8 + riffSize; + if (declaredEnd > bytes.length) return null; + // Walk the chunk list so a truncated or over-declared RIFF is caught too. + let offset = 12; + for (let step = 0; step < MAX_STRUCTURE_STEPS; step += 1) { + if (offset === declaredEnd) return { format: 'webp', consumed: declaredEnd }; + if (offset + 8 > declaredEnd) return null; + const type = bytes.subarray(offset, offset + 4).toString('latin1'); + if (!/^[A-Za-z0-9 ]{4}$/.test(type)) return null; + const size = bytes.readUInt32LE(offset + 4); + if (size > 0x7fffffff) return null; + const padded = size + (size % 2); + const end = offset + 8 + padded; + if (end > declaredEnd) return null; + offset = end; + } + return null; +} + +const PARSERS = [parseJpeg, parsePng, parseWebp]; + +/** + * Parse a supported raster container structurally. + * + * Returns `{ format, consumed, trailingBytes }` when the signature matches a + * supported format and the structure is internally consistent, otherwise null. + * `trailingBytes` is how many supplied bytes the container did not account for. + */ +export function parseImageContainer(bytes) { + if (!Buffer.isBuffer(bytes) || bytes.length < 12) return null; + for (const parse of PARSERS) { + const parsed = parse(bytes); + if (parsed) { + return { ...parsed, trailingBytes: bytes.length - parsed.consumed }; + } + } + return null; +} + +/** + * Canonical acceptance: a supported container whose own structure accounts for + * every supplied byte. Any trailing data is a container polyglot and fails. + */ +export function canonicalImageFormat(bytes) { + const parsed = parseImageContainer(bytes); + if (!parsed || parsed.trailingBytes !== 0) return null; + return parsed.format; +} diff --git a/src/image-ingress.js b/src/image-ingress.js new file mode 100644 index 0000000..c720975 --- /dev/null +++ b/src/image-ingress.js @@ -0,0 +1,299 @@ +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { canonicalImageFormat } from './image-container.js'; + +export const MAX_IMAGE_BYTES = 4 * 1024 * 1024; +export const MAX_IMAGE_DIMENSION = 4096; +// Total-pixel ceiling, bounded independently of the per-side cap so extreme +// aspect ratios cannot smuggle a large pixel budget past a per-dimension check. +export const MAX_IMAGE_PIXELS = MAX_IMAGE_DIMENSION * MAX_IMAGE_DIMENSION; +const REENCODE_TIMEOUT_MS = 15_000; + +// Production runs a pinned, immutable interpreter provisioned outside the +// release tree; TIMMY_PYTHON must be an absolute path so the service never +// depends on PATH resolution. Development falls back to `python3`. +export function resolveInterpreter(env = process.env) { + const configured = String(env.TIMMY_PYTHON || '').trim(); + if (!configured) return 'python3'; + if (!configured.startsWith('/')) { + throw new Error('TIMMY_PYTHON must be an absolute interpreter path.'); + } + return configured; +} + +// The pinned, immutable production image runtime. The service refuses to start +// decode work if the provisioned interpreter is not this exact toolchain: +// a drifting runtime is a server-side provisioning fault, and decoding images +// under an untested Pillow/Pillow-version would silently change the security +// boundary. The deployment smoke gate (scripts/deploy_staging.py) verifies the +// same pin before promoting; this is the in-process fail-closed backstop. +export const PINNED_PYTHON_VERSION = '3.11'; +export const PINNED_PILLOW_VERSION = '12.3.0'; +export const REENCODE_SMOKE_TIMEOUT_MS = 10_000; + +/** + * Verify the provisioned re-encoder runtime matches the pinned, immutable + * production toolchain. Runs the re-encoder with --verify-pin and maps any + * non-zero exit (or a version mismatch) to an IngressUnavailableError so the + * transport layer answers 503 rather than blaming a client photo. + */ +export function verifyReencodeRuntime({ timeoutMs = REENCODE_SMOKE_TIMEOUT_MS } = {}) { + return new Promise((resolve, reject) => { + execFile( + resolveInterpreter(), + [ + join(fileURLToPath(new URL('..', import.meta.url)), 'scripts', 'reencode_image.py'), + '--verify-pin', + ], + { timeout: timeoutMs, windowsHide: true, maxBuffer: 64 * 1024 }, + (error, stdout) => { + if (error) { + // Missing, non-runnable, or version-mismatched runtime: a provisioning + // fault, not the client's problem. + return reject(sanitizedError('unavailable')); + } + let verdict = {}; + try { verdict = JSON.parse(String(stdout).trim()); } catch { /* fall through */ } + if (verdict.ok !== true) { + return reject(sanitizedError('unavailable')); + } + return resolve({ + python: verdict.python, + pillow: verdict.pillow, + pinned: verdict.pinned, + }); + }, + ); + }); +} + +const MAGIC = { + jpeg: (b) => b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff, + png: (b) => b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47, + webp: (b) => b.subarray(0, 4).toString('latin1') === 'RIFF' && b.subarray(8, 12).toString('latin1') === 'WEBP', +}; + +export function sniffImageFormat(bytes) { + if (!Buffer.isBuffer(bytes) || bytes.length < 12) return null; + for (const [format, test] of Object.entries(MAGIC)) { + if (test(bytes)) { + // Signature match is necessary but not sufficient: the container must + // also parse canonically and account for every supplied byte. This + // rejects trailing-data and container polyglots (appended HTML/script in + // any casing, ZIP local headers or EOCD records, other archive + // signatures) without substring scanning, which both misses variants and + // false-positives on compressed entropy data. + return canonicalImageFormat(bytes) === format ? format : null; + } + } + return null; +} + +// Exact supported MIME policy: canonical raster types only. The declared MIME +// is deliberately not trusted for format detection (magic bytes decide that), +// but an unsupported declared type is still refused outright. +const SUPPORTED_MIME = new Set(['image/jpeg', 'image/png', 'image/webp']); + +// Canonical base64 data URL grammar. No whitespace, no newlines, no URL-safe +// alphabet, no extra parameters, exactly one correct padding group. +const DATA_URL_GRAMMAR = /^data:([a-z]+\/[a-z0-9.+-]+);base64,([A-Za-z0-9+/]+={0,2})$/; + +/** + * Decode a canonical base64 raster data URL. + * + * Returns `{ mime, bytes }` only when the value matches the canonical grammar, + * declares a supported MIME type, and re-encodes byte-identically (which is + * what rejects missing or excess padding and non-zero trailing pad bits). + * Returns null otherwise. Nothing is repaired or normalised: a non-canonical + * request is a hand-crafted request and fails closed. + */ +export function decodeImageDataUrl(value) { + if (typeof value !== 'string') return null; + const match = DATA_URL_GRAMMAR.exec(value); + if (!match) return null; + const [, mime, payload] = match; + if (!SUPPORTED_MIME.has(mime)) return null; + if (payload.length % 4 !== 0) return null; + const bytes = Buffer.from(payload, 'base64'); + if (bytes.length === 0) return null; + // Byte-exact canonical round trip: rejects non-canonical padding and any + // non-zero bits in the final partial group. + if (bytes.toString('base64') !== payload) return null; + return { mime, bytes }; +} + +// Errors are sanitized by construction: fixed short strings only. +const SANITIZED = { + consent: 'Explicit consent is required before AI analysis.', + format: 'Upload a JPEG, PNG, or WebP photo. That file is not a supported image.', + size: 'The photo is too large. Use an image under 4 MB.', + dimensions: 'The photo is too large. Maximum dimension is 4096 pixels.', + corrupt: 'The photo is corrupt or malformed. Try a different photo or continue manually.', + unavailable: 'Photo processing is temporarily unavailable. Continue manually.', + capacity: 'Photo processing is temporarily unavailable. Please try again in a moment or continue manually.', +}; + +/** + * A server-side processing fault: the image runtime could not run at all. + * + * This is deliberately a distinct type so the transport layer can answer 503 + * (processing unavailable, manual fallback) instead of 400, which would falsely + * blame the client's photo for being corrupt. + */ +export class IngressUnavailableError extends Error { + constructor(message) { + super(message); + this.name = 'IngressUnavailableError'; + this.ingressUnavailable = true; + } +} + +/** Map an ingress failure to its client-visible HTTP status. */ +export function classifyIngressFailure(error) { + return error instanceof IngressUnavailableError || error?.ingressUnavailable === true ? 503 : 400; +} + +// --- Fail-fast concurrency ceiling around decoder/provider work ------------- +// +// Each decode is a subprocess with a real memory cost, and the service runs +// under MemoryMax=512M. A burst is therefore bounded by a small ceiling and +// excess requests are refused immediately rather than queued: queuing would +// convert a burst into unbounded latency and keep hostile load resident. +export const INGRESS_MAX_CONCURRENCY = Math.max( + 1, + Math.min(4, Number.parseInt(process.env.TIMMY_INGRESS_MAX_CONCURRENCY || '2', 10) || 2), +); + +let activeDecodes = 0; + +/** Current decoder occupancy, for tests and health reporting. */ +export function ingressConcurrencyState() { + return { active: activeDecodes, limit: INGRESS_MAX_CONCURRENCY }; +} + +function acquireDecodeSlot() { + if (activeDecodes >= INGRESS_MAX_CONCURRENCY) return false; + activeDecodes += 1; + return true; +} + +function releaseDecodeSlot() { + activeDecodes = Math.max(0, activeDecodes - 1); +} + +function sanitizedError(key) { + if (key === 'unavailable' || key === 'capacity') return new IngressUnavailableError(SANITIZED[key]); + return new Error(SANITIZED[key]); +} + +// Exit-code contract shared with scripts/reencode_image.py: +// 0 success, 2 rejected hostile/invalid input, 3 processing unavailable. +const REENCODE_EXIT_REJECTED = 2; +const REENCODE_EXIT_UNAVAILABLE = 3; + +function reencode(sourcePath, targetPath, { timeoutMs = REENCODE_TIMEOUT_MS } = {}) { + return new Promise((resolve, reject) => { + execFile( + resolveInterpreter(), + [ + join(fileURLToPath(new URL('..', import.meta.url)), 'scripts', 'reencode_image.py'), + '--in', sourcePath, + '--out', targetPath, + '--max-bytes', String(MAX_IMAGE_BYTES), + '--max-dimension', String(MAX_IMAGE_DIMENSION), + '--max-pixels', String(MAX_IMAGE_PIXELS), + ], + { timeout: timeoutMs, windowsHide: true }, + (error, stdout) => { + if (error) { + // Timeout or signal kill: the runtime did not finish, so this is a + // server-side fault rather than bad client input. + if (error.killed || error.signal) return reject(sanitizedError('unavailable')); + // The interpreter itself could not be executed (missing/not runnable). + if (error.code === 'ENOENT' || error.code === 'EACCES') { + return reject(sanitizedError('unavailable')); + } + let verdict = {}; + try { + verdict = JSON.parse(String(stdout).trim().split('\n').pop() || '{}'); + } catch { verdict = {}; } + // Exit 3 means the image runtime is unavailable. Reporting this as a + // client 400 would falsely blame the user's photo. + if (error.code === REENCODE_EXIT_UNAVAILABLE || verdict.error === 'unavailable') { + return reject(sanitizedError('unavailable')); + } + if (error.code === REENCODE_EXIT_REJECTED) { + if (verdict.error === 'dimensions') return reject(sanitizedError('dimensions')); + return reject(sanitizedError('corrupt')); + } + // Any other exit status is an unexpected runtime failure, not a + // proven statement about the client's input. + return reject(sanitizedError('unavailable')); + } + try { + resolve(JSON.parse(String(stdout).trim())); + } catch { + reject(sanitizedError('unavailable')); + } + }, + ); + }); +} + +export async function validateImageIngress(payload = {}) { + // Cheap, allocation-free checks run before any capacity is taken so hostile + // input can never occupy a decoder slot. + if (payload.consent !== true) throw sanitizedError('consent'); + const decoded = decodeImageDataUrl(payload.imageDataUrl); + if (!decoded) throw sanitizedError('format'); + const bytes = decoded.bytes; + if (bytes.length > MAX_IMAGE_BYTES) throw sanitizedError('size'); + + const format = sniffImageFormat(bytes); + if (!format) throw sanitizedError('format'); + + // Expensive work starts here: bound it with a fail-fast ceiling so a burst + // cannot spawn enough decoders to exceed the service memory budget. + if (!acquireDecodeSlot()) throw sanitizedError('capacity'); + try { + const workDir = await mkdtemp(join(tmpdir(), 'timmy-ingress-')); + try { + const sourcePath = join(workDir, `source.${format}`); + const targetPath = join(workDir, 'reencoded.jpg'); + await writeFile(sourcePath, bytes); + const result = await reencode(sourcePath, targetPath, { + timeoutMs: Number.isFinite(payload.reencodeTimeoutMs) && payload.reencodeTimeoutMs > 0 + ? payload.reencodeTimeoutMs + : REENCODE_TIMEOUT_MS, + }); + let reencoded; + try { + reencoded = await readFile(targetPath); + } catch { + // The subprocess reported success but produced no readable output: a + // server-side fault, not a statement about the client's photo. + throw sanitizedError('unavailable'); + } + return { + imageDataUrl: `data:image/jpeg;base64,${reencoded.toString('base64')}`, + mime: 'image/jpeg', + format: result.format, + width: result.width, + height: result.height, + bytes: reencoded.length, + originalFormat: format, + metadataStripped: result.metadataStripped === true, + }; + } finally { + // Private temp directory cleanup stays unconditional. + await rm(workDir, { recursive: true, force: true }).catch(() => {}); + } + } finally { + // The slot is released on every path, including rejections, so a hostile + // burst cannot permanently exhaust capacity. + releaseDecodeSlot(); + } +} diff --git a/src/rate-limiter.js b/src/rate-limiter.js new file mode 100644 index 0000000..92372cd --- /dev/null +++ b/src/rate-limiter.js @@ -0,0 +1,117 @@ +// Bounded rate limiter for image analysis requests. +// +// Two properties the naive fixed-window version did not have: +// +// 1. Hard-bounded state. Sweeping only *expired* entries lets an attacker with +// many distinct keys grow the map without limit inside one window. State is +// capped at maxKeys; reclamation prefers expired entries, and when none are +// expired the entry closest to expiry is dropped. Currently-blocked +// offenders are never dropped, so flooding cannot reset someone's counter. +// +// 2. No boundary doubling. A fixed window lets a client spend a full budget at +// the end of one window and another immediately after the boundary, i.e. +// 2x the nominal rate in milliseconds. This uses a two-window weighted +// sliding count so the boundary is smooth. +// +// Only counters are stored — never payloads, bodies, or headers. + +const DEFAULT_MAX_KEYS = 10_000; + +export function createRateLimiter({ + windowMs = 60_000, + maxRequests = 10, + maxKeys = DEFAULT_MAX_KEYS, +} = {}) { + // key -> { windowStart, count, previousCount } + const windows = new Map(); + + function slidingCount(entry, now) { + // Weight the previous window by how much of it still overlaps the trailing + // `windowMs` interval ending at `now`. + const elapsed = now - entry.windowStart; + const overlap = Math.max(0, 1 - elapsed / windowMs); + return entry.previousCount * overlap + entry.count; + } + + function isBlocked(entry, now) { + return slidingCount(entry, now) >= maxRequests; + } + + function reclaim(now, protectedKey) { + if (windows.size < maxKeys) return; + // First pass: drop fully expired entries (two windows old, so their + // weighted contribution is zero). + for (const [key, entry] of windows) { + if (key === protectedKey) continue; + if (now - entry.windowStart >= windowMs * 2) windows.delete(key); + } + if (windows.size < maxKeys) return; + // Second pass: hard bound. Drop the oldest entries that are not currently + // blocking anyone, so eviction can never reset an active offender's count. + const candidates = []; + for (const [key, entry] of windows) { + if (key === protectedKey) continue; + if (isBlocked(entry, now)) continue; + candidates.push([key, entry.windowStart]); + } + candidates.sort((a, b) => a[1] - b[1]); + for (const [key] of candidates) { + if (windows.size < maxKeys) break; + windows.delete(key); + } + if (windows.size < maxKeys) return; + // Every retained entry is actively blocked. Drop the oldest of those to + // preserve the hard bound; its window is closest to expiring anyway. + let oldestKey = null; + let oldestStart = Infinity; + for (const [key, entry] of windows) { + if (key === protectedKey) continue; + if (entry.windowStart < oldestStart) { + oldestStart = entry.windowStart; + oldestKey = key; + } + } + if (oldestKey !== null) windows.delete(oldestKey); + } + + function take(key, now = Date.now()) { + let entry = windows.get(key); + if (!entry) { + reclaim(now, key); + entry = { windowStart: now, count: 0, previousCount: 0 }; + windows.set(key, entry); + } else { + const elapsed = now - entry.windowStart; + if (elapsed >= windowMs * 2) { + entry.windowStart = now; + entry.count = 0; + entry.previousCount = 0; + } else if (elapsed >= windowMs) { + entry.windowStart = entry.windowStart + windowMs * Math.floor(elapsed / windowMs); + entry.previousCount = entry.count; + entry.count = 0; + } + } + + if (slidingCount(entry, now) + 1 > maxRequests) { + // Retry after the remaining overlap of the trailing interval. + const elapsed = now - entry.windowStart; + const retryAfterMs = Math.max(1, Math.ceil(windowMs - elapsed)); + return { + allowed: false, + retryAfterMs, + reason: 'Too many photo analyses. Please slow down and try again later.', + }; + } + entry.count += 1; + return { allowed: true, remaining: Math.max(0, maxRequests - Math.ceil(slidingCount(entry, now))) }; + } + + return { + take, + limit: maxRequests, + windowMs, + maxKeys, + size: () => windows.size, + }; +} diff --git a/src/vision-service.js b/src/vision-service.js index 6edb789..c34c141 100644 --- a/src/vision-service.js +++ b/src/vision-service.js @@ -1,4 +1,5 @@ -import { buildVisionRequest, parseVisionResponse, validatePhotoPayload } from './analysis.js'; +import { validateImageIngress } from './image-ingress.js'; +import { buildVisionRequest, parseVisionResponse } from './analysis.js'; function providerEndpoint(baseUrl) { let url; @@ -8,7 +9,9 @@ function providerEndpoint(baseUrl) { } export async function analyzePhoto({ payload, fetchImpl = fetch, config }) { - const photo = validatePhotoPayload(payload); + // Hardened ingress first: consent, magic bytes, limits, safe re-encode, + // metadata stripping. All failures happen before any provider work. + const photo = await validateImageIngress(payload); if (!config?.model) throw new Error('AI analysis is not configured.'); const endpoint = providerEndpoint(config.baseUrl); const response = await fetchImpl(endpoint, { diff --git a/tests/body-timeout.test.js b/tests/body-timeout.test.js new file mode 100644 index 0000000..93bf89a --- /dev/null +++ b/tests/body-timeout.test.js @@ -0,0 +1,132 @@ +// Inbound body-read timeout and stop-on-oversize contract. +// +// A client must not be able to hold a connection open by dribbling a body +// forever, nor make the server drain an enormous payload before rejecting it. +// The server must enforce a body-read timeout and stop/destroy oversized request +// processing as soon as the declared or observed size exceeds the limit, while +// preserving the configured base path and a sanitized 413. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { setTimeout as sleep } from 'node:timers/promises'; +import net from 'node:net'; +import { spawn } from 'node:child_process'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = fileURLToPath(new URL('..', import.meta.url)); +const BASE_PORT = 4199; +const originFor = (port) => `http://127.0.0.1:${port}`; +const BODY_TIMEOUT_MS = 1500; + +let nextPort = BASE_PORT; +let currentPort = BASE_PORT; +async function startServer(env = {}) { + const PORT = nextPort++; + currentPort = PORT; + const workdir = await mkdtemp(join(tmpdir(), 'timmy-body-test-')); + const child = spawn(process.execPath, ['server.mjs'], { + cwd: root, + env: { + ...process.env, + PORT: String(PORT), + TIMMY_VISION_ENABLED: 'true', + TIMMY_BODY_READ_TIMEOUT_MS: String(BODY_TIMEOUT_MS), + ...env, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const origin = originFor(PORT); + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + if (child.exitCode !== null) throw new Error(`server exited ${child.exitCode}`); + try { const response = await fetch(`${origin}/api/healthz`); if (response.ok) break; } catch {} + await sleep(50); + } + return { child, workdir, origin }; +} + +test('a slow-dribbling request body is terminated by the body-read timeout', async () => { + const { child, workdir, origin } = await startServer(); + try { + const socket = net.connect(currentPort, '127.0.0.1'); + await new Promise((resolve) => socket.once('connect', resolve)); + // Declare a large body and then dribble spaces far slower than the timeout. + socket.write( + `POST ${basePath()}/api/analyze HTTP/1.1\r\n` + + `Host: 127.0.0.1\r\nContent-Type: application/json\r\n` + + `Content-Length: ${8 * 1024 * 1024}\r\nConnection: close\r\n\r\n`, + ); + const start = Date.now(); + let written = 0; + const chunk = Buffer.alloc(64 * 1024, 0x20); + let ended = false; + let closeAt = 0; + const writer = setInterval(() => { + if (written >= 7 * 1024 * 1024 || socket.destroyed) { clearInterval(writer); return; } + try { socket.write(chunk); written += chunk.length; } catch { clearInterval(writer); } + }, 400); + socket.on('close', () => { ended = true; closeAt = Date.now(); }); + // Swallow EPIPE: once the server tears down the socket, any in-flight write + // must not crash the test process. + socket.on('error', () => { clearInterval(writer); }); + // Wait well past the timeout to see whether the server kills the slow stream. + await sleep(BODY_TIMEOUT_MS + 2500); + clearInterval(writer); + const elapsed = Date.now() - start; + socket.destroy(); + assert.ok(ended, 'the server must close a slow-dribbling body before it finishes streaming'); + assert.ok(closeAt - start < BODY_TIMEOUT_MS + 2000, + `a slow body was allowed to stream for ${closeAt - start}ms; the timeout is not enforced`); + } finally { + child.kill('SIGTERM'); + await rm(workdir, { recursive: true, force: true }); + } +}); + +test('an oversized request is rejected with a sanitized 413 and not drained', async () => { + const { child, workdir, origin } = await startServer(); + try { + const response = await fetch(`${origin}${basePath()}/api/analyze`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + // No Content-Length: the server must stop once the stream exceeds the cap, + // not wait for the client to finish sending. `duplex: 'half'` is required + // by fetch when a request body is a streaming ReadableStream. + duplex: 'half', + body: new ReadableStream({ + start(controller) { + const blob = new Uint8Array(10 * 1024 * 1024).fill(0x20); + controller.enqueue(blob); + // Keep the stream open so the server must stop it, not us. + setTimeout(() => controller.close(), 5000); + }, + }), + }); + assert.equal(response.status, 413); + const body = await response.text(); + assert.match(body, /too large/i); + assert.doesNotMatch(body, /[A-Za-z0-9+/]{40,}/, 'no payload data in the 413 response'); + } finally { + child.kill('SIGTERM'); + await rm(workdir, { recursive: true, force: true }); + } +}); + +test('the configured base path is preserved on the analyze route after hardening', async () => { + const { child, workdir, origin } = await startServer({ TIMMY_BASE_PATH: '/timmy-staging' }); + try { + const response = await fetch(`${origin}/timmy-staging/api/healthz`); + assert.equal(response.status, 200); + const missing = await fetch(`${origin}/api/healthz`); + assert.equal(missing.status, 404); + } finally { + child.kill('SIGTERM'); + await rm(workdir, { recursive: true, force: true }); + } +}); + +function basePath() { + return process.env.TIMMY_BASE_PATH_TEST || ''; +} diff --git a/tests/ci-workflow.test.js b/tests/ci-workflow.test.js index 07fdb83..5e642b0 100644 --- a/tests/ci-workflow.test.js +++ b/tests/ci-workflow.test.js @@ -17,6 +17,10 @@ test('Gitea CI gates pull requests and main with the reproducible quality suite' assert.match(workflow, /fetch-depth: 0/); assert.match(workflow, /npm ci/); assert.match(workflow, /python3 -m pip install --break-system-packages -r requirements-test\.txt/); + assert.match(workflow, /uses: actions\/setup-python@v5/); + assert.match(workflow, /python-version: ['"]3\.11['"]/); + assert.match(workflow, /TIMMY_PYTHON=.*sys\.executable/); + assert.match(workflow, /sys\.modules\[s\.name\]=m/); assert.match(workflow, /npm test/); assert.match(workflow, /npm run test:ui/); assert.match(workflow, /npm run test:photo/); diff --git a/tests/fixtures/ingress-bomb.png b/tests/fixtures/ingress-bomb.png new file mode 100644 index 0000000..f9ef206 Binary files /dev/null and b/tests/fixtures/ingress-bomb.png differ diff --git a/tests/fixtures/ingress-buried-signatures.jpg b/tests/fixtures/ingress-buried-signatures.jpg new file mode 100644 index 0000000..62e6965 Binary files /dev/null and b/tests/fixtures/ingress-buried-signatures.jpg differ diff --git a/tests/fixtures/ingress-clean.jpg b/tests/fixtures/ingress-clean.jpg new file mode 100644 index 0000000..93a664d Binary files /dev/null and b/tests/fixtures/ingress-clean.jpg differ diff --git a/tests/fixtures/ingress-empty.jpg b/tests/fixtures/ingress-empty.jpg new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/ingress-entropy-control.jpg b/tests/fixtures/ingress-entropy-control.jpg new file mode 100644 index 0000000..cde4056 Binary files /dev/null and b/tests/fixtures/ingress-entropy-control.jpg differ diff --git a/tests/fixtures/ingress-exif.jpg b/tests/fixtures/ingress-exif.jpg new file mode 100644 index 0000000..73153a8 Binary files /dev/null and b/tests/fixtures/ingress-exif.jpg differ diff --git a/tests/fixtures/ingress-garbage.jpg b/tests/fixtures/ingress-garbage.jpg new file mode 100644 index 0000000..0707d54 Binary files /dev/null and b/tests/fixtures/ingress-garbage.jpg differ diff --git a/tests/fixtures/ingress-metadata.png b/tests/fixtures/ingress-metadata.png new file mode 100644 index 0000000..cb52c64 Binary files /dev/null and b/tests/fixtures/ingress-metadata.png differ diff --git a/tests/fixtures/ingress-oversized.jpg b/tests/fixtures/ingress-oversized.jpg new file mode 100644 index 0000000..5e40db7 Binary files /dev/null and b/tests/fixtures/ingress-oversized.jpg differ diff --git a/tests/fixtures/ingress-polyglot.gif b/tests/fixtures/ingress-polyglot.gif new file mode 100644 index 0000000..de73330 Binary files /dev/null and b/tests/fixtures/ingress-polyglot.gif differ diff --git a/tests/fixtures/ingress-script.svg b/tests/fixtures/ingress-script.svg new file mode 100644 index 0000000..f157858 --- /dev/null +++ b/tests/fixtures/ingress-script.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fixtures/ingress-spoofed.html b/tests/fixtures/ingress-spoofed.html new file mode 100644 index 0000000..dd1a1df --- /dev/null +++ b/tests/fixtures/ingress-spoofed.html @@ -0,0 +1 @@ +not an image \ No newline at end of file diff --git a/tests/fixtures/ingress-tail-7z.jpg b/tests/fixtures/ingress-tail-7z.jpg new file mode 100644 index 0000000..9a11c96 Binary files /dev/null and b/tests/fixtures/ingress-tail-7z.jpg differ diff --git a/tests/fixtures/ingress-tail-after-iend.png b/tests/fixtures/ingress-tail-after-iend.png new file mode 100644 index 0000000..d67e741 Binary files /dev/null and b/tests/fixtures/ingress-tail-after-iend.png differ diff --git a/tests/fixtures/ingress-tail-gzip.jpg b/tests/fixtures/ingress-tail-gzip.jpg new file mode 100644 index 0000000..a5d1002 Binary files /dev/null and b/tests/fixtures/ingress-tail-gzip.jpg differ diff --git a/tests/fixtures/ingress-tail-html.jpg b/tests/fixtures/ingress-tail-html.jpg new file mode 100644 index 0000000..cfe6e90 Binary files /dev/null and b/tests/fixtures/ingress-tail-html.jpg differ diff --git a/tests/fixtures/ingress-tail-mixed-script.jpg b/tests/fixtures/ingress-tail-mixed-script.jpg new file mode 100644 index 0000000..6ec985a Binary files /dev/null and b/tests/fixtures/ingress-tail-mixed-script.jpg differ diff --git a/tests/fixtures/ingress-tail-rar.jpg b/tests/fixtures/ingress-tail-rar.jpg new file mode 100644 index 0000000..6cf2fe9 Binary files /dev/null and b/tests/fixtures/ingress-tail-rar.jpg differ diff --git a/tests/fixtures/ingress-tail-single-nul.jpg b/tests/fixtures/ingress-tail-single-nul.jpg new file mode 100644 index 0000000..38c72f6 Binary files /dev/null and b/tests/fixtures/ingress-tail-single-nul.jpg differ diff --git a/tests/fixtures/ingress-tail-upper-script.jpg b/tests/fixtures/ingress-tail-upper-script.jpg new file mode 100644 index 0000000..4c9bcaf Binary files /dev/null and b/tests/fixtures/ingress-tail-upper-script.jpg differ diff --git a/tests/fixtures/ingress-tail-zip-eocd.jpg b/tests/fixtures/ingress-tail-zip-eocd.jpg new file mode 100644 index 0000000..65ad664 Binary files /dev/null and b/tests/fixtures/ingress-tail-zip-eocd.jpg differ diff --git a/tests/fixtures/ingress-tail-zip-local.jpg b/tests/fixtures/ingress-tail-zip-local.jpg new file mode 100644 index 0000000..602a538 Binary files /dev/null and b/tests/fixtures/ingress-tail-zip-local.jpg differ diff --git a/tests/fixtures/ingress-truncated.jpg b/tests/fixtures/ingress-truncated.jpg new file mode 100644 index 0000000..e3eed7c Binary files /dev/null and b/tests/fixtures/ingress-truncated.jpg differ diff --git a/tests/fixtures/ingress-zip-polyglot.jpg b/tests/fixtures/ingress-zip-polyglot.jpg new file mode 100644 index 0000000..7b7191c Binary files /dev/null and b/tests/fixtures/ingress-zip-polyglot.jpg differ diff --git a/tests/image-concurrency.test.js b/tests/image-concurrency.test.js new file mode 100644 index 0000000..21a38be --- /dev/null +++ b/tests/image-concurrency.test.js @@ -0,0 +1,146 @@ +// Concurrency ceiling contract. +// +// The decoder runs as a subprocess and each one costs real memory. The service +// runs under MemoryMax=512M, so a burst must be bounded by a small fail-fast +// ceiling rather than queued indefinitely: excess requests are refused +// immediately with a sanitized retry message, and the number of decoder +// children alive at once never exceeds the ceiling. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { chmod, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { + INGRESS_MAX_CONCURRENCY, + IngressUnavailableError, + ingressConcurrencyState, + validateImageIngress, +} from '../src/image-ingress.js'; + +const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, import.meta.url)); +const clean = await readFile(fixture('ingress-clean.jpg')); +const cleanDataUrl = `data:image/jpeg;base64,${clean.toString('base64')}`; + +// An instrumented stand-in interpreter: it records one marker file per spawn +// while it is alive, so the test can observe the true peak child count instead +// of trusting an internal counter. +async function trackingInterpreter(name, holdSeconds = '0.6') { + const dir = join(tmpdir(), `timmy-ingress-conc-${name}`); + await rm(dir, { recursive: true, force: true }); + await mkdir(join(dir, 'live'), { recursive: true }); + await mkdir(join(dir, 'seen'), { recursive: true }); + const path = join(dir, 'python3'); + await writeFile(path, [ + '#!/bin/sh', + `marker="${dir}/live/$$"`, + `touch "$marker" "${dir}/seen/$$"`, + `sleep ${holdSeconds}`, + 'rm -f "$marker"', + // Emit a valid success verdict and write the expected output file. + 'out=""', + 'while [ $# -gt 0 ]; do', + ' if [ "$1" = "--out" ]; then out="$2"; fi', + ' shift', + 'done', + `cp "${fixture('ingress-clean.jpg')}" "$out"`, + 'echo \'{"ok": true, "format": "jpeg", "width": 64, "height": 64, "bytes": 639, "metadataStripped": true}\'', + ].join('\n')); + await chmod(path, 0o755); + return { path, dir }; +} + +const withPython = async (path, run) => { + const previous = process.env.TIMMY_PYTHON; + process.env.TIMMY_PYTHON = path; + try { + return await run(); + } finally { + if (previous === undefined) delete process.env.TIMMY_PYTHON; + else process.env.TIMMY_PYTHON = previous; + } +}; + +test('the concurrency ceiling is small enough to stay inside the service memory budget', () => { + assert.equal(typeof INGRESS_MAX_CONCURRENCY, 'number'); + assert.ok(INGRESS_MAX_CONCURRENCY >= 1, 'at least one decode must be permitted'); + assert.ok(INGRESS_MAX_CONCURRENCY <= 4, + `ceiling ${INGRESS_MAX_CONCURRENCY} is too high for a 512 MiB service`); + // Worst observed single-decoder peak is ~24 MiB; the whole burst must leave + // ample headroom under MemoryMax=512M alongside the Node process itself. + assert.ok(INGRESS_MAX_CONCURRENCY * 64 * 1024 * 1024 < 512 * 1024 * 1024, + 'the ceiling must bound peak decoder memory well under 512 MiB'); +}); + +test('a burst beyond the ceiling never runs more decoder children than the ceiling', async () => { + const { path, dir } = await trackingInterpreter('peak'); + await withPython(path, async () => { + let peakLive = 0; + const sampler = setInterval(async () => { + try { + const live = await readdir(join(dir, 'live')); + peakLive = Math.max(peakLive, live.length); + } catch { /* directory races are fine */ } + }, 15); + const burst = await Promise.allSettled( + Array.from({ length: 24 }, () => validateImageIngress({ consent: true, imageDataUrl: cleanDataUrl })), + ); + clearInterval(sampler); + const spawned = (await readdir(join(dir, 'seen'))).length; + const rejected = burst.filter((r) => r.status === 'rejected'); + + assert.ok(peakLive <= INGRESS_MAX_CONCURRENCY, + `observed ${peakLive} concurrent decoder children, ceiling is ${INGRESS_MAX_CONCURRENCY}`); + assert.ok(spawned <= INGRESS_MAX_CONCURRENCY, + `a 24-request burst spawned ${spawned} decoders; only ${INGRESS_MAX_CONCURRENCY} may run and the rest must be refused`); + assert.ok(rejected.length >= 24 - INGRESS_MAX_CONCURRENCY, + 'excess requests must be refused rather than queued'); + }); +}); + +test('over-ceiling requests fail fast with a sanitized unavailable message', async () => { + const { path } = await trackingInterpreter('failfast'); + await withPython(path, async () => { + const started = Date.now(); + const burst = await Promise.allSettled( + Array.from({ length: 12 }, () => validateImageIngress({ consent: true, imageDataUrl: cleanDataUrl })), + ); + const rejected = burst.filter((r) => r.status === 'rejected').map((r) => r.reason); + assert.ok(rejected.length > 0, 'a 12-request burst must refuse some requests'); + for (const error of rejected) { + assert.ok(error instanceof IngressUnavailableError, + 'an over-capacity refusal is a server-capacity condition, not corrupt client input'); + assert.match(error.message, /temporarily unavailable|slow down|try again/i); + assert.doesNotMatch(error.message, /corrupt|malformed/i); + assert.doesNotMatch(error.message, /[A-Za-z0-9+/]{40,}/, 'no payload data in refusals'); + assert.ok(error.message.length < 200); + } + // Fail-fast: refusals must not wait for the in-flight decoders to finish. + assert.ok(Date.now() - started < 5000, 'refusals must be immediate, not queued behind decodes'); + }); +}); + +test('capacity is released so later requests succeed after a burst', async () => { + const { path } = await trackingInterpreter('release', '0.05'); + await withPython(path, async () => { + await Promise.allSettled( + Array.from({ length: 12 }, () => validateImageIngress({ consent: true, imageDataUrl: cleanDataUrl })), + ); + const state = ingressConcurrencyState(); + assert.equal(state.active, 0, 'every slot must be released after the burst settles'); + const after = await validateImageIngress({ consent: true, imageDataUrl: cleanDataUrl }); + assert.equal(after.format, 'jpeg'); + }); +}); + +test('rejected input releases its slot too', async () => { + const garbage = await readFile(fixture('ingress-garbage.jpg')); + for (let i = 0; i < INGRESS_MAX_CONCURRENCY + 3; i += 1) { + await assert.rejects(() => validateImageIngress({ + consent: true, imageDataUrl: `data:image/jpeg;base64,${garbage.toString('base64')}`, + })); + } + assert.equal(ingressConcurrencyState().active, 0, + 'a rejected request must not leak a concurrency slot'); +}); diff --git a/tests/image-dataurl.test.js b/tests/image-dataurl.test.js new file mode 100644 index 0000000..cab83ed --- /dev/null +++ b/tests/image-dataurl.test.js @@ -0,0 +1,95 @@ +// Canonical base64 data-URL grammar contract. +// +// The client always produces a canonical `data:;base64,` +// URL (canvas.toDataURL). Anything else is a hand-crafted request, so ingress +// must require the canonical grammar and a byte-exact round trip rather than +// silently repairing missing padding, embedded newlines, or non-zero trailing +// pad bits. The exact supported MIME policy is unchanged: JPEG/PNG/WebP only. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; + +import { decodeImageDataUrl, validateImageIngress } from '../src/image-ingress.js'; + +const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, import.meta.url)); +const clean = await readFile(fixture('ingress-clean.jpg')); +const canonical = clean.toString('base64'); + +const ingest = (imageDataUrl) => validateImageIngress({ consent: true, imageDataUrl }); + +test('canonical base64 data URLs round-trip byte-exactly', () => { + const decoded = decodeImageDataUrl(`data:image/jpeg;base64,${canonical}`); + assert.ok(decoded, 'canonical data URL must decode'); + assert.equal(decoded.mime, 'image/jpeg'); + assert.ok(decoded.bytes.equals(clean), 'decoded bytes must equal the source bytes exactly'); +}); + +test('missing base64 padding is rejected', async () => { + const unpadded = canonical.replace(/=+$/, ''); + assert.notEqual(unpadded, canonical, 'fixture must actually require padding'); + assert.equal(decodeImageDataUrl(`data:image/jpeg;base64,${unpadded}`), null); + await assert.rejects(() => ingest(`data:image/jpeg;base64,${unpadded}`), /not a supported image/i); +}); + +test('excess base64 padding is rejected', async () => { + assert.equal(decodeImageDataUrl(`data:image/jpeg;base64,${canonical}===`), null); + await assert.rejects(() => ingest(`data:image/jpeg;base64,${canonical}====`), /not a supported image/i); +}); + +test('CR and LF inside the base64 payload are rejected', async () => { + const withCrlf = `${canonical.slice(0, 40)}\r\n${canonical.slice(40)}`; + const withLf = `${canonical.slice(0, 40)}\n${canonical.slice(40)}`; + const withCr = `${canonical.slice(0, 40)}\r${canonical.slice(40)}`; + for (const payload of [withCrlf, withLf, withCr]) { + assert.equal(decodeImageDataUrl(`data:image/jpeg;base64,${payload}`), null); + await assert.rejects(() => ingest(`data:image/jpeg;base64,${payload}`), /not a supported image/i); + } +}); + +test('non-canonical encodings are rejected: non-zero trailing pad bits, whitespace, URL-safe alphabet', async () => { + const nonZeroPadBits = `${canonical.slice(0, -2)}/=`; + const withSpace = `${canonical.slice(0, 20)} ${canonical.slice(20)}`; + const urlSafe = canonical.replace(/\+/g, '-').replace(/\//g, '_'); + for (const payload of [nonZeroPadBits, withSpace]) { + assert.equal(decodeImageDataUrl(`data:image/jpeg;base64,${payload}`), null); + await assert.rejects(() => ingest(`data:image/jpeg;base64,${payload}`), /not a supported image/i); + } + if (urlSafe !== canonical) { + assert.equal(decodeImageDataUrl(`data:image/jpeg;base64,${urlSafe}`), null); + } +}); + +test('the supported MIME policy is exact and unchanged', async () => { + for (const mime of ['image/jpeg', 'image/png', 'image/webp']) { + assert.ok(decodeImageDataUrl(`data:${mime};base64,${canonical}`), `${mime} must remain supported`); + } + for (const mime of ['image/gif', 'image/svg+xml', 'text/html', 'application/octet-stream', '']) { + assert.equal(decodeImageDataUrl(`data:${mime};base64,${canonical}`), null, `${mime} must not be supported`); + await assert.rejects(() => ingest(`data:${mime};base64,${canonical}`), /not a supported image/i); + } +}); + +test('malformed data-URL envelopes are rejected without repair', async () => { + const malformed = [ + `data:image/jpeg,${canonical}`, // no base64 token + `data:image/jpeg;base64;${canonical}`, // wrong separator + `DATA:image/jpeg;base64,${canonical}`, // scheme casing is not canonical here + `data:image/jpeg;base64,`, // empty payload + ` data:image/jpeg;base64,${canonical}`, // leading whitespace + `data:image/jpeg;base64,${canonical} `, // trailing whitespace + `data:image/jpeg;charset=utf-8;base64,${canonical}`, // extra parameters + canonical, // bare base64, no envelope + ]; + for (const value of malformed) { + assert.equal(decodeImageDataUrl(value), null, `must reject: ${value.slice(0, 42)}`); + await assert.rejects(() => ingest(value), /not a supported image/i); + } +}); + +test('a canonical supported data URL still completes ingress end to end', async () => { + const result = await ingest(`data:image/jpeg;base64,${canonical}`); + assert.equal(result.format, 'jpeg'); + assert.equal(result.mime, 'image/jpeg'); + assert.match(result.imageDataUrl, /^data:image\/jpeg;base64,[A-Za-z0-9+/]+={0,2}$/); +}); diff --git a/tests/image-ingress.test.js b/tests/image-ingress.test.js new file mode 100644 index 0000000..0e6c1dd --- /dev/null +++ b/tests/image-ingress.test.js @@ -0,0 +1,170 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { readFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; + +import { + MAX_IMAGE_BYTES, + MAX_IMAGE_DIMENSION, + sniffImageFormat, + validateImageIngress, +} from '../src/image-ingress.js'; +import { analyzePhoto } from '../src/vision-service.js'; + +const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, import.meta.url)); +const readFixture = async (name) => await readFile(fixture(name)); + +function dataUrl(bytes, mime) { + return `data:${mime};base64,${bytes.toString('base64')}`; +} + +async function ingestFixture(name, mime = 'image/jpeg', overrides = {}) { + const bytes = await readFixture(name); + return validateImageIngress({ + imageDataUrl: dataUrl(bytes, mime), + consent: true, + ...overrides, + }); +} + +test('clean synthetic JPEG passes ingress and is re-encoded without metadata', async () => { + const result = await ingestFixture('ingress-clean.jpg'); + assert.equal(result.format, 'jpeg'); + assert.ok(result.bytes <= MAX_IMAGE_BYTES); + assert.match(result.imageDataUrl, /^data:image\/jpeg;base64,/); + const exifCount = execFileSync('python3', [ + '-c', + 'from PIL import Image;import sys,base64;print(len(Image.open(__import__("io").BytesIO(base64.b64decode(sys.argv[1]))).getexif()))', + result.imageDataUrl.split(',')[1], + ], { encoding: 'utf8' }); + assert.equal(exifCount.trim(), '0'); +}); + +test('magic bytes are verified independent of declared MIME', async () => { + // HTML payload wearing a JPEG content type must be rejected. + await assert.rejects( + () => ingestFixture('ingress-spoofed.html'), + /not a supported image/i, + ); + // A real PNG declared as JPEG must still be accepted by sniffing, not by MIME. + const png = await ingestFixture('ingress-metadata.png', 'image/jpeg'); + assert.equal(png.originalFormat, 'png'); + assert.equal(png.format, 'jpeg'); +}); + +test('polyglot payloads are rejected', async () => { + await assert.rejects(() => ingestFixture('ingress-polyglot.gif'), /not a supported image|rejected|unsafe/i); + await assert.rejects(() => ingestFixture('ingress-zip-polyglot.jpg'), /rejected|unsafe|corrupt|malformed|not a supported image/i); +}); + +test('malformed and truncated images fail closed with sanitized errors', async () => { + for (const [name, mime] of [['ingress-garbage.jpg', 'image/jpeg'], ['ingress-empty.jpg', 'image/jpeg']]) { + await assert.rejects(() => ingestFixture(name, mime), (error) => { + assert.match(error.message, /upload a jpeg, png, or webp photo/i); + return true; + }); + } + // A truncated JPEG has no EOI, so canonical container parsing rejects it + // before any decode or subprocess spawn. Either sanitized ingress error is + // acceptable; leaking anything else is not. + await assert.rejects(() => ingestFixture('ingress-truncated.jpg'), (error) => { + assert.match(error.message, /not a supported image|corrupt or malformed/i); + return true; + }); +}); + +test('decompression bombs and oversized dimensions fail before provider work', async () => { + await assert.rejects(() => ingestFixture('ingress-bomb.png'), /too large|dimensions/i); + await assert.rejects(() => ingestFixture('ingress-oversized.jpg'), /too large|dimensions/i); + assert.equal(MAX_IMAGE_DIMENSION <= 4096, true); +}); + +test('EXIF and GPS metadata are stripped from the re-encoded image', async () => { + const result = await ingestFixture('ingress-exif.jpg'); + assert.equal(result.metadataStripped, true); + const check = execFileSync('python3', ['-c', + 'from PIL import Image;import sys,base64,io;' + + 'im=Image.open(io.BytesIO(base64.b64decode(sys.argv[1])));' + + 'ex=im.getexif();' + + 'gps=ex.get_ifd(0x8825);' + + 'print("MAKE" if ex.get(0x010F) else "CLEAN", "GPS" if gps else "CLEAN")', + result.imageDataUrl.split(',')[1], + ], { encoding: 'utf8' }); + assert.equal(check.trim(), 'CLEAN CLEAN'); + const pngResult = await ingestFixture('ingress-metadata.png'); + const pngCheck = execFileSync('python3', ['-c', + 'from PIL import Image;import sys,base64,io;' + + 'im=Image.open(io.BytesIO(base64.b64decode(sys.argv[1])));' + + 'info=getattr(im,"text",{}) or {};' + + 'print("TEXT" if info else "CLEAN", im.format.lower())', + pngResult.imageDataUrl.split(',')[1], + ], { encoding: 'utf8' }); + assert.equal(pngCheck.trim(), 'CLEAN jpeg'); +}); + +test('body limit rejects oversized base64 bodies before decoding', async () => { + const huge = Buffer.alloc(MAX_IMAGE_BYTES + 1024, 65); + await assert.rejects( + () => validateImageIngress({ imageDataUrl: dataUrl(huge, 'image/jpeg'), consent: true }), + /under \d+ mb/i, + ); +}); + +test('consent is still required after hardening', async () => { + const bytes = await readFixture('ingress-clean.jpg'); + await assert.rejects( + () => validateImageIngress({ imageDataUrl: dataUrl(bytes, 'image/jpeg') }), + /consent/i, + ); +}); + +test('SVG uploads never pass ingress regardless of extension', async () => { + await assert.rejects( + () => ingestFixture('ingress-script.svg', 'image/svg+xml'), + /not a supported image/i, + ); +}); + +test('errors are sanitized: no image bytes, no base64, no stack in messages', async () => { + const cases = []; + for (const name of ['ingress-spoofed.html', 'ingress-polyglot.gif', 'ingress-truncated.jpg', 'ingress-bomb.png']) { + try { await ingestFixture(name); } catch (error) { cases.push(error.message); } + } + for (const message of cases) { + assert.doesNotMatch(message, /[A-Za-z0-9+/]{40,}/); + assert.doesNotMatch(message, /at\s+\S+\s+\(/); + assert.ok(message.length < 200); + } +}); + +test('rate limiter allows a bounded burst then fails closed with sanitized retry message', async () => { + const { createRateLimiter } = await import('../src/rate-limiter.js'); + const limiter = createRateLimiter(); + const first = limiter.take('client-a', Date.now()); + assert.equal(first.allowed, true); + for (let i = 0; i < limiter.limit - 1; i += 1) { + assert.equal(limiter.take('client-a', Date.now()).allowed, true); + } + const blocked = limiter.take('client-a', Date.now()); + assert.equal(blocked.allowed, false); + assert.match(blocked.reason, /try again later|slow down/i); + assert.ok(blocked.retryAfterMs > 0); + assert.doesNotMatch(blocked.reason, /image|payload|byte/i); + // A different client key is unaffected. + assert.equal(limiter.take('client-b', Date.now()).allowed, true); +}); + +test('analyzePhoto routes through hardened ingress before provider fetch', async () => { + const bytes = await readFixture('ingress-polyglot.gif'); + let providerCalled = false; + await assert.rejects( + () => analyzePhoto({ + payload: { consent: true, imageDataUrl: dataUrl(bytes, 'image/gif') }, + config: { model: 'm', baseUrl: 'http://127.0.0.1:9/v1' }, + fetchImpl: async () => { providerCalled = true; throw new Error('provider reached'); }, + }), + /not a supported image|rejected|unsafe/i, + ); + assert.equal(providerCalled, false); +}); diff --git a/tests/image-polyglot.test.js b/tests/image-polyglot.test.js new file mode 100644 index 0000000..0946815 --- /dev/null +++ b/tests/image-polyglot.test.js @@ -0,0 +1,86 @@ +// Container/polyglot rejection contract. +// +// The requirement is malformed/polyglot REJECTION, not "the re-encoder happens +// to drop the tail". Rejection must come from canonical parsing of the image +// container — the declared structure must consume exactly the supplied bytes — +// so it cannot be evaded by changing keyword casing or archive flavour, and it +// must not fire on arbitrary compressed bytes that merely contain those +// sequences inside legal image structure. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; + +import { validateImageIngress } from '../src/image-ingress.js'; +import { analyzePhoto } from '../src/vision-service.js'; + +const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, import.meta.url)); +const readFixture = (name) => readFile(fixture(name)); +const dataUrl = (bytes, mime = 'image/jpeg') => `data:${mime};base64,${bytes.toString('base64')}`; + +const TRAILING_DATA_FIXTURES = [ + ['ingress-tail-upper-script.jpg', 'uppercase