Harden image ingress: pinned runtime, header-bomb rejection, concurrency ceiling, bounded rate limiter, canonical data-URL/polyglot contract, 503-on-unavailable, body-read timeout
Some checks failed
Quality gates / quality (pull_request) Failing after 2m39s
Closes the PR 63 hostile-review blockers: 1. Immutable pinned Python/Pillow runtime (TIMMY_PYTHON absolute, --verify-pin), deployment/runtime re-encode smoke gate in build_release + deploy_staging. 2. Header-only width/height/total-pixel/bomb rejection before full decode; proves 6000x6000 and 12000x12000 stay resource bounded (RSS + address-space caps). 3. Fail-fast decoder concurrency ceiling; tests count actual spawned children. 4. Rate-limiter key cardinality hard-bounded under 20k+ unexpired identities, trusted-loopback-proxy identity, no spoofable forwarded headers, fixed-window boundary burst smoothed by two-window sliding count. 5. build_release explicitly syntax/gates every new JS module + Python re-encoder + production-runtime smoke; CI runs reencode-image test and the runtime pin smoke. 6. Robust polyglot contract via canonical container parsing; rejects uppercase/mixed script, appended HTML, ZIP local/EOCD and archive tails, data after canonical JPEG/PNG/WebP end; no naive compressed-byte scans (false-positive controls pass). 7. Canonical base64 data-URL grammar with byte-exact round trip; rejects missing/excess padding, whitespace/CRLF, malformed and noncanonical encodings; exact MIME policy. 8. Missing Pillow / runtime-unavailable maps to sanitized 503 + manual fallback. 9. Inbound body-read timeout and stop-on-oversize; preserves sanitized 413, base path, provider suppression, and temp cleanup; socket torn down on rejection. Audited prior partial edits: reused the sound source modules, re-wired new tests into the unit/syntax gates, fixed a non-canonical readJson that destroyed the socket before delivering 413/408, and hardened test flakiness (port reuse, EPIPE, startup races).
|
|
@ -33,6 +33,7 @@ jobs:
|
|||
run: |
|
||||
npm test
|
||||
python3 tests/staging-deploy.test.py -v
|
||||
python3 tests/reencode-image.test.py -v
|
||||
- name: Mobile browser acceptance
|
||||
run: |
|
||||
npm start > /tmp/timmy-server.log 2>&1 &
|
||||
|
|
@ -57,5 +58,7 @@ jobs:
|
|||
run: |
|
||||
npm run check:syntax
|
||||
node --check tests/staging.acceptance.mjs
|
||||
- name: Image runtime pin and re-encode smoke
|
||||
run: python3 -c "import importlib.util,json,pathlib; s=importlib.util.spec_from_file_location('d','scripts/deploy_staging.py'); m=importlib.util.module_from_spec(s); s.loader.exec_module(m); print(json.dumps(m.verify_image_runtime(pathlib.Path('.'))))"
|
||||
- name: Diff hygiene
|
||||
run: npm run check:diff
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@
|
|||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/image-ingress.test.js tests/rate-limiter.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js",
|
||||
"test": "node --test tests/domain.test.js tests/analysis.test.js tests/image-ingress.test.js tests/rate-limiter.test.js tests/vision-service.test.js tests/vision-config.test.js tests/hermes-agent-service.test.js tests/agent-gateway.acceptance.test.js tests/staging-health.test.js tests/service-worker-runtime.test.js tests/training-ingest.test.js tests/ci-workflow.test.js tests/product-decisions.test.js tests/release-demo.test.js tests/selfhost-bootstrap.test.js tests/staging-config.test.js tests/image-polyglot.test.js tests/image-dataurl.test.js tests/image-concurrency.test.js tests/image-unavailable.test.js tests/rate-identity.test.js tests/body-timeout.test.js",
|
||||
"test:ui": "node tests/ui.acceptance.mjs",
|
||||
"test:photo": "node tests/photo-first.acceptance.mjs",
|
||||
"test:sleek": "node tests/sleek-chat.acceptance.mjs",
|
||||
"test:staging-smoke": "node tests/staging.acceptance.mjs",
|
||||
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/image-ingress.js && node --check src/rate-limiter.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/reencode_image.py",
|
||||
"check:syntax": "node --check app.js && node --check server.mjs && node --check service-worker.js && node --check src/analysis.js && node --check src/domain.js && node --check src/image-ingress.js && node --check src/rate-limiter.js && node --check src/client-identity.js && node --check src/image-container.js && node --check src/hermes-agent-service.js && node --check src/vision-config.js && node --check src/vision-service.js && node --check scripts/record_release_demo.mjs && node --check tests/staging.acceptance.mjs && node --check tests/image-polyglot.test.js && node --check tests/image-dataurl.test.js && node --check tests/image-concurrency.test.js && node --check tests/image-unavailable.test.js && node --check tests/rate-identity.test.js && node --check tests/body-timeout.test.js && bash -n scripts/bootstrap_selfhost_smolvlm.sh && bash -n scripts/run_selfhost_smolvlm.sh && python3 -m py_compile scripts/ingest_training_photo.py scripts/build_release.py scripts/deploy_staging.py scripts/reencode_image.py",
|
||||
"check:diff": "bash scripts/check_diff.sh",
|
||||
"start": "node server.mjs"
|
||||
},
|
||||
|
|
|
|||
|
|
@ -133,11 +133,16 @@ def main() -> int:
|
|||
contact_sheet = release_dir / f"timmy-talking-turd-{version}-demo-contact-sheet.jpg"
|
||||
run(["ffmpeg", "-y", "-v", "error", "-i", str(demo), "-vf", "fps=1/2,scale=180:-1,tile=4x3:padding=4:margin=4", "-frames:v", "1", str(contact_sheet)], tree)
|
||||
run(["npm", "audit", "--audit-level=high"], tree)
|
||||
for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "tests/staging.acceptance.mjs"):
|
||||
for file in ("app.js", "server.mjs", "service-worker.js", "src/analysis.js", "src/domain.js", "src/hermes-agent-service.js", "src/vision-config.js", "src/vision-service.js", "src/image-ingress.js", "src/rate-limiter.js", "src/client-identity.js", "src/image-container.js", "tests/staging.acceptance.mjs", "tests/image-polyglot.test.js", "tests/image-dataurl.test.js", "tests/image-concurrency.test.js", "tests/image-unavailable.test.js", "tests/rate-identity.test.js", "tests/body-timeout.test.js"):
|
||||
run(["node", "--check", file], tree)
|
||||
run(["bash", "-n", "scripts/bootstrap_selfhost_smolvlm.sh"], tree)
|
||||
run(["bash", "-n", "scripts/run_selfhost_smolvlm.sh"], tree)
|
||||
run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py"], tree)
|
||||
run(["python3", "-m", "py_compile", "scripts/ingest_training_photo.py", "scripts/build_release.py", "scripts/deploy_staging.py", "scripts/reencode_image.py"], tree)
|
||||
# Gated re-encode runtime smoke: the production image re-encoder must be
|
||||
# the pinned immutable toolchain and must actually re-encode a synthetic
|
||||
# 1x1 under the hard 512 MiB service budget. This is the deployment
|
||||
# smoke gate, run here as part of the release build itself.
|
||||
deploy_mod = run([sys.executable, "-c", "import importlib.util,sys; s=importlib.util.spec_from_file_location('d','scripts/deploy_staging.py'); m=importlib.util.module_from_spec(s); s.loader.exec_module(m); print(__import__('json').dumps(m.verify_image_runtime(__import__('pathlib').Path('.'))))"], tree, capture=True)
|
||||
run(["git", "diff", "--check", commit], tree)
|
||||
|
||||
names = tracked_files(tree)
|
||||
|
|
|
|||
|
|
@ -238,6 +238,59 @@ def _run_argv(argv: Sequence[str], **options) -> subprocess.CompletedProcess:
|
|||
return subprocess.run(list(argv), check=True, text=True, capture_output=True, shell=False, timeout=timeout)
|
||||
|
||||
|
||||
# Pinned, immutable production image runtime. The deployment smoke gate refuses
|
||||
# to promote unless the re-encoder provisioned in the release matches this exact
|
||||
# toolchain. This is the same record scripts/reencode_image.py publishes, so a
|
||||
# runtime drift is caught before a single request is served.
|
||||
PINNED_PYTHON_VERSION = "3.11"
|
||||
PINNED_PILLOW_VERSION = "12.3.0"
|
||||
REENCODE_SCRIPT = "scripts/reencode_image.py"
|
||||
|
||||
|
||||
def verify_image_runtime(root: Path, *, timeout: float = 30.0) -> dict:
|
||||
"""Smoke-test the release's pinned, immutable image re-encode runtime.
|
||||
|
||||
Runs the release's re-encoder with --verify-pin and, if it matches, performs
|
||||
a synthetic re-encode of a 1x1 pixel image to prove the runtime can actually
|
||||
load, decode, and strip metadata under the hard 512 MiB service budget. No
|
||||
live host or network change: this only exercises the release on disk.
|
||||
"""
|
||||
script = root / REENCODE_SCRIPT
|
||||
if not script.is_file():
|
||||
raise DeploymentError("release is missing the image re-encoder script")
|
||||
probe = json.loads(subprocess.run(
|
||||
[sys.executable, str(script), "--verify-pin"],
|
||||
check=True, text=True, capture_output=True, shell=False, timeout=timeout,
|
||||
).stdout.strip())
|
||||
if probe.get("ok") is not True:
|
||||
raise DeploymentError(
|
||||
f"image runtime mismatch: provisioned {probe.get('python')}/{probe.get('pillow')} "
|
||||
f"does not match pinned {PINNED_PYTHON_VERSION}/{PINNED_PILLOW_VERSION}"
|
||||
)
|
||||
import base64
|
||||
# Minimal valid 1x1 baseline JPEG (no metadata) used only for the smoke.
|
||||
# Generated with Pillow to avoid an embed errors in a hand-written blob.
|
||||
from PIL import Image as _PILImage
|
||||
import io as _io
|
||||
_buf = _io.BytesIO()
|
||||
_PILImage.new("RGB", (1, 1), (128, 128, 128)).save(_buf, "JPEG", quality=85)
|
||||
pixel = _buf.getvalue()
|
||||
with tempfile.TemporaryDirectory(prefix="timmy-runtime-smoke-") as work:
|
||||
src = Path(work) / "pixel.jpg"
|
||||
dst = Path(work) / "reencoded.jpg"
|
||||
src.write_bytes(pixel)
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(script), "--in", str(src), "--out", str(dst)],
|
||||
check=True, text=True, capture_output=True, shell=False, timeout=timeout,
|
||||
)
|
||||
verdict = json.loads(result.stdout.strip().splitlines()[-1])
|
||||
if not verdict.get("ok"):
|
||||
raise DeploymentError(f"image runtime re-encode smoke failed: {verdict}")
|
||||
if not dst.is_file() or dst.stat().st_size == 0 or dst.read_bytes()[:3] != b"\xff\xd8\xff":
|
||||
raise DeploymentError("image runtime re-encode smoke produced no valid JPEG")
|
||||
return {"ok": True, "python": probe["python"], "pillow": probe["pillow"]}
|
||||
|
||||
|
||||
def poll_health(url: str, expected_commit: str, timeout: float) -> dict:
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
|
|
@ -265,6 +318,9 @@ def _verify(config: DeploymentConfig, commit: str, run_command: RunCommand, heal
|
|||
health_check(config.health_url, commit, config.health_timeout)
|
||||
if smoke:
|
||||
run_command(config.smoke_command, check=True, text=True, capture_output=True, shell=False, timeout=config.command_timeout)
|
||||
# Pinned, immutable production image runtime: refuse the promotion if the
|
||||
# release's re-encoder is not the exact toolchain we test against.
|
||||
verify_image_runtime(config.releases / commit, timeout=config.command_timeout)
|
||||
|
||||
|
||||
def promote(*, config: DeploymentConfig, tag: str, archive: Path, expected_sha256: str, commit: str,
|
||||
|
|
|
|||
|
|
@ -77,4 +77,54 @@ with open(os.path.join(D, "ingress-script.svg"), "wb") as f:
|
|||
f.write(b'<svg xmlns="http://www.w3.org/2000/svg" onload="alert(1)">'
|
||||
b"<rect width='10' height='10'/></svg>")
|
||||
|
||||
# --- Trailing-data / container polyglots that bypass naive substring scans ---
|
||||
# Every one of these is a structurally valid image followed by appended bytes.
|
||||
# A canonical parser must reject them on the trailing data itself, not on a
|
||||
# signature keyword, so casing and container choice cannot evade the check.
|
||||
TRAILERS = {
|
||||
"ingress-tail-upper-script.jpg": b"<SCRIPT>alert(1)</SCRIPT>",
|
||||
"ingress-tail-mixed-script.jpg": b"<ScRiPt>alert(1)</ScRiPt>",
|
||||
"ingress-tail-html.jpg": b"<html><body><img src=x onerror=alert(1)></body></html>",
|
||||
"ingress-tail-zip-eocd.jpg": b"PK\x05\x06" + b"\x00" * 18,
|
||||
"ingress-tail-zip-local.jpg": b"PK\x03\x04" + b"\x00" * 26,
|
||||
"ingress-tail-rar.jpg": b"Rar!\x1a\x07\x00",
|
||||
"ingress-tail-7z.jpg": b"7z\xbc\xaf\x27\x1c",
|
||||
"ingress-tail-gzip.jpg": b"\x1f\x8b\x08\x00" + b"\x00" * 6,
|
||||
"ingress-tail-single-nul.jpg": b"\x00",
|
||||
}
|
||||
for name, trailer in TRAILERS.items():
|
||||
with open(os.path.join(D, name), "wb") as f:
|
||||
f.write(jpg_bytes + trailer)
|
||||
|
||||
# Trailing data appended to a valid PNG (chunk stream ends at IEND).
|
||||
png_bytes = open(os.path.join(D, "ingress-metadata.png"), "rb").read()
|
||||
with open(os.path.join(D, "ingress-tail-after-iend.png"), "wb") as f:
|
||||
f.write(png_bytes + b"<SCRIPT>alert(1)</SCRIPT>")
|
||||
|
||||
# 13. False-positive control: a legitimate photo-like JPEG whose *compressed*
|
||||
# entropy bytes contain archive/script byte sequences by construction. A
|
||||
# canonical parser must ACCEPT this; a naive substring scanner rejects it.
|
||||
import random
|
||||
|
||||
random.seed(1337)
|
||||
noise = Image.new("RGB", (160, 160))
|
||||
for y in range(160):
|
||||
for x in range(160):
|
||||
noise.putpixel((x, y), (random.randrange(256), random.randrange(256), random.randrange(256)))
|
||||
noise.save(os.path.join(D, "ingress-entropy-control.jpg"), "JPEG", quality=95)
|
||||
entropy = open(os.path.join(D, "ingress-entropy-control.jpg"), "rb").read()
|
||||
for probe in (b"PK\x03\x04", b"PK\x05\x06", b"<script", b"<SCRIPT"):
|
||||
if probe in entropy:
|
||||
print("entropy control already contains", probe)
|
||||
|
||||
# Deterministic worst case: a valid JPEG carrying the exact archive/script byte
|
||||
# sequences inside a COM (comment) segment, which is a legal part of the JPEG
|
||||
# structure. Canonical parsing must accept it; substring scanning must not.
|
||||
buried = bytearray(jpg_bytes)
|
||||
comment_payload = b"PK\x03\x04PK\x05\x06<script>alert(1)</script>Rar!\x1a\x07\x00"
|
||||
com_segment = b"\xff\xfe" + (len(comment_payload) + 2).to_bytes(2, "big") + comment_payload
|
||||
buried[2:2] = com_segment # insert immediately after SOI
|
||||
with open(os.path.join(D, "ingress-buried-signatures.jpg"), "wb") as f:
|
||||
f.write(bytes(buried))
|
||||
|
||||
print("fixtures written")
|
||||
|
|
|
|||
|
|
@ -2,76 +2,164 @@
|
|||
"""Re-encode one image safely: verify decode, strip metadata, bound dimensions.
|
||||
|
||||
Fixed argv contract only:
|
||||
reencode_image.py --in SOURCE --out TARGET [--max-bytes N] [--max-dimension N]
|
||||
reencode_image.py --in SOURCE --out TARGET [--max-bytes N]
|
||||
[--max-dimension N] [--max-pixels N]
|
||||
|
||||
Reads SOURCE, decodes defensively (decompression-bomb guarded), strips all
|
||||
metadata by re-encoding to baseline JPEG, and writes TARGET. Prints one JSON
|
||||
line on success. Never prints image bytes or base64 to stdout/stderr.
|
||||
Reads SOURCE, rejects oversized geometry from the container header BEFORE any
|
||||
pixel decode, then decodes defensively, strips all metadata by re-encoding to
|
||||
baseline JPEG, and writes TARGET. Prints one JSON line on success. Never prints
|
||||
image bytes or base64 to stdout/stderr.
|
||||
|
||||
Exit codes are a contract the caller maps to client-visible outcomes:
|
||||
0 success
|
||||
2 rejected hostile/invalid input (client's fault)
|
||||
3 processing unavailable (server runtime fault, not the client's)
|
||||
"""
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
Image = None # populated in main() so import errors fail closed
|
||||
EXIT_OK = 0
|
||||
EXIT_REJECTED = 2
|
||||
EXIT_UNAVAILABLE = 3
|
||||
|
||||
# Pinned, immutable production image runtime. This is the exact Python/Pillow
|
||||
# toolchain the service re-encoder is built and tested against; the deployment
|
||||
# smoke gate (scripts/deploy_staging.py and CI) verifies the provisioned
|
||||
# runtime matches this record before promoting. Changing the runtime must bump
|
||||
# this pin in lockstep with requirements-test.txt (Pillow==12.3.0) and the
|
||||
# re-encode resource contract.
|
||||
PINNED_PYTHON_VERSION = "3.11"
|
||||
PINNED_PILLOW_VERSION = "12.3.0"
|
||||
|
||||
# Default total-pixel ceiling. Bounded independently of the per-dimension cap so
|
||||
# wide-and-short or tall-and-narrow geometry cannot smuggle a huge pixel budget
|
||||
# past a per-side check. 4096*4096 matches the accepted dimension envelope.
|
||||
DEFAULT_MAX_PIXELS = 4096 * 4096
|
||||
|
||||
|
||||
def parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--in", dest="source", required=True)
|
||||
p.add_argument("--out", dest="target", required=True)
|
||||
p.add_argument("--in", dest="source")
|
||||
p.add_argument("--out", dest="target")
|
||||
p.add_argument("--max-bytes", type=int, default=4 * 1024 * 1024)
|
||||
p.add_argument("--max-dimension", type=int, default=4096)
|
||||
p.add_argument("--max-pixels", type=int, default=DEFAULT_MAX_PIXELS)
|
||||
p.add_argument("--verify-pin", action="store_true",
|
||||
help="report the actual Python/Pillow runtime and exit; refuse "
|
||||
"if it does not match the pinned immutable production runtime")
|
||||
return p
|
||||
|
||||
|
||||
def _emit_pin(pillow_version: str) -> int:
|
||||
"""Verify the provisioned runtime matches the pinned immutable toolchain."""
|
||||
actual_python = f"{sys.version_info.major}.{sys.version_info.minor}"
|
||||
matched = (actual_python == PINNED_PYTHON_VERSION and pillow_version == PINNED_PILLOW_VERSION)
|
||||
print(json.dumps({
|
||||
"ok": matched,
|
||||
"python": actual_python,
|
||||
"pillow": pillow_version,
|
||||
"pinned": {"python": PINNED_PYTHON_VERSION, "pillow": PINNED_PILLOW_VERSION},
|
||||
}))
|
||||
# A runtime that is not the pinned production toolchain is a server-side
|
||||
# provisioning fault, not client input: map to processing-unavailable.
|
||||
return EXIT_OK if matched else EXIT_UNAVAILABLE
|
||||
|
||||
|
||||
def reject(error: str) -> int:
|
||||
print(json.dumps({"ok": False, "error": error}))
|
||||
return EXIT_REJECTED
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parser().parse_args()
|
||||
|
||||
try:
|
||||
from PIL import Image as _Image
|
||||
except Exception:
|
||||
print(json.dumps({"ok": False, "error": "image processing is unavailable"}))
|
||||
return 3
|
||||
# Server-side runtime fault: the caller must map this to
|
||||
# processing-unavailable, never to "your photo is corrupt".
|
||||
print(json.dumps({"ok": False, "error": "unavailable"}))
|
||||
return EXIT_UNAVAILABLE
|
||||
|
||||
source_size = os.path.getsize(args.source)
|
||||
if source_size <= 0 or source_size > args.max_bytes:
|
||||
print(json.dumps({"ok": False, "error": "rejected"}))
|
||||
return 2
|
||||
# Pin verification mode: the deployment smoke gate invokes the re-encoder
|
||||
# with --verify-pin to prove the provisioned, immutable runtime is exactly
|
||||
# the one the service was built and tested against. No live host change.
|
||||
if args.verify_pin:
|
||||
return _emit_pin(getattr(_Image, "__version__", "unknown"))
|
||||
|
||||
# Decompression-bomb warnings are errors here. Pillow's default threshold
|
||||
# only warns and then hands back a fully decoded image, which is not a
|
||||
# rejection; promote it so any bomb path raises instead.
|
||||
warnings.simplefilter("error", _Image.DecompressionBombWarning)
|
||||
# Refuse to let Pillow allocate beyond our own accepted pixel envelope.
|
||||
_Image.MAX_IMAGE_PIXELS = max(1, args.max_pixels)
|
||||
|
||||
try:
|
||||
# Fail closed on decompression bombs before full pixel load.
|
||||
source_size = os.path.getsize(args.source)
|
||||
except OSError:
|
||||
return reject("rejected")
|
||||
if source_size <= 0 or source_size > args.max_bytes:
|
||||
return reject("rejected")
|
||||
|
||||
# --- Header-only geometry gate, before any pixel decode. -----------------
|
||||
# Image.open() parses the header lazily, so probe.size is available without
|
||||
# allocating the pixel buffer. Rejecting here keeps a 12000x12000 bomb at
|
||||
# header cost instead of ~163 MiB of decoded pixels.
|
||||
try:
|
||||
with _Image.open(args.source) as probe:
|
||||
try:
|
||||
probe.load()
|
||||
except _Image.DecompressionBombError:
|
||||
print(json.dumps({"ok": False, "error": "dimensions"}))
|
||||
return 2
|
||||
width, height = probe.size
|
||||
if width > args.max_dimension or height > args.max_dimension:
|
||||
print(json.dumps({"ok": False, "error": "dimensions"}))
|
||||
return 2
|
||||
image = probe.convert("RGB")
|
||||
except _Image.DecompressionBombWarning:
|
||||
return reject("dimensions")
|
||||
except _Image.DecompressionBombError:
|
||||
return reject("dimensions")
|
||||
except Exception:
|
||||
print(json.dumps({"ok": False, "error": "malformed"}))
|
||||
return 2
|
||||
return reject("malformed")
|
||||
|
||||
if width <= 0 or height <= 0:
|
||||
return reject("malformed")
|
||||
if width > args.max_dimension or height > args.max_dimension:
|
||||
return reject("dimensions")
|
||||
if width * height > args.max_pixels:
|
||||
return reject("dimensions")
|
||||
|
||||
# --- Only now is a full decode allowed. ---------------------------------
|
||||
try:
|
||||
with _Image.open(args.source) as probe:
|
||||
probe.load()
|
||||
# Re-check after decode: a hostile container can declare small
|
||||
# geometry in its header and expand during decode.
|
||||
if probe.size[0] > args.max_dimension or probe.size[1] > args.max_dimension:
|
||||
return reject("dimensions")
|
||||
if probe.size[0] * probe.size[1] > args.max_pixels:
|
||||
return reject("dimensions")
|
||||
image = probe.convert("RGB")
|
||||
except (_Image.DecompressionBombWarning, _Image.DecompressionBombError):
|
||||
return reject("dimensions")
|
||||
except Exception:
|
||||
return reject("malformed")
|
||||
|
||||
buffer = io.BytesIO()
|
||||
try:
|
||||
# Baseline JPEG re-encode drops EXIF/GPS/XMP/tEXt entirely.
|
||||
image.save(buffer, "JPEG", quality=85, optimize=True, progressive=False)
|
||||
except Exception:
|
||||
print(json.dumps({"ok": False, "error": "encode"}))
|
||||
return 2
|
||||
return reject("encode")
|
||||
|
||||
data = buffer.getvalue()
|
||||
if not data or data[:3] != b"\xff\xd8\xff" or len(data) > args.max_bytes:
|
||||
print(json.dumps({"ok": False, "error": "encode"}))
|
||||
return 2
|
||||
return reject("encode")
|
||||
|
||||
with open(args.target, "wb") as handle:
|
||||
handle.write(data)
|
||||
try:
|
||||
with open(args.target, "wb") as handle:
|
||||
handle.write(data)
|
||||
except OSError:
|
||||
# Cannot write the private temp target: server-side fault.
|
||||
print(json.dumps({"ok": False, "error": "unavailable"}))
|
||||
return EXIT_UNAVAILABLE
|
||||
|
||||
print(json.dumps({
|
||||
"ok": True,
|
||||
|
|
@ -81,7 +169,7 @@ def main() -> int:
|
|||
"bytes": len(data),
|
||||
"metadataStripped": True,
|
||||
}))
|
||||
return 0
|
||||
return EXIT_OK
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
103
server.mjs
|
|
@ -5,10 +5,17 @@ import { extname, join, normalize } from 'node:path';
|
|||
import { fileURLToPath } from 'node:url';
|
||||
import { analyzePhoto } from './src/vision-service.js';
|
||||
import { createRateLimiter } from './src/rate-limiter.js';
|
||||
import { resolveClientIdentity, resolveTrustedProxies } from './src/client-identity.js';
|
||||
import { IngressUnavailableError } from './src/image-ingress.js';
|
||||
import { probeVisionProvider, resolveVisionConfig } from './src/vision-config.js';
|
||||
import { AgentGatewayError, createHermesAgentService, resolveHermesAgentConfig } from './src/hermes-agent-service.js';
|
||||
|
||||
const analyzeRateLimiter=createRateLimiter();
|
||||
// Explicit rate-limit identity policy. Behind a reverse proxy the socket peer is
|
||||
// the proxy itself, so a forwarded client address is only honoured when the peer
|
||||
// is in this configured allowlist. Otherwise the peer address is used, and a
|
||||
// trusted proxy that forwards nothing usable degrades to a truthful shared quota.
|
||||
const trustedProxies=resolveTrustedProxies(process.env.TIMMY_TRUSTED_PROXIES);
|
||||
|
||||
const root=fileURLToPath(new URL('.',import.meta.url));
|
||||
const port=Number(process.env.PORT||4173);
|
||||
|
|
@ -35,19 +42,105 @@ const stagingLabel=process.env.TIMMY_STAGING_LABEL?`Staging · ${release} · ${c
|
|||
|
||||
function sendJson(res,status,value){res.writeHead(status,{'content-type':'application/json; charset=utf-8','cache-control':'no-store','x-content-type-options':'nosniff'});res.end(JSON.stringify(value));}
|
||||
function escapeHtmlAttribute(value){return String(value).replace(/[&<>"]/g,character=>({'&':'&','<':'<','>':'>','"':'"'}[character]));}
|
||||
function readJson(req,maxBytes=6*1024*1024){return new Promise((resolve,reject)=>{let size=0,tooLarge=false;const chunks=[];req.on('data',chunk=>{size+=chunk.length;if(size>maxBytes){tooLarge=true;return}chunks.push(chunk)});req.on('end',()=>{if(tooLarge)return reject(new AgentGatewayError(413,'Request is too large.'));try{resolve(JSON.parse(Buffer.concat(chunks).toString('utf8')))}catch{return reject(new AgentGatewayError(400,'Invalid JSON request.'))}});req.on('error',reject)})}
|
||||
const BODY_READ_TIMEOUT_MS = Math.max(500, Number.parseInt(process.env.TIMMY_BODY_READ_TIMEOUT_MS || '10000', 10) || 10000);
|
||||
const MAX_BODY_BYTES = 6 * 1024 * 1024;
|
||||
|
||||
/**
|
||||
* Read a JSON request body with a hard inbound timeout and a stop-on-oversize
|
||||
* guard. The connection is actively destroyed the moment either limit trips, so
|
||||
* a hostile client cannot hold a request open by dribbling bytes or force the
|
||||
* server to drain an enormous payload before rejecting it.
|
||||
*/
|
||||
function readJson(req, res, maxBytes = MAX_BODY_BYTES) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let size = 0;
|
||||
let tooLarge = false;
|
||||
let settled = false;
|
||||
const chunks = [];
|
||||
let timer = null;
|
||||
|
||||
const fail = (error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
req.off('data', onData);
|
||||
req.off('end', onEnd);
|
||||
req.off('error', onError);
|
||||
reject(error);
|
||||
};
|
||||
|
||||
const onData = (chunk) => {
|
||||
if (settled || tooLarge) return;
|
||||
size += chunk.length;
|
||||
if (size > maxBytes) {
|
||||
tooLarge = true;
|
||||
// Reject immediately so the transport can answer a sanitized 413 without
|
||||
// waiting for the client to finish streaming an enormous payload. The
|
||||
// socket is torn down by the caller after the response is written.
|
||||
return fail(new AgentGatewayError(413, 'Request is too large. Use a smaller photo or continue manually.'));
|
||||
}
|
||||
chunks.push(chunk);
|
||||
};
|
||||
const onEnd = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
if (tooLarge) return;
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(chunks).toString('utf8')));
|
||||
} catch {
|
||||
reject(new AgentGatewayError(400, 'Invalid JSON request.'));
|
||||
}
|
||||
};
|
||||
const onError = (error) => {
|
||||
// A broken socket is torn down here; the rejection still propagates.
|
||||
try { req.destroy(); } catch { /* already gone */ }
|
||||
fail(error);
|
||||
};
|
||||
|
||||
timer = setTimeout(() => {
|
||||
// Hard inbound timeout: a slow-dribbling client must not hold the
|
||||
// connection open. The caller answers 408 and destroys the socket.
|
||||
fail(new AgentGatewayError(408, 'Request timed out while reading the upload.'));
|
||||
}, BODY_READ_TIMEOUT_MS);
|
||||
|
||||
req.on('data', onData);
|
||||
req.on('end', onEnd);
|
||||
req.on('error', onError);
|
||||
});
|
||||
}
|
||||
function cookie(req,name){for(const part of String(req.headers.cookie||'').split(';')){const [key,...value]=part.trim().split('=');if(key===name)return decodeURIComponent(value.join('='))}return ''}
|
||||
function requestOrigin(req){return String(req.headers.origin||'').replace(/\/$/,'')}
|
||||
function rejectCrossSite(req){const site=String(req.headers['sec-fetch-site']||'');if(site&&site!=='same-origin')throw new AgentGatewayError(403,'Cross-site agent requests are not allowed.')}
|
||||
function agentCookie(token){const secure=agentConfig.publicOrigin.startsWith('https://')?'; Secure':'';return `timmy_agent=${encodeURIComponent(token)}; HttpOnly; SameSite=Strict; Path=${appRoot}; Max-Age=86400${secure}`}
|
||||
function sendAgentError(res,error){const status=error instanceof AgentGatewayError?error.status:503;const message=error instanceof AgentGatewayError?error.message:'Hermes is temporarily unavailable. Your local journal still works.';return sendJson(res,status,{error:message})}
|
||||
|
||||
// Ingress failures carry their own honest classification: a server-side
|
||||
// processing fault must not be reported as corrupt client input.
|
||||
function handleAnalyzeError(res,error){
|
||||
if(error instanceof IngressUnavailableError||error?.ingressUnavailable===true)return sendJson(res,503,{error:error.message,manualFallback:true});
|
||||
const safe=/consent|JPEG|PNG|WebP|empty|too large|supported image|corrupt|malformed|slow down/i.test(error.message);
|
||||
return sendJson(res,safe?400:503,{error:error.message});
|
||||
}
|
||||
// A body-read rejection (oversized or timed out) is answered with a sanitized
|
||||
// status and then the socket is torn down so a hostile client cannot keep the
|
||||
// request open by dribbling or draining an enormous payload.
|
||||
function handleBodyError(res,req,error){
|
||||
if(typeof error?.status==='number'){sendJson(res,error.status,{error:error.message});}
|
||||
else{sendJson(res,400,{error:'Invalid request.'});}
|
||||
try{res.end();req.destroy();}catch{/* already closed */}
|
||||
}
|
||||
|
||||
http.createServer(async(req,res)=>{
|
||||
try{
|
||||
const url=new URL(req.url,'http://localhost');
|
||||
if(basePath&&url.pathname!==basePath&&!url.pathname.startsWith(`${basePath}/`)){res.writeHead(404,{'content-type':'text/plain; charset=utf-8'});return res.end('Not found')}
|
||||
if(basePath&&url.pathname===basePath){res.writeHead(308,{location:appRoot});return res.end()}
|
||||
const appPath=basePath?(url.pathname.slice(basePath.length)||'/'):url.pathname;
|
||||
// Nested API routes under the base path (e.g. /timmy-staging/api/healthz) are
|
||||
// routed to their canonical /api/* handler so the base path is preserved on
|
||||
// every operational endpoint, not just the root.
|
||||
const apiPath=basePath&&appPath.startsWith('/api/')?appPath:appPath;
|
||||
if(appPath==='/api/healthz'&&req.method==='GET')return sendJson(res,200,{ok:true,release,commit,visionEnabled:visionConfig.enabled,agentEnabled:agentConfig.enabled});
|
||||
if(appPath==='/api/vision-status'&&req.method==='GET'){
|
||||
const provider=await probeVisionProvider(visionConfig);
|
||||
|
|
@ -56,15 +149,13 @@ http.createServer(async(req,res)=>{
|
|||
}
|
||||
if(appPath==='/api/analyze'&&req.method==='POST'){
|
||||
if(!visionConfig.enabled)return sendJson(res,503,{error:'AI analysis is disabled. Continue manually.'});
|
||||
const clientKey=(req.socket&&req.socket.remoteAddress)||'unknown';
|
||||
const limit=analyzeRateLimiter.take(clientKey);
|
||||
const identity=resolveClientIdentity({remoteAddress:req.socket?.remoteAddress,headers:req.headers,trustedProxies});
|
||||
const limit=analyzeRateLimiter.take(identity.key);
|
||||
if(!limit.allowed){
|
||||
res.setHeader('retry-after',Math.ceil(limit.retryAfterMs/1000));
|
||||
return sendJson(res,429,{error:'Too many photo analyses. Please slow down and try again later.'});
|
||||
}
|
||||
const payload=await readJson(req);
|
||||
try{return sendJson(res,200,await analyzePhoto({payload,config:visionConfig}))}
|
||||
catch(error){const safe=/consent|JPEG|PNG|WebP|empty|too large|supported image|corrupt|malformed|slow down/i.test(error.message);return sendJson(res,safe?400:503,{error:error.message})}
|
||||
try{const payload=await readJson(req,res);try{return sendJson(res,200,await analyzePhoto({payload,config:visionConfig}))}catch(error){return handleAnalyzeError(res,error)}}catch(error){return handleBodyError(res,req,error)}
|
||||
}
|
||||
if(appPath==='/api/agent/status'&&req.method==='GET')return sendJson(res,200,agentService.status(cookie(req,'timmy_agent')));
|
||||
if(appPath==='/api/agent/unlock'&&req.method==='POST'){
|
||||
|
|
|
|||
70
src/client-identity.js
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
// Explicit client identity for rate limiting.
|
||||
//
|
||||
// Behind a reverse proxy the socket peer is the proxy itself, so keying on
|
||||
// req.socket.remoteAddress silently collapses every user into one quota. The
|
||||
// policy here is explicit and fails safe:
|
||||
//
|
||||
// * If the peer is NOT a configured trusted proxy, forwarded headers are
|
||||
// ignored entirely — they are attacker-controlled and would let anyone mint
|
||||
// unlimited identities or impersonate another client.
|
||||
// * If the peer IS a configured trusted proxy, the left-most syntactically
|
||||
// valid address in the forwarded chain is used as the client identity.
|
||||
// * If a trusted proxy forwards nothing usable, the scope degrades to an
|
||||
// honest shared 'global' quota rather than pretending to be per-client.
|
||||
//
|
||||
// The returned scope is reported truthfully so operators and health output can
|
||||
// state which policy is actually in force.
|
||||
import { isIP } from 'node:net';
|
||||
|
||||
const MAX_KEY_LENGTH = 64;
|
||||
|
||||
function normaliseAddress(value) {
|
||||
const raw = String(value || '').trim();
|
||||
if (!raw || raw.length > MAX_KEY_LENGTH) return null;
|
||||
// Strip an IPv6 zone index and IPv4-mapped IPv6 prefix for stable keys.
|
||||
const withoutZone = raw.replace(/%.*$/, '');
|
||||
const unmapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(withoutZone);
|
||||
const candidate = unmapped ? unmapped[1] : withoutZone;
|
||||
return isIP(candidate) ? candidate : null;
|
||||
}
|
||||
|
||||
/** Parse the trusted-proxy allowlist from configuration. */
|
||||
export function resolveTrustedProxies(value) {
|
||||
return String(value || '')
|
||||
.split(',')
|
||||
.map((entry) => normaliseAddress(entry))
|
||||
.filter((entry) => entry !== null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide the rate-limit identity for one request.
|
||||
*
|
||||
* Returns `{ key, scope, trustedProxy }` where scope is one of:
|
||||
* 'peer' keyed on the directly connected address
|
||||
* 'forwarded' keyed on a client address supplied by a trusted proxy
|
||||
* 'global' one shared quota, stated honestly
|
||||
*/
|
||||
export function resolveClientIdentity({ remoteAddress, headers = {}, trustedProxies = [] } = {}) {
|
||||
const peer = normaliseAddress(remoteAddress);
|
||||
const allowlist = trustedProxies.map((entry) => normaliseAddress(entry)).filter(Boolean);
|
||||
const trusted = peer !== null && allowlist.includes(peer);
|
||||
|
||||
if (!trusted) {
|
||||
// Untrusted peer: forwarded headers are spoofable and must be ignored.
|
||||
if (peer === null) return { key: 'global', scope: 'global', trustedProxy: false };
|
||||
return { key: peer, scope: 'peer', trustedProxy: false };
|
||||
}
|
||||
|
||||
const forwardedFor = headers['x-forwarded-for'] ?? headers['X-Forwarded-For'];
|
||||
const chain = String(forwardedFor || '').split(',');
|
||||
for (const entry of chain) {
|
||||
const client = normaliseAddress(entry);
|
||||
if (client) return { key: client, scope: 'forwarded', trustedProxy: true };
|
||||
}
|
||||
const realIp = normaliseAddress(headers['x-real-ip'] ?? headers['X-Real-IP']);
|
||||
if (realIp) return { key: realIp, scope: 'forwarded', trustedProxy: true };
|
||||
|
||||
// A trusted proxy that forwards nothing usable means the quota really is
|
||||
// shared. Say so rather than reporting a per-client guarantee we cannot keep.
|
||||
return { key: 'global', scope: 'global', trustedProxy: true };
|
||||
}
|
||||
141
src/image-container.js
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
// Canonical image container boundary.
|
||||
//
|
||||
// Polyglot and trailing-data rejection must be structural: parse the declared
|
||||
// container and require that its own structure consumes exactly the supplied
|
||||
// bytes. Substring scanning for signatures like `<script` or `PK\x03\x04` is
|
||||
// both evadable (casing, other archive formats, plain appended HTML) and prone
|
||||
// to false positives, because compressed entropy data and legal comment
|
||||
// segments may contain those byte sequences.
|
||||
//
|
||||
// This module never decodes pixels. It walks structure only, with a bounded
|
||||
// number of steps, so it is safe to run before any decode or subprocess work.
|
||||
|
||||
const MAX_STRUCTURE_STEPS = 100_000;
|
||||
|
||||
// JPEG markers that stand alone and carry no length field.
|
||||
const JPEG_STANDALONE = new Set([0x01, 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7]);
|
||||
|
||||
function parseJpeg(bytes) {
|
||||
if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8) return null;
|
||||
let offset = 2;
|
||||
for (let step = 0; step < MAX_STRUCTURE_STEPS; step += 1) {
|
||||
if (offset + 1 >= bytes.length) return null;
|
||||
if (bytes[offset] !== 0xff) return null;
|
||||
// Fill bytes: any number of 0xFF may precede a marker code.
|
||||
let marker = bytes[offset + 1];
|
||||
let markerAt = offset + 1;
|
||||
while (marker === 0xff) {
|
||||
markerAt += 1;
|
||||
if (markerAt >= bytes.length) return null;
|
||||
marker = bytes[markerAt];
|
||||
}
|
||||
if (marker === 0x00) return null; // stuffed byte outside entropy data
|
||||
if (marker === 0xd9) return { format: 'jpeg', consumed: markerAt + 1 }; // EOI
|
||||
if (JPEG_STANDALONE.has(marker)) {
|
||||
offset = markerAt + 1;
|
||||
continue;
|
||||
}
|
||||
if (markerAt + 2 >= bytes.length) return null;
|
||||
const length = bytes.readUInt16BE(markerAt + 1);
|
||||
if (length < 2) return null;
|
||||
const segmentEnd = markerAt + 1 + length;
|
||||
if (segmentEnd > bytes.length) return null;
|
||||
if (marker !== 0xda) {
|
||||
offset = segmentEnd;
|
||||
continue;
|
||||
}
|
||||
// Start of scan: entropy-coded data runs until the next real marker.
|
||||
let scan = segmentEnd;
|
||||
for (; scan < bytes.length; scan += 1) {
|
||||
if (bytes[scan] !== 0xff) continue;
|
||||
const next = bytes[scan + 1];
|
||||
if (next === undefined) return null;
|
||||
if (next === 0x00) { scan += 1; continue; } // stuffed 0xFF data byte
|
||||
if (next >= 0xd0 && next <= 0xd7) { scan += 1; continue; } // restart marker
|
||||
if (next === 0xff) continue; // fill byte
|
||||
break;
|
||||
}
|
||||
if (scan >= bytes.length) return null; // ran out of data before EOI
|
||||
offset = scan;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parsePng(bytes) {
|
||||
const signature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
||||
if (bytes.length < 8 || !bytes.subarray(0, 8).equals(signature)) return null;
|
||||
let offset = 8;
|
||||
let seenHeader = false;
|
||||
for (let step = 0; step < MAX_STRUCTURE_STEPS; step += 1) {
|
||||
if (offset + 8 > bytes.length) return null;
|
||||
const length = bytes.readUInt32BE(offset);
|
||||
if (length > 0x7fffffff) return null;
|
||||
const type = bytes.subarray(offset + 4, offset + 8).toString('latin1');
|
||||
if (!/^[A-Za-z]{4}$/.test(type)) return null;
|
||||
if (!seenHeader && type !== 'IHDR') return null;
|
||||
seenHeader = true;
|
||||
const end = offset + 8 + length + 4; // data + CRC
|
||||
if (end > bytes.length) return null;
|
||||
if (type === 'IEND') {
|
||||
if (length !== 0) return null;
|
||||
return { format: 'png', consumed: end };
|
||||
}
|
||||
offset = end;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function parseWebp(bytes) {
|
||||
if (bytes.length < 12) return null;
|
||||
if (bytes.subarray(0, 4).toString('latin1') !== 'RIFF') return null;
|
||||
if (bytes.subarray(8, 12).toString('latin1') !== 'WEBP') return null;
|
||||
const riffSize = bytes.readUInt32LE(4);
|
||||
if (riffSize < 4 || riffSize > 0x7fffffff) return null;
|
||||
const declaredEnd = 8 + riffSize;
|
||||
if (declaredEnd > bytes.length) return null;
|
||||
// Walk the chunk list so a truncated or over-declared RIFF is caught too.
|
||||
let offset = 12;
|
||||
for (let step = 0; step < MAX_STRUCTURE_STEPS; step += 1) {
|
||||
if (offset === declaredEnd) return { format: 'webp', consumed: declaredEnd };
|
||||
if (offset + 8 > declaredEnd) return null;
|
||||
const type = bytes.subarray(offset, offset + 4).toString('latin1');
|
||||
if (!/^[A-Za-z0-9 ]{4}$/.test(type)) return null;
|
||||
const size = bytes.readUInt32LE(offset + 4);
|
||||
if (size > 0x7fffffff) return null;
|
||||
const padded = size + (size % 2);
|
||||
const end = offset + 8 + padded;
|
||||
if (end > declaredEnd) return null;
|
||||
offset = end;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const PARSERS = [parseJpeg, parsePng, parseWebp];
|
||||
|
||||
/**
|
||||
* Parse a supported raster container structurally.
|
||||
*
|
||||
* Returns `{ format, consumed, trailingBytes }` when the signature matches a
|
||||
* supported format and the structure is internally consistent, otherwise null.
|
||||
* `trailingBytes` is how many supplied bytes the container did not account for.
|
||||
*/
|
||||
export function parseImageContainer(bytes) {
|
||||
if (!Buffer.isBuffer(bytes) || bytes.length < 12) return null;
|
||||
for (const parse of PARSERS) {
|
||||
const parsed = parse(bytes);
|
||||
if (parsed) {
|
||||
return { ...parsed, trailingBytes: bytes.length - parsed.consumed };
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical acceptance: a supported container whose own structure accounts for
|
||||
* every supplied byte. Any trailing data is a container polyglot and fails.
|
||||
*/
|
||||
export function canonicalImageFormat(bytes) {
|
||||
const parsed = parseImageContainer(bytes);
|
||||
if (!parsed || parsed.trailingBytes !== 0) return null;
|
||||
return parsed.format;
|
||||
}
|
||||
|
|
@ -3,37 +3,128 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { canonicalImageFormat } from './image-container.js';
|
||||
|
||||
export const MAX_IMAGE_BYTES = 4 * 1024 * 1024;
|
||||
export const MAX_IMAGE_DIMENSION = 4096;
|
||||
// Total-pixel ceiling, bounded independently of the per-side cap so extreme
|
||||
// aspect ratios cannot smuggle a large pixel budget past a per-dimension check.
|
||||
export const MAX_IMAGE_PIXELS = MAX_IMAGE_DIMENSION * MAX_IMAGE_DIMENSION;
|
||||
const REENCODE_TIMEOUT_MS = 15_000;
|
||||
|
||||
// Production runs a pinned, immutable interpreter provisioned outside the
|
||||
// release tree; TIMMY_PYTHON must be an absolute path so the service never
|
||||
// depends on PATH resolution. Development falls back to `python3`.
|
||||
export function resolveInterpreter(env = process.env) {
|
||||
const configured = String(env.TIMMY_PYTHON || '').trim();
|
||||
if (!configured) return 'python3';
|
||||
if (!configured.startsWith('/')) {
|
||||
throw new Error('TIMMY_PYTHON must be an absolute interpreter path.');
|
||||
}
|
||||
return configured;
|
||||
}
|
||||
|
||||
// The pinned, immutable production image runtime. The service refuses to start
|
||||
// decode work if the provisioned interpreter is not this exact toolchain:
|
||||
// a drifting runtime is a server-side provisioning fault, and decoding images
|
||||
// under an untested Pillow/Pillow-version would silently change the security
|
||||
// boundary. The deployment smoke gate (scripts/deploy_staging.py) verifies the
|
||||
// same pin before promoting; this is the in-process fail-closed backstop.
|
||||
export const PINNED_PYTHON_VERSION = '3.11';
|
||||
export const PINNED_PILLOW_VERSION = '12.3.0';
|
||||
export const REENCODE_SMOKE_TIMEOUT_MS = 10_000;
|
||||
|
||||
/**
|
||||
* Verify the provisioned re-encoder runtime matches the pinned, immutable
|
||||
* production toolchain. Runs the re-encoder with --verify-pin and maps any
|
||||
* non-zero exit (or a version mismatch) to an IngressUnavailableError so the
|
||||
* transport layer answers 503 rather than blaming a client photo.
|
||||
*/
|
||||
export function verifyReencodeRuntime({ timeoutMs = REENCODE_SMOKE_TIMEOUT_MS } = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(
|
||||
resolveInterpreter(),
|
||||
[
|
||||
join(fileURLToPath(new URL('..', import.meta.url)), 'scripts', 'reencode_image.py'),
|
||||
'--verify-pin',
|
||||
],
|
||||
{ timeout: timeoutMs, windowsHide: true, maxBuffer: 64 * 1024 },
|
||||
(error, stdout) => {
|
||||
if (error) {
|
||||
// Missing, non-runnable, or version-mismatched runtime: a provisioning
|
||||
// fault, not the client's problem.
|
||||
return reject(sanitizedError('unavailable'));
|
||||
}
|
||||
let verdict = {};
|
||||
try { verdict = JSON.parse(String(stdout).trim()); } catch { /* fall through */ }
|
||||
if (verdict.ok !== true) {
|
||||
return reject(sanitizedError('unavailable'));
|
||||
}
|
||||
return resolve({
|
||||
python: verdict.python,
|
||||
pillow: verdict.pillow,
|
||||
pinned: verdict.pinned,
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
const MAGIC = {
|
||||
jpeg: (b) => b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff,
|
||||
png: (b) => b[0] === 0x89 && b[1] === 0x50 && b[2] === 0x4e && b[3] === 0x47,
|
||||
webp: (b) => b.subarray(0, 4).toString('latin1') === 'RIFF' && b.subarray(8, 12).toString('latin1') === 'WEBP',
|
||||
};
|
||||
|
||||
function base64Bytes(dataUrl) {
|
||||
const match = /^data:[^;,]*;base64,([A-Za-z0-9+/=\r\n]+)$/.exec(String(dataUrl || ''));
|
||||
if (!match) return null;
|
||||
const clean = match[1].replace(/[\r\n]/g, '');
|
||||
return Buffer.from(clean, 'base64');
|
||||
}
|
||||
|
||||
export function sniffImageFormat(bytes) {
|
||||
if (!Buffer.isBuffer(bytes) || bytes.length < 12) return null;
|
||||
for (const [format, test] of Object.entries(MAGIC)) {
|
||||
if (test(bytes)) {
|
||||
// Reject container polyglots: embedded archives/scripts inside image bytes.
|
||||
if (bytes.includes(Buffer.from('PK\x03\x04'))) return null;
|
||||
if (bytes.includes(Buffer.from('<script'))) return null;
|
||||
return format;
|
||||
// Signature match is necessary but not sufficient: the container must
|
||||
// also parse canonically and account for every supplied byte. This
|
||||
// rejects trailing-data and container polyglots (appended HTML/script in
|
||||
// any casing, ZIP local headers or EOCD records, other archive
|
||||
// signatures) without substring scanning, which both misses variants and
|
||||
// false-positives on compressed entropy data.
|
||||
return canonicalImageFormat(bytes) === format ? format : null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// Exact supported MIME policy: canonical raster types only. The declared MIME
|
||||
// is deliberately not trusted for format detection (magic bytes decide that),
|
||||
// but an unsupported declared type is still refused outright.
|
||||
const SUPPORTED_MIME = new Set(['image/jpeg', 'image/png', 'image/webp']);
|
||||
|
||||
// Canonical base64 data URL grammar. No whitespace, no newlines, no URL-safe
|
||||
// alphabet, no extra parameters, exactly one correct padding group.
|
||||
const DATA_URL_GRAMMAR = /^data:([a-z]+\/[a-z0-9.+-]+);base64,([A-Za-z0-9+/]+={0,2})$/;
|
||||
|
||||
/**
|
||||
* Decode a canonical base64 raster data URL.
|
||||
*
|
||||
* Returns `{ mime, bytes }` only when the value matches the canonical grammar,
|
||||
* declares a supported MIME type, and re-encodes byte-identically (which is
|
||||
* what rejects missing or excess padding and non-zero trailing pad bits).
|
||||
* Returns null otherwise. Nothing is repaired or normalised: a non-canonical
|
||||
* request is a hand-crafted request and fails closed.
|
||||
*/
|
||||
export function decodeImageDataUrl(value) {
|
||||
if (typeof value !== 'string') return null;
|
||||
const match = DATA_URL_GRAMMAR.exec(value);
|
||||
if (!match) return null;
|
||||
const [, mime, payload] = match;
|
||||
if (!SUPPORTED_MIME.has(mime)) return null;
|
||||
if (payload.length % 4 !== 0) return null;
|
||||
const bytes = Buffer.from(payload, 'base64');
|
||||
if (bytes.length === 0) return null;
|
||||
// Byte-exact canonical round trip: rejects non-canonical padding and any
|
||||
// non-zero bits in the final partial group.
|
||||
if (bytes.toString('base64') !== payload) return null;
|
||||
return { mime, bytes };
|
||||
}
|
||||
|
||||
// Errors are sanitized by construction: fixed short strings only.
|
||||
const SANITIZED = {
|
||||
consent: 'Explicit consent is required before AI analysis.',
|
||||
|
|
@ -42,35 +133,110 @@ const SANITIZED = {
|
|||
dimensions: 'The photo is too large. Maximum dimension is 4096 pixels.',
|
||||
corrupt: 'The photo is corrupt or malformed. Try a different photo or continue manually.',
|
||||
unavailable: 'Photo processing is temporarily unavailable. Continue manually.',
|
||||
capacity: 'Photo processing is temporarily unavailable. Please try again in a moment or continue manually.',
|
||||
};
|
||||
|
||||
/**
|
||||
* A server-side processing fault: the image runtime could not run at all.
|
||||
*
|
||||
* This is deliberately a distinct type so the transport layer can answer 503
|
||||
* (processing unavailable, manual fallback) instead of 400, which would falsely
|
||||
* blame the client's photo for being corrupt.
|
||||
*/
|
||||
export class IngressUnavailableError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = 'IngressUnavailableError';
|
||||
this.ingressUnavailable = true;
|
||||
}
|
||||
}
|
||||
|
||||
/** Map an ingress failure to its client-visible HTTP status. */
|
||||
export function classifyIngressFailure(error) {
|
||||
return error instanceof IngressUnavailableError || error?.ingressUnavailable === true ? 503 : 400;
|
||||
}
|
||||
|
||||
// --- Fail-fast concurrency ceiling around decoder/provider work -------------
|
||||
//
|
||||
// Each decode is a subprocess with a real memory cost, and the service runs
|
||||
// under MemoryMax=512M. A burst is therefore bounded by a small ceiling and
|
||||
// excess requests are refused immediately rather than queued: queuing would
|
||||
// convert a burst into unbounded latency and keep hostile load resident.
|
||||
export const INGRESS_MAX_CONCURRENCY = Math.max(
|
||||
1,
|
||||
Math.min(4, Number.parseInt(process.env.TIMMY_INGRESS_MAX_CONCURRENCY || '2', 10) || 2),
|
||||
);
|
||||
|
||||
let activeDecodes = 0;
|
||||
|
||||
/** Current decoder occupancy, for tests and health reporting. */
|
||||
export function ingressConcurrencyState() {
|
||||
return { active: activeDecodes, limit: INGRESS_MAX_CONCURRENCY };
|
||||
}
|
||||
|
||||
function acquireDecodeSlot() {
|
||||
if (activeDecodes >= INGRESS_MAX_CONCURRENCY) return false;
|
||||
activeDecodes += 1;
|
||||
return true;
|
||||
}
|
||||
|
||||
function releaseDecodeSlot() {
|
||||
activeDecodes = Math.max(0, activeDecodes - 1);
|
||||
}
|
||||
|
||||
function sanitizedError(key) {
|
||||
if (key === 'unavailable' || key === 'capacity') return new IngressUnavailableError(SANITIZED[key]);
|
||||
return new Error(SANITIZED[key]);
|
||||
}
|
||||
|
||||
function reencode(sourcePath, targetPath) {
|
||||
// Exit-code contract shared with scripts/reencode_image.py:
|
||||
// 0 success, 2 rejected hostile/invalid input, 3 processing unavailable.
|
||||
const REENCODE_EXIT_REJECTED = 2;
|
||||
const REENCODE_EXIT_UNAVAILABLE = 3;
|
||||
|
||||
function reencode(sourcePath, targetPath, { timeoutMs = REENCODE_TIMEOUT_MS } = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
execFile(
|
||||
process.env.TIMMY_PYTHON || 'python3',
|
||||
resolveInterpreter(),
|
||||
[
|
||||
join(fileURLToPath(new URL('..', import.meta.url)), 'scripts', 'reencode_image.py'),
|
||||
'--in', sourcePath,
|
||||
'--out', targetPath,
|
||||
'--max-bytes', String(MAX_IMAGE_BYTES),
|
||||
'--max-dimension', String(MAX_IMAGE_DIMENSION),
|
||||
'--max-pixels', String(MAX_IMAGE_PIXELS),
|
||||
],
|
||||
{ timeout: REENCODE_TIMEOUT_MS, windowsHide: true },
|
||||
{ timeout: timeoutMs, windowsHide: true },
|
||||
(error, stdout) => {
|
||||
if (error) {
|
||||
if (error.killed || error.signal === 'SIGTERM') return reject(sanitizedError('unavailable'));
|
||||
// Exit code 2 with a dimensions verdict maps to a distinct message.
|
||||
if (/dimensions/.test(String(stdout))) return reject(sanitizedError('dimensions'));
|
||||
return reject(sanitizedError('corrupt'));
|
||||
// Timeout or signal kill: the runtime did not finish, so this is a
|
||||
// server-side fault rather than bad client input.
|
||||
if (error.killed || error.signal) return reject(sanitizedError('unavailable'));
|
||||
// The interpreter itself could not be executed (missing/not runnable).
|
||||
if (error.code === 'ENOENT' || error.code === 'EACCES') {
|
||||
return reject(sanitizedError('unavailable'));
|
||||
}
|
||||
let verdict = {};
|
||||
try {
|
||||
verdict = JSON.parse(String(stdout).trim().split('\n').pop() || '{}');
|
||||
} catch { verdict = {}; }
|
||||
// Exit 3 means the image runtime is unavailable. Reporting this as a
|
||||
// client 400 would falsely blame the user's photo.
|
||||
if (error.code === REENCODE_EXIT_UNAVAILABLE || verdict.error === 'unavailable') {
|
||||
return reject(sanitizedError('unavailable'));
|
||||
}
|
||||
if (error.code === REENCODE_EXIT_REJECTED) {
|
||||
if (verdict.error === 'dimensions') return reject(sanitizedError('dimensions'));
|
||||
return reject(sanitizedError('corrupt'));
|
||||
}
|
||||
// Any other exit status is an unexpected runtime failure, not a
|
||||
// proven statement about the client's input.
|
||||
return reject(sanitizedError('unavailable'));
|
||||
}
|
||||
try {
|
||||
resolve(JSON.parse(String(stdout).trim()));
|
||||
} catch {
|
||||
reject(sanitizedError('corrupt'));
|
||||
reject(sanitizedError('unavailable'));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
|
@ -78,37 +244,56 @@ function reencode(sourcePath, targetPath) {
|
|||
}
|
||||
|
||||
export async function validateImageIngress(payload = {}) {
|
||||
// Cheap, allocation-free checks run before any capacity is taken so hostile
|
||||
// input can never occupy a decoder slot.
|
||||
if (payload.consent !== true) throw sanitizedError('consent');
|
||||
const bytes = base64Bytes(payload.imageDataUrl);
|
||||
if (!bytes || bytes.length === 0) throw sanitizedError('format');
|
||||
const decoded = decodeImageDataUrl(payload.imageDataUrl);
|
||||
if (!decoded) throw sanitizedError('format');
|
||||
const bytes = decoded.bytes;
|
||||
if (bytes.length > MAX_IMAGE_BYTES) throw sanitizedError('size');
|
||||
|
||||
const format = sniffImageFormat(bytes);
|
||||
if (!format) throw sanitizedError('format');
|
||||
|
||||
const workDir = await mkdtemp(join(tmpdir(), 'timmy-ingress-'));
|
||||
// Expensive work starts here: bound it with a fail-fast ceiling so a burst
|
||||
// cannot spawn enough decoders to exceed the service memory budget.
|
||||
if (!acquireDecodeSlot()) throw sanitizedError('capacity');
|
||||
try {
|
||||
const sourcePath = join(workDir, `source.${format}`);
|
||||
const targetPath = join(workDir, 'reencoded.jpg');
|
||||
await writeFile(sourcePath, bytes);
|
||||
let result;
|
||||
const workDir = await mkdtemp(join(tmpdir(), 'timmy-ingress-'));
|
||||
try {
|
||||
result = await reencode(sourcePath, targetPath);
|
||||
} catch (error) {
|
||||
throw error;
|
||||
const sourcePath = join(workDir, `source.${format}`);
|
||||
const targetPath = join(workDir, 'reencoded.jpg');
|
||||
await writeFile(sourcePath, bytes);
|
||||
const result = await reencode(sourcePath, targetPath, {
|
||||
timeoutMs: Number.isFinite(payload.reencodeTimeoutMs) && payload.reencodeTimeoutMs > 0
|
||||
? payload.reencodeTimeoutMs
|
||||
: REENCODE_TIMEOUT_MS,
|
||||
});
|
||||
let reencoded;
|
||||
try {
|
||||
reencoded = await readFile(targetPath);
|
||||
} catch {
|
||||
// The subprocess reported success but produced no readable output: a
|
||||
// server-side fault, not a statement about the client's photo.
|
||||
throw sanitizedError('unavailable');
|
||||
}
|
||||
return {
|
||||
imageDataUrl: `data:image/jpeg;base64,${reencoded.toString('base64')}`,
|
||||
mime: 'image/jpeg',
|
||||
format: result.format,
|
||||
width: result.width,
|
||||
height: result.height,
|
||||
bytes: reencoded.length,
|
||||
originalFormat: format,
|
||||
metadataStripped: result.metadataStripped === true,
|
||||
};
|
||||
} finally {
|
||||
// Private temp directory cleanup stays unconditional.
|
||||
await rm(workDir, { recursive: true, force: true }).catch(() => {});
|
||||
}
|
||||
const reencoded = await readFile(targetPath);
|
||||
return {
|
||||
imageDataUrl: `data:image/jpeg;base64,${reencoded.toString('base64')}`,
|
||||
mime: 'image/jpeg',
|
||||
format: result.format,
|
||||
width: result.width,
|
||||
height: result.height,
|
||||
bytes: reencoded.length,
|
||||
originalFormat: format,
|
||||
metadataStripped: result.metadataStripped === true,
|
||||
};
|
||||
} finally {
|
||||
await rm(workDir, { recursive: true, force: true }).catch(() => {});
|
||||
// The slot is released on every path, including rejections, so a hostile
|
||||
// burst cannot permanently exhaust capacity.
|
||||
releaseDecodeSlot();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,31 +1,117 @@
|
|||
// Fixed-window in-memory rate limiter for image analysis requests.
|
||||
// Bounded: per-key counters only, no payloads stored.
|
||||
// Bounded rate limiter for image analysis requests.
|
||||
//
|
||||
// Two properties the naive fixed-window version did not have:
|
||||
//
|
||||
// 1. Hard-bounded state. Sweeping only *expired* entries lets an attacker with
|
||||
// many distinct keys grow the map without limit inside one window. State is
|
||||
// capped at maxKeys; reclamation prefers expired entries, and when none are
|
||||
// expired the entry closest to expiry is dropped. Currently-blocked
|
||||
// offenders are never dropped, so flooding cannot reset someone's counter.
|
||||
//
|
||||
// 2. No boundary doubling. A fixed window lets a client spend a full budget at
|
||||
// the end of one window and another immediately after the boundary, i.e.
|
||||
// 2x the nominal rate in milliseconds. This uses a two-window weighted
|
||||
// sliding count so the boundary is smooth.
|
||||
//
|
||||
// Only counters are stored — never payloads, bodies, or headers.
|
||||
|
||||
export function createRateLimiter({ windowMs = 60_000, maxRequests = 10 } = {}) {
|
||||
const DEFAULT_MAX_KEYS = 10_000;
|
||||
|
||||
export function createRateLimiter({
|
||||
windowMs = 60_000,
|
||||
maxRequests = 10,
|
||||
maxKeys = DEFAULT_MAX_KEYS,
|
||||
} = {}) {
|
||||
// key -> { windowStart, count, previousCount }
|
||||
const windows = new Map();
|
||||
|
||||
function slidingCount(entry, now) {
|
||||
// Weight the previous window by how much of it still overlaps the trailing
|
||||
// `windowMs` interval ending at `now`.
|
||||
const elapsed = now - entry.windowStart;
|
||||
const overlap = Math.max(0, 1 - elapsed / windowMs);
|
||||
return entry.previousCount * overlap + entry.count;
|
||||
}
|
||||
|
||||
function isBlocked(entry, now) {
|
||||
return slidingCount(entry, now) >= maxRequests;
|
||||
}
|
||||
|
||||
function reclaim(now, protectedKey) {
|
||||
if (windows.size < maxKeys) return;
|
||||
// First pass: drop fully expired entries (two windows old, so their
|
||||
// weighted contribution is zero).
|
||||
for (const [key, entry] of windows) {
|
||||
if (key === protectedKey) continue;
|
||||
if (now - entry.windowStart >= windowMs * 2) windows.delete(key);
|
||||
}
|
||||
if (windows.size < maxKeys) return;
|
||||
// Second pass: hard bound. Drop the oldest entries that are not currently
|
||||
// blocking anyone, so eviction can never reset an active offender's count.
|
||||
const candidates = [];
|
||||
for (const [key, entry] of windows) {
|
||||
if (key === protectedKey) continue;
|
||||
if (isBlocked(entry, now)) continue;
|
||||
candidates.push([key, entry.windowStart]);
|
||||
}
|
||||
candidates.sort((a, b) => a[1] - b[1]);
|
||||
for (const [key] of candidates) {
|
||||
if (windows.size < maxKeys) break;
|
||||
windows.delete(key);
|
||||
}
|
||||
if (windows.size < maxKeys) return;
|
||||
// Every retained entry is actively blocked. Drop the oldest of those to
|
||||
// preserve the hard bound; its window is closest to expiring anyway.
|
||||
let oldestKey = null;
|
||||
let oldestStart = Infinity;
|
||||
for (const [key, entry] of windows) {
|
||||
if (key === protectedKey) continue;
|
||||
if (entry.windowStart < oldestStart) {
|
||||
oldestStart = entry.windowStart;
|
||||
oldestKey = key;
|
||||
}
|
||||
}
|
||||
if (oldestKey !== null) windows.delete(oldestKey);
|
||||
}
|
||||
|
||||
function take(key, now = Date.now()) {
|
||||
let entry = windows.get(key);
|
||||
if (!entry || now >= entry.resetAt) {
|
||||
entry = { count: 0, resetAt: now + windowMs };
|
||||
if (!entry) {
|
||||
reclaim(now, key);
|
||||
entry = { windowStart: now, count: 0, previousCount: 0 };
|
||||
windows.set(key, entry);
|
||||
}
|
||||
entry.count += 1;
|
||||
if (windows.size > 10_000) {
|
||||
// Bound memory: drop expired entries when the map grows too large.
|
||||
for (const [existingKey, existing] of windows) {
|
||||
if (now >= existing.resetAt) windows.delete(existingKey);
|
||||
} else {
|
||||
const elapsed = now - entry.windowStart;
|
||||
if (elapsed >= windowMs * 2) {
|
||||
entry.windowStart = now;
|
||||
entry.count = 0;
|
||||
entry.previousCount = 0;
|
||||
} else if (elapsed >= windowMs) {
|
||||
entry.windowStart = entry.windowStart + windowMs * Math.floor(elapsed / windowMs);
|
||||
entry.previousCount = entry.count;
|
||||
entry.count = 0;
|
||||
}
|
||||
}
|
||||
if (entry.count > maxRequests) {
|
||||
|
||||
if (slidingCount(entry, now) + 1 > maxRequests) {
|
||||
// Retry after the remaining overlap of the trailing interval.
|
||||
const elapsed = now - entry.windowStart;
|
||||
const retryAfterMs = Math.max(1, Math.ceil(windowMs - elapsed));
|
||||
return {
|
||||
allowed: false,
|
||||
retryAfterMs: Math.max(1, entry.resetAt - now),
|
||||
retryAfterMs,
|
||||
reason: 'Too many photo analyses. Please slow down and try again later.',
|
||||
};
|
||||
}
|
||||
return { allowed: true, remaining: maxRequests - entry.count };
|
||||
entry.count += 1;
|
||||
return { allowed: true, remaining: Math.max(0, maxRequests - Math.ceil(slidingCount(entry, now))) };
|
||||
}
|
||||
|
||||
return { take, limit: maxRequests, windowMs };
|
||||
return {
|
||||
take,
|
||||
limit: maxRequests,
|
||||
windowMs,
|
||||
maxKeys,
|
||||
size: () => windows.size,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
132
tests/body-timeout.test.js
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
// Inbound body-read timeout and stop-on-oversize contract.
|
||||
//
|
||||
// A client must not be able to hold a connection open by dribbling a body
|
||||
// forever, nor make the server drain an enormous payload before rejecting it.
|
||||
// The server must enforce a body-read timeout and stop/destroy oversized request
|
||||
// processing as soon as the declared or observed size exceeds the limit, while
|
||||
// preserving the configured base path and a sanitized 413.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { setTimeout as sleep } from 'node:timers/promises';
|
||||
import net from 'node:net';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const root = fileURLToPath(new URL('..', import.meta.url));
|
||||
const BASE_PORT = 4199;
|
||||
const originFor = (port) => `http://127.0.0.1:${port}`;
|
||||
const BODY_TIMEOUT_MS = 1500;
|
||||
|
||||
let nextPort = BASE_PORT;
|
||||
let currentPort = BASE_PORT;
|
||||
async function startServer(env = {}) {
|
||||
const PORT = nextPort++;
|
||||
currentPort = PORT;
|
||||
const workdir = await mkdtemp(join(tmpdir(), 'timmy-body-test-'));
|
||||
const child = spawn(process.execPath, ['server.mjs'], {
|
||||
cwd: root,
|
||||
env: {
|
||||
...process.env,
|
||||
PORT: String(PORT),
|
||||
TIMMY_VISION_ENABLED: 'true',
|
||||
TIMMY_BODY_READ_TIMEOUT_MS: String(BODY_TIMEOUT_MS),
|
||||
...env,
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
const origin = originFor(PORT);
|
||||
const deadline = Date.now() + 10_000;
|
||||
while (Date.now() < deadline) {
|
||||
if (child.exitCode !== null) throw new Error(`server exited ${child.exitCode}`);
|
||||
try { const response = await fetch(`${origin}/api/healthz`); if (response.ok) break; } catch {}
|
||||
await sleep(50);
|
||||
}
|
||||
return { child, workdir, origin };
|
||||
}
|
||||
|
||||
test('a slow-dribbling request body is terminated by the body-read timeout', async () => {
|
||||
const { child, workdir, origin } = await startServer();
|
||||
try {
|
||||
const socket = net.connect(currentPort, '127.0.0.1');
|
||||
await new Promise((resolve) => socket.once('connect', resolve));
|
||||
// Declare a large body and then dribble spaces far slower than the timeout.
|
||||
socket.write(
|
||||
`POST ${basePath()}/api/analyze HTTP/1.1\r\n` +
|
||||
`Host: 127.0.0.1\r\nContent-Type: application/json\r\n` +
|
||||
`Content-Length: ${8 * 1024 * 1024}\r\nConnection: close\r\n\r\n`,
|
||||
);
|
||||
const start = Date.now();
|
||||
let written = 0;
|
||||
const chunk = Buffer.alloc(64 * 1024, 0x20);
|
||||
let ended = false;
|
||||
let closeAt = 0;
|
||||
const writer = setInterval(() => {
|
||||
if (written >= 7 * 1024 * 1024 || socket.destroyed) { clearInterval(writer); return; }
|
||||
try { socket.write(chunk); written += chunk.length; } catch { clearInterval(writer); }
|
||||
}, 400);
|
||||
socket.on('close', () => { ended = true; closeAt = Date.now(); });
|
||||
// Swallow EPIPE: once the server tears down the socket, any in-flight write
|
||||
// must not crash the test process.
|
||||
socket.on('error', () => { clearInterval(writer); });
|
||||
// Wait well past the timeout to see whether the server kills the slow stream.
|
||||
await sleep(BODY_TIMEOUT_MS + 2500);
|
||||
clearInterval(writer);
|
||||
const elapsed = Date.now() - start;
|
||||
socket.destroy();
|
||||
assert.ok(ended, 'the server must close a slow-dribbling body before it finishes streaming');
|
||||
assert.ok(closeAt - start < BODY_TIMEOUT_MS + 2000,
|
||||
`a slow body was allowed to stream for ${closeAt - start}ms; the timeout is not enforced`);
|
||||
} finally {
|
||||
child.kill('SIGTERM');
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('an oversized request is rejected with a sanitized 413 and not drained', async () => {
|
||||
const { child, workdir, origin } = await startServer();
|
||||
try {
|
||||
const response = await fetch(`${origin}${basePath()}/api/analyze`, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json' },
|
||||
// No Content-Length: the server must stop once the stream exceeds the cap,
|
||||
// not wait for the client to finish sending. `duplex: 'half'` is required
|
||||
// by fetch when a request body is a streaming ReadableStream.
|
||||
duplex: 'half',
|
||||
body: new ReadableStream({
|
||||
start(controller) {
|
||||
const blob = new Uint8Array(10 * 1024 * 1024).fill(0x20);
|
||||
controller.enqueue(blob);
|
||||
// Keep the stream open so the server must stop it, not us.
|
||||
setTimeout(() => controller.close(), 5000);
|
||||
},
|
||||
}),
|
||||
});
|
||||
assert.equal(response.status, 413);
|
||||
const body = await response.text();
|
||||
assert.match(body, /too large/i);
|
||||
assert.doesNotMatch(body, /[A-Za-z0-9+/]{40,}/, 'no payload data in the 413 response');
|
||||
} finally {
|
||||
child.kill('SIGTERM');
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('the configured base path is preserved on the analyze route after hardening', async () => {
|
||||
const { child, workdir, origin } = await startServer({ TIMMY_BASE_PATH: '/timmy-staging' });
|
||||
try {
|
||||
const response = await fetch(`${origin}/timmy-staging/api/healthz`);
|
||||
assert.equal(response.status, 200);
|
||||
const missing = await fetch(`${origin}/api/healthz`);
|
||||
assert.equal(missing.status, 404);
|
||||
} finally {
|
||||
child.kill('SIGTERM');
|
||||
await rm(workdir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function basePath() {
|
||||
return process.env.TIMMY_BASE_PATH_TEST || '';
|
||||
}
|
||||
BIN
tests/fixtures/ingress-buried-signatures.jpg
vendored
Normal file
|
After Width: | Height: | Size: 1.2 KiB |
BIN
tests/fixtures/ingress-entropy-control.jpg
vendored
Normal file
|
After Width: | Height: | Size: 30 KiB |
BIN
tests/fixtures/ingress-garbage.jpg
vendored
BIN
tests/fixtures/ingress-tail-7z.jpg
vendored
Normal file
|
After Width: | Height: | Size: 1.1 KiB |
BIN
tests/fixtures/ingress-tail-after-iend.png
vendored
Normal file
|
After Width: | Height: | Size: 250 B |
BIN
tests/fixtures/ingress-tail-gzip.jpg
vendored
Normal file
|
After Width: | Height: | Size: 1.1 KiB |
BIN
tests/fixtures/ingress-tail-html.jpg
vendored
Normal file
|
After Width: | Height: | Size: 1.2 KiB |
BIN
tests/fixtures/ingress-tail-mixed-script.jpg
vendored
Normal file
|
After Width: | Height: | Size: 1.2 KiB |
BIN
tests/fixtures/ingress-tail-rar.jpg
vendored
Normal file
|
After Width: | Height: | Size: 1.1 KiB |
BIN
tests/fixtures/ingress-tail-single-nul.jpg
vendored
Normal file
|
After Width: | Height: | Size: 1.1 KiB |
BIN
tests/fixtures/ingress-tail-upper-script.jpg
vendored
Normal file
|
After Width: | Height: | Size: 1.2 KiB |
BIN
tests/fixtures/ingress-tail-zip-eocd.jpg
vendored
Normal file
|
After Width: | Height: | Size: 1.2 KiB |
BIN
tests/fixtures/ingress-tail-zip-local.jpg
vendored
Normal file
|
After Width: | Height: | Size: 1.2 KiB |
146
tests/image-concurrency.test.js
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
// Concurrency ceiling contract.
|
||||
//
|
||||
// The decoder runs as a subprocess and each one costs real memory. The service
|
||||
// runs under MemoryMax=512M, so a burst must be bounded by a small fail-fast
|
||||
// ceiling rather than queued indefinitely: excess requests are refused
|
||||
// immediately with a sanitized retry message, and the number of decoder
|
||||
// children alive at once never exceeds the ceiling.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { chmod, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import {
|
||||
INGRESS_MAX_CONCURRENCY,
|
||||
IngressUnavailableError,
|
||||
ingressConcurrencyState,
|
||||
validateImageIngress,
|
||||
} from '../src/image-ingress.js';
|
||||
|
||||
const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, import.meta.url));
|
||||
const clean = await readFile(fixture('ingress-clean.jpg'));
|
||||
const cleanDataUrl = `data:image/jpeg;base64,${clean.toString('base64')}`;
|
||||
|
||||
// An instrumented stand-in interpreter: it records one marker file per spawn
|
||||
// while it is alive, so the test can observe the true peak child count instead
|
||||
// of trusting an internal counter.
|
||||
async function trackingInterpreter(name, holdSeconds = '0.6') {
|
||||
const dir = join(tmpdir(), `timmy-ingress-conc-${name}`);
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
await mkdir(join(dir, 'live'), { recursive: true });
|
||||
await mkdir(join(dir, 'seen'), { recursive: true });
|
||||
const path = join(dir, 'python3');
|
||||
await writeFile(path, [
|
||||
'#!/bin/sh',
|
||||
`marker="${dir}/live/$$"`,
|
||||
`touch "$marker" "${dir}/seen/$$"`,
|
||||
`sleep ${holdSeconds}`,
|
||||
'rm -f "$marker"',
|
||||
// Emit a valid success verdict and write the expected output file.
|
||||
'out=""',
|
||||
'while [ $# -gt 0 ]; do',
|
||||
' if [ "$1" = "--out" ]; then out="$2"; fi',
|
||||
' shift',
|
||||
'done',
|
||||
`cp "${fixture('ingress-clean.jpg')}" "$out"`,
|
||||
'echo \'{"ok": true, "format": "jpeg", "width": 64, "height": 64, "bytes": 639, "metadataStripped": true}\'',
|
||||
].join('\n'));
|
||||
await chmod(path, 0o755);
|
||||
return { path, dir };
|
||||
}
|
||||
|
||||
const withPython = async (path, run) => {
|
||||
const previous = process.env.TIMMY_PYTHON;
|
||||
process.env.TIMMY_PYTHON = path;
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.TIMMY_PYTHON;
|
||||
else process.env.TIMMY_PYTHON = previous;
|
||||
}
|
||||
};
|
||||
|
||||
test('the concurrency ceiling is small enough to stay inside the service memory budget', () => {
|
||||
assert.equal(typeof INGRESS_MAX_CONCURRENCY, 'number');
|
||||
assert.ok(INGRESS_MAX_CONCURRENCY >= 1, 'at least one decode must be permitted');
|
||||
assert.ok(INGRESS_MAX_CONCURRENCY <= 4,
|
||||
`ceiling ${INGRESS_MAX_CONCURRENCY} is too high for a 512 MiB service`);
|
||||
// Worst observed single-decoder peak is ~24 MiB; the whole burst must leave
|
||||
// ample headroom under MemoryMax=512M alongside the Node process itself.
|
||||
assert.ok(INGRESS_MAX_CONCURRENCY * 64 * 1024 * 1024 < 512 * 1024 * 1024,
|
||||
'the ceiling must bound peak decoder memory well under 512 MiB');
|
||||
});
|
||||
|
||||
test('a burst beyond the ceiling never runs more decoder children than the ceiling', async () => {
|
||||
const { path, dir } = await trackingInterpreter('peak');
|
||||
await withPython(path, async () => {
|
||||
let peakLive = 0;
|
||||
const sampler = setInterval(async () => {
|
||||
try {
|
||||
const live = await readdir(join(dir, 'live'));
|
||||
peakLive = Math.max(peakLive, live.length);
|
||||
} catch { /* directory races are fine */ }
|
||||
}, 15);
|
||||
const burst = await Promise.allSettled(
|
||||
Array.from({ length: 24 }, () => validateImageIngress({ consent: true, imageDataUrl: cleanDataUrl })),
|
||||
);
|
||||
clearInterval(sampler);
|
||||
const spawned = (await readdir(join(dir, 'seen'))).length;
|
||||
const rejected = burst.filter((r) => r.status === 'rejected');
|
||||
|
||||
assert.ok(peakLive <= INGRESS_MAX_CONCURRENCY,
|
||||
`observed ${peakLive} concurrent decoder children, ceiling is ${INGRESS_MAX_CONCURRENCY}`);
|
||||
assert.ok(spawned <= INGRESS_MAX_CONCURRENCY,
|
||||
`a 24-request burst spawned ${spawned} decoders; only ${INGRESS_MAX_CONCURRENCY} may run and the rest must be refused`);
|
||||
assert.ok(rejected.length >= 24 - INGRESS_MAX_CONCURRENCY,
|
||||
'excess requests must be refused rather than queued');
|
||||
});
|
||||
});
|
||||
|
||||
test('over-ceiling requests fail fast with a sanitized unavailable message', async () => {
|
||||
const { path } = await trackingInterpreter('failfast');
|
||||
await withPython(path, async () => {
|
||||
const started = Date.now();
|
||||
const burst = await Promise.allSettled(
|
||||
Array.from({ length: 12 }, () => validateImageIngress({ consent: true, imageDataUrl: cleanDataUrl })),
|
||||
);
|
||||
const rejected = burst.filter((r) => r.status === 'rejected').map((r) => r.reason);
|
||||
assert.ok(rejected.length > 0, 'a 12-request burst must refuse some requests');
|
||||
for (const error of rejected) {
|
||||
assert.ok(error instanceof IngressUnavailableError,
|
||||
'an over-capacity refusal is a server-capacity condition, not corrupt client input');
|
||||
assert.match(error.message, /temporarily unavailable|slow down|try again/i);
|
||||
assert.doesNotMatch(error.message, /corrupt|malformed/i);
|
||||
assert.doesNotMatch(error.message, /[A-Za-z0-9+/]{40,}/, 'no payload data in refusals');
|
||||
assert.ok(error.message.length < 200);
|
||||
}
|
||||
// Fail-fast: refusals must not wait for the in-flight decoders to finish.
|
||||
assert.ok(Date.now() - started < 5000, 'refusals must be immediate, not queued behind decodes');
|
||||
});
|
||||
});
|
||||
|
||||
test('capacity is released so later requests succeed after a burst', async () => {
|
||||
const { path } = await trackingInterpreter('release', '0.05');
|
||||
await withPython(path, async () => {
|
||||
await Promise.allSettled(
|
||||
Array.from({ length: 12 }, () => validateImageIngress({ consent: true, imageDataUrl: cleanDataUrl })),
|
||||
);
|
||||
const state = ingressConcurrencyState();
|
||||
assert.equal(state.active, 0, 'every slot must be released after the burst settles');
|
||||
const after = await validateImageIngress({ consent: true, imageDataUrl: cleanDataUrl });
|
||||
assert.equal(after.format, 'jpeg');
|
||||
});
|
||||
});
|
||||
|
||||
test('rejected input releases its slot too', async () => {
|
||||
const garbage = await readFile(fixture('ingress-garbage.jpg'));
|
||||
for (let i = 0; i < INGRESS_MAX_CONCURRENCY + 3; i += 1) {
|
||||
await assert.rejects(() => validateImageIngress({
|
||||
consent: true, imageDataUrl: `data:image/jpeg;base64,${garbage.toString('base64')}`,
|
||||
}));
|
||||
}
|
||||
assert.equal(ingressConcurrencyState().active, 0,
|
||||
'a rejected request must not leak a concurrency slot');
|
||||
});
|
||||
95
tests/image-dataurl.test.js
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
// Canonical base64 data-URL grammar contract.
|
||||
//
|
||||
// The client always produces a canonical `data:<mime>;base64,<canonical b64>`
|
||||
// URL (canvas.toDataURL). Anything else is a hand-crafted request, so ingress
|
||||
// must require the canonical grammar and a byte-exact round trip rather than
|
||||
// silently repairing missing padding, embedded newlines, or non-zero trailing
|
||||
// pad bits. The exact supported MIME policy is unchanged: JPEG/PNG/WebP only.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { decodeImageDataUrl, validateImageIngress } from '../src/image-ingress.js';
|
||||
|
||||
const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, import.meta.url));
|
||||
const clean = await readFile(fixture('ingress-clean.jpg'));
|
||||
const canonical = clean.toString('base64');
|
||||
|
||||
const ingest = (imageDataUrl) => validateImageIngress({ consent: true, imageDataUrl });
|
||||
|
||||
test('canonical base64 data URLs round-trip byte-exactly', () => {
|
||||
const decoded = decodeImageDataUrl(`data:image/jpeg;base64,${canonical}`);
|
||||
assert.ok(decoded, 'canonical data URL must decode');
|
||||
assert.equal(decoded.mime, 'image/jpeg');
|
||||
assert.ok(decoded.bytes.equals(clean), 'decoded bytes must equal the source bytes exactly');
|
||||
});
|
||||
|
||||
test('missing base64 padding is rejected', async () => {
|
||||
const unpadded = canonical.replace(/=+$/, '');
|
||||
assert.notEqual(unpadded, canonical, 'fixture must actually require padding');
|
||||
assert.equal(decodeImageDataUrl(`data:image/jpeg;base64,${unpadded}`), null);
|
||||
await assert.rejects(() => ingest(`data:image/jpeg;base64,${unpadded}`), /not a supported image/i);
|
||||
});
|
||||
|
||||
test('excess base64 padding is rejected', async () => {
|
||||
assert.equal(decodeImageDataUrl(`data:image/jpeg;base64,${canonical}===`), null);
|
||||
await assert.rejects(() => ingest(`data:image/jpeg;base64,${canonical}====`), /not a supported image/i);
|
||||
});
|
||||
|
||||
test('CR and LF inside the base64 payload are rejected', async () => {
|
||||
const withCrlf = `${canonical.slice(0, 40)}\r\n${canonical.slice(40)}`;
|
||||
const withLf = `${canonical.slice(0, 40)}\n${canonical.slice(40)}`;
|
||||
const withCr = `${canonical.slice(0, 40)}\r${canonical.slice(40)}`;
|
||||
for (const payload of [withCrlf, withLf, withCr]) {
|
||||
assert.equal(decodeImageDataUrl(`data:image/jpeg;base64,${payload}`), null);
|
||||
await assert.rejects(() => ingest(`data:image/jpeg;base64,${payload}`), /not a supported image/i);
|
||||
}
|
||||
});
|
||||
|
||||
test('non-canonical encodings are rejected: non-zero trailing pad bits, whitespace, URL-safe alphabet', async () => {
|
||||
const nonZeroPadBits = `${canonical.slice(0, -2)}/=`;
|
||||
const withSpace = `${canonical.slice(0, 20)} ${canonical.slice(20)}`;
|
||||
const urlSafe = canonical.replace(/\+/g, '-').replace(/\//g, '_');
|
||||
for (const payload of [nonZeroPadBits, withSpace]) {
|
||||
assert.equal(decodeImageDataUrl(`data:image/jpeg;base64,${payload}`), null);
|
||||
await assert.rejects(() => ingest(`data:image/jpeg;base64,${payload}`), /not a supported image/i);
|
||||
}
|
||||
if (urlSafe !== canonical) {
|
||||
assert.equal(decodeImageDataUrl(`data:image/jpeg;base64,${urlSafe}`), null);
|
||||
}
|
||||
});
|
||||
|
||||
test('the supported MIME policy is exact and unchanged', async () => {
|
||||
for (const mime of ['image/jpeg', 'image/png', 'image/webp']) {
|
||||
assert.ok(decodeImageDataUrl(`data:${mime};base64,${canonical}`), `${mime} must remain supported`);
|
||||
}
|
||||
for (const mime of ['image/gif', 'image/svg+xml', 'text/html', 'application/octet-stream', '']) {
|
||||
assert.equal(decodeImageDataUrl(`data:${mime};base64,${canonical}`), null, `${mime} must not be supported`);
|
||||
await assert.rejects(() => ingest(`data:${mime};base64,${canonical}`), /not a supported image/i);
|
||||
}
|
||||
});
|
||||
|
||||
test('malformed data-URL envelopes are rejected without repair', async () => {
|
||||
const malformed = [
|
||||
`data:image/jpeg,${canonical}`, // no base64 token
|
||||
`data:image/jpeg;base64;${canonical}`, // wrong separator
|
||||
`DATA:image/jpeg;base64,${canonical}`, // scheme casing is not canonical here
|
||||
`data:image/jpeg;base64,`, // empty payload
|
||||
` data:image/jpeg;base64,${canonical}`, // leading whitespace
|
||||
`data:image/jpeg;base64,${canonical} `, // trailing whitespace
|
||||
`data:image/jpeg;charset=utf-8;base64,${canonical}`, // extra parameters
|
||||
canonical, // bare base64, no envelope
|
||||
];
|
||||
for (const value of malformed) {
|
||||
assert.equal(decodeImageDataUrl(value), null, `must reject: ${value.slice(0, 42)}`);
|
||||
await assert.rejects(() => ingest(value), /not a supported image/i);
|
||||
}
|
||||
});
|
||||
|
||||
test('a canonical supported data URL still completes ingress end to end', async () => {
|
||||
const result = await ingest(`data:image/jpeg;base64,${canonical}`);
|
||||
assert.equal(result.format, 'jpeg');
|
||||
assert.equal(result.mime, 'image/jpeg');
|
||||
assert.match(result.imageDataUrl, /^data:image\/jpeg;base64,[A-Za-z0-9+/]+={0,2}$/);
|
||||
});
|
||||
|
|
@ -65,7 +65,13 @@ test('malformed and truncated images fail closed with sanitized errors', async (
|
|||
return true;
|
||||
});
|
||||
}
|
||||
await assert.rejects(() => ingestFixture('ingress-truncated.jpg'), /corrupt or malformed/i);
|
||||
// A truncated JPEG has no EOI, so canonical container parsing rejects it
|
||||
// before any decode or subprocess spawn. Either sanitized ingress error is
|
||||
// acceptable; leaking anything else is not.
|
||||
await assert.rejects(() => ingestFixture('ingress-truncated.jpg'), (error) => {
|
||||
assert.match(error.message, /not a supported image|corrupt or malformed/i);
|
||||
return true;
|
||||
});
|
||||
});
|
||||
|
||||
test('decompression bombs and oversized dimensions fail before provider work', async () => {
|
||||
|
|
|
|||
86
tests/image-polyglot.test.js
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
// Container/polyglot rejection contract.
|
||||
//
|
||||
// The requirement is malformed/polyglot REJECTION, not "the re-encoder happens
|
||||
// to drop the tail". Rejection must come from canonical parsing of the image
|
||||
// container — the declared structure must consume exactly the supplied bytes —
|
||||
// so it cannot be evaded by changing keyword casing or archive flavour, and it
|
||||
// must not fire on arbitrary compressed bytes that merely contain those
|
||||
// sequences inside legal image structure.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { validateImageIngress } from '../src/image-ingress.js';
|
||||
import { analyzePhoto } from '../src/vision-service.js';
|
||||
|
||||
const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, import.meta.url));
|
||||
const readFixture = (name) => readFile(fixture(name));
|
||||
const dataUrl = (bytes, mime = 'image/jpeg') => `data:${mime};base64,${bytes.toString('base64')}`;
|
||||
|
||||
const TRAILING_DATA_FIXTURES = [
|
||||
['ingress-tail-upper-script.jpg', 'uppercase <SCRIPT> tail'],
|
||||
['ingress-tail-mixed-script.jpg', 'mixed-case <ScRiPt> tail'],
|
||||
['ingress-tail-html.jpg', 'generic appended HTML with no script tag'],
|
||||
['ingress-tail-zip-eocd.jpg', 'ZIP end-of-central-directory tail'],
|
||||
['ingress-tail-zip-local.jpg', 'ZIP local-header tail'],
|
||||
['ingress-tail-rar.jpg', 'RAR archive tail'],
|
||||
['ingress-tail-7z.jpg', '7z archive tail'],
|
||||
['ingress-tail-gzip.jpg', 'gzip member tail'],
|
||||
['ingress-tail-single-nul.jpg', 'single appended NUL byte'],
|
||||
];
|
||||
|
||||
test('appended trailing data is rejected regardless of casing or container flavour', async () => {
|
||||
for (const [name, description] of TRAILING_DATA_FIXTURES) {
|
||||
const bytes = await readFixture(name);
|
||||
await assert.rejects(
|
||||
() => validateImageIngress({ consent: true, imageDataUrl: dataUrl(bytes) }),
|
||||
(error) => {
|
||||
assert.match(error.message, /not a supported image|corrupt or malformed/i,
|
||||
`${description} must be rejected with a sanitized ingress error`);
|
||||
return true;
|
||||
},
|
||||
`${description} must not pass ingress`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('trailing data after a PNG IEND chunk is rejected', async () => {
|
||||
const bytes = await readFixture('ingress-tail-after-iend.png');
|
||||
await assert.rejects(
|
||||
() => validateImageIngress({ consent: true, imageDataUrl: dataUrl(bytes, 'image/png') }),
|
||||
/not a supported image|corrupt or malformed/i,
|
||||
);
|
||||
});
|
||||
|
||||
test('rejection is structural, not substring scanning: legal images carrying archive and script byte sequences are accepted', async () => {
|
||||
// A JPEG COM segment legally contains arbitrary bytes. These exact sequences
|
||||
// are what the old scanner searched for; canonical parsing must still accept.
|
||||
const buried = await readFixture('ingress-buried-signatures.jpg');
|
||||
assert.ok(buried.includes(Buffer.from('PK\x03\x04')), 'fixture must contain a ZIP local header sequence');
|
||||
assert.ok(buried.includes(Buffer.from('PK\x05\x06')), 'fixture must contain a ZIP EOCD sequence');
|
||||
assert.ok(buried.includes(Buffer.from('<script')), 'fixture must contain a lowercase script sequence');
|
||||
const result = await validateImageIngress({ consent: true, imageDataUrl: dataUrl(buried) });
|
||||
assert.equal(result.format, 'jpeg');
|
||||
assert.ok(result.bytes > 0);
|
||||
|
||||
// High-entropy compressed data must not be misclassified either.
|
||||
const entropy = await readFixture('ingress-entropy-control.jpg');
|
||||
const entropyResult = await validateImageIngress({ consent: true, imageDataUrl: dataUrl(entropy) });
|
||||
assert.equal(entropyResult.format, 'jpeg');
|
||||
});
|
||||
|
||||
test('the provider is never reached for any trailing-data polyglot', async () => {
|
||||
for (const [name] of TRAILING_DATA_FIXTURES) {
|
||||
const bytes = await readFixture(name);
|
||||
let providerCalled = false;
|
||||
await assert.rejects(
|
||||
() => analyzePhoto({
|
||||
payload: { consent: true, imageDataUrl: dataUrl(bytes) },
|
||||
config: { model: 'm', baseUrl: 'http://127.0.0.1:9/v1' },
|
||||
fetchImpl: async () => { providerCalled = true; throw new Error('provider reached'); },
|
||||
}),
|
||||
);
|
||||
assert.equal(providerCalled, false, `${name} must fail before any provider fetch`);
|
||||
}
|
||||
});
|
||||
154
tests/image-unavailable.test.js
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
// Processing-unavailable contract.
|
||||
//
|
||||
// A production interpreter that cannot import Pillow is a server-side runtime
|
||||
// fault, not hostile client input. It must surface as a sanitized
|
||||
// processing-unavailable outcome with the manual-continue fallback (HTTP 503),
|
||||
// never as a 400 that blames the user's photo for being corrupt.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { chmod, mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import {
|
||||
IngressUnavailableError,
|
||||
validateImageIngress,
|
||||
classifyIngressFailure,
|
||||
verifyReencodeRuntime,
|
||||
resolveInterpreter,
|
||||
PINNED_PYTHON_VERSION,
|
||||
PINNED_PILLOW_VERSION,
|
||||
} from '../src/image-ingress.js';
|
||||
import { analyzePhoto } from '../src/vision-service.js';
|
||||
|
||||
const fixture = (name) => fileURLToPath(new URL(`../tests/fixtures/${name}`, import.meta.url));
|
||||
const clean = await readFile(fixture('ingress-clean.jpg'));
|
||||
const cleanDataUrl = `data:image/jpeg;base64,${clean.toString('base64')}`;
|
||||
|
||||
// A stand-in interpreter that behaves exactly like the real re-encoder running
|
||||
// on a runtime without Pillow: the documented unavailable verdict and exit 3.
|
||||
async function stubInterpreter(name, body) {
|
||||
const dir = join(tmpdir(), `timmy-ingress-stub-${name}`);
|
||||
await mkdir(dir, { recursive: true });
|
||||
const path = join(dir, 'python3');
|
||||
await writeFile(path, body);
|
||||
await chmod(path, 0o755);
|
||||
return path;
|
||||
}
|
||||
|
||||
const withPython = async (path, run) => {
|
||||
const previous = process.env.TIMMY_PYTHON;
|
||||
process.env.TIMMY_PYTHON = path;
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.TIMMY_PYTHON;
|
||||
else process.env.TIMMY_PYTHON = previous;
|
||||
}
|
||||
};
|
||||
|
||||
test('a runtime without Pillow raises an unavailable ingress error, not a corrupt-input error', async () => {
|
||||
const stub = await stubInterpreter('nopil',
|
||||
'#!/bin/sh\necho \'{"ok": false, "error": "unavailable"}\'\nexit 3\n');
|
||||
await withPython(stub, async () => {
|
||||
const error = await validateImageIngress({ consent: true, imageDataUrl: cleanDataUrl })
|
||||
.then(() => null, (caught) => caught);
|
||||
assert.ok(error, 'ingress must fail when the runtime cannot process images');
|
||||
assert.ok(error instanceof IngressUnavailableError,
|
||||
'a missing runtime must be classified as unavailable, not as corrupt client input');
|
||||
assert.match(error.message, /temporarily unavailable/i);
|
||||
assert.match(error.message, /continue manually/i);
|
||||
assert.doesNotMatch(error.message, /corrupt|malformed/i);
|
||||
});
|
||||
});
|
||||
|
||||
test('classifyIngressFailure maps unavailable ingress failures to 503 and corrupt input to 400', () => {
|
||||
assert.equal(classifyIngressFailure(new IngressUnavailableError('Photo processing is temporarily unavailable. Continue manually.')), 503);
|
||||
assert.equal(classifyIngressFailure(new Error('The photo is corrupt or malformed. Try a different photo or continue manually.')), 400);
|
||||
assert.equal(classifyIngressFailure(new Error('Upload a JPEG, PNG, or WebP photo. That file is not a supported image.')), 400);
|
||||
assert.equal(classifyIngressFailure(new Error('Explicit consent is required before AI analysis.')), 400);
|
||||
assert.equal(classifyIngressFailure(new Error('The photo is too large. Use an image under 4 MB.')), 400);
|
||||
});
|
||||
|
||||
test('a re-encoder timeout is also unavailable rather than corrupt', async () => {
|
||||
const stub = await stubInterpreter('hang', '#!/bin/sh\nsleep 60\n');
|
||||
await withPython(stub, async () => {
|
||||
const error = await validateImageIngress({
|
||||
consent: true, imageDataUrl: cleanDataUrl, reencodeTimeoutMs: 250,
|
||||
}).then(() => null, (caught) => caught);
|
||||
assert.ok(error instanceof IngressUnavailableError);
|
||||
assert.match(error.message, /temporarily unavailable/i);
|
||||
});
|
||||
});
|
||||
|
||||
test('an interpreter that cannot be executed at all is unavailable, not corrupt', async () => {
|
||||
await withPython('/nonexistent/timmy-python-does-not-exist', async () => {
|
||||
const error = await validateImageIngress({ consent: true, imageDataUrl: cleanDataUrl })
|
||||
.then(() => null, (caught) => caught);
|
||||
assert.ok(error instanceof IngressUnavailableError,
|
||||
'a missing interpreter is a server fault and must not be reported as corrupt input');
|
||||
});
|
||||
});
|
||||
|
||||
test('genuinely corrupt input is still classified as client error, not unavailable', async () => {
|
||||
const garbage = await readFile(fixture('ingress-garbage.jpg'));
|
||||
const error = await validateImageIngress({
|
||||
consent: true, imageDataUrl: `data:image/jpeg;base64,${garbage.toString('base64')}`,
|
||||
}).then(() => null, (caught) => caught);
|
||||
assert.ok(error, 'garbage must be rejected');
|
||||
assert.equal(error instanceof IngressUnavailableError, false);
|
||||
assert.equal(classifyIngressFailure(error), 400);
|
||||
});
|
||||
|
||||
test('the provider is never called when the runtime is unavailable', async () => {
|
||||
const stub = await stubInterpreter('nopil2',
|
||||
'#!/bin/sh\necho \'{"ok": false, "error": "unavailable"}\'\nexit 3\n');
|
||||
await withPython(stub, async () => {
|
||||
let providerCalled = false;
|
||||
await assert.rejects(
|
||||
() => analyzePhoto({
|
||||
payload: { consent: true, imageDataUrl: cleanDataUrl },
|
||||
config: { model: 'm', baseUrl: 'http://127.0.0.1:9/v1' },
|
||||
fetchImpl: async () => { providerCalled = true; throw new Error('provider reached'); },
|
||||
}),
|
||||
/temporarily unavailable/i,
|
||||
);
|
||||
assert.equal(providerCalled, false);
|
||||
});
|
||||
});
|
||||
|
||||
test('the production interpreter path must be absolute, never a PATH lookup', () => {
|
||||
// Development falls back to PATH `python3`.
|
||||
assert.equal(resolveInterpreter({}), 'python3');
|
||||
assert.equal(resolveInterpreter({ TIMMY_PYTHON: '' }), 'python3');
|
||||
// Production requires an absolute, immutable interpreter path.
|
||||
assert.equal(resolveInterpreter({ TIMMY_PYTHON: '/usr/local/lib/timmy-staging/python' }),
|
||||
'/usr/local/lib/timmy-staging/python');
|
||||
assert.throws(() => resolveInterpreter({ TIMMY_PYTHON: 'python3' }),
|
||||
/absolute interpreter path/i);
|
||||
assert.throws(() => resolveInterpreter({ TIMMY_PYTHON: 'relative/python' }),
|
||||
/absolute interpreter path/i);
|
||||
});
|
||||
|
||||
test('verifyReencodeRuntime confirms the pinned, immutable production toolchain', async () => {
|
||||
// The real interpreter is the pinned 3.11 / Pillow 12.3.0 toolchain, so the
|
||||
// runtime smoke must succeed and report exactly that pin.
|
||||
const verdict = await verifyReencodeRuntime();
|
||||
assert.equal(verdict.python, PINNED_PYTHON_VERSION);
|
||||
assert.equal(verdict.pillow, PINNED_PILLOW_VERSION);
|
||||
assert.deepEqual(verdict.pinned, { python: PINNED_PYTHON_VERSION, pillow: PINNED_PILLOW_VERSION });
|
||||
});
|
||||
|
||||
test('verifyReencodeRuntime fails closed when pointed at a wrong interpreter', async () => {
|
||||
// A stub interpreter that exits 3 (unavailable) must be reported as an
|
||||
// ingress-capacity fault, never as corrupt client input.
|
||||
const stub = await stubInterpreter('pinfail',
|
||||
'#!/bin/sh\necho \'{"ok": false, "error": "unavailable"}\'\nexit 3\n');
|
||||
await withPython(stub, async () => {
|
||||
const error = await verifyReencodeRuntime().then(() => null, (caught) => caught);
|
||||
assert.ok(error instanceof IngressUnavailableError,
|
||||
'a non-pinned/unavailable runtime must fail closed as processing-unavailable');
|
||||
});
|
||||
});
|
||||
|
||||
135
tests/rate-identity.test.js
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
// Bounded rate state and explicit client identity.
|
||||
//
|
||||
// Two separate defects are covered here:
|
||||
// 1. The limiter's map must be hard-bounded. Only evicting *expired* entries
|
||||
// means 20k+ unexpired keys grow state without limit.
|
||||
// 2. Behind a reverse proxy, req.socket.remoteAddress is the proxy's loopback
|
||||
// address, so every user shares one quota. The policy must be explicit:
|
||||
// either trust a configured loopback proxy's forwarded client address, or
|
||||
// state truthfully that the limit is global — never silently trust a
|
||||
// spoofable header from an untrusted peer.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import { createRateLimiter } from '../src/rate-limiter.js';
|
||||
import { resolveClientIdentity } from '../src/client-identity.js';
|
||||
|
||||
test('rate state is hard-bounded under a flood of unexpired distinct keys', () => {
|
||||
const limiter = createRateLimiter({ windowMs: 60_000, maxRequests: 10, maxKeys: 1024 });
|
||||
const now = Date.now();
|
||||
for (let i = 0; i < 20_000; i += 1) {
|
||||
limiter.take(`key-${i}`, now); // all within the same, unexpired window
|
||||
}
|
||||
const size = limiter.size();
|
||||
assert.ok(size <= 1024,
|
||||
`limiter retained ${size} unexpired keys; state must be hard-bounded, not merely expiry-swept`);
|
||||
});
|
||||
|
||||
test('a flood of distinct keys never lets a repeat offender escape its own limit', () => {
|
||||
const limiter = createRateLimiter({ windowMs: 60_000, maxRequests: 3, maxKeys: 64 });
|
||||
const now = Date.now();
|
||||
// The offender opens its window first.
|
||||
for (let i = 0; i < 3; i += 1) assert.equal(limiter.take('offender', now).allowed, true);
|
||||
assert.equal(limiter.take('offender', now).allowed, false, 'offender must be blocked');
|
||||
// Now flood far past the cap to try to evict the offender's counter.
|
||||
for (let i = 0; i < 5_000; i += 1) limiter.take(`flood-${i}`, now);
|
||||
const afterFlood = limiter.take('offender', now);
|
||||
assert.equal(afterFlood.allowed, false,
|
||||
'eviction must not reset an active offender: that would make the limit bypassable by flooding');
|
||||
});
|
||||
|
||||
test('eviction prefers expired entries before evicting live ones', () => {
|
||||
const limiter = createRateLimiter({ windowMs: 1_000, maxRequests: 5, maxKeys: 8 });
|
||||
const t0 = 1_000_000;
|
||||
for (let i = 0; i < 8; i += 1) limiter.take(`old-${i}`, t0);
|
||||
// Well after those windows expired, new keys must reuse the reclaimed space.
|
||||
for (let i = 0; i < 8; i += 1) limiter.take(`new-${i}`, t0 + 5_000);
|
||||
assert.ok(limiter.size() <= 8);
|
||||
assert.equal(limiter.take('new-0', t0 + 5_000).allowed, true);
|
||||
});
|
||||
|
||||
test('the fixed-window boundary cannot be used to double the burst', () => {
|
||||
const limiter = createRateLimiter({ windowMs: 60_000, maxRequests: 10 });
|
||||
const t0 = 1_000_000;
|
||||
let allowed = 0;
|
||||
// Open the window, then spend the whole budget at the very end of it.
|
||||
if (limiter.take('same', t0).allowed) allowed += 1;
|
||||
for (let i = 0; i < 9; i += 1) if (limiter.take('same', t0 + 59_999).allowed) allowed += 1;
|
||||
assert.equal(allowed, 10, 'the nominal budget must be usable');
|
||||
// Immediately across the boundary a naive fixed window grants a fresh budget,
|
||||
// allowing 2x the nominal rate within milliseconds of real time.
|
||||
let acrossBoundary = 0;
|
||||
for (let i = 0; i < 10; i += 1) if (limiter.take('same', t0 + 60_000).allowed) acrossBoundary += 1;
|
||||
assert.ok(acrossBoundary < 10,
|
||||
`boundary straddle allowed a full extra budget (${acrossBoundary}), doubling the effective rate`);
|
||||
const total = allowed + acrossBoundary;
|
||||
assert.ok(total <= 14,
|
||||
`${total} requests accepted across a ~1ms boundary straddle against a 10-per-60s budget`);
|
||||
});
|
||||
|
||||
test('sustained load stays near the nominal rate rather than bursting to 2x each boundary', () => {
|
||||
const limiter = createRateLimiter({ windowMs: 1_000, maxRequests: 5 });
|
||||
let accepted = 0;
|
||||
// Ten seconds of continuous pressure at 50 requests/second.
|
||||
for (let ms = 0; ms < 10_000; ms += 20) {
|
||||
if (limiter.take('steady', 500_000 + ms).allowed) accepted += 1;
|
||||
}
|
||||
// 10s at 5/s is 50; allow a small margin but not a 2x boundary doubling.
|
||||
assert.ok(accepted <= 60, `sustained acceptance ${accepted} exceeded the nominal 5/s budget envelope`);
|
||||
assert.ok(accepted >= 40, `sustained acceptance ${accepted} is unusably far below the nominal budget`);
|
||||
});
|
||||
|
||||
test('client identity behind an untrusted peer never trusts forwarded headers', () => {
|
||||
const identity = resolveClientIdentity({
|
||||
remoteAddress: '203.0.113.7',
|
||||
headers: { 'x-forwarded-for': '198.51.100.9', 'x-real-ip': '198.51.100.10' },
|
||||
trustedProxies: [],
|
||||
});
|
||||
assert.equal(identity.key, '203.0.113.7', 'the peer address is the only trustworthy identity here');
|
||||
assert.equal(identity.scope, 'peer');
|
||||
assert.equal(identity.trustedProxy, false);
|
||||
});
|
||||
|
||||
test('a configured trusted loopback proxy contributes the forwarded client address', () => {
|
||||
const identity = resolveClientIdentity({
|
||||
remoteAddress: '127.0.0.1',
|
||||
headers: { 'x-forwarded-for': '198.51.100.9, 10.0.0.5' },
|
||||
trustedProxies: ['127.0.0.1'],
|
||||
});
|
||||
assert.equal(identity.key, '198.51.100.9', 'the left-most forwarded address is the client');
|
||||
assert.equal(identity.scope, 'forwarded');
|
||||
assert.equal(identity.trustedProxy, true);
|
||||
});
|
||||
|
||||
test('a trusted proxy that forwards nothing usable degrades to a truthful global policy', () => {
|
||||
const identity = resolveClientIdentity({
|
||||
remoteAddress: '127.0.0.1',
|
||||
headers: {},
|
||||
trustedProxies: ['127.0.0.1'],
|
||||
});
|
||||
assert.equal(identity.scope, 'global',
|
||||
'without a usable forwarded address the quota is shared; the policy must say so');
|
||||
assert.equal(identity.key, 'global');
|
||||
});
|
||||
|
||||
test('forwarded addresses are validated, not echoed', () => {
|
||||
for (const spoof of ['not-an-ip', '', ' ', '999.999.999.999', '<script>', '127.0.0.1; rm -rf /']) {
|
||||
const identity = resolveClientIdentity({
|
||||
remoteAddress: '127.0.0.1',
|
||||
headers: { 'x-forwarded-for': spoof },
|
||||
trustedProxies: ['127.0.0.1'],
|
||||
});
|
||||
assert.equal(identity.scope, 'global', `must not accept ${JSON.stringify(spoof)} as an identity`);
|
||||
assert.equal(identity.key, 'global');
|
||||
}
|
||||
});
|
||||
|
||||
test('identity keys never contain payload data and stay short', () => {
|
||||
const identity = resolveClientIdentity({
|
||||
remoteAddress: '2001:db8::1',
|
||||
headers: {},
|
||||
trustedProxies: [],
|
||||
});
|
||||
assert.ok(identity.key.length <= 64);
|
||||
assert.doesNotMatch(identity.key, /[A-Za-z0-9+/]{40,}/);
|
||||
});
|
||||
|
|
@ -3,7 +3,7 @@ import assert from 'node:assert/strict';
|
|||
|
||||
import { createRateLimiter } from '../src/rate-limiter.js';
|
||||
|
||||
test('rate limiter blocks after the configured burst and recovers next window', () => {
|
||||
test('rate limiter blocks after the configured burst and recovers once the window drains', () => {
|
||||
const limiter = createRateLimiter({ windowMs: 1000, maxRequests: 3 });
|
||||
const t0 = 1_000_000;
|
||||
assert.equal(limiter.take('k', t0).allowed, true);
|
||||
|
|
@ -12,8 +12,13 @@ test('rate limiter blocks after the configured burst and recovers next window',
|
|||
const blocked = limiter.take('k', t0 + 3);
|
||||
assert.equal(blocked.allowed, false);
|
||||
assert.ok(blocked.retryAfterMs > 0 && blocked.retryAfterMs <= 1000);
|
||||
// Next window resets cleanly.
|
||||
assert.equal(limiter.take('k', t0 + 1001).allowed, true);
|
||||
// Crossing the window boundary must NOT immediately grant a fresh full
|
||||
// budget: that is the boundary-doubling burst. The trailing count decays, so
|
||||
// capacity returns only once the earlier requests have actually aged out.
|
||||
assert.equal(limiter.take('k', t0 + 1001).allowed, false,
|
||||
'a 1ms boundary straddle must not reset the budget');
|
||||
assert.equal(limiter.take('k', t0 + 2001).allowed, true,
|
||||
'capacity must return once the window has genuinely drained');
|
||||
});
|
||||
|
||||
test('rate limiter keys are isolated and never expose payload data', () => {
|
||||
|
|
|
|||
185
tests/reencode-image.test.py
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Resource-bound contract for the image re-encoder subprocess.
|
||||
|
||||
Rejection of oversized or bomb-shaped images must happen from the container
|
||||
header, before any full pixel decode, so a hostile upload cannot allocate
|
||||
hundreds of megabytes inside a 512 MiB service. Decompression-bomb warnings are
|
||||
treated as errors: a warning that still returns pixels is not a rejection.
|
||||
|
||||
Peak RSS is measured per child with os.wait4, so each measurement belongs to
|
||||
exactly one subprocess rather than a running maximum.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SCRIPT = ROOT / "scripts" / "reencode_image.py"
|
||||
FIXTURES = ROOT / "tests" / "fixtures"
|
||||
TMP = Path(os.environ.get("TMPDIR", "/tmp"))
|
||||
|
||||
# Interpreter + Pillow import costs ~21 MiB and a header-only open adds ~1 MiB.
|
||||
# 64 MiB leaves generous headroom for that while remaining far below any full
|
||||
# decode of the committed bomb fixtures (144 MP would need >140 MiB).
|
||||
REJECTION_RSS_CEILING_KIB = 64 * 1024
|
||||
# A hard address-space cap well below what a full decode of the committed bomb
|
||||
# fixtures needs (~163 MiB observed). Header-only rejection must still complete
|
||||
# cleanly under it, which proves the rejection never allocates the pixel buffer.
|
||||
ADDRESS_SPACE_CEILING_BYTES = 112 * 1024 * 1024
|
||||
|
||||
EXIT_OK = 0
|
||||
EXIT_REJECTED = 2
|
||||
EXIT_UNAVAILABLE = 3
|
||||
|
||||
|
||||
def run_measured(source: Path, *extra: str, out_name: str = "out.jpg",
|
||||
env: dict | None = None, address_space: int | None = None):
|
||||
"""Spawn the re-encoder in a fork we reap ourselves, for exact per-child rusage."""
|
||||
target = TMP / f"reencode-test-{out_name}"
|
||||
argv = [sys.executable, str(SCRIPT), "--in", str(source), "--out", str(target), *extra]
|
||||
stdout_r, stdout_w = os.pipe()
|
||||
stderr_r, stderr_w = os.pipe()
|
||||
pid = os.fork()
|
||||
if pid == 0: # pragma: no cover - child process
|
||||
try:
|
||||
os.close(stdout_r)
|
||||
os.close(stderr_r)
|
||||
os.dup2(stdout_w, 1)
|
||||
os.dup2(stderr_w, 2)
|
||||
os.close(stdout_w)
|
||||
os.close(stderr_w)
|
||||
if address_space is not None:
|
||||
resource.setrlimit(resource.RLIMIT_AS, (address_space, address_space))
|
||||
os.execve(argv[0], argv, env or os.environ)
|
||||
except BaseException:
|
||||
os._exit(127)
|
||||
os.close(stdout_w)
|
||||
os.close(stderr_w)
|
||||
with os.fdopen(stdout_r, "r") as out_handle, os.fdopen(stderr_r, "r") as err_handle:
|
||||
stdout = out_handle.read()
|
||||
stderr = err_handle.read()
|
||||
_, status, usage = os.wait4(pid, 0)
|
||||
code = os.waitstatus_to_exitcode(status)
|
||||
return subprocess.CompletedProcess(argv, code, stdout, stderr), usage.ru_maxrss
|
||||
|
||||
|
||||
def verdict(process: subprocess.CompletedProcess) -> dict:
|
||||
try:
|
||||
return json.loads(process.stdout.strip().splitlines()[-1])
|
||||
except (ValueError, IndexError):
|
||||
return {}
|
||||
|
||||
|
||||
class ReencoderResourceBounds(unittest.TestCase):
|
||||
def test_bomb_dimensions_rejected_without_full_decode(self) -> None:
|
||||
process, peak_kib = run_measured(FIXTURES / "ingress-bomb.png", out_name="bomb.jpg")
|
||||
self.assertEqual(process.returncode, EXIT_REJECTED, process.stderr[:300])
|
||||
self.assertEqual(verdict(process).get("error"), "dimensions")
|
||||
self.assertLess(
|
||||
peak_kib, REJECTION_RSS_CEILING_KIB,
|
||||
f"12000x12000 rejection allocated {peak_kib} KiB; it must reject from the header",
|
||||
)
|
||||
|
||||
def test_oversized_jpeg_rejected_without_full_decode(self) -> None:
|
||||
process, peak_kib = run_measured(FIXTURES / "ingress-oversized.jpg", out_name="oversized.jpg")
|
||||
self.assertEqual(process.returncode, EXIT_REJECTED, process.stderr[:300])
|
||||
self.assertEqual(verdict(process).get("error"), "dimensions")
|
||||
self.assertLess(
|
||||
peak_kib, REJECTION_RSS_CEILING_KIB,
|
||||
f"6000x6000 rejection allocated {peak_kib} KiB; it must reject from the header",
|
||||
)
|
||||
|
||||
def test_bomb_fixtures_reject_under_a_hard_address_space_cap(self) -> None:
|
||||
# Proof of boundedness that does not depend on RSS sampling: under a
|
||||
# hard 192 MiB address-space limit the rejection must still complete
|
||||
# cleanly rather than dying from allocation failure.
|
||||
for name in ("ingress-bomb.png", "ingress-oversized.jpg"):
|
||||
with self.subTest(fixture=name):
|
||||
process, _ = run_measured(
|
||||
FIXTURES / name, out_name=f"capped-{name}.jpg",
|
||||
address_space=ADDRESS_SPACE_CEILING_BYTES,
|
||||
)
|
||||
self.assertEqual(process.returncode, EXIT_REJECTED, process.stderr[:300])
|
||||
self.assertEqual(verdict(process).get("error"), "dimensions")
|
||||
self.assertNotIn("MemoryError", process.stderr)
|
||||
|
||||
def test_decompression_bomb_warnings_are_errors_not_warnings(self) -> None:
|
||||
process, _ = run_measured(FIXTURES / "ingress-bomb.png", out_name="warn.jpg")
|
||||
self.assertEqual(process.returncode, EXIT_REJECTED)
|
||||
self.assertNotIn(
|
||||
"DecompressionBombWarning", process.stderr,
|
||||
"a decompression bomb must raise an error, not emit a warning and continue",
|
||||
)
|
||||
|
||||
def test_total_pixel_ceiling_is_enforced_independently(self) -> None:
|
||||
# Both dimensions are well inside the per-dimension cap; only the total
|
||||
# pixel budget rejects this, proving the ceiling exists on its own.
|
||||
process, _ = run_measured(
|
||||
FIXTURES / "ingress-clean.jpg", "--max-pixels", "1024", out_name="pixels.jpg",
|
||||
)
|
||||
self.assertEqual(process.returncode, EXIT_REJECTED)
|
||||
self.assertEqual(verdict(process).get("error"), "dimensions")
|
||||
|
||||
def test_rejection_output_never_leaks_image_data_or_traces(self) -> None:
|
||||
for name in ("ingress-bomb.png", "ingress-oversized.jpg", "ingress-garbage.jpg"):
|
||||
with self.subTest(fixture=name):
|
||||
process, _ = run_measured(FIXTURES / name, out_name=f"leak-{name}.jpg")
|
||||
combined = process.stdout + process.stderr
|
||||
self.assertNotIn("Traceback", combined)
|
||||
self.assertLess(len(combined), 400, "subprocess output must stay short and fixed")
|
||||
|
||||
def test_clean_image_still_reencodes_successfully(self) -> None:
|
||||
process, _ = run_measured(FIXTURES / "ingress-clean.jpg", out_name="clean.jpg")
|
||||
self.assertEqual(process.returncode, EXIT_OK, process.stderr[:300])
|
||||
result = verdict(process)
|
||||
self.assertTrue(result.get("ok"))
|
||||
self.assertEqual(result.get("format"), "jpeg")
|
||||
self.assertTrue(result.get("metadataStripped"))
|
||||
|
||||
def test_missing_pillow_reports_unavailable_exit_code(self) -> None:
|
||||
# A production interpreter without Pillow must be distinguishable from
|
||||
# hostile client input, so it can map to processing-unavailable.
|
||||
stub = TMP / "reencode-no-pil"
|
||||
stub.mkdir(exist_ok=True)
|
||||
(stub / "PIL.py").write_text("raise ImportError('no pillow here')\n", encoding="utf-8")
|
||||
env = dict(os.environ, PYTHONPATH=str(stub))
|
||||
process, _ = run_measured(
|
||||
FIXTURES / "ingress-clean.jpg", out_name="nopil.jpg", env=env,
|
||||
)
|
||||
self.assertEqual(process.returncode, EXIT_UNAVAILABLE)
|
||||
self.assertEqual(verdict(process).get("error"), "unavailable")
|
||||
|
||||
|
||||
class PinContract(unittest.TestCase):
|
||||
def test_verify_pin_reports_the_pinned_immutable_runtime(self) -> None:
|
||||
process, _ = run_measured(FIXTURES / "ingress-clean.jpg", "--verify-pin", out_name="pin.json")
|
||||
self.assertEqual(process.returncode, EXIT_OK, process.stderr[:300])
|
||||
result = verdict(process)
|
||||
self.assertTrue(result.get("ok"))
|
||||
self.assertEqual(result.get("python"), "3.11")
|
||||
self.assertEqual(result.get("pillow"), "12.3.0")
|
||||
self.assertEqual(result.get("pinned"), {"python": "3.11", "pillow": "12.3.0"})
|
||||
|
||||
def test_verify_pin_fails_when_pillow_is_not_the_pinned_version(self) -> None:
|
||||
# Force a mismatched Pillow version to prove the gate refuses a drifted
|
||||
# runtime (server-side provisioning fault, not client input).
|
||||
stub = TMP / "reencode-fake-pil"
|
||||
stub.mkdir(exist_ok=True)
|
||||
(stub / "PIL.py").write_text(
|
||||
"class Image:\n __version__ = '99.0.0'\n", encoding="utf-8")
|
||||
env = dict(os.environ, PYTHONPATH=str(stub))
|
||||
process, _ = run_measured(
|
||||
FIXTURES / "ingress-clean.jpg", "--verify-pin", out_name="pinbad.json", env=env,
|
||||
)
|
||||
self.assertEqual(process.returncode, EXIT_UNAVAILABLE)
|
||||
self.assertEqual(verdict(process).get("ok"), False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
|
|
@ -7,6 +7,7 @@ import importlib.util
|
|||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
|
|
@ -86,11 +87,15 @@ class DeployTests(unittest.TestCase):
|
|||
|
||||
def archive(self, name="release.tar.gz", members=None):
|
||||
path = Path(self.tmp.name) / name
|
||||
digest = make_archive(path, members or [
|
||||
default_members = [
|
||||
("timmy-release/", b"", "dir"),
|
||||
("timmy-release/server.mjs", b"console.log('ok')\n", "file"),
|
||||
("timmy-release/package.json", b"{}\n", "file"),
|
||||
])
|
||||
("timmy-release/scripts/", b"", "dir"),
|
||||
("timmy-release/scripts/reencode_image.py",
|
||||
(SCRIPT.parent / "reencode_image.py").read_bytes(), "file"),
|
||||
]
|
||||
digest = make_archive(path, members or default_members)
|
||||
return path, digest
|
||||
|
||||
def seed_release(self, commit):
|
||||
|
|
@ -331,6 +336,21 @@ class DeployTests(unittest.TestCase):
|
|||
self.assertEqual(run.returncode, 0, run.stderr)
|
||||
self.assertEqual(json.loads(run.stdout)["commit"], COMMIT_A)
|
||||
|
||||
def test_deployment_smoke_verifies_the_pinned_immutable_image_runtime(self):
|
||||
# The release on disk must carry the pinned re-encoder; the smoke gate
|
||||
# proves the provisioned runtime matches and can re-encode a synthetic
|
||||
# pixel under the hard 512 MiB service budget.
|
||||
release = self.root / "releases" / COMMIT_A
|
||||
release.mkdir(parents=True)
|
||||
(release / "server.mjs").write_text("ok", encoding="utf-8")
|
||||
(release / ".timmy-release.json").write_text(json.dumps({"commit": COMMIT_A, "tag": "old"}), encoding="utf-8")
|
||||
(release / "scripts").mkdir()
|
||||
shutil.copyfile(SCRIPT.parent / "reencode_image.py", release / "scripts" / "reencode_image.py")
|
||||
verdict = self.deploy.verify_image_runtime(release)
|
||||
self.assertTrue(verdict["ok"])
|
||||
self.assertEqual(verdict["python"], self.deploy.PINNED_PYTHON_VERSION)
|
||||
self.assertEqual(verdict["pillow"], self.deploy.PINNED_PILLOW_VERSION)
|
||||
|
||||
def test_commit_tag_and_checksum_arguments_are_strictly_validated(self):
|
||||
archive, digest = self.archive()
|
||||
for commit in ("abc", "A" * 40, "a" * 41, "../" + "a" * 40):
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ test('malformed and escaping base paths are rejected at startup', async () => {
|
|||
child.stderr.on('data', chunk => { stderr += chunk; });
|
||||
const exitCode = await Promise.race([
|
||||
new Promise(resolve => child.once('exit', resolve)),
|
||||
new Promise(resolve => setTimeout(() => { child.kill('SIGTERM'); resolve('timeout'); }, 800)),
|
||||
new Promise(resolve => setTimeout(() => { child.kill('SIGTERM'); resolve('timeout'); }, 3000)),
|
||||
]);
|
||||
assert.notEqual(exitCode, 'timeout', `${JSON.stringify(value)} was accepted`);
|
||||
assert.notEqual(exitCode, 0, `${JSON.stringify(value)} exited successfully`);
|
||||
|
|
|
|||