From e6c2478271119984b5290c8d9cfcd6c3fd74cdf0 Mon Sep 17 00:00:00 2001 From: hotragn Date: Wed, 16 Sep 2026 15:49:00 -0400 Subject: [PATCH 1/2] [Bugfix][Multimodal] Tolerate malformed EXIF in Molmo 2 image preprocessing molmo2's own exif_transpose() calls ImageOps.exif_transpose bare, while vllm.multimodal.image.normalize_image suppresses the read. ImageMediaIO primes Pillow's lazy EXIF state, so the second read in get_image_replacement_molmo2 raises SyntaxError out of the multi-modal processor. Reuse normalize_image instead of keeping a second copy of the logic, so the tolerance #56576 added for hashing also holds here. Co-Authored-By: Claude Opus 5 Signed-off-by: hotragn --- .../multimodal/processing/test_molmo2.py | 47 +++++++++++++++++++ vllm/model_executor/models/molmo2.py | 4 +- 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/tests/models/multimodal/processing/test_molmo2.py b/tests/models/multimodal/processing/test_molmo2.py index 5633d47a97dc..02e9bf15c4fc 100644 --- a/tests/models/multimodal/processing/test_molmo2.py +++ b/tests/models/multimodal/processing/test_molmo2.py @@ -1,16 +1,22 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import struct +from io import BytesIO from types import SimpleNamespace import pytest import torch +from PIL import Image as PILImage +from PIL import ImageOps from vllm.config.multimodal import VideoDummyOptions from vllm.model_executor.models.molmo2 import ( Molmo2DummyInputsBuilder, build_flat_image_bool_length, + exif_transpose, ) +from vllm.multimodal.media.image import ImageMediaIO def test_build_flat_image_bool_length_matches_molmoweb_processor_tokens(): @@ -88,3 +94,44 @@ def test_dummy_video_num_frames_override_honors_min_of_two( video, _metadata = data["video"][0] assert video.shape[0] == expected_frames + + +def _malformed_exif_jpeg() -> bytes: + """A JPEG whose APP1/Exif segment carries an invalid TIFF header.""" + buf = BytesIO() + PILImage.new("RGB", (64, 48)).save(buf, "JPEG") + jpg = buf.getvalue() + rest = jpg[2:] + rest = rest[2 + struct.unpack(">H", rest[2:4])[0] :] + payload = b"Exif\x00\x00XXXX\x00\x00\x00\x08" + bytes(32) + return b"\xff\xd8\xff\xe1" + struct.pack(">H", len(payload) + 2) + payload + rest + + +def _served_image() -> PILImage.Image: + """An image with malformed EXIF, loaded the way a served request loads it. + + ``ImageMediaIO.load_bytes`` runs a suppressed ``exif_transpose`` and then + ``load()``, which is what primes Pillow's lazy EXIF state so the *next* + read of it raises. + """ + return ImageMediaIO(image_mode="RGB").load_bytes(_malformed_exif_jpeg()).media + + +def test_exif_transpose_tolerates_malformed_exif(): + """A malformed EXIF header must not fail the request in the prompt update. + + ``get_image_replacement_molmo2`` transposes the served image again, so a + bare ``ImageOps.exif_transpose`` turns it into a ``SyntaxError`` escaping the + multi-modal processor. This is the same payload #56576 made + ``MultiModalHasher`` tolerate; hashing runs first, so that fix moved the + failure here rather than removing it. + """ + # Precondition, on its own image: Pillow raises only on the first read of + # the poisoned state, so every case below needs a freshly loaded image or it + # would pass by coincidence. + with pytest.raises(Exception): # noqa: B017 - Pillow raises SyntaxError + ImageOps.exif_transpose(_served_image()) + + assert exif_transpose(_served_image()) is not None + assert exif_transpose([_served_image(), None])[0] is not None + assert exif_transpose(None) is None diff --git a/vllm/model_executor/models/molmo2.py b/vllm/model_executor/models/molmo2.py index 4c0ce0868e2d..e653a17ffdae 100644 --- a/vllm/model_executor/models/molmo2.py +++ b/vllm/model_executor/models/molmo2.py @@ -11,7 +11,6 @@ import torch import torch.nn as nn import torch.nn.functional as F -from PIL import ImageOps from PIL.Image import Image from transformers import ( BaseImageProcessor, @@ -54,6 +53,7 @@ ) from vllm.model_executor.models.module_mapping import MultiModelKeys from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.image import normalize_image from vllm.multimodal.inputs import ( MultiModalFieldConfig, MultiModalKwargsItems, @@ -1299,7 +1299,7 @@ def exif_transpose( exif_transpose(img) if isinstance(img, Image) else img for img in images ] elif images is not None and isinstance(images, Image): - images = ImageOps.exif_transpose(images) + images = normalize_image(images) return images From 5a8c4e8c45794ce005980a6002e50f6de4c0c60c Mon Sep 17 00:00:00 2001 From: hotragn Date: Sat, 19 Sep 2026 16:41:18 -0400 Subject: [PATCH 2/2] Remove the regression test per review Co-Authored-By: Claude Opus 5 Signed-off-by: hotragn --- .../multimodal/processing/test_molmo2.py | 47 ------------------- 1 file changed, 47 deletions(-) diff --git a/tests/models/multimodal/processing/test_molmo2.py b/tests/models/multimodal/processing/test_molmo2.py index 7f4c8d38d83c..68d3c8163cb7 100644 --- a/tests/models/multimodal/processing/test_molmo2.py +++ b/tests/models/multimodal/processing/test_molmo2.py @@ -1,22 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import struct -from io import BytesIO from types import SimpleNamespace import pytest import torch -from PIL import Image as PILImage -from PIL import ImageOps from vllm.config.multimodal import MultiModalDummyOptions, VideoDummyOptions from vllm.model_executor.models.molmo2 import ( Molmo2DummyInputsBuilder, build_flat_image_bool_length, - exif_transpose, ) -from vllm.multimodal.media.image import ImageMediaIO def test_build_flat_image_bool_length_matches_molmoweb_processor_tokens(): @@ -96,44 +90,3 @@ def test_dummy_video_num_frames_override_honors_min_of_two( video, _metadata = data["video"][0] assert video.shape[0] == expected_frames - - -def _malformed_exif_jpeg() -> bytes: - """A JPEG whose APP1/Exif segment carries an invalid TIFF header.""" - buf = BytesIO() - PILImage.new("RGB", (64, 48)).save(buf, "JPEG") - jpg = buf.getvalue() - rest = jpg[2:] - rest = rest[2 + struct.unpack(">H", rest[2:4])[0] :] - payload = b"Exif\x00\x00XXXX\x00\x00\x00\x08" + bytes(32) - return b"\xff\xd8\xff\xe1" + struct.pack(">H", len(payload) + 2) + payload + rest - - -def _served_image() -> PILImage.Image: - """An image with malformed EXIF, loaded the way a served request loads it. - - ``ImageMediaIO.load_bytes`` runs a suppressed ``exif_transpose`` and then - ``load()``, which is what primes Pillow's lazy EXIF state so the *next* - read of it raises. - """ - return ImageMediaIO(image_mode="RGB").load_bytes(_malformed_exif_jpeg()).media - - -def test_exif_transpose_tolerates_malformed_exif(): - """A malformed EXIF header must not fail the request in the prompt update. - - ``get_image_replacement_molmo2`` transposes the served image again, so a - bare ``ImageOps.exif_transpose`` turns it into a ``SyntaxError`` escaping the - multi-modal processor. This is the same payload #56576 made - ``MultiModalHasher`` tolerate; hashing runs first, so that fix moved the - failure here rather than removing it. - """ - # Precondition, on its own image: Pillow raises only on the first read of - # the poisoned state, so every case below needs a freshly loaded image or it - # would pass by coincidence. - with pytest.raises(Exception): # noqa: B017 - Pillow raises SyntaxError - ImageOps.exif_transpose(_served_image()) - - assert exif_transpose(_served_image()) is not None - assert exif_transpose([_served_image(), None])[0] is not None - assert exif_transpose(None) is None