diff --git a/python/sglang/srt/multimodal/inkling/image_processing.py b/python/sglang/srt/multimodal/inkling/image_processing.py index 1a99d45e3ee2..3fa7f01f02d3 100644 --- a/python/sglang/srt/multimodal/inkling/image_processing.py +++ b/python/sglang/srt/multimodal/inkling/image_processing.py @@ -1,252 +1,259 @@ -"""HuggingFace-convention image processor for Inkling models.""" - -from __future__ import annotations - -import io -import math -from typing import List, Optional, Union - -import numpy as np -import torch -from numba import njit -from transformers.image_processing_utils import BaseImageProcessor, BatchFeature -from transformers.image_utils import ImageInput - -IMAGE_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32) -IMAGE_STD = np.array([0.26862954, 0.2613026, 0.2757771], dtype=np.float32) -PAD_RAW_VALUE = np.float32(-1.0 / 255.0) -PAD_NORM = (np.full((3,), PAD_RAW_VALUE, dtype=np.float32) - IMAGE_MEAN) / IMAGE_STD - - -def _validate_image_rescale( - rescale_image_frac: Optional[float], - rescale_image_max_upscaled_long_edge: Optional[int], -) -> None: - if rescale_image_frac is not None and ( - not math.isfinite(rescale_image_frac) or rescale_image_frac <= 0 - ): - raise ValueError( - "rescale_image_frac must be positive and finite or None, " - f"got {rescale_image_frac}" - ) - if rescale_image_max_upscaled_long_edge is None: - return - if rescale_image_max_upscaled_long_edge <= 0: - raise ValueError( - "rescale_image_max_upscaled_long_edge must be positive or None, " - f"got {rescale_image_max_upscaled_long_edge}" - ) - if rescale_image_frac is None or rescale_image_frac <= 1.0: - raise ValueError( - "rescale_image_max_upscaled_long_edge requires rescale_image_frac > 1, " - f"got {rescale_image_frac}" - ) - - -def _scaled_image_dimensions( - width: int, - height: int, - rescale_image_frac: Optional[float], - rescale_image_max_upscaled_long_edge: Optional[int], -) -> tuple[int, int]: - """Return the long-edge-scaled ``(width, height)``.""" - if rescale_image_frac is None: - return width, height - - long_edge = max(width, height) - if long_edge == 0: - return width, height - - target_long_edge = float(long_edge) * rescale_image_frac - if rescale_image_max_upscaled_long_edge is not None: - # The cap limits growth but never shrinks an image already above it. - effective_cap = max(rescale_image_max_upscaled_long_edge, long_edge) - target_long_edge = min(target_long_edge, float(effective_cap)) - - ratio = target_long_edge / float(long_edge) - if ratio == 1.0: - return width, height - - # Use half-away-from-zero rounding for positive dimensions. Python's round() - # uses ties-to-even, which can produce a different output size at exact halves. - def scale(value: int) -> int: - return max(1, math.floor(float(value) * ratio + 0.5)) - - return scale(width), scale(height) - - -def _load_image_bytes(image) -> bytes: - """Coerce a single image input into raw PNG/JPEG bytes for preprocessing.""" - if isinstance(image, (bytes, bytearray, memoryview)): - return bytes(image) - if isinstance(image, str): - if image.startswith(("http://", "https://", "data:")): - raise ValueError( - "InklingImageProcessor received a URL/data: image; resolve it to bytes " - "upstream (e.g. via SGLang load_mm_data) before preprocessing." - ) - path = image[len("file://") :] if image.startswith("file://") else image - with open(path, "rb") as f: - return f.read() - - from PIL import Image - - if isinstance(image, torch.Tensor): - arr = image.detach().cpu().numpy() - image = Image.fromarray(arr.astype("uint8") if arr.dtype != np.uint8 else arr) - elif not isinstance(image, Image.Image): - image = Image.fromarray(image) - if image.mode != "RGB": - image = image.convert("RGB") - buf = io.BytesIO() - image.save(buf, format="PNG") - return buf.getvalue() - - -@njit(cache=True) -def _fill_patches_numba( - arr: np.ndarray, - patch_size: int, - patches: np.ndarray, - mean: np.ndarray, - std: np.ndarray, - pad_norm: np.ndarray, -) -> None: - h = arr.shape[0] - w = arr.shape[1] - nph = (h + patch_size - 1) // patch_size - npw = w // patch_size + 1 - inv255 = np.float32(1.0 / 255.0) - - for k in range(nph * npw): - i = k // npw - j = k - i * npw - y_base = i * patch_size - x_base = j * patch_size - - for y in range(patch_size): - iy = y_base + y - for x in range(patch_size): - ix = x_base + x - if iy < h and ix < w: - for c in range(3): - raw = np.float32(arr[iy, ix, c]) * inv255 - patches[k, y, x, c] = (raw - mean[c]) / std[c] - else: - for c in range(3): - patches[k, y, x, c] = pad_norm[c] - - -def _encode_image_bytes( - image_bytes: bytes, - *, - patch_size: int, - rescale_image_frac: Optional[float], - rescale_image_max_upscaled_long_edge: Optional[int], -) -> torch.Tensor: - if patch_size <= 0: - raise ValueError("patch_size must be greater than zero") - _validate_image_rescale( - rescale_image_frac, - rescale_image_max_upscaled_long_edge, - ) - - from PIL import Image - - image = Image.open(io.BytesIO(image_bytes)).convert("RGB") - scaled_size = _scaled_image_dimensions( - *image.size, - rescale_image_frac=rescale_image_frac, - rescale_image_max_upscaled_long_edge=rescale_image_max_upscaled_long_edge, - ) - if scaled_size != image.size: - image = image.resize(scaled_size, resample=Image.Resampling.LANCZOS) - arr = np.array(image, dtype=np.uint8, copy=True) - height, width, _ = arr.shape - - nph = (height + patch_size - 1) // patch_size - npw = width // patch_size + 1 - num_patches = nph * npw - - patches = np.empty((num_patches, patch_size, patch_size, 3), dtype=np.float32) - _fill_patches_numba(arr, patch_size, patches, IMAGE_MEAN, IMAGE_STD, PAD_NORM) - - return ( - torch.from_numpy(patches) - .to(torch.bfloat16) - .view(num_patches, 1, patch_size, patch_size, 3) - .expand(num_patches, 2, patch_size, patch_size, 3) - ) - - -class InklingImageProcessor(BaseImageProcessor): - r"""Turn raw images into ``vision_patches_bthwc`` for Inkling hMLP. - - ``rescale_image_frac`` scales the long edge while preserving aspect ratio. - ``rescale_image_max_upscaled_long_edge`` optionally caps only upscaling and - therefore requires a scale factor greater than one. The defaults, ``2.0`` and - ``2048``, grow images toward a 2048-pixel long edge by at most 2x, while leaving - images already at or above 2048 unchanged. - """ - - model_input_names = ["vision_patches_bthwc"] - - def __init__( - self, - patch_size: int = 40, - rescale_image_frac: Optional[float] = 2.0, - rescale_image_max_upscaled_long_edge: Optional[int] = 2048, - **kwargs, - ): - if patch_size <= 0: - raise ValueError("patch_size must be greater than zero") - _validate_image_rescale( - rescale_image_frac, - rescale_image_max_upscaled_long_edge, - ) - super().__init__(**kwargs) - self.patch_size = patch_size - self.rescale_image_frac = rescale_image_frac - self.rescale_image_max_upscaled_long_edge = rescale_image_max_upscaled_long_edge - - def _encode_one(self, image) -> torch.Tensor: - return _encode_image_bytes( - _load_image_bytes(image), - patch_size=self.patch_size, - rescale_image_frac=self.rescale_image_frac, - rescale_image_max_upscaled_long_edge=self.rescale_image_max_upscaled_long_edge, - ) - - def preprocess( - self, - images: Union[ImageInput, List], - return_tensors: Optional[str] = "pt", - **kwargs, - ) -> BatchFeature: - del return_tensors, kwargs - if not isinstance(images, (list, tuple)): - images = [images] - - per_image_patches: List[torch.Tensor] = [] - num_patches: List[int] = [] - num_tokens: List[int] = [] - for img in images: - vp = self._encode_one(img) - n_patches = int(vp.shape[0]) - per_image_patches.append(vp) - num_patches.append(n_patches) - num_tokens.append(n_patches) - - if len(per_image_patches) == 1: - vision_patches_bthwc = per_image_patches[0] - elif per_image_patches: - vision_patches_bthwc = torch.cat(per_image_patches, dim=0) - else: - vision_patches_bthwc = torch.empty(0) - - data = { - "vision_patches_bthwc": vision_patches_bthwc, - "num_patches": num_patches, - "num_tokens": num_tokens, - } - return BatchFeature(data=data, tensor_type=None) +"""HuggingFace-convention image processor for Inkling models.""" + +from __future__ import annotations + +import io +import math +from typing import List, Optional, Union + +import numpy as np +import torch +from numba import njit +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature +from transformers.image_utils import ImageInput + +IMAGE_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32) +IMAGE_STD = np.array([0.26862954, 0.2613026, 0.2757771], dtype=np.float32) +PAD_RAW_VALUE = np.float32(-1.0 / 255.0) +PAD_NORM = (np.full((3,), PAD_RAW_VALUE, dtype=np.float32) - IMAGE_MEAN) / IMAGE_STD + + +def _validate_image_rescale( + rescale_image_frac: Optional[float], + rescale_image_max_upscaled_long_edge: Optional[int], +) -> None: + if rescale_image_frac is not None and ( + not math.isfinite(rescale_image_frac) or rescale_image_frac <= 0 + ): + raise ValueError( + "rescale_image_frac must be positive and finite or None, " + f"got {rescale_image_frac}" + ) + if rescale_image_max_upscaled_long_edge is None: + return + if rescale_image_max_upscaled_long_edge <= 0: + raise ValueError( + "rescale_image_max_upscaled_long_edge must be positive or None, " + f"got {rescale_image_max_upscaled_long_edge}" + ) + if rescale_image_frac is None or rescale_image_frac <= 1.0: + raise ValueError( + "rescale_image_max_upscaled_long_edge requires rescale_image_frac > 1, " + f"got {rescale_image_frac}" + ) + + +def _scaled_image_dimensions( + width: int, + height: int, + rescale_image_frac: Optional[float], + rescale_image_max_upscaled_long_edge: Optional[int], +) -> tuple[int, int]: + """Return the long-edge-scaled ``(width, height)``.""" + if rescale_image_frac is None: + return width, height + + long_edge = max(width, height) + if long_edge == 0: + return width, height + + target_long_edge = float(long_edge) * rescale_image_frac + if rescale_image_max_upscaled_long_edge is not None: + # The cap limits growth but never shrinks an image already above it. + effective_cap = max(rescale_image_max_upscaled_long_edge, long_edge) + target_long_edge = min(target_long_edge, float(effective_cap)) + + ratio = target_long_edge / float(long_edge) + if ratio == 1.0: + return width, height + + # Use half-away-from-zero rounding for positive dimensions. Python's round() + # uses ties-to-even, which can produce a different output size at exact halves. + def scale(value: int) -> int: + return max(1, math.floor(float(value) * ratio + 0.5)) + + return scale(width), scale(height) + + +def _load_image_bytes(image) -> bytes: + """Coerce a single image input into raw PNG/JPEG bytes for preprocessing.""" + if isinstance(image, (bytes, bytearray, memoryview)): + return bytes(image) + if isinstance(image, str): + if image.startswith(("http://", "https://", "data:")): + raise ValueError( + "InklingImageProcessor received a URL/data: image; resolve it to bytes " + "upstream (e.g. via SGLang load_mm_data) before preprocessing." + ) + path = image[len("file://") :] if image.startswith("file://") else image + with open(path, "rb") as f: + return f.read() + + from PIL import Image + + if isinstance(image, torch.Tensor): + arr = image.detach().cpu().numpy() + image = Image.fromarray(arr.astype("uint8") if arr.dtype != np.uint8 else arr) + elif not isinstance(image, Image.Image): + image = Image.fromarray(image) + if image.mode != "RGB": + image = image.convert("RGB") + buf = io.BytesIO() + image.save(buf, format="PNG") + return buf.getvalue() + + +@njit(cache=True) +def _fill_patches_numba( + arr: np.ndarray, + patch_size: int, + patches: np.ndarray, + mean: np.ndarray, + std: np.ndarray, + pad_norm: np.ndarray, +) -> None: + h = arr.shape[0] + w = arr.shape[1] + nph = (h + patch_size - 1) // patch_size + npw = w // patch_size + 1 + inv255 = np.float32(1.0 / 255.0) + + for k in range(nph * npw): + i = k // npw + j = k - i * npw + y_base = i * patch_size + x_base = j * patch_size + + for y in range(patch_size): + iy = y_base + y + for x in range(patch_size): + ix = x_base + x + if iy < h and ix < w: + for c in range(3): + raw = np.float32(arr[iy, ix, c]) * inv255 + patches[k, y, x, c] = (raw - mean[c]) / std[c] + else: + for c in range(3): + patches[k, y, x, c] = pad_norm[c] + + +def _encode_image_bytes( + image_bytes: bytes, + *, + patch_size: int, + rescale_image_frac: Optional[float], + rescale_image_max_upscaled_long_edge: Optional[int], +) -> torch.Tensor: + if patch_size <= 0: + raise ValueError("patch_size must be greater than zero") + _validate_image_rescale( + rescale_image_frac, + rescale_image_max_upscaled_long_edge, + ) + + from PIL import Image, UnidentifiedImageError + + try: + image = Image.open(io.BytesIO(image_bytes)).convert("RGB") + except UnidentifiedImageError as e: + # Undecodable client-supplied image bytes are a 400, not a 500: + # UnidentifiedImageError is unclassified by default and would escape + # to the server fault path. Mirror BaseMultimodalProcessor's + # CLIENT_MEDIA_EXCEPTIONS handling. + raise ValueError(f"Could not decode image bytes: {e}") from e + scaled_size = _scaled_image_dimensions( + *image.size, + rescale_image_frac=rescale_image_frac, + rescale_image_max_upscaled_long_edge=rescale_image_max_upscaled_long_edge, + ) + if scaled_size != image.size: + image = image.resize(scaled_size, resample=Image.Resampling.LANCZOS) + arr = np.array(image, dtype=np.uint8, copy=True) + height, width, _ = arr.shape + + nph = (height + patch_size - 1) // patch_size + npw = width // patch_size + 1 + num_patches = nph * npw + + patches = np.empty((num_patches, patch_size, patch_size, 3), dtype=np.float32) + _fill_patches_numba(arr, patch_size, patches, IMAGE_MEAN, IMAGE_STD, PAD_NORM) + + return ( + torch.from_numpy(patches) + .to(torch.bfloat16) + .view(num_patches, 1, patch_size, patch_size, 3) + .expand(num_patches, 2, patch_size, patch_size, 3) + ) + + +class InklingImageProcessor(BaseImageProcessor): + r"""Turn raw images into ``vision_patches_bthwc`` for Inkling hMLP. + + ``rescale_image_frac`` scales the long edge while preserving aspect ratio. + ``rescale_image_max_upscaled_long_edge`` optionally caps only upscaling and + therefore requires a scale factor greater than one. The defaults, ``2.0`` and + ``2048``, grow images toward a 2048-pixel long edge by at most 2x, while leaving + images already at or above 2048 unchanged. + """ + + model_input_names = ["vision_patches_bthwc"] + + def __init__( + self, + patch_size: int = 40, + rescale_image_frac: Optional[float] = 2.0, + rescale_image_max_upscaled_long_edge: Optional[int] = 2048, + **kwargs, + ): + if patch_size <= 0: + raise ValueError("patch_size must be greater than zero") + _validate_image_rescale( + rescale_image_frac, + rescale_image_max_upscaled_long_edge, + ) + super().__init__(**kwargs) + self.patch_size = patch_size + self.rescale_image_frac = rescale_image_frac + self.rescale_image_max_upscaled_long_edge = rescale_image_max_upscaled_long_edge + + def _encode_one(self, image) -> torch.Tensor: + return _encode_image_bytes( + _load_image_bytes(image), + patch_size=self.patch_size, + rescale_image_frac=self.rescale_image_frac, + rescale_image_max_upscaled_long_edge=self.rescale_image_max_upscaled_long_edge, + ) + + def preprocess( + self, + images: Union[ImageInput, List], + return_tensors: Optional[str] = "pt", + **kwargs, + ) -> BatchFeature: + del return_tensors, kwargs + if not isinstance(images, (list, tuple)): + images = [images] + + per_image_patches: List[torch.Tensor] = [] + num_patches: List[int] = [] + num_tokens: List[int] = [] + for img in images: + vp = self._encode_one(img) + n_patches = int(vp.shape[0]) + per_image_patches.append(vp) + num_patches.append(n_patches) + num_tokens.append(n_patches) + + if len(per_image_patches) == 1: + vision_patches_bthwc = per_image_patches[0] + elif per_image_patches: + vision_patches_bthwc = torch.cat(per_image_patches, dim=0) + else: + vision_patches_bthwc = torch.empty(0) + + data = { + "vision_patches_bthwc": vision_patches_bthwc, + "num_patches": num_patches, + "num_tokens": num_tokens, + } + return BatchFeature(data=data, tensor_type=None) diff --git a/python/sglang/srt/multimodal/processors/inkling.py b/python/sglang/srt/multimodal/processors/inkling.py index 803e973228ea..9cc195ac9d6a 100644 --- a/python/sglang/srt/multimodal/processors/inkling.py +++ b/python/sglang/srt/multimodal/processors/inkling.py @@ -1,317 +1,332 @@ -# Copyright 2026 SGLang Team -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== -"""SGLang multimodal processor for Inkling models. - -Adapts the HF-convention feature extraction in ``sglang.srt.multimodal.inkling`` to -SGLang's ``BaseMultimodalProcessor``: expands the MM placeholder tokens in a -pre-rendered ``input_ids`` (the chat renderer is a separate workstream) and attaches -per-item features as ``MultimodalDataItem``s. A modality is enabled only when its -``*_config.decoder_dmodel`` is set, so text-only checkpoints disable both towers. -""" - -from __future__ import annotations - -import base64 -import logging -from collections.abc import Mapping -from typing import Any, Dict, List, Optional, Tuple, Type, Union - -import torch - -from sglang.srt.environ import envs -from sglang.srt.managers.schedule_batch import ( - Modality, - MultimodalDataItem, - MultimodalProcessorOutput, -) -from sglang.srt.models.inkling import InklingForConditionalGeneration -from sglang.srt.multimodal.inkling import ( - InklingAudioFeatureExtractor, - InklingImageProcessor, - InklingProcessor, -) -from sglang.srt.multimodal.processors.base_processor import ( - BaseMultimodalProcessor as SGLangBaseProcessor, -) -from sglang.srt.parser.inkling_tokenizer import ( - AUDIO_END, -) -from sglang.srt.parser.inkling_tokenizer import AUDIO_TOKEN_ID as INKLING_AUDIO_TOKEN_ID -from sglang.srt.parser.inkling_tokenizer import IMAGE_TOKEN_ID as INKLING_IMAGE_TOKEN_ID -from sglang.srt.parser.inkling_tokenizer import ( - INKLING_SPECIAL_TOKEN_IDS, -) -from sglang.srt.utils.common import download_remote_media - -logger = logging.getLogger(__name__) - - -def _cfg(obj, name, default=None): - return getattr(obj, name, default) if obj is not None else default - - -def _resolve_media_item(item): - """Resolve a request media item to raw bytes for Inkling preprocessing. - - The OpenAI chat path hands ImageData/AudioData objects or data:/http(s) URLs; the - /generate path hands raw bytes or local file paths. Resolve URLs to bytes here - (data: base64 -> exact original bytes; http(s) -> download) and pass bytes / file - paths / PIL through unchanged for the per-modality byte loader to handle. - """ - url = None - if isinstance(item, str): - url = item - elif isinstance(item, Mapping): - url = item.get("url") - elif hasattr(item, "url"): - url = getattr(item, "url") - if not isinstance(url, str): - return item - if url.startswith("data:"): - header, _, payload = url.partition(",") - return base64.b64decode(payload) if ";base64" in header else payload.encode() - if url.startswith(("http://", "https://")): - return download_remote_media(url, timeout=30) - return url # plain path / file:// -> handled by the per-modality byte loader - - -_MISSING = object() - - -def _require(obj, name, *, where): - """Read a config field that MUST be present — no silent default. - - A wrong-but-silent fallback here corrupts model inputs (e.g. encoding with a - different dmel grid than the model de-bins with yields garbage audio embeddings - and no error), so fail loudly instead. - """ - val = getattr(obj, name, _MISSING) if obj is not None else _MISSING - if val is _MISSING or val is None: - raise ValueError( - f"InklingMultimodalProcessor: required config field {where}.{name!r} is " - f"missing. It must be set in the model config so preprocessing matches " - f"the model. Add it to config.json." - ) - return val - - -class InklingMultimodalProcessor(SGLangBaseProcessor): - # import_processors() registers this for the Inkling arch. Text-only checkpoints leave - # both towers disabled (gated on *_config.decoder_dmodel), so it is a no-op there. - models: List[Type] = [InklingForConditionalGeneration] - - def __init__(self, hf_config, server_args, _processor, *args, **kwargs): - super().__init__(hf_config, server_args, _processor, *args, **kwargs) - - vision_config = _cfg(hf_config, "vision_config") - audio_config = _cfg(hf_config, "audio_config") - # InklingMMConfig always builds default vision/audio sub-configs (decoder_dmodel=None) - # even for the text-only model, so gate "enabled" on decoder_dmodel being set, not - # on the sub-config existing — else a text-only checkpoint trips the checks below. - vision_enabled = _cfg(vision_config, "decoder_dmodel") is not None - audio_enabled = _cfg(audio_config, "decoder_dmodel") is not None - - patch_size = _cfg(vision_config, "patch_size", 40) - if envs.SGLANG_INKLING_RS_MM_PREPROCESS.get(): - try: - from sglang.srt.multimodal.inkling.image_processing_rust import ( - InklingRustImageProcessor, - ) - - image_processor = InklingRustImageProcessor(patch_size=patch_size) - logger.info("Using Rust-accelerated Inkling image processor") - except ImportError: - logger.warning( - "SGLANG_INKLING_RS_MM_PREPROCESS=1 but " - "sglang.srt.rust_extensions._multimodal is not available; " - "falling back to the default image processor." - ) - image_processor = InklingImageProcessor(patch_size=patch_size) - else: - image_processor = InklingImageProcessor(patch_size=patch_size) - # The dmel grid used here at encode time must equal what the model de-bins with - # at decode (InklingAudio reads it from audio_config); require it when audio - # is enabled rather than guessing a default that would silently corrupt embeds. - if audio_enabled: - audio_params = { - "n_mels": _require(audio_config, "n_mel_bins", where="audio_config"), - "num_dmel_bins": _require( - audio_config, "mel_vocab_size", where="audio_config" - ), - "dmel_min_value": _require( - audio_config, "dmel_min_value", where="audio_config" - ), - "dmel_max_value": _require( - audio_config, "dmel_max_value", where="audio_config" - ), - } - else: - audio_params = {} - audio_extractor = InklingAudioFeatureExtractor(params=audio_params) - - # Inkling's placeholder ids are protocol constants. Older tensor-IO checkpoints may - # omit them from config.json, so fall back to the local renderer constants when - # the corresponding tower is enabled. - self.IMAGE_TOKEN_ID = _cfg(hf_config, "image_token_id") - self.AUDIO_TOKEN_ID = _cfg(hf_config, "audio_token_id") - self.AUDIO_END_TOKEN_ID = _cfg(hf_config, "audio_end_token_id") - if vision_enabled and self.IMAGE_TOKEN_ID is None: - self.IMAGE_TOKEN_ID = INKLING_IMAGE_TOKEN_ID - if audio_enabled and self.AUDIO_TOKEN_ID is None: - self.AUDIO_TOKEN_ID = INKLING_AUDIO_TOKEN_ID - if audio_enabled and self.AUDIO_END_TOKEN_ID is None: - self.AUDIO_END_TOKEN_ID = INKLING_SPECIAL_TOKEN_IDS[AUDIO_END] - - self.inkling_processor = InklingProcessor( - image_processor=image_processor, - audio_feature_extractor=audio_extractor, - tokenizer=self._tokenizer, - ) - - # ---- core (pure, testable) ------------------------------------------ - - def assemble( - self, - input_ids: List[int], - image_data: Optional[List] = None, - audio_data: Optional[List] = None, - ) -> MultimodalProcessorOutput: - """Expand single MM placeholders in ``input_ids`` into per-item token - blocks and attach features. Walks ``input_ids`` left-to-right, consuming - ``image_data`` / ``audio_data`` in encounter order. - """ - image_data = image_data or [] - audio_data = audio_data or [] - - # One placeholder per media item (expanded below); a count mismatch (incl. a - # None token id absent from config) must fail loudly, not drop media silently. - n_img_ph = ( - sum(1 for t in input_ids if t == self.IMAGE_TOKEN_ID) - if self.IMAGE_TOKEN_ID is not None - else 0 - ) - n_aud_ph = ( - sum(1 for t in input_ids if t == self.AUDIO_TOKEN_ID) - if self.AUDIO_TOKEN_ID is not None - else 0 - ) - if n_img_ph != len(image_data): - raise ValueError( - f"InklingMultimodalProcessor: {n_img_ph} image placeholder token(s) in " - f"input_ids but {len(image_data)} image(s) provided; counts must match." - ) - if n_aud_ph != len(audio_data): - raise ValueError( - f"InklingMultimodalProcessor: {n_aud_ph} audio placeholder token(s) in " - f"input_ids but {len(audio_data)} audio(s) provided; counts must match." - ) - - img_feat = ( - self.inkling_processor.process_images(image_data) if image_data else None - ) - aud_feat = ( - self.inkling_processor.process_audios(audio_data) if audio_data else None - ) - - # Rust processor returns content_hashes; original processor does not. - img_hashes = img_feat.get("content_hashes") if img_feat else None - - out_ids: List[int] = [] - image_items: List[Tuple[int, int, torch.Tensor]] = [] # (start, end, feature) - audio_items: List[Tuple[int, int, torch.Tensor]] = [] - i_img = i_aud = 0 - - for tok in input_ids: - if self.IMAGE_TOKEN_ID is not None and tok == self.IMAGE_TOKEN_ID: - # hMLP folds each patch's interior to channel depth -> one token per - # patch, so num_tokens == num_patches. Assert in case a fold is ever added. - n_tokens = img_feat["num_tokens"][i_img] - n_patches = img_feat["num_patches"][i_img] - assert n_tokens == n_patches, ( - f"num_tokens ({n_tokens}) != num_patches ({n_patches}); the hMLP " - f"emits one token per patch. Drive the placeholder count and the " - f"feature row count from a single source if a fold is added." - ) - start = len(out_ids) - out_ids.extend([self.IMAGE_TOKEN_ID] * n_tokens) - # patches for this image are the i_img-th contiguous slice - base = sum(img_feat["num_patches"][:i_img]) - feat = img_feat["vision_patches_bthwc"][base : base + n_patches] - image_items.append((start, start + n_tokens - 1, feat)) - i_img += 1 - elif self.AUDIO_TOKEN_ID is not None and tok == self.AUDIO_TOKEN_ID: - n_tokens = aud_feat["num_audio_tokens"][i_aud] - start = len(out_ids) - out_ids.extend([self.AUDIO_TOKEN_ID] * n_tokens) - feat = aud_feat["dmel_bins"][i_aud] - audio_items.append((start, start + n_tokens - 1, feat)) - i_aud += 1 - else: - out_ids.append(tok) - - mm_items: List[MultimodalDataItem] = [] - for idx, (start, end, feat) in enumerate(image_items): - mm_items.append( - MultimodalDataItem( - modality=Modality.IMAGE, - feature=feat, - offsets=[(start, end)], - hash=img_hashes[idx] if img_hashes else None, - ) - ) - for start, end, feat in audio_items: - mm_items.append( - MultimodalDataItem( - modality=Modality.AUDIO, feature=feat, offsets=[(start, end)] - ) - ) - - return MultimodalProcessorOutput( - input_ids=out_ids, - mm_items=mm_items, - im_token_id=self.IMAGE_TOKEN_ID, - audio_token_id=self.AUDIO_TOKEN_ID, - audio_end_id=self.AUDIO_END_TOKEN_ID, - ) - - # ---- SGLang entrypoint ---------------------------------------------- - - async def process_mm_data_async( - self, - image_data: Optional[List[Union[str, bytes, Dict]]] = None, - audio_data: Optional[List[Union[str, bytes, Dict]]] = None, - input_text: str = "", - request_obj: Any = None, - *args, - **kwargs, - ) -> Optional[MultimodalProcessorOutput]: - input_ids = getattr(request_obj, "input_ids", None) - if input_ids is None: - if self._tokenizer is None: - raise ValueError( - "InklingMultimodalProcessor v1 requires pre-rendered input_ids " - "(request_obj.input_ids); the custom Inkling chat renderer is a " - "separate workstream. No tokenizer available to render text." - ) - input_ids = self._tokenizer(input_text).input_ids - if isinstance(input_ids, torch.Tensor): - input_ids = input_ids.flatten().tolist() - - # Resolve request media (data:/http URLs, ImageData objects) to bytes so the - # Inkling preprocessors can consume them; bytes / paths pass through unchanged. - if image_data: - image_data = [_resolve_media_item(it) for it in image_data] - if audio_data: - audio_data = [_resolve_media_item(it) for it in audio_data] - return self.assemble(list(input_ids), image_data, audio_data) +# Copyright 2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""SGLang multimodal processor for Inkling models. + +Adapts the HF-convention feature extraction in ``sglang.srt.multimodal.inkling`` to +SGLang's ``BaseMultimodalProcessor``: expands the MM placeholder tokens in a +pre-rendered ``input_ids`` (the chat renderer is a separate workstream) and attaches +per-item features as ``MultimodalDataItem``s. A modality is enabled only when its +``*_config.decoder_dmodel`` is set, so text-only checkpoints disable both towers. +""" + +from __future__ import annotations + +import base64 +import logging +from collections.abc import Mapping +from typing import Any, Dict, List, Optional, Tuple, Type, Union + +import torch + +from sglang.srt.environ import envs +from sglang.srt.managers.schedule_batch import ( + Modality, + MultimodalDataItem, + MultimodalProcessorOutput, +) +from sglang.srt.models.inkling import InklingForConditionalGeneration +from sglang.srt.multimodal.inkling import ( + InklingAudioFeatureExtractor, + InklingImageProcessor, + InklingProcessor, +) +from sglang.srt.multimodal.processors.base_processor import ( + BaseMultimodalProcessor as SGLangBaseProcessor, +) +from sglang.srt.parser.inkling_tokenizer import ( + AUDIO_END, +) +from sglang.srt.parser.inkling_tokenizer import AUDIO_TOKEN_ID as INKLING_AUDIO_TOKEN_ID +from sglang.srt.parser.inkling_tokenizer import IMAGE_TOKEN_ID as INKLING_IMAGE_TOKEN_ID +from sglang.srt.parser.inkling_tokenizer import ( + INKLING_SPECIAL_TOKEN_IDS, +) +from sglang.srt.utils.common import ( + CLIENT_MEDIA_EXCEPTIONS, + download_remote_media, +) + +logger = logging.getLogger(__name__) + + +def _cfg(obj, name, default=None): + return getattr(obj, name, default) if obj is not None else default + + +def _resolve_media_item(item): + """Resolve a request media item to raw bytes for Inkling preprocessing. + + The OpenAI chat path hands ImageData/AudioData objects or data:/http(s) URLs; the + /generate path hands raw bytes or local file paths. Resolve URLs to bytes here + (data: base64 -> exact original bytes; http(s) -> download) and pass bytes / file + paths / PIL through unchanged for the per-modality byte loader to handle. + """ + url = None + if isinstance(item, str): + url = item + elif isinstance(item, Mapping): + url = item.get("url") + elif hasattr(item, "url"): + url = getattr(item, "url") + if not isinstance(url, str): + return item + if not url.strip(): + # An empty media URL is a client request error, not a missing local + # file; raise ValueError so it follows the same 400 path as every + # other VLM processor (BaseMultimodalProcessor._load_single_item), + # instead of surfacing as FileNotFoundError / HTTP 500 later. + raise ValueError("Error while resolving media: media URL must not be empty.") + if url.startswith("data:"): + header, _, payload = url.partition(",") + return base64.b64decode(payload) if ";base64" in header else payload.encode() + if url.startswith(("http://", "https://")): + try: + return download_remote_media(url, timeout=30) + except CLIENT_MEDIA_EXCEPTIONS as e: + # Network/DNS/timeout/oversized payloads are client errors, not + # server faults; mirror the base processor's classification so the + # OpenAI layer returns 400 rather than 500. + raise ValueError(f"Error while downloading media URL {url}: {e}") from e + return url # plain path / file:// -> handled by the per-modality byte loader + + +_MISSING = object() + + +def _require(obj, name, *, where): + """Read a config field that MUST be present — no silent default. + + A wrong-but-silent fallback here corrupts model inputs (e.g. encoding with a + different dmel grid than the model de-bins with yields garbage audio embeddings + and no error), so fail loudly instead. + """ + val = getattr(obj, name, _MISSING) if obj is not None else _MISSING + if val is _MISSING or val is None: + raise ValueError( + f"InklingMultimodalProcessor: required config field {where}.{name!r} is " + f"missing. It must be set in the model config so preprocessing matches " + f"the model. Add it to config.json." + ) + return val + + +class InklingMultimodalProcessor(SGLangBaseProcessor): + # import_processors() registers this for the Inkling arch. Text-only checkpoints leave + # both towers disabled (gated on *_config.decoder_dmodel), so it is a no-op there. + models: List[Type] = [InklingForConditionalGeneration] + + def __init__(self, hf_config, server_args, _processor, *args, **kwargs): + super().__init__(hf_config, server_args, _processor, *args, **kwargs) + + vision_config = _cfg(hf_config, "vision_config") + audio_config = _cfg(hf_config, "audio_config") + # InklingMMConfig always builds default vision/audio sub-configs (decoder_dmodel=None) + # even for the text-only model, so gate "enabled" on decoder_dmodel being set, not + # on the sub-config existing — else a text-only checkpoint trips the checks below. + vision_enabled = _cfg(vision_config, "decoder_dmodel") is not None + audio_enabled = _cfg(audio_config, "decoder_dmodel") is not None + + patch_size = _cfg(vision_config, "patch_size", 40) + if envs.SGLANG_INKLING_RS_MM_PREPROCESS.get(): + try: + from sglang.srt.multimodal.inkling.image_processing_rust import ( + InklingRustImageProcessor, + ) + + image_processor = InklingRustImageProcessor(patch_size=patch_size) + logger.info("Using Rust-accelerated Inkling image processor") + except ImportError: + logger.warning( + "SGLANG_INKLING_RS_MM_PREPROCESS=1 but " + "sglang.srt.rust_extensions._multimodal is not available; " + "falling back to the default image processor." + ) + image_processor = InklingImageProcessor(patch_size=patch_size) + else: + image_processor = InklingImageProcessor(patch_size=patch_size) + # The dmel grid used here at encode time must equal what the model de-bins with + # at decode (InklingAudio reads it from audio_config); require it when audio + # is enabled rather than guessing a default that would silently corrupt embeds. + if audio_enabled: + audio_params = { + "n_mels": _require(audio_config, "n_mel_bins", where="audio_config"), + "num_dmel_bins": _require( + audio_config, "mel_vocab_size", where="audio_config" + ), + "dmel_min_value": _require( + audio_config, "dmel_min_value", where="audio_config" + ), + "dmel_max_value": _require( + audio_config, "dmel_max_value", where="audio_config" + ), + } + else: + audio_params = {} + audio_extractor = InklingAudioFeatureExtractor(params=audio_params) + + # Inkling's placeholder ids are protocol constants. Older tensor-IO checkpoints may + # omit them from config.json, so fall back to the local renderer constants when + # the corresponding tower is enabled. + self.IMAGE_TOKEN_ID = _cfg(hf_config, "image_token_id") + self.AUDIO_TOKEN_ID = _cfg(hf_config, "audio_token_id") + self.AUDIO_END_TOKEN_ID = _cfg(hf_config, "audio_end_token_id") + if vision_enabled and self.IMAGE_TOKEN_ID is None: + self.IMAGE_TOKEN_ID = INKLING_IMAGE_TOKEN_ID + if audio_enabled and self.AUDIO_TOKEN_ID is None: + self.AUDIO_TOKEN_ID = INKLING_AUDIO_TOKEN_ID + if audio_enabled and self.AUDIO_END_TOKEN_ID is None: + self.AUDIO_END_TOKEN_ID = INKLING_SPECIAL_TOKEN_IDS[AUDIO_END] + + self.inkling_processor = InklingProcessor( + image_processor=image_processor, + audio_feature_extractor=audio_extractor, + tokenizer=self._tokenizer, + ) + + # ---- core (pure, testable) ------------------------------------------ + + def assemble( + self, + input_ids: List[int], + image_data: Optional[List] = None, + audio_data: Optional[List] = None, + ) -> MultimodalProcessorOutput: + """Expand single MM placeholders in ``input_ids`` into per-item token + blocks and attach features. Walks ``input_ids`` left-to-right, consuming + ``image_data`` / ``audio_data`` in encounter order. + """ + image_data = image_data or [] + audio_data = audio_data or [] + + # One placeholder per media item (expanded below); a count mismatch (incl. a + # None token id absent from config) must fail loudly, not drop media silently. + n_img_ph = ( + sum(1 for t in input_ids if t == self.IMAGE_TOKEN_ID) + if self.IMAGE_TOKEN_ID is not None + else 0 + ) + n_aud_ph = ( + sum(1 for t in input_ids if t == self.AUDIO_TOKEN_ID) + if self.AUDIO_TOKEN_ID is not None + else 0 + ) + if n_img_ph != len(image_data): + raise ValueError( + f"InklingMultimodalProcessor: {n_img_ph} image placeholder token(s) in " + f"input_ids but {len(image_data)} image(s) provided; counts must match." + ) + if n_aud_ph != len(audio_data): + raise ValueError( + f"InklingMultimodalProcessor: {n_aud_ph} audio placeholder token(s) in " + f"input_ids but {len(audio_data)} audio(s) provided; counts must match." + ) + + img_feat = ( + self.inkling_processor.process_images(image_data) if image_data else None + ) + aud_feat = ( + self.inkling_processor.process_audios(audio_data) if audio_data else None + ) + + # Rust processor returns content_hashes; original processor does not. + img_hashes = img_feat.get("content_hashes") if img_feat else None + + out_ids: List[int] = [] + image_items: List[Tuple[int, int, torch.Tensor]] = [] # (start, end, feature) + audio_items: List[Tuple[int, int, torch.Tensor]] = [] + i_img = i_aud = 0 + + for tok in input_ids: + if self.IMAGE_TOKEN_ID is not None and tok == self.IMAGE_TOKEN_ID: + # hMLP folds each patch's interior to channel depth -> one token per + # patch, so num_tokens == num_patches. Assert in case a fold is ever added. + n_tokens = img_feat["num_tokens"][i_img] + n_patches = img_feat["num_patches"][i_img] + assert n_tokens == n_patches, ( + f"num_tokens ({n_tokens}) != num_patches ({n_patches}); the hMLP " + f"emits one token per patch. Drive the placeholder count and the " + f"feature row count from a single source if a fold is added." + ) + start = len(out_ids) + out_ids.extend([self.IMAGE_TOKEN_ID] * n_tokens) + # patches for this image are the i_img-th contiguous slice + base = sum(img_feat["num_patches"][:i_img]) + feat = img_feat["vision_patches_bthwc"][base : base + n_patches] + image_items.append((start, start + n_tokens - 1, feat)) + i_img += 1 + elif self.AUDIO_TOKEN_ID is not None and tok == self.AUDIO_TOKEN_ID: + n_tokens = aud_feat["num_audio_tokens"][i_aud] + start = len(out_ids) + out_ids.extend([self.AUDIO_TOKEN_ID] * n_tokens) + feat = aud_feat["dmel_bins"][i_aud] + audio_items.append((start, start + n_tokens - 1, feat)) + i_aud += 1 + else: + out_ids.append(tok) + + mm_items: List[MultimodalDataItem] = [] + for idx, (start, end, feat) in enumerate(image_items): + mm_items.append( + MultimodalDataItem( + modality=Modality.IMAGE, + feature=feat, + offsets=[(start, end)], + hash=img_hashes[idx] if img_hashes else None, + ) + ) + for start, end, feat in audio_items: + mm_items.append( + MultimodalDataItem( + modality=Modality.AUDIO, feature=feat, offsets=[(start, end)] + ) + ) + + return MultimodalProcessorOutput( + input_ids=out_ids, + mm_items=mm_items, + im_token_id=self.IMAGE_TOKEN_ID, + audio_token_id=self.AUDIO_TOKEN_ID, + audio_end_id=self.AUDIO_END_TOKEN_ID, + ) + + # ---- SGLang entrypoint ---------------------------------------------- + + async def process_mm_data_async( + self, + image_data: Optional[List[Union[str, bytes, Dict]]] = None, + audio_data: Optional[List[Union[str, bytes, Dict]]] = None, + input_text: str = "", + request_obj: Any = None, + *args, + **kwargs, + ) -> Optional[MultimodalProcessorOutput]: + input_ids = getattr(request_obj, "input_ids", None) + if input_ids is None: + if self._tokenizer is None: + raise ValueError( + "InklingMultimodalProcessor v1 requires pre-rendered input_ids " + "(request_obj.input_ids); the custom Inkling chat renderer is a " + "separate workstream. No tokenizer available to render text." + ) + input_ids = self._tokenizer(input_text).input_ids + if isinstance(input_ids, torch.Tensor): + input_ids = input_ids.flatten().tolist() + + # Resolve request media (data:/http URLs, ImageData objects) to bytes so the + # Inkling preprocessors can consume them; bytes / paths pass through unchanged. + if image_data: + image_data = [_resolve_media_item(it) for it in image_data] + if audio_data: + audio_data = [_resolve_media_item(it) for it in audio_data] + return self.assemble(list(input_ids), image_data, audio_data) diff --git a/test/registered/unit/multimodal/test_inkling_media_errors.py b/test/registered/unit/multimodal/test_inkling_media_errors.py new file mode 100644 index 000000000000..1d9c01759886 --- /dev/null +++ b/test/registered/unit/multimodal/test_inkling_media_errors.py @@ -0,0 +1,102 @@ +"""Bad-media classification for the Inkling multimodal processor. + +Inkling resolves request media through its own ``_resolve_media_item`` and decodes +image bytes through ``_encode_image_bytes`` instead of +``BaseMultimodalProcessor._load_single_item``. Client-supplied media that cannot +be resolved or decoded must raise ``ValueError`` (the 400 path, classified by +``CLIENT_MEDIA_EXCEPTIONS``), not bubble up as raw connection / decode errors +that the server reports as HTTP 500. See issue #40897. +""" + +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=11, suite="base-a-test-cpu") + +import base64 +import binascii +import unittest +from unittest.mock import patch + +import requests +from sglang.srt.multimodal.inkling.image_processing import _encode_image_bytes +from sglang.srt.multimodal.processors.inkling import _resolve_media_item +from sglang.test.test_utils import CustomTestCase + + +class TestResolveMediaItem(CustomTestCase): + def test_empty_url_is_client_error(self): + for item in ("", {"url": ""}, {"url": " "}): + with self.subTest(item=item), self.assertRaises(ValueError): + _resolve_media_item(item) + + def test_unreachable_http_url_is_client_error(self): + # A connection failure while fetching the remote media is the client's + # bad URL, not a server fault; the raw ConnectionError must be wrapped. + cases = ( + requests.exceptions.ConnectionError("dns failure"), + requests.exceptions.Timeout("read timed out"), + requests.exceptions.TooManyRedirects("loop"), + ) + for exc in cases: + with self.subTest(exc_type=type(exc).__name__), patch( + "sglang.srt.multimodal.processors.inkling.download_remote_media", + side_effect=exc, + ): + with self.assertRaises(ValueError) as ctx: + _resolve_media_item( + {"url": "https://unreachable.example/x.png"} + ) + self.assertIs(ctx.exception.__cause__, exc) + + def test_invalid_data_base64_is_client_error(self): + # binascii.Error is a ValueError subclass, so this was already on the + # 400 path; lock it in so the resolution refactor does not regress it. + with self.assertRaises(binascii.Error): + _resolve_media_item("data:image/png;base64,!!!not-base64!!!") + + def test_valid_data_url_round_trip(self): + payload = b"\x89PNG\r\n\x1a\n" + url = "data:image/png;base64," + base64.b64encode(payload).decode() + self.assertEqual(_resolve_media_item(url), payload) + + def test_remote_fetch_result_passes_through(self): + with patch( + "sglang.srt.multimodal.processors.inkling.download_remote_media", + return_value=b"remote-bytes", + ): + self.assertEqual( + _resolve_media_item("https://media.host/x.png"), b"remote-bytes" + ) + + def test_local_path_unchanged(self): + # Plain file paths are left for the per-modality byte loader. + self.assertEqual(_resolve_media_item("/local/img.png"), "/local/img.png") + self.assertEqual(_resolve_media_item(b"already-bytes"), b"already-bytes") + + +class TestEncodeImageBytes(CustomTestCase): + def test_undecodable_image_bytes_is_client_error(self): + with self.assertRaises(ValueError) as ctx: + _encode_image_bytes( + b"definitely not an image", + patch_size=14, + rescale_image_frac=None, + rescale_image_max_upscaled_long_edge=None, + ) + self.assertIn("Could not decode image bytes", str(ctx.exception)) + + def test_undecodable_image_bytes_cause_is_pil_error(self): + from PIL import UnidentifiedImageError + + with self.assertRaises(ValueError) as ctx: + _encode_image_bytes( + b"definitely not an image", + patch_size=14, + rescale_image_frac=None, + rescale_image_max_upscaled_long_edge=None, + ) + self.assertIsInstance(ctx.exception.__cause__, UnidentifiedImageError) + + +if __name__ == "__main__": + unittest.main()