From 0c5ee22d66381dbdbb24fc235974bc1334f11c48 Mon Sep 17 00:00:00 2001 From: liusy58 Date: Sun, 28 Dec 2025 22:01:33 +0800 Subject: [PATCH 1/2] Fix OOM by offloading multimodal features to CPU after embedding --- python/sglang/srt/managers/mm_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index 1d8e48fbd951..0300610389b2 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -1079,6 +1079,13 @@ def general_mm_embed_routine( kwargs["input_deepstack_embeds"] = other_info["input_deepstack_embeds"] # once used, mm_inputs is useless, considering chunked-prefill is disabled for multimodal models # just being defensive here + if mm_inputs_list: + for mm_input_obj in mm_inputs_list: + if mm_input_obj and hasattr(mm_input_obj, "mm_items"): + for mm_item in mm_input_obj.mm_items: + feature = getattr(mm_item, "feature", None) + if isinstance(feature, torch.Tensor) and feature.is_cuda: + mm_item.feature = feature.to("cpu", non_blocking=True) forward_batch.mm_inputs = None else: input_embeds = embed_tokens(input_ids) From 90ce745b27b3cd6dd301cae5d33f88e6d07192da Mon Sep 17 00:00:00 2001 From: liusy58 Date: Wed, 31 Dec 2025 17:11:11 +0800 Subject: [PATCH 2/2] add notes --- python/sglang/srt/managers/mm_utils.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/managers/mm_utils.py b/python/sglang/srt/managers/mm_utils.py index 0300610389b2..6bbbf51cc2aa 100644 --- a/python/sglang/srt/managers/mm_utils.py +++ b/python/sglang/srt/managers/mm_utils.py @@ -1077,8 +1077,14 @@ def general_mm_embed_routine( # add for qwen3_vl deepstack if use_deepstack: kwargs["input_deepstack_embeds"] = other_info["input_deepstack_embeds"] - # once used, mm_inputs is useless, considering chunked-prefill is disabled for multimodal models - # just being defensive here + # Offload GPU features to CPU instead of discarding them to balance memory + # efficiency and data persistence. + # In chunked-prefill, a request is processed across multiple batches, and + # the original multimodal data must remain accessible until the entire + # prefill phase is complete. Since the multimodal embedding cache is + # best-effort, offloading to CPU ensures we have a reliable fallback + # if a cache miss occurs in subsequent chunks, while still freeing up + # critical GPU memory. if mm_inputs_list: for mm_input_obj in mm_inputs_list: if mm_input_obj and hasattr(mm_input_obj, "mm_items"):