From b7ea2dc5010e01c81b1581b93facdf67fc41df5e Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Tue, 21 Apr 2026 13:16:31 -0700 Subject: [PATCH 1/2] fix(vllm): cap gpu_memory_utilization for encoder-only vision model load The encode worker's handler creates a nested vllm.LLM() via load_vision_model() to load only the vision submodule. Without an explicit gpu_memory_utilization kwarg it falls back to vLLM's default 0.9, which fails the free-memory precheck on <=24 GiB GPUs when the outer worker (e.g. PD in single-GPU mode) is already holding memory: ValueError: Free memory on device cuda:0 (19.5/22.03 GiB) on startup is less than desired GPU memory utilization (0.9, 19.83 GiB). vLLM's precheck in gpu_worker.request_memory() runs before kv_cache_memory_bytes takes effect, so capping kv-cache alone isn't enough. 0.2 is generous for an encoder-only load (64 MiB KV stub, ~1 GiB ViT weights for Qwen3-VL-2B) and safe on every GPU class we run on. Fixes OPS-4445: unskips test_serve_deployment[mm_e_pd_qwen3-vl-2b] test_serve_deployment[mm_epd_qwen3-vl-2b] test_serve_deployment[mm_epd_qwen3-vl-2b-video] Co-Authored-By: Claude Opus 4.7 (1M context) --- components/src/dynamo/vllm/multimodal_utils/model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/src/dynamo/vllm/multimodal_utils/model.py b/components/src/dynamo/vllm/multimodal_utils/model.py index 186cc6e390ca..0403735a3a31 100644 --- a/components/src/dynamo/vllm/multimodal_utils/model.py +++ b/components/src/dynamo/vllm/multimodal_utils/model.py @@ -173,6 +173,9 @@ def load_vision_model(model_id: str, enforce_eager: bool = False) -> torch.nn.Mo vllm_model = LLM( model=model_id, enforce_eager=enforce_eager, + # vLLM's free-memory precheck runs before kv_cache_memory_bytes applies; + # default 0.9 fails on <=24 GiB GPUs when another worker shares the device. + gpu_memory_utilization=0.2, kv_cache_memory_bytes=1024 * 1024 * 64, # 64MB KV cache for vLLM to complete the init lifecycle, encoder-only doesn't require KV cache. From 7d1b42e46065f5d3322977c7dfa1f9c21e348f45 Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Tue, 21 Apr 2026 14:34:29 -0700 Subject: [PATCH 2/2] test(serve): re-enable qwen3-vl-2b e_pd/epd/epd-video multimodal disagg tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three tests were skipped in #8443 while OPS-4445 was investigated. The preceding commit fixes the encode worker's nested vllm.LLM() memory precheck; re-enable the tests under pre_merge so they gate regressions. Leaves the qwen2-audio-7b post_merge skip in place — that is a distinct failure on the agg topology (no encode worker, not addressed here). Co-Authored-By: Claude Opus 4.7 (1M context) --- tests/serve/multimodal_profiles/vllm.py | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/tests/serve/multimodal_profiles/vllm.py b/tests/serve/multimodal_profiles/vllm.py index dd203339f215..db25472de5c4 100644 --- a/tests/serve/multimodal_profiles/vllm.py +++ b/tests/serve/multimodal_profiles/vllm.py @@ -29,24 +29,12 @@ profiled_vram_gib=9.6, ), "e_pd": TopologyConfig( - marks=[ - pytest.mark.skip( - reason="vLLM engine core init fails on disagg e_pd. " - "https://linear.app/nvidia/issue/OPS-4445" - ), - pytest.mark.pre_merge, - ], + marks=[pytest.mark.pre_merge], timeout_s=340, single_gpu=True, ), "epd": TopologyConfig( - marks=[ - pytest.mark.skip( - reason="vLLM engine core init fails on disagg epd. " - "https://linear.app/nvidia/issue/OPS-4445" - ), - pytest.mark.pre_merge, - ], + marks=[pytest.mark.pre_merge], timeout_s=300, single_gpu=True, ), @@ -68,13 +56,7 @@ delayed_start=60, ), "epd": TopologyConfig( - marks=[ - pytest.mark.skip( - reason="vLLM engine core init fails on disagg epd. " - "https://linear.app/nvidia/issue/OPS-4445" - ), - pytest.mark.pre_merge, - ], + marks=[pytest.mark.pre_merge], timeout_s=600, delayed_start=60, single_gpu=True,