From 3d7ef816466907c3fea10f3f86f007125533fc99 Mon Sep 17 00:00:00 2001 From: Mick Qian Date: Fri, 23 Oct 2026 11:48:25 +0800 Subject: [PATCH] [diffusion] Auto mode: skip the residency calibration probe until a planner consumes it Auto mode ran a full-shape warmup probe and raised the warmup to four steps to calibrate component residency, but nothing applies the result: only the measurement half (#37916) is merged, while the planner and apply PRs (#37917/#37918/#35335) are open. The probe still cost startup time and dropped the IPC/Ulysses staging buffers afterwards, which hung a 4-GPU CFG+Ulysses Cosmos3 startup on GB300. The args gate now ends with "residency planner not available", so the warmup builder emits the bounded warmup only. The planner PR flips this line when it lands. The debug residency hint no longer claims that auto applies adjustments. Co-Authored-By: Claude Opus 5.5 --- .../runtime/managers/gpu_worker.py | 4 +- .../runtime/server_args/auto_tune.py | 3 +- .../test/unit/test_auto_residency.py | 57 +++++++++++++++++-- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index fcb31631c8f9..35cd7c1a0f1f 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -547,9 +547,7 @@ def do_mem_analysis(self, output_batch: OutputBatch): residency_hint = ( f" Components that can remain on GPU: {can_stay_resident}. " - "Make it explicit with --component-residency =resident; " - "--performance-mode auto with server warmup applies safe " - "adjustments automatically." + "Make it explicit with --component-residency =resident." if can_stay_resident else "" ) diff --git a/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py b/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py index 12bfc720118e..cf012ceee519 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py +++ b/python/sglang/multimodal_gen/runtime/server_args/auto_tune.py @@ -93,7 +93,8 @@ def auto_residency_args_skip_reason(server_args: ServerArgs) -> str | None: return "dynamic batching enabled" if not current_platform.is_cuda(): return "requires CUDA" - return None + # nothing applies calibrated placements yet, so warmup skips the probe and step floor + return "residency planner not available" PERFORMANCE_MODES = ("manual", "auto", "speed", "memory") diff --git a/python/sglang/multimodal_gen/test/unit/test_auto_residency.py b/python/sglang/multimodal_gen/test/unit/test_auto_residency.py index 31299282eb35..42f6b141c942 100644 --- a/python/sglang/multimodal_gen/test/unit/test_auto_residency.py +++ b/python/sglang/multimodal_gen/test/unit/test_auto_residency.py @@ -7,18 +7,24 @@ from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType from sglang.multimodal_gen.configs.pipeline_configs.longlive2 import LongLive2T2VConfig +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams from sglang.multimodal_gen.runtime.managers.memory_managers.auto_residency import ( ACTIVATION_EXTRAPOLATION_MARGIN, GIB_BYTES, WarmupMemoryRecord, estimate_default_workload_peak_bytes, ) +from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.warmup_request_builder import ( SERVER_WARMUP_MAX_VIDEO_FRAMES, + SERVER_WARMUP_VIDEO_STEPS, _resolve_auto_residency_warmup_shape, _resolve_warmup_num_frames, + build_warmup_reqs, ) +PLANNER_UNAVAILABLE = "residency planner not available" + def _record( *, @@ -455,8 +461,9 @@ def test_fixed_loading_paths_still_calibrate_other_components( ): monkeypatch.delenv("SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY", raising=False) monkeypatch.delenv("SGLANG_CACHE_DIT_ENABLED", raising=False) + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) reason = self._skip_reason(self._base_args(**overrides)) - assert reason is None or reason == "requires CUDA" + assert reason == PLANNER_UNAVAILABLE def test_cache_dit_excluded(self, monkeypatch): monkeypatch.delenv("SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY", raising=False) @@ -472,15 +479,53 @@ def test_cache_dit_excluded(self, monkeypatch): {"use_fsdp_inference": True}, ], ) - def test_parallel_paths_reach_platform_gate(self, monkeypatch, overrides): + def test_parallel_paths_reach_planner_gate(self, monkeypatch, overrides): monkeypatch.delenv("SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY", raising=False) monkeypatch.delenv("SGLANG_CACHE_DIT_ENABLED", raising=False) + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) reason = self._skip_reason(self._base_args(**overrides)) - assert reason is None or reason == "requires CUDA" + assert reason == PLANNER_UNAVAILABLE - def test_eligible_path_reaches_platform_gate(self, monkeypatch): + def test_eligible_path_reaches_planner_gate(self, monkeypatch): monkeypatch.delenv("SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY", raising=False) monkeypatch.delenv("SGLANG_CACHE_DIT_ENABLED", raising=False) + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) reason = self._skip_reason(self._base_args()) - # on a CUDA host everything passes; CPU CI stops at the platform gate - assert reason is None or reason == "requires CUDA" + assert reason == PLANNER_UNAVAILABLE + + def test_eligible_server_warmup_runs_no_probe_or_step_floor(self, monkeypatch): + # no calibration consumer yet: one bounded warmup, not probe + 4-step floor + monkeypatch.delenv("SGLANG_DIFFUSION_DISABLE_AUTO_RESIDENCY", raising=False) + monkeypatch.delenv("SGLANG_CACHE_DIT_ENABLED", raising=False) + monkeypatch.setattr(current_platform, "is_cuda", lambda: True) + args = self._base_args( + warmup_steps=1, + enable_cfg_parallel=False, + num_gpus=1, + pipeline_config=SimpleNamespace( + task_type=ModelTaskType.T2V, + supports_auto_residency=True, + adjust_num_frames=lambda value: value, + vae_stride=None, + vae_scale_factor=None, + vae_config=SimpleNamespace(arch_config=None), + ), + is_arg_explicitly_set=lambda _name: False, + ) + monkeypatch.setattr( + "sglang.multimodal_gen.runtime.warmup_request_builder.get_model_sampling_defaults", + lambda _args: SamplingParams( + width=1280, + height=720, + num_frames=81, + num_inference_steps=35, + adjust_frames=False, + supported_resolutions=[(1280, 720), (832, 480)], + ), + ) + reqs = build_warmup_reqs( + args, warmup_resolutions=None, server_based_warmup=True + ) + assert len(reqs) == 1 + assert not reqs[0].extra.get("auto_residency_full_shape_probe") + assert reqs[0].num_inference_steps == SERVER_WARMUP_VIDEO_STEPS