diff --git a/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx b/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx index c1bdfc424146..4da75d45ca14 100644 --- a/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx +++ b/docs/cookbook/diffusion/LongLive/LongLive-2.0.mdx @@ -25,6 +25,11 @@ Please refer to the [official SGLang-diffusion installation guide](/docs/sglang- sglang serve --model-path Rabinovich/LongLive-2.0-5B-Diffusers ``` +In `auto` mode, GPUs with at least 60 GiB available keep the DiT, text encoder, +and VAE resident. This uses about 44 GiB on H200 for the 832x480 preset and +avoids moving encoder and decoder layers from host memory on every request. +Smaller GPUs retain the layerwise-offload defaults. + If the GPU runs out of memory, move the text encoder, VAE, and DiT to CPU between stages: ```bash Command diff --git a/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py b/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py index dcd23479fdea..209a484ab877 100644 --- a/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py +++ b/python/sglang/multimodal_gen/configs/pipeline_configs/longlive2.py @@ -6,6 +6,9 @@ from sglang.multimodal_gen.configs.models import DiTConfig from sglang.multimodal_gen.configs.models.dits.longlive2 import LongLive2VideoConfig from sglang.multimodal_gen.configs.pipeline_configs.base import ModelTaskType +from sglang.multimodal_gen.configs.pipeline_configs.model_deployment_config import ( + ModelDeploymentConfig, +) from sglang.multimodal_gen.configs.pipeline_configs.wan import Wan2_2_TI2V_5B_Config from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -28,6 +31,13 @@ class LongLive2T2VConfig(Wan2_2_TI2V_5B_Config): dit_config: DiTConfig = field(default_factory=LongLive2VideoConfig) + def get_model_deployment_config(self) -> ModelDeploymentConfig: + return ModelDeploymentConfig( + dit_layerwise_offload_modes=("memory",), + keep_resident_min_available_gb=60, + keep_resident_components=("dit", "text_encoder", "vae"), + ) + def adjust_num_frames(self, num_frames: int) -> int: num_frames = super().adjust_num_frames(num_frames) vae_scale_factor_temporal = self.vae_config.arch_config.scale_factor_temporal diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index 60cc7373b8ca..634f9c186bca 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -2713,8 +2713,8 @@ "expected_e2e_ms": 5648.49, "expected_avg_denoise_ms": 477.49, "expected_median_denoise_ms": 56.74, - "load_peak_vram_mb": 16110.0, - "runtime_peak_vram_mb": 45756.0, + "load_peak_vram_mb": 32892.0, + "runtime_peak_vram_mb": 61190.0, "estimated_full_test_time_s": 153.1 }, "longlive2_i2v": { @@ -2731,8 +2731,8 @@ "expected_e2e_ms": 9086.81, "expected_avg_denoise_ms": 492.02, "expected_median_denoise_ms": 151.9, - "load_peak_vram_mb": 16110.0, - "runtime_peak_vram_mb": 45756.0, + "load_peak_vram_mb": 32892.0, + "runtime_peak_vram_mb": 62510.0, "estimated_full_test_time_s": 149.4 }, "lingbot_video_moe_t2v": { diff --git a/python/sglang/multimodal_gen/test/unit/test_server_args.py b/python/sglang/multimodal_gen/test/unit/test_server_args.py index 6a73359c95d1..8bfe3ca298ea 100644 --- a/python/sglang/multimodal_gen/test/unit/test_server_args.py +++ b/python/sglang/multimodal_gen/test/unit/test_server_args.py @@ -23,6 +23,9 @@ from sglang.multimodal_gen.configs.pipeline_configs.longcat_image import ( LongCatImagePipelineConfig, ) +from sglang.multimodal_gen.configs.pipeline_configs.longlive2 import ( + LongLive2T2VConfig, +) from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import ( LTX2PipelineConfig, LTX23PipelineConfig, @@ -1389,6 +1392,7 @@ def test_pipeline_configs_declare_auto_tune_hints(self): lingbot_deployment = LingBotWorldCausalDMDConfig().get_model_deployment_config() ltx_deployment = LTX2PipelineConfig().get_model_deployment_config() ltx23_config = LTX23PipelineConfig() + longlive_deployment = LongLive2T2VConfig().get_model_deployment_config() sana_wm_deployment = SanaWMPipelineConfig().get_model_deployment_config() self.assertIsNone(qwen_deployment.fsdp_auto_min_available_memory_gb) @@ -1423,6 +1427,11 @@ def test_pipeline_configs_declare_auto_tune_hints(self): self.assertEqual(ltx_deployment.get_auto_cfg_parallel_degree(4), 1) self.assertEqual(ltx_deployment.get_auto_cfg_parallel_degree(8), 1) self.assertEqual(ltx_deployment.get_auto_cfg_parallel_degree(2), 2) + self.assertEqual(longlive_deployment.keep_resident_min_available_gb, 60) + self.assertEqual( + longlive_deployment.keep_resident_components, + ("dit", "text_encoder", "vae"), + ) self.assertFalse( LTX2PipelineConfig().dit_config.arch_config.enable_packed_qkv_input_a2a ) @@ -1455,6 +1464,25 @@ def test_pipeline_configs_declare_auto_tune_hints(self): self.assertEqual(qwen_deployment.keep_resident_components, ("vae",)) self.assertIsNone(qwen_deployment.keep_resident_min_available_gb) + def test_longlive_residency_scales_with_available_memory(self): + high_memory_args = self._from_dict_with_pipeline_config( + LongLive2T2VConfig(), + memory_gb=80, + kwargs={"performance_mode": "auto"}, + ) + high_memory_offload = high_memory_args.layerwise_offload_components or [] + self.assertNotIn("text_encoder", high_memory_offload) + self.assertNotIn("vae", high_memory_offload) + + constrained_args = self._from_dict_with_pipeline_config( + LongLive2T2VConfig(), + memory_gb=50, + kwargs={"performance_mode": "auto"}, + ) + constrained_offload = constrained_args.layerwise_offload_components or [] + self.assertIn("text_encoder", constrained_offload) + self.assertIn("vae", constrained_offload) + def test_qwen_ar_generation_residency_scales_with_available_memory(self): pipeline_configs = ( QwenImageLayeredPipelineConfig(),