diff --git a/docs/source/models/visual-generation.md b/docs/source/models/visual-generation.md index ee5185ccaa42..941e7df4b473 100644 --- a/docs/source/models/visual-generation.md +++ b/docs/source/models/visual-generation.md @@ -18,6 +18,7 @@ TensorRT-LLM **VisualGen** provides a unified inference stack for diffusion mode - Sparse attention support: see [VisualGen Sparse Attention](../visual-gen/features/sparse-attention.md). - Multi-GPU parallelism (CFG parallel, Ulysses sequence parallel, Tensor parallelism). - **Step caching** — two runtime caching backends (**TeaCache** and **Cache-DiT**) that skip transformer computation on steps where the step-to-step change is small. +- CPU offloading to reduce peak GPU memory usage. - `trtllm-serve` integration with OpenAI-compatible API endpoints for image and video generation. ## Supported Models @@ -54,21 +55,21 @@ Models are auto-detected from the checkpoint directory. Diffusers-format models ### Feature Matrix -| Model | FP8 blockwise | NVFP4 | TeaCache | Cache-DiT | CFG Parallelism | Ulysses Parallelism | Parallel VAE | CUDA Graph | torch.compile | trtllm-serve | Attention2D | Ring Attention | Tensor Parallelism | VSA | -|---|---|---|---|---|---|---|---|---|---|---|--|--|--|--| -| **FLUX.1** | Yes | Yes | Yes | Yes | No [^1] | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | No | -| **FLUX.2** | Yes | Yes | Yes | Yes | No [^1] | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | No | -| **Wan 2.1** | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | -| **Wan 2.1 VSA** [^2] | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | -| **Wan 2.2** | Yes | Yes | Yes [^3] | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | -| **FastWan 2.2** | Yes | Yes | No | No | No [^7] | No | No | Yes | Yes | Yes | No | No | No | No | -| **LTX-2** | Yes | Yes | Yes [^4] | Yes | Yes | Yes | No | No | Yes | Yes | Yes | Yes | No | No | -| **Qwen-Image** | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | Yes | Yes | No | No | -| **Qwen-Image-Layered** [^6] | No | No | No | No | No | No | No | Yes | Yes | Yes | No | No | No | No | -| **Qwen-Image-Edit-2511** | Yes | Yes | No | No | Yes | No | No | Yes | Yes | Yes | No | No | No | No | -| **Cosmos3** | Yes | Yes | No | No | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | No | -| **HunyuanVideo 1.5** | Yes | Yes | No | No | No | No | No | No | No | Yes | No | No | No | No | -| **GlmImage** | Yes | Yes | No | No | No | No | No | No | No | Yes | No | No | No | No | +| Model | FP8 blockwise | NVFP4 | TeaCache | Cache-DiT | CPU Offloading | CFG Parallelism | Ulysses Parallelism | Parallel VAE | CUDA Graph | torch.compile | trtllm-serve | Attention2D | Ring Attention | Tensor Parallelism | VSA | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| +| **FLUX.1** | Yes | Yes | Yes | Yes | No | No [^1] | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | No | +| **FLUX.2** | Yes | Yes | Yes | Yes | No | No [^1] | Yes | No | Yes | Yes | Yes | Yes | Yes | Yes | No | +| **Wan 2.1** | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | +| **Wan 2.1 VSA** [^2] | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | +| **Wan 2.2** | Yes | Yes | Yes [^3] | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | +| **FastWan 2.2** | Yes | Yes | No | No | No | No [^7] | No | No | Yes | Yes | Yes | No | No | No | No | +| **LTX-2** | Yes | Yes | Yes [^4] | Yes | No | Yes | Yes | No | No | Yes | Yes | Yes | Yes | No | No | +| **Qwen-Image** | Yes | Yes | Yes | Yes | No | Yes | Yes | No | Yes | Yes | Yes | Yes | Yes | No | No | +| **Qwen-Image-Layered** [^6] | No | No | No | No | No | No | No | No | Yes | Yes | Yes | No | No | No | No | +| **Qwen-Image-Edit-2511** | Yes | Yes | No | No | No | Yes | No | No | Yes | Yes | Yes | No | No | No | No | +| **Cosmos3** | Yes | Yes | No | No | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | No | +| **HunyuanVideo 1.5** | Yes | Yes | No | No | No | No | No | No | No | No | Yes | No | No | No | No | +| **GlmImage** | Yes | Yes | No | No | No | No | No | No | No | No | Yes | No | No | No | No | [^1]: FLUX models use embedded guidance and do not have a separate negative prompt path, so CFG parallelism is not applicable. @@ -317,6 +318,9 @@ attention_config: vsa_sparsity: 0.90 ``` +### CPU Offloading + +CPU offloading stages move selected Wan and Cosmos3 T2V pipeline components between CPU and GPU to reduce peak GPU memory usage; enable it with `cpu_offload_config.enable: true`. ### Multi-GPU Parallelism @@ -330,6 +334,7 @@ Configured under `VisualGenArgs.parallel_config`. Modes can be combined: - **Attention2D** (`attn2d_size: [N, M]`): Shards the sequence axis across an `N × M` device mesh (CP degree = `N · M`; total SP degree = `N · M · ulysses_size`). - **Ring Attention** (`ring_size: N`): Shards the sequence axis across a 1D ring of `N` ranks, streaming K/V blocks (CP degree = `N`; total SP degree = `N · ulysses_size`; mutually exclusive with Attention2D). - **Tensor Parallelism** (`tp_size: N`): Splits attention heads and transformer MLPs across GPUs for faster compute and reduced memory usage. + ## Developer Guide ### Architecture Overview diff --git a/tensorrt_llm/_torch/modules/linear.py b/tensorrt_llm/_torch/modules/linear.py index 8c326ee9db64..23fbca3ad9ec 100644 --- a/tensorrt_llm/_torch/modules/linear.py +++ b/tensorrt_llm/_torch/modules/linear.py @@ -898,11 +898,14 @@ def rescale_fused_weights(self, module: Linear): """ weight_mode = module.weights_loading_config.weight_mode shard_key_to_index = weight_mode.shard_key_to_index + compute_device = module.weight.device + if compute_device.type == "cpu" and torch.cuda.is_available(): + compute_device = torch.device("cuda") # Handle input_scale if hasattr(module, "has_static_input_scale"): # Compute max and replace input_scale with a new parameter - max_input_scale = module.tmp_input_scales.max() + max_input_scale = module.tmp_input_scales.to(compute_device).max() module.input_scale.data.copy_(max_input_scale) # Also update inv_input_scale for static quantization if hasattr( @@ -916,13 +919,13 @@ def rescale_fused_weights(self, module: Linear): module.inv_input_scale = None # Compute max weight_scale - max_weight_scale = module.tmp_weight_scales.max() + max_weight_scale = module.tmp_weight_scales.to(compute_device).max() module.weight_scale.data.copy_(max_weight_scale) # Rescale each weight shard: (weight * original_scale) / max_scale for shard_key in weight_mode.shard_keys: idx = shard_key_to_index[shard_key] - original_scale = module.tmp_weight_scales[idx] + original_scale = module.tmp_weight_scales[idx].to(compute_device) # Get shard position from mapping shard_offset, shard_size = module.fused_weight_shard_indices_mapping[ @@ -931,12 +934,14 @@ def rescale_fused_weights(self, module: Linear): # Rescale: FP8 -> BF16 -> multiply by original_scale -> divide by max_scale -> FP8 weight_shard = module.weight.data[shard_offset:shard_offset + shard_size] - rescaled_weight = weight_shard.to(module.dtype).mul_(original_scale) + rescaled_weight = weight_shard.to(compute_device).to( + module.dtype).mul_(original_scale) rescaled_weight = rescaled_weight.div_( max_weight_scale.to(rescaled_weight.device)).to( torch.float8_e4m3fn) module.weight.data[shard_offset:shard_offset + - shard_size] = rescaled_weight + shard_size] = rescaled_weight.to( + module.weight.device) delattr(module, "tmp_input_scales") delattr(module, "tmp_weight_scales") diff --git a/tensorrt_llm/_torch/visual_gen/config.py b/tensorrt_llm/_torch/visual_gen/config.py index bc89766c680f..26ad53454c9d 100644 --- a/tensorrt_llm/_torch/visual_gen/config.py +++ b/tensorrt_llm/_torch/visual_gen/config.py @@ -35,6 +35,7 @@ CacheConfig, CacheDiTConfig, CompilationConfig, + CpuOffloadConfig, CudaGraphConfig, ParallelConfig, RuntimeLoRAConfig, @@ -109,6 +110,7 @@ class DiffusionModelConfig(_VisualGenConfigBase): # Unified parallelism mapping copied from the owning pipeline config. visual_gen_mapping: Optional[Any] = None # VisualGenMapping (lazy import) + device: str = "cuda" dynamic_weight_quant: bool = False @@ -119,6 +121,7 @@ class DiffusionModelConfig(_VisualGenConfigBase): compilation: CompilationConfig = PydanticField(default_factory=CompilationConfig) torch_compile: TorchCompileConfig = PydanticField(default_factory=TorchCompileConfig) cuda_graph: CudaGraphConfig = PydanticField(default_factory=CudaGraphConfig) + cpu_offload_config: CpuOffloadConfig = PydanticField(default_factory=CpuOffloadConfig) attention: AttentionConfig = PydanticField(default_factory=AttentionConfig) attention_metadata_state: Optional[Dict[str, Any]] = None parallel: ParallelConfig = PydanticField(default_factory=ParallelConfig) @@ -174,6 +177,7 @@ class DiffusionPipelineConfig(_VisualGenConfigBase): # Unified parallelism mapping (populated by setup_visual_gen_mapping) visual_gen_mapping: Optional[Any] = None # VisualGenMapping (lazy import) + device: str = "cuda" dynamic_weight_quant: bool = False @@ -184,6 +188,7 @@ class DiffusionPipelineConfig(_VisualGenConfigBase): compilation: CompilationConfig = PydanticField(default_factory=CompilationConfig) torch_compile: TorchCompileConfig = PydanticField(default_factory=TorchCompileConfig) cuda_graph: CudaGraphConfig = PydanticField(default_factory=CudaGraphConfig) + cpu_offload_config: CpuOffloadConfig = PydanticField(default_factory=CpuOffloadConfig) attention: AttentionConfig = PydanticField(default_factory=AttentionConfig) attention_metadata_state: Optional[Dict[str, Any]] = None parallel: ParallelConfig = PydanticField(default_factory=ParallelConfig) @@ -242,12 +247,14 @@ def _make_model_config( extra_attrs=_model_config_value(self.extra_attrs), # Topology mappings carry distributed process-group handles. visual_gen_mapping=_model_config_value(self.visual_gen_mapping, deep_copy=False), + device=_model_config_value(self.device), dynamic_weight_quant=_model_config_value(self.dynamic_weight_quant), quant_config=_model_config_value(self.quant_config), quant_config_dict=_model_config_value(self.quant_config_dict), compilation=_model_config_value(self.compilation), torch_compile=_model_config_value(self.torch_compile), cuda_graph=_model_config_value(self.cuda_graph), + cpu_offload_config=_model_config_value(self.cpu_offload_config), attention=_model_config_value(self.attention), attention_metadata_state=_model_config_value(self.attention_metadata_state), parallel=_model_config_value(self.parallel), @@ -462,8 +469,9 @@ def from_pretrained( Args: checkpoint_dir: Path to checkpoint args: VisualGenArgs containing user config - - (compilation, torch_compile, cuda_graph, pipeline, attention, parallel, teacache, - cache_backend, cache_dit) + - (compilation_config, torch_compile_config, cuda_graph_config, + cpu_offload_config, pipeline_config, attention_config, parallel_config, + cache_config) **kwargs: Additional config options (e.g., mapping) """ kwargs.pop("trust_remote_code", None) @@ -472,6 +480,7 @@ def from_pretrained( compilation_cfg = args.compilation_config if args else CompilationConfig() torch_compile_cfg = args.torch_compile_config if args else TorchCompileConfig() cuda_graph_cfg = args.cuda_graph_config if args else CudaGraphConfig() + offload_cfg = args.cpu_offload_config if args else CpuOffloadConfig() attention_cfg = args.attention_config if args else AttentionConfig() parallel_cfg = args.parallel_config if args else ParallelConfig() cache_cfg = args.cache_config if args else None @@ -622,6 +631,8 @@ def from_pretrained( create_attention_metadata_state() if attention_cfg.backend == "TRTLLM" else None ) + device = kwargs.pop("device", "cuda") + pipeline_config = cls( quant_config=quant_config, quant_config_dict=quant_config_dict, @@ -631,10 +642,12 @@ def from_pretrained( compilation=compilation_cfg, torch_compile=torch_compile_cfg, cuda_graph=cuda_graph_cfg, + cpu_offload_config=offload_cfg, attention=attention_cfg, attention_metadata_state=attention_metadata_state, parallel=parallel_cfg, cache=cache_cfg, + device=device, runtime_lora=runtime_lora_cfg, enable_layerwise_nvtx_marker=enable_layerwise_nvtx_marker, skip_create_weights_in_init=True, diff --git a/tensorrt_llm/_torch/visual_gen/executor.py b/tensorrt_llm/_torch/visual_gen/executor.py index 00ba095e7ad3..050d09cdafbd 100644 --- a/tensorrt_llm/_torch/visual_gen/executor.py +++ b/tensorrt_llm/_torch/visual_gen/executor.py @@ -227,6 +227,21 @@ def _detect_external_launch() -> Optional[Tuple[int, int, int, str, int]]: return None +def _cuda_memory_logging_enabled() -> bool: + """Whether per-request CUDA peak-memory logging is enabled. + + This is a development-only knob exposed as an environment variable + (rather than a public ``VisualGenArgs`` field) to keep the engine + config surface clean, mirroring the nsys trace knob + ``TLLM_PROFILE_VISUAL_GEN_START_STOP``. + """ + return os.environ.get("TLLM_VISUAL_GEN_LOG_CUDA_MEMORY", "0").strip().lower() in ( + "1", + "true", + "yes", + ) + + @dataclass class DiffusionRequest: """Request for diffusion inference. @@ -437,6 +452,9 @@ def _merge_defaults(self, req: DiffusionRequest): def process_request(self, req: DiffusionRequest): """Process a single request.""" + log_cuda_memory = _cuda_memory_logging_enabled() + if log_cuda_memory: + self._reset_cuda_peak_memory_stats() try: self._merge_defaults(req) # Include request preparation in executor-side generation latency. @@ -458,7 +476,10 @@ def process_request(self, req: DiffusionRequest): f"Warmed-up shapes: {self.pipeline._warmed_up_shapes}" ) output = self.pipeline.run_inference(req) + if log_cuda_memory: + self._log_cuda_peak_memory(req.request_id) generation = time.perf_counter() - generation_start # seconds + if self.rank == 0: # CUDA IPC handles are invalid within the producing process, so # a same-process client takes the media via in-process handoff. @@ -471,6 +492,8 @@ def process_request(self, req: DiffusionRequest): ) ) except Exception as e: + if log_cuda_memory: + self._log_cuda_peak_memory(req.request_id) logger.error(f"Worker {self.device_id}: Error: {e}") logger.error(traceback.format_exc()) if self.rank == 0: @@ -482,6 +505,36 @@ def process_request(self, req: DiffusionRequest): ) ) + def _reset_cuda_peak_memory_stats(self) -> None: + """Reset CUDA peak memory stats for this worker if CUDA is available.""" + if not torch.cuda.is_available(): + return + + try: + torch.cuda.reset_peak_memory_stats(self.device_id) + except RuntimeError as e: + logger.warning( + f"Worker {self.device_id} rank {self.rank}: " + f"Unable to reset CUDA peak memory stats: {e}" + ) + + def _log_cuda_peak_memory(self, request_id: int) -> None: + """Log peak CUDA memory observed for one request.""" + if not torch.cuda.is_available(): + return + + try: + peak_allocated = torch.cuda.max_memory_allocated(self.device_id) + logger.info( + f"Worker {self.device_id} rank {self.rank}: " + f"Request {request_id} peak CUDA memory: {peak_allocated / 2**30:.2f} GiB" + ) + except RuntimeError as e: + logger.warning( + f"Worker {self.device_id} rank {self.rank}: " + f"Unable to log CUDA peak memory for request {request_id}: {e}" + ) + def run_diffusion_worker( rank: int, diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py index 494a415ad99a..a78e1bfba4a5 100644 --- a/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/pipeline_cosmos3.py @@ -22,6 +22,7 @@ import PIL.Image import torch +import torch.nn as nn from diffusers.utils.torch_utils import randn_tensor from diffusers.video_processor import VideoProcessor from transformers import AutoTokenizer @@ -136,6 +137,22 @@ def default_negative_prompt(output_type: str) -> str: TRTLLM_DISABLE_COSMOS3_GUARDRAILS = os.environ.get("TRTLLM_DISABLE_COSMOS3_GUARDRAILS", "0") == "1" +# Public offload component names for the two transformer towers. The "reasoner" +# (understanding) pathway is the causal language model that processes text; the +# "generator" (generation) pathway is the stack of cross-attention layers that +# produces video tokens. Only the heavy decoder-layer ModuleLists are offloaded; +# the small shared embeddings/projections/norms stay resident on GPU. +COSMOS3_REASONER_OFFLOAD_COMPONENT = "reasoner" +COSMOS3_GENERATOR_OFFLOAD_COMPONENT = "generator" +# Opt-in guardrail offload components (rank 0 only): the Qwen3Guard text checker and +# the RetinaFace video face-blur model. Kept out of the CPU defaults below. +COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT = "text_guardrail" +COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT = "video_guardrail" +_COSMOS3_DEFAULT_OFFLOAD_STAGES = ( + (COSMOS3_REASONER_OFFLOAD_COMPONENT,), + (COSMOS3_GENERATOR_OFFLOAD_COMPONENT,), +) + # ``W,H`` bucket names the reference builds requests from. A request there starts # as a (resolution, bucket) pair and the bucket string is carried into the prompt # verbatim; we only ever see the resolved height/width, so map back to the nearest @@ -355,6 +372,69 @@ def _init_transformer(self) -> None: model_config = self.pipeline_config.model_configs["transformer"] self.transformer = Cosmos3VFMTransformer(model_config) + # ========================================================================= + # Offloading + # ========================================================================= + + def default_offload_stages(self) -> tuple[tuple[str, ...], ...]: + """Offload the reasoner and generator towers as separate stages. + + Only invoked when ``cpu_offload_config.enable`` is true (the base class + short-circuits before calling this). Stages are held in CPU storage + while inactive and moved onto the pipeline GPU when active. + """ + return _COSMOS3_DEFAULT_OFFLOAD_STAGES + + def offload_pipeline_components(self) -> dict[str, nn.Module]: + """Expose the two transformer towers, VAE, and guardrails as offload components. + + Cosmos3 packs both pathways into a single ``transformer`` module, so the + default ``BasePipeline.offload_pipeline_components`` (which looks for a + ``transformer.blocks`` ModuleList) does not apply. We expose the heavy + decoder-layer ModuleLists of each tower individually so they can be + brought on/off the GPU independently. The opt-in guardrail components wrap + the underlying safety nn.Modules (loaded on rank 0 only). + """ + components: dict[str, nn.Module] = {} + + transformer = getattr(self, "transformer", None) + if transformer is not None: + language_model = getattr(transformer, "language_model", None) + reasoner_layers = ( + getattr(language_model, "layers", None) if language_model is not None else None + ) + if reasoner_layers is not None: + components[COSMOS3_REASONER_OFFLOAD_COMPONENT] = reasoner_layers + + generator_layers = getattr(transformer, "gen_layers", None) + if generator_layers is not None: + components[COSMOS3_GENERATOR_OFFLOAD_COMPONENT] = generator_layers + + vae = getattr(self, PipelineComponent.VAE.value, None) + if vae is not None: + components[PipelineComponent.VAE.value] = vae + + # Guardrails (rank 0 only). CosmosSafetyChecker is an nn.Module but its + # GuardrailRunner children are plain objects, so expose the real safety + # nn.Modules (Qwen3Guard, RetinaFaceFilter) wrapped in a ModuleList. + safety_checker = getattr(self, "safety_checker", None) + if safety_checker is not None: + for component_name, runner in ( + (COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT, safety_checker.text_guardrail), + (COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT, safety_checker.video_guardrail), + ): + modules = [m for m in runner.models if isinstance(m, nn.Module)] + if modules: + components[component_name] = nn.ModuleList(modules) + + return components + + def extra_offload_component_names(self) -> set[str]: + # Guardrails load on rank 0 only; treat their names as valid on all ranks + # so explicit multi-GPU stages don't fail during validation. Other ranks + # drop them later via the offloader's stage filtering. + return {COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT, COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT} + def load_weights(self, weights: dict) -> None: if self.transformer is not None and hasattr(self.transformer, "load_weights"): transformer_weights = weights.get("transformer", weights) @@ -406,9 +486,14 @@ def load_standard_components( if PipelineComponent.VAE not in skip_components: logger.info("Loading VAE...") + vae_device = ( + torch.device("cpu") + if PipelineComponent.VAE.value in self.offloader.requested_components() + else device + ) self.vae = load_wan_vae( checkpoint_dir, - device, + vae_device, dtype=torch.bfloat16, ) @@ -1064,7 +1149,8 @@ def _decode_latents_raw(self, latents): scaling_factor = getattr(self.vae.config, "scaling_factor", 1.0) latents = latents / scaling_factor - return self.vae.decode(latents, return_dict=False)[0] + with self.offloader.context_if_requested(PipelineComponent.VAE.value): + return self.vae.decode(latents, return_dict=False)[0] @nvtx_range("_decode_latents", color="blue") def _decode_latents(self, latents): @@ -1107,7 +1193,8 @@ def _encode_video_tensor(self, video_tensor: torch.Tensor) -> torch.Tensor: ) video = video_tensor.to(device=self.device, dtype=self.vae.dtype) - latent = self.vae.encode(video).latent_dist.mode() + with self.offloader.context_if_requested(PipelineComponent.VAE.value): + latent = self.vae.encode(video).latent_dist.mode() if hasattr(self.vae.config, "latents_mean") and hasattr(self.vae.config, "latents_std"): latents_mean = ( @@ -1659,12 +1746,13 @@ def forward( prompts_to_check = list(prompt) if negative_prompt is not None: prompts_to_check.append(negative_prompt) - for p in prompts_to_check: - is_safe = self.safety_checker.check_text_safety(p) - if not is_safe: - logger.warning("Text guardrail blocked prompt") - text_blocked.fill_(1) - break + with self.offloader.context_if_requested(COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT): + for p in prompts_to_check: + is_safe = self.safety_checker.check_text_safety(p) + if not is_safe: + logger.warning("Text guardrail blocked prompt") + text_blocked.fill_(1) + break if torch.distributed.is_available() and torch.distributed.is_initialized(): torch.distributed.broadcast(text_blocked, src=0) @@ -2067,6 +2155,7 @@ def forward_fn( fps=frame_rate, noisy_frame_mask=velocity_mask, audio_latents=current_audio, + offload_context=self.offloader.context_if_requested, action_latents=current_action, action_domain_ids=action_domain_ids, action_noisy_mask=action_velocity_mask, @@ -2194,7 +2283,8 @@ def post_step_fn(step_latents, step_extra_stream_latents): logger.info(f"Total pipeline time: {time.time() - pipeline_start:.2f}s") if use_guardrails and self.safety_checker is not None: - video = check_video_safety(video, self.safety_checker) + with self.offloader.context_if_requested(COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT): + video = check_video_safety(video, self.safety_checker) timer.mark_end() @@ -2359,6 +2449,7 @@ def run_branch( text_ids=text_ids, text_mask=text_mask, control_latents=branch_control_latents, + offload_context=self.offloader.context_if_requested, **shared_kwargs, ) branch_caches[cache_key] = ( @@ -2796,7 +2887,8 @@ def _forward_transfer( # input tensor's device. video = host_output if self.rank == 0 and use_guardrails and self.safety_checker is not None: - video = check_video_safety(video, self.safety_checker) + with self.offloader.context_if_requested(COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT): + video = check_video_safety(video, self.safety_checker) # Back to the device the other pipelines hand back, now that denoising # has released its working set. if video is not None: diff --git a/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py b/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py index 4eb2ebd0dbd1..24124ebce4ce 100644 --- a/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py +++ b/tensorrt_llm/_torch/visual_gen/models/cosmos3/transformer_cosmos3.py @@ -14,8 +14,9 @@ # limitations under the License. import math +from contextlib import nullcontext from dataclasses import dataclass -from typing import Optional, Tuple, TypeVar +from typing import Callable, ContextManager, Optional, Tuple, TypeVar import torch import torch.nn as nn @@ -59,6 +60,11 @@ def apply_pretrained_config_compat_defaults( return pretrained_config +def _noop_offload_context(_tower_name: str) -> ContextManager: + """Default offload context: no staging, used when callers don't offload.""" + return nullcontext() + + COSMOS3_EDGE_BACKBONE_TYPE = "cosmos3_edge_nemotron_dense" @@ -1454,6 +1460,7 @@ def forward( text_ids: Optional[torch.Tensor] = None, text_mask: Optional[torch.Tensor] = None, video_shape: Optional[Tuple[int, int, int]] = None, + offload_context: Callable[[str], ContextManager] = _noop_offload_context, fps: float | None = None, noisy_frame_mask: torch.Tensor | None = None, audio_latents: Optional[torch.Tensor] = None, @@ -1477,6 +1484,12 @@ def forward( text_ids: [B, S_text] tokenized text input text_mask: [B, S_text] attention mask for text (1=real, 0=pad) video_shape: (T, H, W) in latent space + offload_context: Callable supplied by the pipeline that maps a tower + name ("reasoner" for the understanding pathway, "generator" for + the generation pathway) to a context manager staging that tower's + weights onto the GPU for the duration of its execution. The + pipeline returns a no-op context for towers that are not being + offloaded, so the transformer itself stays offload-agnostic. fps: video frame rate; when provided, temporal mRoPE positions are scaled to reflect real time (FPS modulation). noisy_frame_mask: Optional [B, 1, T, 1, 1] mask where 1=noisy (add @@ -1560,12 +1573,13 @@ def forward( num_vision_items=len(control_lantent_list) + 1, share_vision_temporal_positions=transfer_share_vision_temporal_positions, ) - cached_kv_full = self.language_model( - text_ids, - text_mask, - freqs_und, - timestep=timestep, - ) + with offload_context("reasoner"): + cached_kv_full = self.language_model( + text_ids, + text_mask, + freqs_und, + timestep=timestep, + ) self.cached_freqs_gen = freqs_gen if self.sharder.is_active: @@ -1708,27 +1722,28 @@ def forward( sin = self.sharder.shard(sin, dim=1, pad_to_multiple=True) freqs_gen = (cos, sin) - for i, layer in enumerate(self.gen_layers): - k_und, v_und = self.cached_kv[i] - if not self.sharder.is_active: - k_und = k_und[:, :max_real_len] - v_und = v_und[:, :max_real_len] - hidden_gen = layer( - hidden_gen, - k_und, - v_und, - freqs_gen, - timestep=timestep, - real_text_lens=real_text_lens, - ) - else: - hidden_gen = layer( - hidden_gen, - k_und, - v_und, - freqs_gen, - timestep=timestep, - ) + with offload_context("generator"): + for i, layer in enumerate(self.gen_layers): + k_und, v_und = self.cached_kv[i] + if not self.sharder.is_active: + k_und = k_und[:, :max_real_len] + v_und = v_und[:, :max_real_len] + hidden_gen = layer( + hidden_gen, + k_und, + v_und, + freqs_gen, + timestep=timestep, + real_text_lens=real_text_lens, + ) + else: + hidden_gen = layer( + hidden_gen, + k_und, + v_und, + freqs_gen, + timestep=timestep, + ) hidden_gen = self.sharder.gather(hidden_gen, dim=1, unpad_to=S_gen) diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py index f686b61753e4..148cf659da22 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan.py @@ -98,6 +98,16 @@ "fused fingers, motionless image, cluttered background, three legs, many people in the background, walking backward" ) +_WAN_SINGLE_TRANSFORMER_OFFLOAD_STAGES = ( + ("text_encoder",), + ("transformer",), +) +_WAN_TWO_TRANSFORMER_OFFLOAD_STAGES = ( + ("text_encoder",), + ("transformer",), + ("transformer_2",), +) + @register_pipeline( "WanPipeline", @@ -168,10 +178,10 @@ def dtype(self): @property def device(self): - return self.transformer.device + return super().device @property - def transformer_components(self) -> list: + def transformer_components(self) -> list[str]: """Return list of transformer components this pipeline needs.""" if self.transformer_2 is not None: return ["transformer", "transformer_2"] @@ -219,6 +229,17 @@ def _init_transformer(self) -> None: model_config=self.pipeline_config.model_configs["transformer_2"] ) + def default_offload_stages(self) -> tuple[tuple[str, ...], ...]: + """Return WAN offload stages for the loaded transformer topology. + + Only invoked when ``cpu_offload_config.enable`` is true (the base + class short-circuits before calling this). Stages are held in CPU + storage while inactive and moved onto the pipeline GPU when active. + """ + if self.transformer_2 is not None: + return _WAN_TWO_TRANSFORMER_OFFLOAD_STAGES + return _WAN_SINGLE_TRANSFORMER_OFFLOAD_STAGES + def load_standard_components( self, checkpoint_dir: str, @@ -255,17 +276,27 @@ def load_standard_components( if PipelineComponent.TEXT_ENCODER not in skip_components: logger.info("Loading text encoder...") + text_encoder_device = ( + torch.device("cpu") + if PipelineComponent.TEXT_ENCODER.value in self.offloader.requested_components() + else device + ) self.text_encoder = UMT5EncoderModel.from_pretrained( checkpoint_dir, subfolder=PipelineComponent.TEXT_ENCODER, torch_dtype=self.pipeline_config.torch_dtype, - ).to(device) + ).to(text_encoder_device) if PipelineComponent.VAE not in skip_components: logger.info("Loading VAE...") + vae_device = ( + torch.device("cpu") + if PipelineComponent.VAE.value in self.offloader.requested_components() + else device + ) self.vae = load_wan_vae( checkpoint_dir, - device, + vae_device, dtype=self.pipeline_config.torch_dtype, ) @@ -570,7 +601,9 @@ def forward_fn( # Extract scalar timestep current_t = timestep if timestep.dim() == 0 else timestep[0] - # Select model based on timestep (if two-stage denoising is enabled) + # Select the transformer for this timestep and keep its public + # component name paired for optional offload staging. + transformer_component_name = "transformer" if boundary_timestep is not None and self.transformer_2 is not None: if current_t >= boundary_timestep: current_model = self.transformer @@ -578,6 +611,7 @@ def forward_fn( else: current_model = self.transformer_2 model_name = "transformer_2 (low-noise)" + transformer_component_name = "transformer_2" # Log when switching models if last_model_used[0] != model_name: @@ -603,29 +637,30 @@ def forward_fn( # get_timestep_embedding requires a 1-D input). timestep = current_t.reshape(1).expand(latents.shape[0]) - if _vsa_active and _vsa_builder is not None: - # latents: [B, C, T_latent, H_latent, W_latent] - raw_latent_shape = (latents.shape[2], latents.shape[3], latents.shape[4]) - vsa_metadata = _vsa_builder.build( - current_timestep=_vsa_step_counter[0], - raw_latent_shape=raw_latent_shape, - patch_size=_vsa_patch_size, - vsa_sparsity=_vsa_sparsity, - device=latents.device, - ) - _vsa_step_counter[0] += 1 - with set_vsa_forward_context(vsa_metadata): - return current_model( - hidden_states=latents, - timestep=timestep / self.scheduler.config.num_train_timesteps, - encoder_hidden_states=encoder_hidden_states, + with self.offloader.context_if_requested(transformer_component_name): + if _vsa_active and _vsa_builder is not None: + # latents: [B, C, T_latent, H_latent, W_latent] + raw_latent_shape = (latents.shape[2], latents.shape[3], latents.shape[4]) + vsa_metadata = _vsa_builder.build( + current_timestep=_vsa_step_counter[0], + raw_latent_shape=raw_latent_shape, + patch_size=_vsa_patch_size, + vsa_sparsity=_vsa_sparsity, + device=latents.device, ) - - return current_model( - hidden_states=latents, - timestep=timestep / self.scheduler.config.num_train_timesteps, - encoder_hidden_states=encoder_hidden_states, - ) + _vsa_step_counter[0] += 1 + with set_vsa_forward_context(vsa_metadata): + return current_model( + hidden_states=latents, + timestep=timestep / self.scheduler.config.num_train_timesteps, + encoder_hidden_states=encoder_hidden_states, + ) + + return current_model( + hidden_states=latents, + timestep=timestep / self.scheduler.config.num_train_timesteps, + encoder_hidden_states=encoder_hidden_states, + ) # Pin reference image to latent after each scheduler step (Wan 2.2 5B I2V only). # post_step_fn also carries the denoise loop's side-stream latents (Cosmos3 @@ -684,7 +719,10 @@ def get_embeds(texts): input_ids = text_inputs.input_ids.to(self.device) attention_mask = text_inputs.attention_mask.to(self.device) - embeds = self.text_encoder(input_ids, attention_mask=attention_mask).last_hidden_state + with self.offloader.context_if_requested(PipelineComponent.TEXT_ENCODER.value): + embeds = self.text_encoder( + input_ids, attention_mask=attention_mask + ).last_hidden_state embeds = embeds.to(self.dtype) # Zero-out padded tokens based on mask @@ -775,9 +813,10 @@ def _prepare_latents_wan22_5B_i2v( ) # Encode video condition through VAE - latent_condition = retrieve_latents(self.vae.encode(image), sample_mode="argmax").to( - self.dtype - ) + with self.offloader.context_if_requested(PipelineComponent.VAE.value): + latent_condition = retrieve_latents(self.vae.encode(image), sample_mode="argmax").to( + self.dtype + ) if batch_size > 1: latent_condition = latent_condition.repeat(batch_size, 1, 1, 1, 1) @@ -833,7 +872,8 @@ def _decode_latents(self, latents: torch.Tensor) -> torch.Tensor: latents = latents / scaling_factor # VAE decode: returns (B, C, T, H, W) - video = self.vae.decode(latents, return_dict=False)[0] + with self.offloader.context_if_requested(PipelineComponent.VAE.value): + video = self.vae.decode(latents, return_dict=False)[0] # Post-process video tensor: (B, C, T, H, W) -> (B, T, H, W, C) video = postprocess_video_tensor(video) diff --git a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py index f83d5be29f1b..d8318fe02fe8 100644 --- a/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py +++ b/tensorrt_llm/_torch/visual_gen/models/wan/pipeline_wan_i2v.py @@ -157,7 +157,7 @@ def device(self): return self.transformer.device @property - def transformer_components(self) -> list: + def transformer_components(self) -> list[str]: if self.transformer_2 is not None: return ["transformer", "transformer_2"] return ["transformer"] diff --git a/tensorrt_llm/_torch/visual_gen/offloading.py b/tensorrt_llm/_torch/visual_gen/offloading.py new file mode 100644 index 000000000000..c7eff0ca7714 --- /dev/null +++ b/tensorrt_llm/_torch/visual_gen/offloading.py @@ -0,0 +1,698 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Component-offloading utilities for visual generation pipelines. + +Terminology used throughout this module: + +- **component**: a named, public sub-model of the pipeline (e.g. ``text_encoder``, + ``transformer``, ``vae``). +- **stage**: one step of the offload schedule — a single component, or a group + of components that are co-resident on the GPU and run together before being + evicted back to CPU. + +The offload path keeps model loading and quantization unchanged: weights are +loaded into the modules first, then the components of each stage are copied into +packed CPU storage. At runtime one stage at a time is brought onto a reusable GPU +arena and the original module parameters/buffers are rebound to views of that +storage. +""" + +import time +import weakref +from contextlib import AbstractContextManager, nullcontext +from dataclasses import dataclass +from typing import TYPE_CHECKING, Iterator, Mapping, Sequence + +import torch +import torch.nn as nn + +from tensorrt_llm.logger import logger + +if TYPE_CHECKING: + from tensorrt_llm._torch.visual_gen.pipeline import BasePipeline + + +def _align_offset(offset: int, alignment: int = 256) -> int: + return ((offset + alignment - 1) // alignment) * alignment + + +def transformer_component_offload_name(component_name: str, transformer_prefix: str) -> str: + """Map an internal transformer attribute to its public offload component name. + + Transformer component names follow the diffusers ``model_index.json`` keys + such as ``transformer`` and ``transformer_2``. + """ + del transformer_prefix + return component_name + + +def _format_bytes(num_bytes: int) -> str: + return f"{num_bytes / (1024**3):.2f} GiB" + + +# FlashInfer and other custom kernels can require tensor data pointers to be at +# least 16-byte aligned even for smaller dtypes such as BF16. +_PACKED_TENSOR_ALIGNMENT = 16 + + +OffloadPipelineStage = tuple[str, ...] + + +@dataclass +class _FlatTensorSpec: + owner: nn.Module + name: str + qualified_name: str + is_parameter: bool + shape: tuple[int, ...] + stride: tuple[int, ...] + dtype: torch.dtype + requires_grad: bool + persistent: bool + offset: int + nbytes: int + + +@dataclass +class _StageLayout: + """Packed storage layout and rebound views for one offload stage.""" + + name: str + nbytes: int + specs: list[_FlatTensorSpec] + cpu_storage: torch.Tensor | None = None + cpu_views: tuple[nn.Parameter | torch.Tensor, ...] = () + gpu_views: tuple[nn.Parameter | torch.Tensor, ...] = () + + +class ModuleOffloadManager: + """Pack component stages into CPU storage and bring one stage onto the GPU. + + The manager owns packed byte buffers: + - each layout owns ``cpu_storage`` for one offloaded stage. + - ``gpu_arena`` is reused for whichever stage is currently active. + + Initializing the manager packs and rebinds one stage at a time. This + requires enough host memory to allocate the current stage's packed CPU + storage before that stage's original tensors are released. + """ + + def __init__( + self, + stages: Mapping[str, nn.Module], + device: torch.device | str, + pin_memory: bool = True, + ) -> None: + if not stages: + raise ValueError("At least one offload stage must be provided") + + self.stages = dict(stages) + self.device = torch.device(device) + self.pin_memory = pin_memory + self.gpu_arena: torch.Tensor | None = None + self.layouts: dict[str, _StageLayout] = {} + self.active_stage_name: str | None = None + + for name, module in self.stages.items(): + if not name: + raise ValueError("Offload stage names must be non-empty") + if not isinstance(module, nn.Module): + raise TypeError(f"Offload stage '{name}' must contain an nn.Module") + + @staticmethod + def _owner_and_name(root: nn.Module, qualified_name: str) -> tuple[nn.Module, str]: + if "." not in qualified_name: + return root, qualified_name + module_path, name = qualified_name.rsplit(".", 1) + return root.get_submodule(module_path), name + + @staticmethod + def _tensor_nbytes(tensor: torch.Tensor) -> int: + return tensor.numel() * tensor.element_size() + + @staticmethod + def _storage_key(tensor: torch.Tensor) -> tuple[int, int] | None: + if tensor.numel() == 0: + return None + storage_offset_bytes = tensor.storage_offset() * tensor.element_size() + return tensor.untyped_storage().data_ptr(), storage_offset_bytes + + def _get_alias_spec( + self, + seen_tensors: dict[tuple[int, int], _FlatTensorSpec], + tensor: torch.Tensor, + display_name: str, + ) -> _FlatTensorSpec | None: + key = self._storage_key(tensor) + if key is None: + return None + canonical = seen_tensors.get(key) + if canonical is None: + return None + if self._tensor_nbytes(tensor) != canonical.nbytes or tensor.dtype != canonical.dtype: + raise ValueError( + "Shared parameters or buffers with different sizes or dtypes are " + f"not supported by ModuleOffloadManager: '{display_name}' aliases " + f"'{canonical.qualified_name}'" + ) + return canonical + + def _build_spec( + self, + stage_name: str, + stage_module: nn.Module, + qualified_name: str, + tensor: torch.Tensor, + is_parameter: bool, + offset: int, + ) -> _FlatTensorSpec: + display_name = f"{stage_name}.{qualified_name}" + if not tensor.is_contiguous(): + raise ValueError( + f"Cannot offload non-contiguous tensor '{display_name}' " + f"with stride {tuple(tensor.stride())}" + ) + + owner, name = self._owner_and_name(stage_module, qualified_name) + return _FlatTensorSpec( + owner=owner, + name=name, + qualified_name=display_name, + is_parameter=is_parameter, + shape=tuple(tensor.shape), + stride=tuple(tensor.stride()), + dtype=tensor.dtype, + requires_grad=tensor.requires_grad if is_parameter else False, + persistent=True if is_parameter else name not in owner._non_persistent_buffers_set, + offset=offset, + nbytes=self._tensor_nbytes(tensor), + ) + + def _iter_stage_tensors( + self, stage_module: nn.Module + ) -> Iterator[tuple[str, torch.Tensor, bool]]: + for qualified_name, param in stage_module.named_parameters( + recurse=True, + remove_duplicate=False, + ): + yield qualified_name, param.detach(), True + for qualified_name, buffer in stage_module.named_buffers( + recurse=True, + remove_duplicate=False, + ): + yield qualified_name, buffer.detach(), False + + def _append_layout_spec( + self, + stage_name: str, + stage_module: nn.Module, + qualified_name: str, + tensor: torch.Tensor, + is_parameter: bool, + offset: int, + seen_tensors: dict[tuple[int, int], _FlatTensorSpec], + specs: list[_FlatTensorSpec], + ) -> int: + """Append a tensor spec and return the next stage-local byte offset. + + This handles three layout concerns in one place: alias reuse, packed + tensor alignment, and spec construction. The offset is local to this + stage and is shared by the CPU storage and reusable GPU arena views. + """ + display_name = f"{stage_name}.{qualified_name}" + alias = self._get_alias_spec(seen_tensors, tensor, display_name) + if alias is None: + offset = _align_offset(offset, _PACKED_TENSOR_ALIGNMENT) + + spec = self._build_spec( + stage_name=stage_name, + stage_module=stage_module, + qualified_name=qualified_name, + tensor=tensor, + is_parameter=is_parameter, + offset=alias.offset if alias is not None else offset, + ) + specs.append(spec) + + if alias is not None: + return offset + + key = self._storage_key(tensor) + if key is not None: + seen_tensors[key] = spec + return offset + spec.nbytes + + def _collect_stage_layout(self, stage_name: str, stage_module: nn.Module) -> _StageLayout: + """Build the packed storage layout for one named component stage.""" + offset = 0 + specs: list[_FlatTensorSpec] = [] + seen_tensors: dict[tuple[int, int], _FlatTensorSpec] = {} + + for qualified_name, tensor, is_parameter in self._iter_stage_tensors(stage_module): + offset = self._append_layout_spec( + stage_name=stage_name, + stage_module=stage_module, + qualified_name=qualified_name, + tensor=tensor, + is_parameter=is_parameter, + offset=offset, + seen_tensors=seen_tensors, + specs=specs, + ) + + if not specs: + raise ValueError(f"Offload stage '{stage_name}' has no parameters or buffers") + + return _StageLayout( + name=stage_name, + nbytes=_align_offset(offset), + specs=specs, + ) + + def _copy_stage_to_cpu_storage(self, layout: _StageLayout) -> None: + if layout.cpu_storage is None: + raise RuntimeError( + f"CPU storage for offload stage '{layout.name}' has not been allocated" + ) + for spec in layout.specs: + if spec.nbytes == 0: + continue + try: + tensor = getattr(spec.owner, spec.name).detach() + tensor_bytes = tensor.reshape(-1).view(torch.uint8).cpu() + layout.cpu_storage.narrow(0, spec.offset, spec.nbytes).copy_(tensor_bytes) + except RuntimeError as e: + raise RuntimeError( + f"Failed to copy offload tensor '{spec.qualified_name}' " + f"({_format_bytes(spec.nbytes)}, shape={spec.shape}, dtype={spec.dtype}) " + f"to packed CPU storage at offset {spec.offset}." + ) from e + + def _stage_size_summary(self) -> str: + return ", ".join( + f"{name}={_format_bytes(layout.nbytes)}" for name, layout in self.layouts.items() + ) + + def _cuda_allocation_hint(self) -> str: + if self.device.type != "cuda": + return "" + return ( + " If this is due to CUDA memory fragmentation, try setting " + "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True before starting the process." + ) + + def _allocate_cpu_storage(self, num_bytes: int, stage_name: str | None = None) -> torch.Tensor: + try: + return torch.empty( + num_bytes, + dtype=torch.uint8, + device="cpu", + pin_memory=self.pin_memory, + ) + except RuntimeError as e: + stage_context = f", stage='{stage_name}'" if stage_name is not None else "" + raise RuntimeError( + "Failed to allocate packed CPU storage for visual generation offload " + f"({_format_bytes(num_bytes)}, {num_bytes} bytes{stage_context}, " + f"pin_memory={self.pin_memory}, stages=[{self._stage_size_summary()}])." + ) from e + + def _allocate_gpu_arena(self, num_bytes: int) -> torch.Tensor: + try: + return torch.empty(num_bytes, dtype=torch.uint8, device=self.device) + except RuntimeError as e: + raise RuntimeError( + "Failed to allocate GPU arena for visual generation offload " + f"({_format_bytes(num_bytes)}, {num_bytes} bytes, " + f"device={self.device}, stages=[{self._stage_size_summary()}])." + f"{self._cuda_allocation_hint()}" + ) from e + + def _make_views( + self, + layout: _StageLayout, + storage: torch.Tensor, + ) -> tuple[nn.Parameter | torch.Tensor, ...]: + views: list[nn.Parameter | torch.Tensor] = [] + for spec in layout.specs: + view = storage.narrow(0, spec.offset, spec.nbytes).view(spec.dtype) + view = view.as_strided(spec.shape, spec.stride) + if spec.is_parameter: + views.append(nn.Parameter(view, requires_grad=spec.requires_grad)) + else: + views.append(view) + return tuple(views) + + def _bind_views( + self, + layout: _StageLayout, + views: tuple[nn.Parameter | torch.Tensor, ...], + ) -> None: + for spec, view in zip(layout.specs, views, strict=True): + if spec.is_parameter: + if not isinstance(view, nn.Parameter): + raise TypeError( + f"Expected offload view '{spec.name}' to be an nn.Parameter, " + f"got {type(view).__name__}" + ) + spec.owner.register_parameter(spec.name, view) + else: + if not isinstance(view, torch.Tensor): + raise TypeError( + f"Expected offload view '{spec.name}' to be a torch.Tensor, " + f"got {type(view).__name__}" + ) + spec.owner.register_buffer(spec.name, view, persistent=spec.persistent) + + def initialize(self) -> None: + """Allocate packed storage, copy current tensors, and bind CPU views.""" + if self.layouts: + raise RuntimeError("ModuleOffloadManager has already been initialized") + + start_time = time.time() + for name, module in self.stages.items(): + layout = self._collect_stage_layout(name, module) + self.layouts[name] = layout + + total_cpu_bytes = sum(layout.nbytes for layout in self.layouts.values()) + max_gpu_bytes = max(layout.nbytes for layout in self.layouts.values()) + logger.info( + "Module offload storage layout: " + f"cpu_total={_format_bytes(total_cpu_bytes)}, " + f"gpu_arena={_format_bytes(max_gpu_bytes)}, " + f"stages=[{self._stage_size_summary()}], device={self.device}" + ) + + # Pack and rebind one stage at a time. This keeps setup simple and fast: + # offloading requires enough host memory to allocate one stage's packed + # CPU storage before that stage's original tensors are released. + for layout in self.layouts.values(): + logger.info( + "Module offload packing stage into CPU storage: " + f"{layout.name} ({_format_bytes(layout.nbytes)})" + ) + layout.cpu_storage = self._allocate_cpu_storage(layout.nbytes, stage_name=layout.name) + self._copy_stage_to_cpu_storage(layout) + layout.cpu_views = self._make_views(layout, layout.cpu_storage) + self._rebind_to_cpu(layout.name) + + self.gpu_arena = self._allocate_gpu_arena(max_gpu_bytes) + for layout in self.layouts.values(): + layout.gpu_views = self._make_views(layout, self.gpu_arena) + + logger.info(f"Module offload setup completed in {time.time() - start_time:.2f}s") + + def _get_layout(self, name: str) -> _StageLayout: + try: + return self.layouts[name] + except KeyError as e: + raise KeyError( + f"Unknown offload stage '{name}'. Available stages: {sorted(self.layouts)}" + ) from e + + def stage(self, name: str) -> None: + """Bring one offload stage onto the GPU arena and rebind its tensors.""" + layout = self._get_layout(name) + if self.active_stage_name == name: + return + if layout.cpu_storage is None or self.gpu_arena is None: + raise RuntimeError("ModuleOffloadManager must be initialized before staging") + + if self.active_stage_name is not None: + self._rebind_to_cpu(self.active_stage_name) + self.active_stage_name = None + + src = layout.cpu_storage.narrow(0, 0, layout.nbytes) + dst = self.gpu_arena.narrow(0, 0, layout.nbytes) + try: + dst.copy_(src, non_blocking=layout.cpu_storage.is_pinned()) + if self.device.type == "cuda": + torch.cuda.synchronize(self.device) + except RuntimeError as e: + raise RuntimeError( + f"Failed to bring offload stage '{name}' ({_format_bytes(layout.nbytes)}) " + f"onto {self.device}" + ) from e + self._rebind_to_gpu(name) + self.active_stage_name = name + + def _rebind_to_cpu(self, name: str) -> None: + layout = self._get_layout(name) + if not layout.cpu_views: + raise RuntimeError("ModuleOffloadManager must be initialized before staging") + self._bind_views(layout, layout.cpu_views) + + def _rebind_to_gpu(self, name: str) -> None: + layout = self._get_layout(name) + if not layout.gpu_views: + raise RuntimeError("ModuleOffloadManager must be initialized before staging") + self._bind_views(layout, layout.gpu_views) + + +class OffloadPipeline: + """Bring offload stages onto the GPU explicitly from model call-site contexts. + + This class intentionally does not use forward hooks. Pipeline code must wrap + the relevant call site with ``with self.offloader.context("stage")`` so staging + happens before the model invocation and outside any later CUDA graph capture. + """ + + def __init__( + self, + stages: Sequence[Sequence[str] | str], + components: Mapping[str, nn.Module], + device: torch.device | str, + pin_memory: bool = True, + ) -> None: + if not stages: + raise ValueError("At least one offload pipeline stage must be provided") + + self.stages = tuple( + (stage,) if isinstance(stage, str) else tuple(stage) for stage in stages + ) + self.components = dict(components) + self.device = torch.device(device) + self.pin_memory = pin_memory + self.manager = ModuleOffloadManager( + stages=self._build_stage_modules(), + device=self.device, + pin_memory=self.pin_memory, + ) + + def _build_stage_modules(self) -> dict[str, nn.Module]: + stage_modules: dict[str, nn.Module] = {} + for stage in self.stages: + stage_name = self._stage_name(stage) + if not stage: + raise ValueError("Offload pipeline stages must have at least one component") + if stage_name in stage_modules: + raise ValueError(f"Duplicate offload pipeline stage: {stage_name}") + + modules: list[nn.Module] = [] + for component_name in stage: + try: + component = self.components[component_name] + except KeyError as e: + raise KeyError( + f"Unknown offload pipeline component '{component_name}' for stage " + f"'{stage_name}'. Available components: {sorted(self.components)}" + ) from e + modules.append(component) + + stage_module = modules[0] if len(modules) == 1 else nn.ModuleList(modules) + stage_modules[stage_name] = stage_module + + return stage_modules + + def initialize(self) -> None: + """Allocate and populate backing storage for all configured stages.""" + self.manager.initialize() + + @staticmethod + def _stage_name(stage: Sequence[str] | str) -> str: + return stage if isinstance(stage, str) else "+".join(stage) + + def context(self, stage_name: str): + """Bring stage ``stage_name`` onto the GPU and return a no-op context manager.""" + self.manager.stage(stage_name) + # The active stage intentionally stays resident after the call site. + # The next stage() call rebinds it back to CPU before bringing another + # stage on, and cleanup() handles the final rebind when the pipeline exits. + return nullcontext() + + def cleanup(self) -> None: + """Return the active stage to CPU-backed views.""" + if self.manager.active_stage_name is not None: + self.manager._rebind_to_cpu(self.manager.active_stage_name) + self.manager.active_stage_name = None + + +class PipelineOffloader: + """Resolve, build, and drive component offloading for a visual-gen pipeline. + + Keeps ``BasePipeline`` free of offload bookkeeping: the pipeline only supplies + model-specific hooks (``offload_pipeline_components``, ``default_offload_stages``, + ``extra_offload_component_names``), while this coordinator resolves configured + stages, validates/filters them, builds the backing :class:`OffloadPipeline`, + and brings stages on/off the GPU at explicit call sites. + """ + + def __init__(self, pipeline: "BasePipeline") -> None: + # Hold the owning pipeline weakly: the pipeline owns this offloader + # (``pipeline.offloader``), so a strong back-reference would form a + # reference cycle and keep the whole pipeline (including its GPU + # weights) alive until a ``gc.collect()`` runs, defeating prompt + # release on ``del pipeline``. + self._pipeline_ref = weakref.ref(pipeline) + self._offload_pipeline: OffloadPipeline | None = None + self._stage_name_by_component: dict[str, str] = {} + + @property + def _pipeline(self) -> "BasePipeline": + pipeline = self._pipeline_ref() + if pipeline is None: + raise ReferenceError("PipelineOffloader's owning pipeline has already been released") + return pipeline + + @property + def offload_pipeline(self) -> OffloadPipeline | None: + """The backing :class:`OffloadPipeline`, or ``None`` if not initialized.""" + return self._offload_pipeline + + @staticmethod + def _normalize_stages( + stages: list[str | list[str]], + ) -> tuple[OffloadPipelineStage, ...]: + """Normalize user-configured offload stages to tuple form.""" + normalized_stages: list[OffloadPipelineStage] = [] + for stage in stages: + components = (stage,) if isinstance(stage, str) else tuple(stage) + if not components: + raise ValueError("Offload stages must contain at least one component") + normalized_stages.append(components) + return tuple(normalized_stages) + + def stages(self) -> tuple[OffloadPipelineStage, ...]: + """Return offload stages resolved from the pipeline configuration.""" + cpu_offload_config = getattr(self._pipeline.pipeline_config, "cpu_offload_config", None) + if cpu_offload_config is None or not cpu_offload_config.enable: + return () + configured_stages = getattr(cpu_offload_config, "stages", None) + if configured_stages is not None: + return self._normalize_stages(configured_stages) + return self._pipeline.default_offload_stages() + + def requested_components(self) -> set[str]: + """Return all component names referenced by the configured stages.""" + return {component for stage in self.stages() for component in stage} + + def validate_configured_stages( + self, + stages: tuple[OffloadPipelineStage, ...], + available_components: Mapping[str, nn.Module], + ) -> None: + """Validate user-configured stages against this pipeline's component names.""" + known = dict(available_components) + for name in self._pipeline.extra_offload_component_names(): + known.setdefault(name, None) + unknown_components = sorted( + {component for stage in stages for component in stage if component not in known} + ) + if unknown_components: + available_names = ", ".join(sorted(known)) or "" + unknown_names = ", ".join(unknown_components) + raise ValueError( + f"Unknown cpu_offload_config.stages entries for this model: {unknown_names}. " + f"Available components: {available_names}." + ) + + @staticmethod + def filter_available_stages( + stages: tuple[OffloadPipelineStage, ...], + available_components: Mapping[str, nn.Module], + ) -> tuple[OffloadPipelineStage, ...]: + """Drop stage components that are unavailable for the loaded pipeline.""" + return tuple( + tuple(component for component in stage if component in available_components) + for stage in stages + if any(component in available_components for component in stage) + ) + + def initialize(self) -> None: + """Create and initialize the offload pipeline after weights are loaded.""" + configured_stages = self.stages() + if not configured_stages or self._offload_pipeline is not None: + return + + pipeline = self._pipeline + available_components = pipeline.offload_pipeline_components() + cpu_offload_config = getattr(pipeline.pipeline_config, "cpu_offload_config", None) + has_user_stages = ( + cpu_offload_config is not None + and getattr(cpu_offload_config, "stages", None) is not None + ) + if has_user_stages: + self.validate_configured_stages(configured_stages, available_components) + + stages = self.filter_available_stages(configured_stages, available_components) + if not stages: + return + + if pipeline._cuda_graph_runners: + raise NotImplementedError( + "CUDA graphs are not supported with visual generation offloading yet. " + "Disable either cuda_graph_config.enable or cpu_offload_config.enable." + ) + + pin_memory = bool(getattr(cpu_offload_config, "pin_memory", True)) + stage_summary = " -> ".join("+".join(stage) for stage in stages) + logger.info(f"{pipeline.__class__.__name__} offload pipeline enabled: {stage_summary}") + offload_pipeline = OffloadPipeline( + stages=stages, + components=available_components, + device=torch.device(pipeline.device), + pin_memory=pin_memory, + ) + offload_pipeline.initialize() + self._offload_pipeline = offload_pipeline + self._stage_name_by_component = { + component: "+".join(stage) for stage in stages for component in stage + } + + def cleanup(self) -> None: + """Tear down the offload pipeline and reset state.""" + if self._offload_pipeline is not None: + self._offload_pipeline.cleanup() + self._offload_pipeline = None + self._stage_name_by_component = {} + + def context(self, component_name: str, enable: bool = True) -> AbstractContextManager: + """Bring the stage containing ``component_name`` onto the GPU for one call site.""" + if not enable: + return nullcontext() + if self.stages() and self._offload_pipeline is None: + raise RuntimeError( + "Visual generation offloading is configured but the offload pipeline " + "has not been initialized. Call initialize_offload_pipeline() after " + "loading weights." + ) + if self._offload_pipeline is None: + return nullcontext() + stage_name = self._stage_name_by_component.get(component_name, component_name) + return self._offload_pipeline.context(stage_name) + + def context_if_requested(self, component_name: str) -> AbstractContextManager: + """Return an offload context for ``component_name``, active only if configured.""" + return self.context(component_name, enable=component_name in self.requested_components()) diff --git a/tensorrt_llm/_torch/visual_gen/pipeline.py b/tensorrt_llm/_torch/visual_gen/pipeline.py index 2cf24173699c..d4978872e471 100644 --- a/tensorrt_llm/_torch/visual_gen/pipeline.py +++ b/tensorrt_llm/_torch/visual_gen/pipeline.py @@ -36,6 +36,7 @@ from .cuda_graph_runner import CUDAGraphRunner, CUDAGraphRunnerConfig, SharedGraphPool from .mapping import _VisualGenAutotuneDist from .modules.vae.parallel_vae_interface import ParallelVAEFactory +from .offloading import OffloadPipelineStage, PipelineOffloader, transformer_component_offload_name from .profiler import VisualGenProfiler @@ -97,7 +98,9 @@ def __init__(self, pipeline_config: "DiffusionPipelineConfig"): self.pipeline_config = pipeline_config self.config = pipeline_config.primary_pretrained_config self.mapping: Mapping = getattr(pipeline_config, "mapping", None) or Mapping() + self._device = torch.device(getattr(pipeline_config, "device", "cuda")) self._cuda_graph_runners: Dict[str, CUDAGraphRunner] = {} + self.offloader = PipelineOffloader(self) self._parallel_vae_enabled: bool = False self._warmed_up_shapes: Set[tuple] = set() self._runtime_lora_applications: List[Any] = [] @@ -160,6 +163,12 @@ def _setup_cuda_graphs(self): if not self.pipeline_config.cuda_graph.enable: return + if self.offloader.stages(): + raise NotImplementedError( + "CUDA graphs are not supported with visual generation offloading yet. " + "Disable either cuda_graph_config.enable or cpu_offload_config.enable." + ) + if len(self.transformer_components) > 1: logger.info( "CUDA graph runner: multiple transformer components, using shared graph pool" @@ -209,12 +218,12 @@ def dtype(self): @property def device(self): - return self.transformer.device + return self._device @property - def transformer_components(self) -> list: - """Return list of transformer components this pipeline needs.""" - return [PipelineComponent.TRANSFORMER] if self.transformer is not None else [] + def transformer_components(self) -> list[str]: + """Return names of transformer components this pipeline needs.""" + return [PipelineComponent.TRANSFORMER.value] if self.transformer is not None else [] def warmup_cache_key(self, height: int, width: int, num_frames: int) -> tuple: """Return the cache key for a given warmup shape. @@ -419,6 +428,59 @@ def post_load_weights(self) -> None: if self.transformer is not None and hasattr(self.transformer, "post_load_weights"): self.transformer.post_load_weights() + def default_offload_stages(self) -> tuple[OffloadPipelineStage, ...]: + """Return model-specific offload stages requested by shorthand config. + + Each stage is a tuple of component names from ``offload_pipeline_components``. + Components in the same stage are brought onto the GPU together. + """ + return () + + def extra_offload_component_names(self) -> set[str]: + """Return component names that are valid in configured stages but may be absent. + + These are accepted during stage validation even when missing from + ``offload_pipeline_components`` (e.g. rank-0-only components that other ranks + drop later). Defaults to an empty set; models override as needed. + """ + return set() + + def offload_pipeline_components(self) -> dict[str, nn.Module]: + """Expose standard component subtrees for offloading. + + This default assumes the common diffusers-style layout: a ``text_encoder``, + a ``vae``, and one or more transformers each exposing their decoder layers + as ``.blocks``. Models that deviate from this naming/structure (e.g. Cosmos3, + whose towers live inside a single ``transformer`` module) must override this + to expose their own offloadable components. + + NOTE: this attribute-name based detection is convenient but brittle; if we + extend offloading to more model families it may be worth replacing with an + explicit per-model declaration instead of relying on these conventions. + """ + components: dict[str, nn.Module] = {} + text_encoder = getattr(self, PipelineComponent.TEXT_ENCODER.value, None) + if text_encoder is not None: + components[PipelineComponent.TEXT_ENCODER.value] = text_encoder + + vae = getattr(self, PipelineComponent.VAE.value, None) + if vae is not None: + components[PipelineComponent.VAE.value] = vae + + for component_name in self.transformer_components: + transformer = getattr(self, component_name, None) + blocks = getattr(transformer, "blocks", None) if transformer is not None else None + if blocks is not None: + public_name = transformer_component_offload_name( + component_name, PipelineComponent.TRANSFORMER.value + ) + components[public_name] = blocks + return components + + def initialize_offload_pipeline(self) -> None: + """Create and initialize the offload pipeline after weights are loaded.""" + self.offloader.initialize() + def _setup_runtime_lora(self) -> None: runtime_lora = getattr(self.pipeline_config, "runtime_lora", None) if runtime_lora is None or self._runtime_lora_applications: @@ -1356,6 +1418,7 @@ def cleanup(self): """Call before dist.destroy_process_group().""" self._profiler.close_window() + self.offloader.cleanup() for name, runner in self._cuda_graph_runners.items(): logger.info(f"Releasing CUDA graphs for {name}") runner.clear() diff --git a/tensorrt_llm/_torch/visual_gen/pipeline_loader.py b/tensorrt_llm/_torch/visual_gen/pipeline_loader.py index 99202ca9d46c..b1d95d284105 100644 --- a/tensorrt_llm/_torch/visual_gen/pipeline_loader.py +++ b/tensorrt_llm/_torch/visual_gen/pipeline_loader.py @@ -255,9 +255,10 @@ def load( with MetaInitMode(): pipeline = AutoPipeline.from_config(config, checkpoint_dir) - # Convert meta tensors to CUDA tensors - self._materialize_meta_tensors(pipeline) - pipeline.to(self.device) + # Convert meta tensors to their runtime devices. Offloaded submodules + # stay on CPU until they are explicitly staged. + cpu_offload_modules = self._get_load_time_cpu_offload_modules(pipeline) + self._materialize_meta_tensors(pipeline, cpu_offload_modules) # ===================================================================== # STEP 3: Load Transformer Weights @@ -298,6 +299,7 @@ def load( if hasattr(pipeline, "post_load_weights"): pipeline.post_load_weights() pipeline._setup_runtime_lora() + pipeline.initialize_offload_pipeline() if config.torch_compile.enable: torch._dynamo.config.cache_size_limit = 128 @@ -335,20 +337,83 @@ def load( ) return pipeline - def _materialize_meta_tensors(self, module: torch.nn.Module) -> None: + def _get_load_time_cpu_offload_modules(self, pipeline: "BasePipeline") -> list[torch.nn.Module]: + """Return offloaded modules that should materialize on CPU at load time. + + The offload manager is initialized only after weights and quantization + hooks run, but MetaInit materialization happens before loading. This + helper mirrors the pipeline's configured stages so offloaded towers never + need to materialize on GPU first. + """ + available_components = pipeline.offload_pipeline_components() + modules: list[torch.nn.Module] = [] + seen: set[int] = set() + + for stage in pipeline.offloader.filter_available_stages( + pipeline.offloader.stages(), available_components + ): + for component in stage: + offload_module = available_components[component] + module_id = id(offload_module) + if module_id not in seen: + modules.append(offload_module) + seen.add(module_id) + + return modules + + def _materialize_meta_tensors( + self, + module: torch.nn.Module, + cpu_offload_modules: Optional[list[torch.nn.Module]] = None, + ) -> None: """ - Convert meta tensors to CUDA tensors. + Convert meta tensors to tensors on their runtime devices. Meta tensors are placeholders that don't allocate GPU memory. After model structure is defined, we materialize them to real tensors. + Submodules listed in ``cpu_offload_modules`` are materialized on CPU and + skipped by the final move to ``self.device`` so checkpoint loading never + needs the full offloaded tower resident on GPU. """ - memo = {} + cpu_offload_modules = cpu_offload_modules or [] + meta_device = torch.device("meta") + + cpu_memo = {} + + def init_cpu_tensor(t: torch.Tensor) -> torch.Tensor: + if t.device != meta_device: + return t.to("cpu") + if t not in cpu_memo: + cpu_memo[t] = torch.empty_like(t, device="cpu") + return cpu_memo[t] + + for offload_module in cpu_offload_modules: + offload_module._apply(init_cpu_tensor) + + cpu_tensor_ids = self._collect_module_tensor_ids(cpu_offload_modules) + target_memo = {} def init_meta_tensor(t: torch.Tensor) -> torch.Tensor: - if t.device != torch.device("meta"): + if id(t) in cpu_tensor_ids: return t - if t not in memo: - memo[t] = torch.empty_like(t, device=self.device) - return memo[t] + if t.device != meta_device: + return t.to(self.device) + if t not in target_memo: + target_memo[t] = torch.empty_like(t, device=self.device) + return target_memo[t] module._apply(init_meta_tensor) + + @staticmethod + def _collect_module_tensor_ids(modules: list[torch.nn.Module]) -> set[int]: + """Collect parameter and buffer object IDs owned by ``modules``.""" + tensor_ids: set[int] = set() + for module in modules: + for child in module.modules(): + for param in child._parameters.values(): + if param is not None: + tensor_ids.add(id(param)) + for buffer in child._buffers.values(): + if buffer is not None: + tensor_ids.add(id(buffer)) + return tensor_ids diff --git a/tensorrt_llm/visual_gen/__init__.py b/tensorrt_llm/visual_gen/__init__.py index 90dab7d201c4..71d13d91bb17 100644 --- a/tensorrt_llm/visual_gen/__init__.py +++ b/tensorrt_llm/visual_gen/__init__.py @@ -43,6 +43,7 @@ CacheConfig, CacheDiTConfig, CompilationConfig, + CpuOffloadConfig, CudaGraphConfig, ParallelConfig, QuantAttentionConfig, @@ -68,6 +69,7 @@ "CacheConfig": "tensorrt_llm.visual_gen.args", "CacheDiTConfig": "tensorrt_llm.visual_gen.args", "CompilationConfig": "tensorrt_llm.visual_gen.args", + "CpuOffloadConfig": "tensorrt_llm.visual_gen.args", "CudaGraphConfig": "tensorrt_llm.visual_gen.args", "ParallelConfig": "tensorrt_llm.visual_gen.args", "QuantAttentionConfig": "tensorrt_llm.visual_gen.args", @@ -119,6 +121,7 @@ def __dir__(): # Cross-cutting sub-configs "CompilationConfig", "CudaGraphConfig", + "CpuOffloadConfig", "TorchCompileConfig", "ParallelConfig", "AttentionConfig", diff --git a/tensorrt_llm/visual_gen/args.py b/tensorrt_llm/visual_gen/args.py index bd742ac16922..833765d2e77a 100644 --- a/tensorrt_llm/visual_gen/args.py +++ b/tensorrt_llm/visual_gen/args.py @@ -478,6 +478,57 @@ class CudaGraphConfig(StrictBaseModel): enable: bool = Field(False, status="prototype") +class CpuOffloadConfig(StrictBaseModel): + """Configuration for offloading visual-generation model components to CPU. + + A diffusion pipeline is a sequence of large *components* (text encoder, + denoising transformer, VAE, optional guardrails, ...) that run one after + another, so only one is needed on the GPU at any moment. Offloading + exploits this: weights are loaded and quantized normally, then held in + packed host (CPU) storage and brought onto the GPU one *stage* at a time, + rebinding each module's parameters/buffers to a reusable GPU arena while + that stage runs and evicting it back to CPU afterwards. This lowers peak + GPU memory to roughly the largest single stage rather than the whole + pipeline, letting larger models fit on limited VRAM, at the cost of + host<->device copies between stages (mitigated by ``pin_memory``). + + Terminology (used consistently across these fields): + + - **component**: a named, public sub-model of the pipeline, e.g. + ``text_encoder``, ``transformer``, ``vae``. + - **stage**: one step of the offload schedule — a single component, or a + group of components that are co-resident on the GPU and run together + before being evicted. For example, a stage ``["text_encoder", "transformer"]`` + keeps both components on GPU until that stage completes. The ordered list + of stages is set by ``stages``. + """ + + enable: bool = Field( + False, + status="prototype", + description="Enable offloading of model components to CPU between pipeline stages.", + ) + pin_memory: bool = Field( + True, + status="prototype", + description="Allocate pinned CPU storage for faster host-to-device copies.", + ) + stages: Optional[List[Union[str, List[str]]]] = Field( + default=None, + status="prototype", + description=( + "Optional ordered list of stages, named with model-specific public component " + "names. Each entry is a single component, or a list of components that are " + "co-resident on the GPU and run together as one stage. For example, " + "[['text_encoder', 'transformer'], 'vae'] keeps text_encoder and " + "transformer co-resident before moving to vae. Example components: " + "['text_encoder', 'transformer', 'vae'] for Wan T2V, or " + "['reasoner', 'generator', 'text_guardrail', 'video_guardrail', 'vae'] for Cosmos3. " + "If omitted, the model chooses default stages." + ), + ) + + class CompilationConfig(StrictBaseModel): """Configuration for torch.compile / CUDA graph warmup shapes. @@ -624,6 +675,10 @@ class VisualGenArgs(StrictBaseModel): default_factory=CudaGraphConfig, status="prototype", ) + cpu_offload_config: CpuOffloadConfig = Field( + default_factory=CpuOffloadConfig, + status="prototype", + ) attention_config: AttentionConfig = Field( default_factory=AttentionConfig, status="prototype", diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index 78d9fed4c35a..4329785365ba 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -269,6 +269,7 @@ l0_b200: - unittest/_torch/visual_gen/test_wan21_t2v_teacache_user_coefficients.py - unittest/_torch/visual_gen/test_wan22_i2v_teacache.py - unittest/_torch/visual_gen/test_wan22_t2v_teacache.py + - unittest/_torch/visual_gen/test_offloading.py - unittest/_torch/visual_gen/test_wan_transformer.py - unittest/_torch/visual_gen/test_cosmos3_action.py - unittest/_torch/visual_gen/test_cosmos3_transformer.py @@ -425,6 +426,11 @@ l0_b200: - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_edge_i2v_lpips_against_golden TIMEOUT (15) - examples/visual_gen/test_visual_gen_cosmos3.py::test_cosmos3_edge_t2i_lpips_against_golden TIMEOUT (15) - examples/visual_gen/test_visual_gen_wan.py::test_fastwan_lpips_against_golden TIMEOUT (10) + # Heavyweight CPU-offload accuracy tests (each loads the full pipeline twice: + # offload + baseline). Kept in post_merge to bound the B200 pre_merge wall-time. + - unittest/_torch/visual_gen/test_wan22_t2v_offload.py::TestWan22_A14B_Offload::test_wan22_offload_matches_baseline + - unittest/_torch/visual_gen/test_wan22_t2v_offload.py::TestWan22_A14B_Offload::test_wan22_fp8_offload_matches_baseline + - unittest/_torch/visual_gen/test_cosmos3_t2v_offload.py::TestCosmos3Offload::test_cosmos3_offload_matches_baseline - examples/visual_gen/test_visual_gen_hunyuan.py::test_hunyuan_t2v_lpips_against_golden TIMEOUT (10) - visual_gen/test_visual_gen_benchmark.py::test_offline_benchmark - visual_gen/test_visual_gen_benchmark.py::test_online_benchmark[openai-videos] diff --git a/tests/unittest/_torch/visual_gen/test_cosmos3_distilled.py b/tests/unittest/_torch/visual_gen/test_cosmos3_distilled.py index 368af6943542..d44e80a797e2 100644 --- a/tests/unittest/_torch/visual_gen/test_cosmos3_distilled.py +++ b/tests/unittest/_torch/visual_gen/test_cosmos3_distilled.py @@ -81,10 +81,10 @@ def _base_policy() -> Cosmos3SamplingPolicy: def _bare_pipeline(**attrs) -> Cosmos3OmniMoTPipeline: - """A pipeline instance without heavyweight __init__; ``rank``/``dtype``/ - ``device`` are BasePipeline properties and must not be set here.""" + """A pipeline instance without heavyweight ``__init__``.""" pipeline = object.__new__(Cosmos3OmniMoTPipeline) defaults = dict( + _device=torch.device("cpu"), audio_gen=False, action_gen=False, has_action_weights=False, diff --git a/tests/unittest/_torch/visual_gen/test_cosmos3_edge.py b/tests/unittest/_torch/visual_gen/test_cosmos3_edge.py index 9feed9e2dad2..76ad52c7e3fb 100644 --- a/tests/unittest/_torch/visual_gen/test_cosmos3_edge.py +++ b/tests/unittest/_torch/visual_gen/test_cosmos3_edge.py @@ -750,6 +750,8 @@ def test_qwen3_synthetic_checkpoint_loads(self): def _bare_pipeline(family: str) -> Cosmos3OmniMoTPipeline: pipeline = object.__new__(Cosmos3OmniMoTPipeline) + # __new__ skips BasePipeline.__init__, which is where ``_device`` is set. + pipeline._device = torch.device("cpu") pipeline.family = family pipeline.sampling = Cosmos3SamplingPolicy() pipeline.audio_gen = False @@ -959,6 +961,7 @@ def _pipeline( self, family: str = "nemotron_dense", checkpoint_shift: float = 3.0 ) -> "pipeline_module.Cosmos3OmniMoTPipeline": pipeline = object.__new__(pipeline_module.Cosmos3OmniMoTPipeline) + pipeline._device = torch.device("cpu") pipeline.family = family pipeline.sampling = SimpleNamespace( checkpoint_flow_shift=checkpoint_shift, diff --git a/tests/unittest/_torch/visual_gen/test_cosmos3_pipeline.py b/tests/unittest/_torch/visual_gen/test_cosmos3_pipeline.py index f49af5e7473f..149bc3be2c7c 100644 --- a/tests/unittest/_torch/visual_gen/test_cosmos3_pipeline.py +++ b/tests/unittest/_torch/visual_gen/test_cosmos3_pipeline.py @@ -60,6 +60,7 @@ from tensorrt_llm._torch.visual_gen.models.cosmos3.transformer_cosmos3 import QWEN3_RECIPE from tensorrt_llm._torch.visual_gen.models.wan.vae_loader import TRTLLM_USE_DIFFUSER_VAE_ENV from tensorrt_llm._torch.visual_gen.models.wan.wan_vae import WanVAE +from tensorrt_llm._torch.visual_gen.offloading import PipelineOffloader from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader from tensorrt_llm.visual_gen.args import TorchCompileConfig, VisualGenArgs @@ -291,7 +292,9 @@ def _make_test_image() -> PIL.Image.Image: @pytest.fixture def cosmos3_format_pipeline(): """Minimal pipeline for prompt formatting helpers (no checkpoint).""" - return Cosmos3OmniMoTPipeline.__new__(Cosmos3OmniMoTPipeline) + pipeline = Cosmos3OmniMoTPipeline.__new__(Cosmos3OmniMoTPipeline) + pipeline._device = torch.device("cpu") + return pipeline def _format_prompt_with_metadata( @@ -877,6 +880,7 @@ def test_v2v_keep_last_smoke(self, cosmos3_pipeline): def test_v2v_flow_shift_override_request_path(self): pipeline = Cosmos3OmniMoTPipeline.__new__(Cosmos3OmniMoTPipeline) + pipeline._device = torch.device("cpu") pipeline.transformer = SimpleNamespace(device=torch.device("cpu")) pipeline.audio_gen = False # Bypassing __init__ means the generation-default family has to be set @@ -946,6 +950,7 @@ def test_v2v_rebuilds_the_audio_scheduler_too(self): the checkpoint's flow shift / Karras sigmas and the streams step on different schedules.""" pipeline = Cosmos3OmniMoTPipeline.__new__(Cosmos3OmniMoTPipeline) + pipeline._device = torch.device("cpu") pipeline.transformer = SimpleNamespace(device=torch.device("cpu")) pipeline.audio_gen = True # Bypassing __init__ means the generation-default family has to be set @@ -1007,6 +1012,7 @@ def test_action_video_is_not_classified_as_v2v(self): the system prompt, so the same frame would tokenize differently depending on whether it was passed as an image or a one-frame clip.""" pipeline = Cosmos3OmniMoTPipeline.__new__(Cosmos3OmniMoTPipeline) + pipeline._device = torch.device("cpu") pipeline.transformer = SimpleNamespace( device=torch.device("cpu"), num_embodiment_domains=32 ) @@ -1064,6 +1070,7 @@ def test_action_restores_the_checkpoint_flow_shift(self): instance outlives the request. An action request that followed a V2V one would otherwise keep V2V's uniform sigmas instead of the checkpoint's.""" pipeline = Cosmos3OmniMoTPipeline.__new__(Cosmos3OmniMoTPipeline) + pipeline._device = torch.device("cpu") pipeline.transformer = SimpleNamespace( device=torch.device("cpu"), num_embodiment_domains=32 ) @@ -1176,6 +1183,7 @@ def test_transfer_rejects_an_image_reference(self): from tensorrt_llm._torch.visual_gen.models.cosmos3.transfer import resolve_transfer_config pipeline = Cosmos3OmniMoTPipeline.__new__(Cosmos3OmniMoTPipeline) + pipeline._device = torch.device("cpu") pipeline.transformer = SimpleNamespace(device=torch.device("cpu")) pipeline.action_gen = False # __new__ skips __init__, where the real pipeline resolves this. @@ -1230,6 +1238,7 @@ def test_transfer_use_system_prompt_defaults_off(self): from tensorrt_llm._torch.visual_gen.models.cosmos3.transfer import resolve_transfer_config pipeline = Cosmos3OmniMoTPipeline.__new__(Cosmos3OmniMoTPipeline) + pipeline._device = torch.device("cpu") pipeline.transformer = SimpleNamespace(device=torch.device("cpu")) pipeline.action_gen = False # __new__ skips __init__, where the real pipeline resolves this. @@ -1602,6 +1611,10 @@ class TestCosmos3TextGuardrailBlocked: @staticmethod def _blocked_pipeline(): pipeline = Cosmos3OmniMoTPipeline.__new__(Cosmos3OmniMoTPipeline) + # __new__ skips BasePipeline.__init__, which sets the device and offloader. + pipeline._device = torch.device("cpu") + pipeline.pipeline_config = SimpleNamespace(cpu_offload_config=None) + pipeline.offloader = PipelineOffloader(pipeline) pipeline.transformer = SimpleNamespace(device=torch.device("cpu")) pipeline.audio_gen = False pipeline.audio_scheduler = None @@ -1609,10 +1622,10 @@ def _blocked_pipeline(): # All-None policy is the documented pre-load placeholder. pipeline.sampling = Cosmos3SamplingPolicy() pipeline.default_use_system_prompt = False - # ``rank`` and ``device`` are read-only properties: rank resolves to 0 - # with no distributed init, device comes off the transformer stub. - # The guardrail model is not under test, so a checker that reports - # "unsafe" for any input drives the path without an unsafe prompt. + # ``rank`` is a read-only property that resolves to 0 with no + # distributed init. The guardrail model is not under test, so a + # checker that reports "unsafe" for any input drives the path + # without an unsafe prompt. pipeline.safety_checker = SimpleNamespace(check_text_safety=lambda _prompt: False) pipeline._scheduler_for = lambda *args, **kwargs: None return pipeline @@ -1641,7 +1654,7 @@ def test_fp8_load_and_t2v(self): checkpoint = _require_checkpoint() pipeline = _load_pipeline(checkpoint, quant_config=COSMOS3_FP8_QUANT_CONFIG) try: - assert pipeline.transformer.model_config.quant_config.quant_algo is not None + assert pipeline.pipeline_config.quant_config.quant_algo is not None result = _run_forward(pipeline, image=None, num_frames=NUM_FRAMES) _assert_valid_video(result.video, num_frames=NUM_FRAMES) assert result.frame_rate == FRAME_RATE diff --git a/tests/unittest/_torch/visual_gen/test_cosmos3_t2v_offload.py b/tests/unittest/_torch/visual_gen/test_cosmos3_t2v_offload.py new file mode 100644 index 000000000000..c43fd275d723 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_cosmos3_t2v_offload.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pipeline-level integration tests for Cosmos3 T2V CPU offloading. + +Loads the full model and calls pipeline.forward() with cpu_offload_config.enable=True; +offloaded output is checked against the non-offload TRT-LLM baseline. CUDA graphs +with offloading must raise. These skip unless the checkpoint is available. + +Fast, GPU-free offload unit tests (including the Cosmos3-specific offload wiring) +live in test_offloading.py. + +Run: + pytest tests/unittest/_torch/visual_gen/test_cosmos3_t2v_offload.py -v -s + +Override checkpoint path: + DIFFUSION_MODEL_PATH_COSMOS3=/path/to/cosmos3 \\ + pytest tests/unittest/_torch/visual_gen/test_cosmos3_t2v_offload.py -v -s +""" + +import os + +os.environ["TLLM_DISABLE_MPI"] = "1" +# Cosmos3 guardrails require the cosmos_guardrail package (NVIDIA Open Model +# License), which is not installed in CI. Disable guardrails by default so the +# test runs without it: the guardrail offload stages below stay valid for +# validation and are filtered out at runtime when the safety checker is absent. +# Set this to "0" locally (with cosmos_guardrail installed) to also exercise +# guardrail offloading. +os.environ.setdefault("TRTLLM_DISABLE_COSMOS3_GUARDRAILS", "1") + +import gc +from pathlib import Path + +import pytest +import torch +import torch.nn.functional as F + +from tensorrt_llm._torch.visual_gen.models.cosmos3.pipeline_cosmos3 import ( + COSMOS3_GENERATOR_OFFLOAD_COMPONENT, + COSMOS3_REASONER_OFFLOAD_COMPONENT, + COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT, + COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT, +) + + +@pytest.fixture(autouse=True, scope="module") +def _cleanup_mpi_env(): + yield + os.environ.pop("TLLM_DISABLE_MPI", None) + os.environ.pop("TRTLLM_DISABLE_COSMOS3_GUARDRAILS", None) + + +@pytest.fixture(autouse=True) +def _cleanup_gpu(): + gc.collect() + torch.cuda.empty_cache() + yield + gc.collect() + torch.cuda.empty_cache() + + +def _llm_models_root() -> Path: + if "LLM_MODELS_ROOT" in os.environ: + root = Path(os.environ["LLM_MODELS_ROOT"]) + else: + root = Path("/home/scratch.trt_llm_data_ci/llm-models/") + if not root.exists(): + root = Path("/scratch.trt_llm_data/llm-models/") + assert root.exists(), ( + "Set LLM_MODELS_ROOT or ensure /home/scratch.trt_llm_data_ci/llm-models/ is accessible." + ) + return root + + +def _checkpoint(env_var: str, default_name: str) -> str: + return os.environ.get(env_var) or str(_llm_models_root() / default_name) + + +COSMOS3_PATH = _checkpoint("DIFFUSION_MODEL_PATH_COSMOS3", "Cosmos3-Nano") + +# Mirrors a validated manual offload run; reduced steps keep the test fast. +INFER_PROMPT = "A cute cat playing piano." +# None -> Cosmos3 uses its safe default negative prompt and skips the guardrail +# blocklist check on it (an empty string is flagged as "Input is empty"). +INFER_NEGATIVE_PROMPT = None +INFER_HEIGHT = 720 +INFER_WIDTH = 1080 +INFER_NUM_FRAMES = 33 +INFER_NUM_STEPS = 4 +INFER_GUIDANCE_SCALE = 6.0 +INFER_SEED = 42 +# Exercise both transformer towers, the VAE, and both guardrails. +INFER_OFFLOAD_STAGES = [ + COSMOS3_REASONER_OFFLOAD_COMPONENT, + COSMOS3_GENERATOR_OFFLOAD_COMPONENT, + "vae", + COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT, + COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT, +] +COS_SIM_THRESHOLD = 0.99 + + +def _make_pipeline(checkpoint_path: str, *, enable_offload: bool = False): + from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader + from tensorrt_llm.visual_gen.args import CpuOffloadConfig as ArgsOffloadConfig + from tensorrt_llm.visual_gen.args import TorchCompileConfig, VisualGenArgs + + if not os.path.exists(checkpoint_path): + pytest.skip(f"Checkpoint not found: {checkpoint_path}") + args = VisualGenArgs( + model=checkpoint_path, + torch_compile_config=TorchCompileConfig(enable=False), + cpu_offload_config=ArgsOffloadConfig( + enable=enable_offload, + stages=INFER_OFFLOAD_STAGES if enable_offload else None, + ), + ) + pipeline = PipelineLoader(args).load(skip_warmup=True) + assert pipeline.transformer is not None, "Expected a loaded Cosmos3 transformer" + return pipeline + + +def _capture_video(pipe) -> torch.Tensor: + with torch.no_grad(): + result = pipe.forward( + prompt=INFER_PROMPT, + negative_prompt=INFER_NEGATIVE_PROMPT, + height=INFER_HEIGHT, + width=INFER_WIDTH, + num_frames=INFER_NUM_FRAMES, + num_inference_steps=INFER_NUM_STEPS, + guidance_scale=INFER_GUIDANCE_SCALE, + seed=INFER_SEED, + ) + assert result.video is not None, "Pipeline returned no video (guardrail blocked?)" + return result.video.float() / 255.0 + + +@pytest.fixture +def cosmos3_offload_pipeline(): + pipeline = _make_pipeline(COSMOS3_PATH, enable_offload=True) + yield pipeline + del pipeline + gc.collect() + torch.cuda.empty_cache() + + +@pytest.mark.integration +class TestCosmos3Offload: + """Cosmos3 T2V with CPU offloading of both towers, VAE, and guardrails.""" + + def test_cosmos3_offload_forward(self, cosmos3_offload_pipeline): + pipeline = cosmos3_offload_pipeline + assert pipeline.offloader.stages(), ( + "cpu_offload_config.enable=True but offloader.stages() is empty" + ) + assert pipeline.offloader.offload_pipeline is not None, ( + "offload pipeline was not initialized after load" + ) + _capture_video(pipeline) + print(f"\n ===== Offload — Cosmos3 {INFER_HEIGHT}x{INFER_WIDTH} =====") + print(f" stages: {pipeline.offloader.stages()}") + + def test_cosmos3_offload_matches_baseline(self): + offload_pipe = _make_pipeline(COSMOS3_PATH, enable_offload=True) + offload_video = _capture_video(offload_pipe) + del offload_pipe + gc.collect() + torch.cuda.empty_cache() + + baseline_pipe = _make_pipeline(COSMOS3_PATH, enable_offload=False) + baseline_video = _capture_video(baseline_pipe) + del baseline_pipe + gc.collect() + torch.cuda.empty_cache() + + assert offload_video.shape == baseline_video.shape, ( + f"shape mismatch offload {offload_video.shape} vs baseline {baseline_video.shape}" + ) + a_flat = offload_video.float().cpu().reshape(-1) + b_flat = baseline_video.float().cpu().reshape(-1) + cos_sim = ( + F.cosine_similarity(a_flat.unsqueeze(0), b_flat.unsqueeze(0)).clamp(-1.0, 1.0).item() + ) + print( + f"\n ===== Offload accuracy — Cosmos3 =====\n" + f" cosine_similarity={cos_sim:.6f} (threshold={COS_SIM_THRESHOLD})" + ) + assert cos_sim >= COS_SIM_THRESHOLD, ( + f"offload vs baseline cosine similarity {cos_sim:.6f} < {COS_SIM_THRESHOLD}" + ) + + +@pytest.mark.integration +class TestCosmos3OffloadCudaGraphRaisesError: + """CUDA graphs plus offloading must raise NotImplementedError on pipeline load.""" + + def test_cosmos3_raises_if_cuda_graph_and_offload_enabled(self): + from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader + from tensorrt_llm.visual_gen.args import CpuOffloadConfig as ArgsOffloadConfig + from tensorrt_llm.visual_gen.args import CudaGraphConfig, TorchCompileConfig, VisualGenArgs + + if not os.path.exists(COSMOS3_PATH): + pytest.skip(f"Checkpoint not found: {COSMOS3_PATH}") + args = VisualGenArgs( + model=COSMOS3_PATH, + torch_compile_config=TorchCompileConfig(enable=False), + cuda_graph_config=CudaGraphConfig(enable=True), + cpu_offload_config=ArgsOffloadConfig(enable=True, stages=INFER_OFFLOAD_STAGES), + ) + with pytest.raises( + NotImplementedError, + match="CUDA graphs are not supported with visual generation offloading", + ): + PipelineLoader(args).load(skip_warmup=True) diff --git a/tests/unittest/_torch/visual_gen/test_cosmos3_transfer.py b/tests/unittest/_torch/visual_gen/test_cosmos3_transfer.py index 2102d701867b..83a11058519e 100644 --- a/tests/unittest/_torch/visual_gen/test_cosmos3_transfer.py +++ b/tests/unittest/_torch/visual_gen/test_cosmos3_transfer.py @@ -13,6 +13,7 @@ import os import pickle +from contextlib import nullcontext from pathlib import Path from types import SimpleNamespace @@ -49,6 +50,7 @@ QWEN3_RECIPE, TransformerOutput, ) +from tensorrt_llm._torch.visual_gen.offloading import PipelineOffloader from tensorrt_llm._torch.visual_gen.output import CudaPhaseTimer from tensorrt_llm.media.decoding import VideoStreamInfo from tensorrt_llm.visual_gen.params import VisualGenParams @@ -157,8 +159,15 @@ def forward(self, *, hidden_states, timestep, raw_timestep, text_ids, text_mask, del text_mask token = int(text_ids.reshape(-1)[0].item()) if text_ids.numel() else 0 control_latents = kwargs.get("control_latents") + offload_context = kwargs.get("offload_context") torch.testing.assert_close(timestep, raw_timestep / self.calls_num_train_timesteps) - self.calls.append({"token": token, "has_control": control_latents is not None}) + self.calls.append( + { + "token": token, + "has_control": control_latents is not None, + "offload_context": offload_context, + } + ) if self.cached_kv is None: marker = torch.tensor([token], dtype=torch.float32) self.cached_kv = [(marker, marker + 100)] @@ -233,6 +242,8 @@ def _started_timer() -> CudaPhaseTimer: def _make_pipeline(sampling=None): pipeline = Cosmos3OmniMoTPipeline.__new__(Cosmos3OmniMoTPipeline) nn.Module.__init__(pipeline) + # __new__ skips BasePipeline.__init__, which is where ``_device`` is set. + pipeline._device = torch.device("cpu") pipeline.transformer = StubTransformer() # __new__ skips __init__, which is where the real pipeline resolves this # from the transformer config; the mode-defaults tables are keyed on it. @@ -240,7 +251,8 @@ def _make_pipeline(sampling=None): pipeline.scheduler = StubScheduler() pipeline.sampling = sampling or StubSamplingPolicy() pipeline.safety_checker = None - pipeline.pipeline_config = SimpleNamespace(torch_dtype=torch.float32) + pipeline.pipeline_config = SimpleNamespace(torch_dtype=torch.float32, cpu_offload_config=None) + pipeline.offloader = PipelineOffloader(pipeline) pipeline.vae_scale_factor_temporal = 4 pipeline._guidance_scale = None pipeline._num_timesteps = None @@ -715,6 +727,20 @@ def test_applies_control_and_text_cfg(self): ] torch.testing.assert_close(result, torch.full_like(latents, 254.0)) + def test_passes_offload_context_to_each_transformer_branch(self): + pipeline = _make_pipeline() + + def offload_context(_component_name): + return nullcontext() + + pipeline.offloader.context_if_requested = offload_context + self._run(pipeline, timesteps=[7], guidance_scale=3.0, control_guidance=1.5) + + assert len(pipeline.transformer.calls) == 3 + assert all( + call["offload_context"] is offload_context for call in pipeline.transformer.calls + ) + def test_skips_idle_cfg_branches(self): control_only = _make_pipeline() result, latents = self._run( @@ -1183,6 +1209,12 @@ def test_output_is_screened_when_guardrails_are_on(self, monkeypatch): pipeline = _make_pipeline() seen = {} + def offload_context(component_name): + seen["offload_component"] = component_name + return nullcontext() + + pipeline.offloader.context_if_requested = offload_context + class Checker: pass @@ -1194,6 +1226,9 @@ class Checker: ) self._run(pipeline, monkeypatch, use_guardrails=True) assert seen.get("called"), "transfer returned generated video unscreened" + assert ( + seen["offload_component"] == pipeline_module.COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT + ) def test_phase_timings_are_populated(self, monkeypatch): output = self._run(_make_pipeline(), monkeypatch) diff --git a/tests/unittest/_torch/visual_gen/test_cosmos3_transformer.py b/tests/unittest/_torch/visual_gen/test_cosmos3_transformer.py index 5a53683ab9f1..90fb2634c5e1 100644 --- a/tests/unittest/_torch/visual_gen/test_cosmos3_transformer.py +++ b/tests/unittest/_torch/visual_gen/test_cosmos3_transformer.py @@ -734,7 +734,7 @@ def test_load_fp8_quantization(self, quant_algo: str): ) pipeline = PipelineLoader(args).load(skip_warmup=True, skip_components=_SKIP_AUX) try: - assert pipeline.transformer.model_config.quant_config.quant_algo is not None + assert pipeline.pipeline_config.quant_config.quant_algo is not None transformer = pipeline.transformer c = transformer.latent_channel_size hs, ts, text_ids, text_mask, video_shape = _cosmos3_inputs( diff --git a/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py b/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py index d0bb3fe43a69..ffd2a6e77df2 100644 --- a/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py +++ b/tests/unittest/_torch/visual_gen/test_ltx2_pipeline.py @@ -1288,6 +1288,9 @@ def forward(self, x): return self.lin(x) pipeline = object.__new__(ltx2_two_stages.LTX2TwoStagesPipeline) + # __init__ is bypassed here; BasePipeline.device now reads self._device + # (set in __init__), so initialize it explicitly for the warmup path. + pipeline._device = torch.device("cuda") pipeline.pipeline_config = DiffusionPipelineConfig( cuda_graph=CudaGraphConfig(enable=True), torch_compile=TorchCompileConfig(enable=False), diff --git a/tests/unittest/_torch/visual_gen/test_model_loader.py b/tests/unittest/_torch/visual_gen/test_model_loader.py index cff7570a7604..27f5c640f78e 100644 --- a/tests/unittest/_torch/visual_gen/test_model_loader.py +++ b/tests/unittest/_torch/visual_gen/test_model_loader.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Test PipelineLoader with VisualGenArgs API.""" import json @@ -127,6 +130,7 @@ def test_dual_transformer_checkpoint_creates_distinct_model_configs(tmp_path): def test_pipeline_loader_applies_runtime_lora_after_post_load_hooks(monkeypatch): from tensorrt_llm._torch.visual_gen.config import DiffusionPipelineConfig + from tensorrt_llm._torch.visual_gen.offloading import PipelineOffloader from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader from tensorrt_llm.visual_gen.args import RuntimeLoRAConfig, VisualGenArgs @@ -138,10 +142,14 @@ class FakePipeline(nn.Module): def __init__(self, config): super().__init__() self.pipeline_config = config + self.offloader = PipelineOffloader(self) def to(self, device): return self + def offload_pipeline_components(self): + return {} + def load_transformer_weights(self, checkpoint_dir): events.append("load_transformer_weights") return {} @@ -158,6 +166,9 @@ def post_load_weights(self): def _setup_runtime_lora(self): events.append("runtime_lora") + def initialize_offload_pipeline(self): + self.offloader.initialize() + def torch_compile(self): pass @@ -181,7 +192,11 @@ def _setup_cache_acceleration(self): "tensorrt_llm._torch.visual_gen.pipeline_loader.AutoPipeline.from_config", lambda config, checkpoint_dir: FakePipeline(config), ) - monkeypatch.setattr(PipelineLoader, "_materialize_meta_tensors", lambda self, pipeline: None) + monkeypatch.setattr( + PipelineLoader, + "_materialize_meta_tensors", + lambda self, pipeline, cpu_offload_modules=None: None, + ) PipelineLoader(args, device="cpu").load(skip_warmup=True) diff --git a/tests/unittest/_torch/visual_gen/test_offloading.py b/tests/unittest/_torch/visual_gen/test_offloading.py new file mode 100644 index 000000000000..0b9d337beb00 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_offloading.py @@ -0,0 +1,606 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for visual generation module parameter offloading.""" + +from contextlib import nullcontext +from types import SimpleNamespace +from typing import Optional +from unittest.mock import patch + +import pytest +import torch +import torch.nn as nn + +from tensorrt_llm._torch.visual_gen.config import CpuOffloadConfig, VisualGenArgs +from tensorrt_llm._torch.visual_gen.models.cosmos3.pipeline_cosmos3 import ( + COSMOS3_GENERATOR_OFFLOAD_COMPONENT, + COSMOS3_REASONER_OFFLOAD_COMPONENT, + COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT, + COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT, + Cosmos3OmniMoTPipeline, +) +from tensorrt_llm._torch.visual_gen.offloading import ModuleOffloadManager, OffloadPipeline +from tensorrt_llm._torch.visual_gen.pipeline import BasePipeline + + +class _ToyModule(nn.Module): + def __init__(self, weight_value: float, bias_value: float) -> None: + super().__init__() + self.weight = nn.Parameter(torch.full((2, 2), weight_value)) + self.register_buffer("bias", torch.full((2,), bias_value)) + + +class _AlignmentToyModule(nn.Module): + def __init__(self) -> None: + super().__init__() + self.prefix = nn.Parameter(torch.ones((1,), dtype=torch.bfloat16)) + self.alignment_sensitive = nn.Parameter(torch.ones((3,), dtype=torch.bfloat16)) + + +class _ToyTransformer(nn.Module): + def __init__(self) -> None: + super().__init__() + self.blocks = _ToyModule(weight_value=3.0, bias_value=30.0) + + @property + def device(self): + return next(self.parameters()).device + + +class _OffloadCudaGraphPipeline(BasePipeline): + def _init_transformer(self) -> None: + self.transformer = _ToyModule(weight_value=1.0, bias_value=10.0) + + def default_offload_stages(self) -> tuple[tuple[str, ...], ...]: + return (("transformer",),) + + +class _CustomOffloadPipeline(BasePipeline): + def _init_transformer(self) -> None: + self.transformer = _ToyTransformer() + self.text_encoder = _ToyModule(weight_value=1.0, bias_value=10.0) + self.vae = _ToyModule(weight_value=2.0, bias_value=20.0) + + +def _make_config( + *, + offload_enable: bool = True, + offload_stages: list[str | list[str]] | None = None, + device: str = "cuda", +): + return SimpleNamespace( + primary_pretrained_config=SimpleNamespace(), + device=device, + cuda_graph=SimpleNamespace(enable=False), + torch_compile=SimpleNamespace(enable=False), + cpu_offload_config=CpuOffloadConfig( + enable=offload_enable, + stages=offload_stages, + ), + ) + + +def test_pipeline_device_uses_config_not_transformer_parameter_device(): + pipeline = _CustomOffloadPipeline(_make_config(device="cuda:3")) + + assert pipeline.transformer.device == torch.device("cpu") + assert pipeline.device == torch.device("cuda:3") + + +def _make_manager() -> tuple[ModuleOffloadManager, _ToyModule, _ToyModule]: + stage_a = _ToyModule(weight_value=1.0, bias_value=10.0) + stage_b = _ToyModule(weight_value=2.0, bias_value=20.0) + manager = ModuleOffloadManager( + stages={ + "stage_a": stage_a, + "stage_b": stage_b, + }, + device="cpu", + pin_memory=False, + ) + manager.initialize() + return manager, stage_a, stage_b + + +def _storage_ptr(tensor: torch.Tensor) -> int: + return tensor.untyped_storage().data_ptr() + + +def test_cuda_graphs_with_offload_raise_not_implemented(): + config = _make_config() + config.cuda_graph.enable = True + + with pytest.raises( + NotImplementedError, + match="CUDA graphs are not supported with visual generation offloading", + ): + _OffloadCudaGraphPipeline(config) + + +def test_offload_context_requires_initialized_pipeline_when_configured(): + pipeline = _OffloadCudaGraphPipeline(_make_config()) + + with pipeline.offloader.context("transformer", enable=False): + pass + + with pytest.raises( + RuntimeError, + match="offload pipeline has not been initialized", + ): + with pipeline.offloader.context("transformer"): + pass + + +def test_configured_offload_stages_override_model_defaults_and_expose_vae(): + pipeline = _CustomOffloadPipeline( + _make_config( + offload_stages=[ + "text_encoder", + ["transformer", "vae"], + "transformer_2", + ] + ) + ) + + assert pipeline.offloader.stages() == ( + ("text_encoder",), + ("transformer", "vae"), + ("transformer_2",), + ) + + components = pipeline.offload_pipeline_components() + assert components["text_encoder"] is pipeline.text_encoder + assert components["transformer"] is pipeline.transformer.blocks + assert components["vae"] is pipeline.vae + + assert pipeline.offloader.filter_available_stages(pipeline.offloader.stages(), components) == ( + ("text_encoder",), + ("transformer", "vae"), + ) + + +def test_offload_context_resolves_component_to_stageed_stage(): + pipeline = _CustomOffloadPipeline(_make_config(offload_stages=[["transformer", "vae"]])) + pipeline.initialize_offload_pipeline() + + with pipeline.offloader.context("vae"): + assert pipeline.offloader.offload_pipeline is not None + assert pipeline.offloader.offload_pipeline.manager.active_stage_name == "transformer+vae" + + pipeline.cleanup() + + +def test_initialize_rejects_configured_stage_components_that_are_unavailable_for_model(): + pipeline = _CustomOffloadPipeline( + _make_config(offload_stages=[["transformer", "transformer_2"]]) + ) + + with pytest.raises( + ValueError, + match=r"Unknown cpu_offload_config\.stages entries.*transformer_2.*transformer", + ): + pipeline.initialize_offload_pipeline() + + +def test_visual_gen_args_loads_yaml_offload_stages(tmp_path): + config_path = tmp_path / "visual_gen.yaml" + config_path.write_text( + """ +cpu_offload_config: + enable: true + stages: + - text_encoder + - [transformer, vae] +""", + encoding="utf-8", + ) + + args = VisualGenArgs.from_yaml(config_path) + + assert args.cpu_offload_config.enable is True + assert args.cpu_offload_config.stages == [ + "text_encoder", + ["transformer", "vae"], + ] + + +def test_visual_gen_args_allows_model_specific_offload_stage_names(): + args = VisualGenArgs( + cpu_offload_config={ + "enable": True, + "stages": ["future_model_stage"], + } + ) + + assert args.cpu_offload_config.stages == ["future_model_stage"] + + +def test_pipeline_rejects_unknown_offload_stage_names_for_this_model(): + pipeline = _CustomOffloadPipeline(_make_config(offload_stages=["transformer.blocks"])) + + with pytest.raises( + ValueError, + match=r"Unknown cpu_offload_config\.stages entries.*transformer\.blocks.*transformer", + ): + pipeline.initialize_offload_pipeline() + + +def test_initialize_reports_cpu_storage_allocation_context(): + stage = _ToyModule(weight_value=1.0, bias_value=10.0) + manager = ModuleOffloadManager( + stages={"stage": stage}, + device="cpu", + pin_memory=False, + ) + gpu_arena = torch.empty(1024, dtype=torch.uint8, device="cpu") + + def fail_cpu_storage(*args, **kwargs): + raise RuntimeError("cpu allocation failed") + + with ( + patch.object(manager, "_allocate_gpu_arena", return_value=gpu_arena), + patch("torch.empty", side_effect=fail_cpu_storage), + ): + with pytest.raises( + RuntimeError, + match="Failed to allocate packed CPU storage for visual generation offload", + ) as exc_info: + manager.initialize() + + message = str(exc_info.value) + assert "Failed to allocate packed CPU storage for visual generation offload" in message + assert "pin_memory=False" in message + assert "stages=[stage=" in message + + +def test_initialize_reports_cuda_arena_allocation_hint(): + stage = _ToyModule(weight_value=1.0, bias_value=10.0) + manager = ModuleOffloadManager( + stages={"stage": stage}, + device="cuda", + pin_memory=False, + ) + original_empty = torch.empty + + def fail_cuda_arena(*args, **kwargs): + device = torch.device(kwargs.get("device", "cpu")) + if device.type == "cuda": + raise RuntimeError("cuda allocation failed") + return original_empty(*args, **kwargs) + + with patch("torch.empty", side_effect=fail_cuda_arena): + with pytest.raises( + RuntimeError, + match="Failed to allocate GPU arena for visual generation offload", + ) as exc_info: + manager.initialize() + + message = str(exc_info.value) + assert "Failed to allocate GPU arena for visual generation offload" in message + assert "device=cuda" in message + assert "stages=[stage=" in message + assert "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True" in message + + +def test_initialize_packs_stage_before_rebinding_to_cpu(): + stage_a = _ToyModule(weight_value=1.0, bias_value=10.0) + stage_b = _ToyModule(weight_value=2.0, bias_value=20.0) + manager = ModuleOffloadManager( + stages={ + "stage_a": stage_a, + "stage_b": stage_b, + }, + device="cpu", + pin_memory=False, + ) + events = [] + original_rebind_to_cpu = manager._rebind_to_cpu + original_allocate_cpu_storage = manager._allocate_cpu_storage + + def record_rebind_to_cpu(name): + events.append(("rebind_cpu", name)) + return original_rebind_to_cpu(name) + + def record_allocate_cpu_storage(num_bytes, stage_name=None): + events.append(("allocate_cpu", stage_name)) + return original_allocate_cpu_storage(num_bytes, stage_name=stage_name) + + with ( + patch.object(manager, "_rebind_to_cpu", side_effect=record_rebind_to_cpu), + patch.object( + manager, + "_allocate_cpu_storage", + side_effect=record_allocate_cpu_storage, + ), + ): + manager.initialize() + + assert events.index(("allocate_cpu", "stage_a")) < events.index(("rebind_cpu", "stage_a")) + assert events.index(("allocate_cpu", "stage_b")) < events.index(("rebind_cpu", "stage_b")) + + +def test_packed_tensor_views_are_sufficiently_aligned(): + stage = _AlignmentToyModule() + manager = ModuleOffloadManager( + stages={"stage": stage}, + device="cpu", + pin_memory=False, + ) + manager.initialize() + + layout = manager.layouts["stage"] + for spec in layout.specs: + assert spec.offset % 16 == 0 + + manager.stage("stage") + assert stage.prefix.data_ptr() % 16 == 0 + assert stage.alignment_sensitive.data_ptr() % 16 == 0 + + +def test_offload_pipeline_context_stages_requested_stage(): + stage_a = _ToyModule(weight_value=1.0, bias_value=10.0) + stage_b = _ToyModule(weight_value=2.0, bias_value=20.0) + pipeline = OffloadPipeline( + stages=(("stage_a",), ("stage_b",)), + components={ + "stage_a": stage_a, + "stage_b": stage_b, + }, + device="cpu", + pin_memory=False, + ) + pipeline.initialize() + + assert pipeline.manager.active_stage_name is None + with pipeline.context("stage_a"): + assert pipeline.manager.active_stage_name == "stage_a" + assert pipeline.manager.active_stage_name == "stage_a" + + with pipeline.context("stage_b"): + assert pipeline.manager.active_stage_name == "stage_b" + + +def test_inactive_stage_stays_cpu_backed_after_staging_another_stage(): + manager, stage_a, stage_b = _make_manager() + assert manager.gpu_arena is not None + stage_a_cpu_storage = manager.layouts["stage_a"].cpu_storage + stage_b_cpu_storage = manager.layouts["stage_b"].cpu_storage + assert stage_a_cpu_storage is not None + assert stage_b_cpu_storage is not None + stage_a_cpu_storage_ptr = _storage_ptr(stage_a_cpu_storage) + stage_b_cpu_storage_ptr = _storage_ptr(stage_b_cpu_storage) + gpu_arena_ptr = _storage_ptr(manager.gpu_arena) + + manager.stage("stage_a") + + assert manager.active_stage_name == "stage_a" + assert _storage_ptr(stage_a.weight) == gpu_arena_ptr + assert _storage_ptr(stage_a.bias) == gpu_arena_ptr + assert _storage_ptr(stage_b.weight) == stage_b_cpu_storage_ptr + assert _storage_ptr(stage_b.bias) == stage_b_cpu_storage_ptr + torch.testing.assert_close(stage_b.weight, torch.full((2, 2), 2.0)) + torch.testing.assert_close(stage_b.bias, torch.full((2,), 20.0)) + + manager.stage("stage_b") + + assert manager.active_stage_name == "stage_b" + assert _storage_ptr(stage_a.weight) == stage_a_cpu_storage_ptr + assert _storage_ptr(stage_a.bias) == stage_a_cpu_storage_ptr + assert _storage_ptr(stage_b.weight) == gpu_arena_ptr + assert _storage_ptr(stage_b.bias) == gpu_arena_ptr + torch.testing.assert_close(stage_a.weight, torch.full((2, 2), 1.0)) + torch.testing.assert_close(stage_a.bias, torch.full((2,), 10.0)) + + +def test_state_dict_reads_correct_inactive_stage_data(): + manager, stage_a, stage_b = _make_manager() + + manager.stage("stage_a") + stage_b_state = stage_b.state_dict() + torch.testing.assert_close(stage_b_state["weight"], torch.full((2, 2), 2.0)) + torch.testing.assert_close(stage_b_state["bias"], torch.full((2,), 20.0)) + + manager.stage("stage_b") + stage_a_state = stage_a.state_dict() + torch.testing.assert_close(stage_a_state["weight"], torch.full((2, 2), 1.0)) + torch.testing.assert_close(stage_a_state["bias"], torch.full((2,), 10.0)) + + +def test_rebinding_reuses_cached_view_objects(): + manager, stage_a, _ = _make_manager() + cpu_weight = stage_a.weight + cpu_bias = stage_a.bias + + manager.stage("stage_a") + gpu_weight = stage_a.weight + gpu_bias = stage_a.bias + + assert gpu_weight is not cpu_weight + assert gpu_bias is not cpu_bias + + manager.stage("stage_b") + assert stage_a.weight is cpu_weight + assert stage_a.bias is cpu_bias + + manager.stage("stage_a") + assert stage_a.weight is gpu_weight + assert stage_a.bias is gpu_bias + + +# ============================================================================ +# Cosmos3-specific offload wiring (no GPU / no checkpoint) +# ============================================================================ + + +class _FakeRunner: + """Stand-in for cosmos_guardrail.GuardrailRunner (a plain object).""" + + def __init__(self, models: list) -> None: + self._models = models + + @property + def models(self) -> list: + return self._models + + +class _FakeSafetyChecker: + def __init__(self, text_models: list, video_models: list) -> None: + self.text_guardrail = _FakeRunner(text_models) + self.video_guardrail = _FakeRunner(video_models) + + +class _FakeCosmos3Transformer(nn.Module): + def __init__(self) -> None: + super().__init__() + self.language_model = nn.Module() + self.language_model.layers = nn.ModuleList([nn.Linear(2, 2)]) + self.gen_layers = nn.ModuleList([nn.Linear(2, 2)]) + + +class _Cosmos3OffloadUnitPipeline(Cosmos3OmniMoTPipeline): + """Cosmos3 pipeline with a fake transformer to avoid loading the model.""" + + def _init_transformer(self) -> None: + self.transformer = _FakeCosmos3Transformer() + + +def _make_cosmos3_unit_config( + *, + offload_enable: bool = True, + offload_stages: Optional[list] = None, + device: str = "cpu", +): + return SimpleNamespace( + primary_pretrained_config=SimpleNamespace(), + device=device, + cuda_graph=SimpleNamespace(enable=False), + torch_compile=SimpleNamespace(enable=False), + cpu_offload_config=CpuOffloadConfig(enable=offload_enable, stages=offload_stages), + ) + + +def _make_cosmos3_unit_pipeline(**config_kwargs) -> _Cosmos3OffloadUnitPipeline: + return _Cosmos3OffloadUnitPipeline(_make_cosmos3_unit_config(**config_kwargs)) + + +def test_cosmos3_offload_components_expose_towers_vae_and_guardrails(): + pipeline = _make_cosmos3_unit_pipeline() + pipeline.vae = nn.Linear(2, 2) + text_model = nn.Linear(2, 2) + video_model = nn.Linear(2, 2) + pipeline.safety_checker = _FakeSafetyChecker( + text_models=[text_model], video_models=[video_model] + ) + + components = pipeline.offload_pipeline_components() + + assert ( + components[COSMOS3_REASONER_OFFLOAD_COMPONENT] is pipeline.transformer.language_model.layers + ) + assert components[COSMOS3_GENERATOR_OFFLOAD_COMPONENT] is pipeline.transformer.gen_layers + assert components["vae"] is pipeline.vae + + text_component = components[COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT] + video_component = components[COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT] + assert isinstance(text_component, nn.ModuleList) and list(text_component) == [text_model] + assert isinstance(video_component, nn.ModuleList) and list(video_component) == [video_model] + + +def test_cosmos3_video_encoding_uses_vae_offload_context(): + pipeline = _make_cosmos3_unit_pipeline(offload_enable=False) + pipeline.pipeline_config.torch_dtype = torch.float32 + video = torch.zeros(1, 3, 1, 2, 2) + pipeline.vae = SimpleNamespace( + dtype=torch.float32, + config=SimpleNamespace(scaling_factor=1.0), + encode=lambda value: SimpleNamespace(latent_dist=SimpleNamespace(mode=lambda: value)), + ) + + with patch.object( + pipeline.offloader, + "context_if_requested", + return_value=nullcontext(), + ) as offload_context: + encoded = pipeline._encode_video_tensor(video) + + offload_context.assert_called_once_with("vae") + assert torch.equal(encoded, video) + + +def test_cosmos3_guardrail_components_absent_without_safety_checker(): + pipeline = _make_cosmos3_unit_pipeline() + # safety_checker is loaded on rank 0 only; emulate ranks where it is missing. + components = pipeline.offload_pipeline_components() + + assert COSMOS3_REASONER_OFFLOAD_COMPONENT in components + assert COSMOS3_GENERATOR_OFFLOAD_COMPONENT in components + assert COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT not in components + assert COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT not in components + + +def test_cosmos3_guardrail_components_wrap_only_nn_modules(): + pipeline = _make_cosmos3_unit_pipeline() + nn_model = nn.Linear(2, 2) + # Blocklist is a plain object (not nn.Module) and must be skipped. + pipeline.safety_checker = _FakeSafetyChecker( + text_models=[object(), nn_model], video_models=[object()] + ) + + components = pipeline.offload_pipeline_components() + + assert list(components[COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT]) == [nn_model] + # video guardrail has no nn.Module members -> no component registered. + assert COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT not in components + + +def test_cosmos3_default_offload_stages_exclude_guardrails(): + pipeline = _make_cosmos3_unit_pipeline() + + assert pipeline.default_offload_stages() == ( + (COSMOS3_REASONER_OFFLOAD_COMPONENT,), + (COSMOS3_GENERATOR_OFFLOAD_COMPONENT,), + ) + + +def test_cosmos3_validate_accepts_guardrail_names_even_when_not_loaded(): + # Emulates a non-rank-0 process: guardrail components are configured but absent + # from offload_pipeline_components. Validation must not raise on the guardrails. + pipeline = _make_cosmos3_unit_pipeline( + offload_stages=[ + COSMOS3_REASONER_OFFLOAD_COMPONENT, + COSMOS3_TEXT_GUARDRAIL_OFFLOAD_COMPONENT, + COSMOS3_VIDEO_GUARDRAIL_OFFLOAD_COMPONENT, + ] + ) + available_components = { + COSMOS3_REASONER_OFFLOAD_COMPONENT: pipeline.transformer.language_model.layers + } + + # Guardrail names are tolerated even though they are not in available_components. + pipeline.offloader.validate_configured_stages(pipeline.offloader.stages(), available_components) + + +def test_cosmos3_validate_still_rejects_unknown_component_names(): + pipeline = _make_cosmos3_unit_pipeline(offload_stages=["not_a_real_component"]) + available_components = { + COSMOS3_REASONER_OFFLOAD_COMPONENT: pipeline.transformer.language_model.layers + } + + with pytest.raises( + ValueError, match=r"Unknown cpu_offload_config\.stages entries.*not_a_real_component" + ): + pipeline.offloader.validate_configured_stages( + pipeline.offloader.stages(), available_components + ) diff --git a/tests/unittest/_torch/visual_gen/test_wan22_t2v_offload.py b/tests/unittest/_torch/visual_gen/test_wan22_t2v_offload.py new file mode 100644 index 000000000000..f99119715d45 --- /dev/null +++ b/tests/unittest/_torch/visual_gen/test_wan22_t2v_offload.py @@ -0,0 +1,257 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for Wan 2.2 T2V CPU offloading using the full pipeline. + +Wan 2.2 two-stage T2V is tested with cpu_offload_config.enable=True: + - Wan2.2-T2V-A14B-Diffusers 480x832 + +Offloaded output is checked against the non-offload TRT-LLM baseline. HuggingFace parity +with offloading is in test_wan22_t2v_pipeline.py::test_cosine_similarity_with_offload. +FP8 dynamic quantization is checked against the TRT-LLM baseline only. +CUDA graphs with offloading are not supported (test_wan22_raises_if_cuda_graph_and_offload_enabled). +CUDA-graph correctness is in test_wan22_t2v_pipeline.py. + +Each test loads all pipeline components and calls pipeline.forward(). + +Run all: + pytest tests/unittest/_torch/visual_gen/test_wan22_t2v_offload.py -v -s + +Override checkpoint path: + DIFFUSION_MODEL_PATH_WAN22_T2V=/path/to/wan22 \\ + pytest tests/unittest/_torch/visual_gen/test_wan22_t2v_offload.py -v -s +""" + +import os + +os.environ["TLLM_DISABLE_MPI"] = "1" + +import gc +from pathlib import Path +from typing import Optional + +import pytest +import torch +import torch.nn.functional as F + +from tensorrt_llm._torch.visual_gen.pipeline_loader import PipelineLoader +from tensorrt_llm.visual_gen.args import ( + CpuOffloadConfig, + CudaGraphConfig, + TorchCompileConfig, + VisualGenArgs, +) + + +@pytest.fixture(autouse=True, scope="module") +def _cleanup_mpi_env(): + yield + os.environ.pop("TLLM_DISABLE_MPI", None) + + +@pytest.fixture(autouse=True) +def _cleanup_gpu(): + gc.collect() + torch.cuda.empty_cache() + yield + gc.collect() + torch.cuda.empty_cache() + + +# ============================================================================ +# Path helpers +# ============================================================================ + + +def _llm_models_root() -> Path: + if "LLM_MODELS_ROOT" in os.environ: + root = Path(os.environ["LLM_MODELS_ROOT"]) + else: + root = Path("/home/scratch.trt_llm_data_ci/llm-models/") + if not root.exists(): + root = Path("/scratch.trt_llm_data/llm-models/") + assert root.exists(), ( + "Set LLM_MODELS_ROOT or ensure /home/scratch.trt_llm_data_ci/llm-models/ is accessible." + ) + return root + + +def _checkpoint(env_var: str, default_name: str) -> str: + return os.environ.get(env_var) or str(_llm_models_root() / default_name) + + +WAN22_A14B_PATH = _checkpoint("DIFFUSION_MODEL_PATH_WAN22_T2V", "Wan2.2-T2V-A14B-Diffusers") + +INFER_PROMPT = "A cat sitting on a sunny windowsill watching birds outside." +INFER_NEGATIVE_PROMPT = "" +INFER_NUM_FRAMES = 9 +INFER_NUM_STEPS = 4 +INFER_SEED = 42 +INFER_GUIDANCE_SCALE = 4.0 +COS_SIM_THRESHOLD = 0.99 + + +# ============================================================================ +# Pipeline fixture factory +# ============================================================================ + + +def _make_pipeline( + checkpoint_path: str, + *, + enable_offload: bool = False, + quant_config: Optional[dict] = None, +): + if not os.path.exists(checkpoint_path): + pytest.skip(f"Checkpoint not found: {checkpoint_path}") + args = VisualGenArgs( + model=checkpoint_path, + torch_compile_config=TorchCompileConfig(enable=False), + cpu_offload_config=CpuOffloadConfig(enable=enable_offload), + quant_config=quant_config, + ) + pipeline = PipelineLoader(args).load(skip_warmup=True) + assert pipeline.transformer_2 is not None, ( + "Expected two-stage Wan 2.2 pipeline (transformer_2 should not be None)" + ) + return pipeline + + +def _require_fp8_quant_ops() -> None: + try: + _ = torch.ops.tensorrt_llm.quantize_e4m3_per_tensor + _ = torch.ops.tensorrt_llm.quantize_e4m3_activation + except (AttributeError, RuntimeError) as e: + pytest.skip(f"FP8 quantization ops not available: {e}") + + +@pytest.fixture +def wan22_offload_pipeline(): + pipeline = _make_pipeline(WAN22_A14B_PATH, enable_offload=True) + yield pipeline + del pipeline + torch.cuda.empty_cache() + + +# ============================================================================ +# Shared assertion helpers +# ============================================================================ + + +def _assert_offload_forward(pipeline, *, model: str = "") -> None: + assert pipeline.offloader.stages(), ( + f"{model}: cpu_offload_config.enable=True but offloader.stages() is empty" + ) + assert pipeline.offloader.offload_pipeline is not None, ( + f"{model}: offload pipeline was not initialized after load" + ) + + with torch.no_grad(): + pipeline.forward( + prompt=INFER_PROMPT, + negative_prompt=INFER_NEGATIVE_PROMPT, + height=480, + width=832, + num_frames=INFER_NUM_FRAMES, + num_inference_steps=INFER_NUM_STEPS, + guidance_scale=INFER_GUIDANCE_SCALE, + seed=INFER_SEED, + ) + + print(f"\n ===== Offload — Wan 2.2 {model} 480x832 =====") + print(f" stages: {pipeline.offloader.stages()}") + print(" =================================================") + + +def _assert_offload_matches_baseline( + checkpoint_path: str, + *, + model: str, + quant_config: Optional[dict] = None, +) -> None: + def _capture_video(pipe) -> torch.Tensor: + with torch.no_grad(): + result = pipe.forward( + prompt=INFER_PROMPT, + negative_prompt=INFER_NEGATIVE_PROMPT, + height=480, + width=832, + num_frames=INFER_NUM_FRAMES, + num_inference_steps=INFER_NUM_STEPS, + guidance_scale=INFER_GUIDANCE_SCALE, + seed=INFER_SEED, + ) + return result.video.float() / 255.0 + + offload_pipe = _make_pipeline(checkpoint_path, enable_offload=True, quant_config=quant_config) + offload_video = _capture_video(offload_pipe) + del offload_pipe + gc.collect() + torch.cuda.empty_cache() + + baseline_pipe = _make_pipeline(checkpoint_path, enable_offload=False, quant_config=quant_config) + baseline_video = _capture_video(baseline_pipe) + del baseline_pipe + gc.collect() + torch.cuda.empty_cache() + + assert offload_video.shape == baseline_video.shape, ( + f"{model}: shape mismatch offload {offload_video.shape} vs baseline {baseline_video.shape}" + ) + a_flat = offload_video.float().cpu().reshape(-1) + b_flat = baseline_video.float().cpu().reshape(-1) + cos_sim = F.cosine_similarity(a_flat.unsqueeze(0), b_flat.unsqueeze(0)).clamp(-1.0, 1.0).item() + print( + f"\n ===== Offload accuracy — {model} =====\n" + f" cosine_similarity={cos_sim:.6f} (threshold={COS_SIM_THRESHOLD})\n" + f" ========================================================" + ) + assert cos_sim >= COS_SIM_THRESHOLD, ( + f"{model}: offload vs baseline cosine similarity {cos_sim:.6f} < {COS_SIM_THRESHOLD}" + ) + + +# ============================================================================ +# Tests +# ============================================================================ + + +@pytest.mark.integration +@pytest.mark.wan_t2v +class TestWan22_A14B_Offload: + """Wan2.2-T2V-A14B 480x832 two-stage with CPU offloading.""" + + def test_wan22_offload(self, wan22_offload_pipeline): + _assert_offload_forward(wan22_offload_pipeline, model="T2V-A14B") + + def test_wan22_offload_matches_baseline(self): + _assert_offload_matches_baseline(WAN22_A14B_PATH, model="Wan2.2-T2V-A14B") + + def test_wan22_fp8_offload_matches_baseline(self): + _require_fp8_quant_ops() + _assert_offload_matches_baseline( + WAN22_A14B_PATH, + model="Wan2.2-T2V-A14B FP8", + quant_config={"quant_algo": "FP8", "dynamic": True}, + ) + + +@pytest.mark.integration +@pytest.mark.wan_t2v +class TestWanT2VOffloadCudaGraphRaisesError: + """CUDA graphs plus offloading must raise NotImplementedError on pipeline load.""" + + def test_wan22_raises_if_cuda_graph_and_offload_enabled(self): + if not os.path.exists(WAN22_A14B_PATH): + pytest.skip(f"Checkpoint not found: {WAN22_A14B_PATH}") + args = VisualGenArgs( + model=WAN22_A14B_PATH, + torch_compile_config=TorchCompileConfig(enable=False), + cuda_graph_config=CudaGraphConfig(enable=True), + cpu_offload_config=CpuOffloadConfig(enable=True), + ) + with pytest.raises( + NotImplementedError, + match="CUDA graphs are not supported with visual generation offloading", + ): + PipelineLoader(args).load(skip_warmup=True) diff --git a/tests/unittest/_torch/visual_gen/test_wan22_t2v_pipeline.py b/tests/unittest/_torch/visual_gen/test_wan22_t2v_pipeline.py index 9a4b36a7de8f..97abe4b574a6 100644 --- a/tests/unittest/_torch/visual_gen/test_wan22_t2v_pipeline.py +++ b/tests/unittest/_torch/visual_gen/test_wan22_t2v_pipeline.py @@ -13,6 +13,9 @@ Model tested: - Wan2.2-T2V-A14B-Diffusers (480x832, 33 frames) +Also covers offloading vs HuggingFace. Offload-specific baseline and CUDA graph +incompatibility tests are in test_wan22_t2v_offload.py. + Run: pytest tests/unittest/_torch/visual_gen/test_wan22_t2v_pipeline.py -v -s @@ -39,6 +42,7 @@ from tensorrt_llm.visual_gen.args import ( AttentionConfig, CacheDiTConfig, + CpuOffloadConfig, TorchCompileConfig, VisualGenArgs, ) @@ -90,15 +94,20 @@ def _checkpoint(env_var: str, default_name: str) -> str: # ============================================================================ -def _load_trtllm_pipeline(checkpoint_path: str): +def _load_trtllm_pipeline( + checkpoint_path: str, + *, + enable_offload: bool = False, +): """Load TRTLLM WanPipeline (two-stage) without torch.compile or warmup.""" if not os.path.exists(checkpoint_path): pytest.skip(f"Checkpoint not found: {checkpoint_path}") - args = VisualGenArgs( + kwargs = dict( model=checkpoint_path, torch_compile_config=TorchCompileConfig(enable=False), + cpu_offload_config=CpuOffloadConfig(enable=enable_offload), ) - return PipelineLoader(args).load(skip_warmup=True) + return PipelineLoader(VisualGenArgs(**kwargs)).load(skip_warmup=True) def _load_hf_pipeline(checkpoint_path: str): @@ -183,10 +192,18 @@ def _assert_pipeline_matches_hf( num_frames: int, guidance_scale: float, model_label: str, + *, + enable_offload: bool = False, ) -> None: """Run TRTLLM and HF pipelines sequentially, compare decoded video output.""" # --- TRTLLM --- - trtllm_pipe = _load_trtllm_pipeline(checkpoint_path) + trtllm_pipe = _load_trtllm_pipeline( + checkpoint_path, + enable_offload=enable_offload, + ) + if enable_offload: + assert trtllm_pipe.offloader.stages(), f"{model_label}: offload stages must be configured" + assert trtllm_pipe.offloader.offload_pipeline is not None # Confirm two-stage denoising is active (Wan 2.2 specific sanity check) assert trtllm_pipe.transformer_2 is not None, ( @@ -271,6 +288,17 @@ def test_cosine_similarity(self): model_label="Wan2.2-T2V-A14B", ) + def test_cosine_similarity_with_offload(self): + _assert_pipeline_matches_hf( + checkpoint_path=WAN22_A14B_PATH, + height=480, + width=832, + num_frames=9, + guidance_scale=4.0, + model_label="Wan2.2-T2V-A14B (offload)", + enable_offload=True, + ) + # ============================================================================ # Two-stage feature fixtures (class-scoped: each A14B pipeline pins tens of GB)