From 58d15ba8bc3e7dd227a617aab1fc5b9dc93fd835 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Wed, 22 Apr 2026 21:56:05 -0700 Subject: [PATCH 1/3] cuda graph embedding and output layer --- megatron/core/models/hybrid/hybrid_block.py | 37 ++------------------- megatron/core/models/hybrid/hybrid_model.py | 36 +++++++++++++++++++- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/megatron/core/models/hybrid/hybrid_block.py b/megatron/core/models/hybrid/hybrid_block.py index 5494d531e52..6d20bcdd6e5 100644 --- a/megatron/core/models/hybrid/hybrid_block.py +++ b/megatron/core/models/hybrid/hybrid_block.py @@ -25,7 +25,7 @@ from megatron.core.transformer import TransformerConfig from megatron.core.transformer.enums import CudaGraphScope from megatron.core.transformer.identity_op import IdentityOp -from megatron.core.transformer.module import GraphableMegatronModule, MegatronModule +from megatron.core.transformer.module import MegatronModule from megatron.core.transformer.spec_utils import ModuleSpec, build_module from megatron.core.transformer.transformer_layer import TransformerLayer from megatron.core.transformer.utils import sharded_state_dict_default @@ -47,7 +47,7 @@ class HybridStackSubmodules: mtp_block_spec: Optional[ModuleSpec] = None -class HybridStack(GraphableMegatronModule, MegatronModule): +class HybridStack(MegatronModule): """ Constructor for the HybridStack class. @@ -206,39 +206,6 @@ def mamba_state_shapes_per_request(self) -> Optional[Tuple[Tuple[int], Tuple[int return layer.mamba_state_shapes_per_request() return None - def _should_call_local_cudagraph(self, *args, **kwargs): - """ - Check if we should call the local cudagraph path. - """ - if ( - not self.training - and hasattr(self, 'cudagraph_manager') - and kwargs['attention_mask'] is None - and ( - kwargs.get('inference_context') is not None - or kwargs.get('inference_params') is not None - ) - and CudaGraphScope.full_iteration_inference in self.config.cuda_graph_scope - ): - if kwargs['inference_context'].is_static_batching(): - using_cuda_graph = kwargs['inference_context'].is_decode_only() - else: - using_cuda_graph = kwargs['inference_context'].using_cuda_graph_this_step() - - if using_cuda_graph: - return True - return False - - def __call__(self, *args, **kwargs): - if self._should_call_local_cudagraph(*args, **kwargs): - kwargs['hidden_states'] = ( - kwargs['hidden_states'].unwrap() - if isinstance(kwargs['hidden_states'], WrappedTensor) - else kwargs['hidden_states'] - ) - return super().__call__(*args, **kwargs)[0] - return super().__call__(*args, **kwargs) - def forward( self, hidden_states: Union[Tensor, WrappedTensor], diff --git a/megatron/core/models/hybrid/hybrid_model.py b/megatron/core/models/hybrid/hybrid_model.py index f1b3c102634..97759dd0435 100644 --- a/megatron/core/models/hybrid/hybrid_model.py +++ b/megatron/core/models/hybrid/hybrid_model.py @@ -32,11 +32,13 @@ is_using_quantization_scales, log_single_rank, ) +from megatron.core.transformer.module import GraphableMegatronModule +from megatron.core.transformer.enums import CudaGraphScope logger = logging.getLogger(__name__) -class HybridModel(LanguageModule): +class HybridModel(LanguageModule, GraphableMegatronModule): """Hybrid language model. Args: @@ -320,6 +322,38 @@ def preprocess_for_fine_grained_offloading(self): off_interface.mark_not_offloadable(param) self.disable_param_offloading = False + def _should_call_local_cudagraph(self, *args, **kwargs): + """ + Check if we should call the local cudagraph path. + """ + if ( + not self.training + and hasattr(self, 'cudagraph_manager') + and ( + kwargs.get('inference_context') is not None + or kwargs.get('inference_params') is not None + ) + and CudaGraphScope.full_iteration_inference in self.config.cuda_graph_scope + ): + if kwargs['inference_context'].is_static_batching(): + using_cuda_graph = kwargs['inference_context'].is_decode_only() + else: + using_cuda_graph = kwargs['inference_context'].using_cuda_graph_this_step() + + if using_cuda_graph: + return True + return False + + def __call__(self, *args, **kwargs): + if self._should_call_local_cudagraph(*args, **kwargs): + return super().__call__(*args, **kwargs)[0] + return super().__call__(*args, **kwargs) + + def create_mcore_cudagraph_manager(self, config): + if CudaGraphScope.full_iteration_inference in config.cuda_graph_scope: + from megatron.core.transformer.cuda_graphs import CudaGraphManager + self.cudagraph_manager = CudaGraphManager(config) + def forward( self, input_ids: Tensor, From 0a81c4d0939f7620ead55fb376755c430610799b Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Fri, 24 Apr 2026 09:19:24 -0700 Subject: [PATCH 2/3] format --- megatron/core/models/hybrid/hybrid_model.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/megatron/core/models/hybrid/hybrid_model.py b/megatron/core/models/hybrid/hybrid_model.py index 5210453dc04..b575792e81b 100644 --- a/megatron/core/models/hybrid/hybrid_model.py +++ b/megatron/core/models/hybrid/hybrid_model.py @@ -20,7 +20,8 @@ from megatron.core.quantization.utils import get_quant_config_or_none from megatron.core.tensor_parallel import gather_from_sequence_parallel_region from megatron.core.transformer import TransformerConfig -from megatron.core.transformer.enums import ModelType +from megatron.core.transformer.enums import CudaGraphScope, ModelType +from megatron.core.transformer.module import GraphableMegatronModule from megatron.core.transformer.multi_token_prediction import ( MultiTokenPredictionBlock, mtp_on_this_rank, @@ -33,8 +34,6 @@ is_using_quantization_scales, log_single_rank, ) -from megatron.core.transformer.module import GraphableMegatronModule -from megatron.core.transformer.enums import CudaGraphScope logger = logging.getLogger(__name__) @@ -368,10 +367,14 @@ def __call__(self, *args, **kwargs): if self._should_call_local_cudagraph(*args, **kwargs): return super().__call__(*args, **kwargs)[0] return super().__call__(*args, **kwargs) - + def create_mcore_cudagraph_manager(self, config): + """ + Create the cudagraph manager for the full iteration inference scope + """ if CudaGraphScope.full_iteration_inference in config.cuda_graph_scope: from megatron.core.transformer.cuda_graphs import CudaGraphManager + self.cudagraph_manager = CudaGraphManager(config) def forward( From 9fd995b99a4220f77b235e7b7f06fff2d5b1c398 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Fri, 24 Apr 2026 10:59:04 -0700 Subject: [PATCH 3/3] fix unit test --- tests/unit_tests/inference/engines/test_dynamic_engine.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit_tests/inference/engines/test_dynamic_engine.py b/tests/unit_tests/inference/engines/test_dynamic_engine.py index b23e9562242..7b310c6210f 100644 --- a/tests/unit_tests/inference/engines/test_dynamic_engine.py +++ b/tests/unit_tests/inference/engines/test_dynamic_engine.py @@ -600,8 +600,11 @@ def test_simple(self, model_provider, num_cuda_graphs, cuda_graph_scope) -> None assert env.engine.context.cuda_graph_batch_dimensions_list model = env.engine.controller.inference_wrapped_model.model if cuda_graph_scope == [CudaGraphScope.full_iteration_inference]: - # check if cudagraph runners are created at the decoder level - assert model.decoder.cudagraph_manager.cudagraph_runners + # hybrid models attach cudagraph_manager to the model; others attach to the decoder + if model_provider == "hybrid": + assert model.cudagraph_manager.cudagraph_runners + else: + assert model.decoder.cudagraph_manager.cudagraph_runners else: # check if cudagraph runners are created at the layer level for layer in model.decoder.layers: