From 7c09c7024bae7612b18a863f2542a02d1adb2494 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Mon, 19 Jan 2026 03:00:53 -0800 Subject: [PATCH 01/14] init commit --- megatron/core/transformer/cuda_graphs.py | 7 +++++- megatron/core/transformer/module.py | 22 +++++++++++++++---- .../core/transformer/transformer_layer.py | 2 ++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/megatron/core/transformer/cuda_graphs.py b/megatron/core/transformer/cuda_graphs.py index ec02555233b..ac49d1d28d0 100644 --- a/megatron/core/transformer/cuda_graphs.py +++ b/megatron/core/transformer/cuda_graphs.py @@ -1994,10 +1994,15 @@ def cuda_graph_set_manual_hooks(self): Set CUDA Graph manual hooks for the modules that contain direct parameters and are covered by cudagraphs. """ + make_forward_pre_hook_func = model_chunk._make_forward_pre_hook + if self.config.delay_wgrad_compute: + make_backward_post_hook_func = model_chunk._make_backward_post_hook + else: + make_backward_post_hook_func = None for chunk_number, layers in enumerate(self.callables_per_chunk): model_chunk = self.model[chunk_number] for layer in layers: - layer.setup_manual_hooks(model_chunk._make_forward_pre_hook) + layer.setup_manual_hooks(make_forward_pre_hook_func, make_backward_post_hook_func) def delete_cuda_graphs(self): """ diff --git a/megatron/core/transformer/module.py b/megatron/core/transformer/module.py index d68f34ffd0b..77e3fa1f504 100644 --- a/megatron/core/transformer/module.py +++ b/megatron/core/transformer/module.py @@ -217,24 +217,38 @@ def get_layer_static_inputs(self, seq_length, micro_batch_size): ) return static_inputs - def setup_manual_hooks(self, make_hook_func): + def setup_manual_hooks(self, make_forward_pre_hook_func, make_backward_post_hook_func): """ Set CUDA Graph manual hooks for the submodules that contain direct parameters and are covered by cudagraphs. """ - self.cuda_graph_manual_hooks = [] + self.cuda_graph_manual_pre_hooks = [] + self.cuda_graph_manual_post_hooks = [] # Select the modules who contain direct parameters and are covered by cudagraphs. # Add these modules to the `cuda_graph_manual_hooks` because their hooks will not # be automatically triggered when they go through the CUDA Graph path. param_modules = {} + params = {} for submodule in self._get_submodules_under_cudagraphs(): for module in submodule.modules(): if next(module.parameters(recurse=False), None) is not None: # Module contains direct parameters. param_modules[id(module)] = module - for module in param_modules.values(): - self.cuda_graph_manual_hooks.append((make_hook_func(), (module,))) + + # Add parameters to `params` to avoid duplicate parameters. + if not hasattr(module, "register_wgrad_accumulation_and_reduce_hooks"): + continue + for param in module.parameters(recurse=False): + params[id(param)] = param + + # Save forward pre-hooks and backward post-hooks. + if make_forward_pre_hook_func is not None: + for module in param_modules.values(): + self.cuda_graph_manual_pre_hooks.append((make_forward_pre_hook_func(), (module,))) + if make_backward_post_hook_func is not None: + for param in params.values(): + self.cuda_graph_manual_post_hooks.append(make_backward_post_hook_func(param)) def _get_submodules_under_cudagraphs(self): """ diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index ce90aaf357a..fcdbcc4884e 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -1090,6 +1090,8 @@ def backward_dw_cudagraph(self, microbatch_idx): if not hasattr(self.cuda_graphs[cg_index], 'backward_dw'): return self.cuda_graphs[cg_index].backward_dw() + for hook in self.cuda_graph_manual_post_hooks: + hook() def __call__(self, *args, **kwargs): if self._should_call_local_cudagraph(*args, **kwargs): From 448b526a44d13e15df7097d4263cf31906f83cb1 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Mon, 19 Jan 2026 03:09:29 -0800 Subject: [PATCH 02/14] fix --- megatron/core/transformer/cuda_graphs.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/megatron/core/transformer/cuda_graphs.py b/megatron/core/transformer/cuda_graphs.py index ac49d1d28d0..177a7ce0d29 100644 --- a/megatron/core/transformer/cuda_graphs.py +++ b/megatron/core/transformer/cuda_graphs.py @@ -1994,13 +1994,13 @@ def cuda_graph_set_manual_hooks(self): Set CUDA Graph manual hooks for the modules that contain direct parameters and are covered by cudagraphs. """ - make_forward_pre_hook_func = model_chunk._make_forward_pre_hook - if self.config.delay_wgrad_compute: - make_backward_post_hook_func = model_chunk._make_backward_post_hook - else: - make_backward_post_hook_func = None for chunk_number, layers in enumerate(self.callables_per_chunk): model_chunk = self.model[chunk_number] + make_forward_pre_hook_func = model_chunk._make_forward_pre_hook + if self.config.delay_wgrad_compute: + make_backward_post_hook_func = model_chunk._make_backward_post_hook + else: + make_backward_post_hook_func = None for layer in layers: layer.setup_manual_hooks(make_forward_pre_hook_func, make_backward_post_hook_func) From ae2821b950ac0786ce3ccd509cd011c1595f3eba Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Wed, 21 Jan 2026 01:39:44 -0800 Subject: [PATCH 03/14] fix dup call for layer_norm_weight --- megatron/core/extensions/transformer_engine.py | 4 ++++ megatron/core/transformer/module.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py index d823e42b0bc..8f76749a825 100644 --- a/megatron/core/extensions/transformer_engine.py +++ b/megatron/core/extensions/transformer_engine.py @@ -915,6 +915,10 @@ def backward_dw(self): if self.config.delay_wgrad_compute: super().backward_dw() + def get_backward_dw_params(self): + """Get the parameters that need backward wgrad compute.""" + return [getattr(self, weight_name) for weight_name in self.weight_names] + class TEColumnParallelLinear(TELinear): """Wrapper for the Transformer-Engine's `Linear` layer diff --git a/megatron/core/transformer/module.py b/megatron/core/transformer/module.py index 77e3fa1f504..89d9953114d 100644 --- a/megatron/core/transformer/module.py +++ b/megatron/core/transformer/module.py @@ -240,7 +240,11 @@ def setup_manual_hooks(self, make_forward_pre_hook_func, make_backward_post_hook if not hasattr(module, "register_wgrad_accumulation_and_reduce_hooks"): continue for param in module.parameters(recurse=False): - params[id(param)] = param + if hasattr(module, "get_backward_dw_params"): + for param in module.get_backward_dw_params(): + params[id(param)] = param + else: + params[id(param)] = param # Save forward pre-hooks and backward post-hooks. if make_forward_pre_hook_func is not None: From c88d549865e795ac80d0d8be7744c76996efc590 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Wed, 21 Jan 2026 19:03:58 -0800 Subject: [PATCH 04/14] add assertion for grad accumulation fusion --- megatron/core/transformer/transformer_config.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 18cea44c51f..8cb420a137f 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -1867,6 +1867,12 @@ def __post_init__(self): 'CUDA graph scope on moe and mlp is not ' 'supported with overlap_moe_expert_parallel_comm' ) + if self.delay_wgrad_compute: + assert self.gradient_accumulation_fusion, ( + 'CUDA graph with delay_wgrad_compute is only supported when ' + 'gradient_accumulation_fusion is enabled, since gradient accumulation ' + 'implementation does not use static memory.' + ) # Check delay_wgrad_compute compatibility if self.delay_wgrad_compute: From 91f1b31d31f4bbd254dcb5ec653f8104bf1b5efe Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Wed, 21 Jan 2026 19:36:14 -0800 Subject: [PATCH 05/14] fix --- megatron/core/transformer/cuda_graphs.py | 3 ++- megatron/core/transformer/module.py | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/megatron/core/transformer/cuda_graphs.py b/megatron/core/transformer/cuda_graphs.py index 177a7ce0d29..a02e9ccf9ce 100644 --- a/megatron/core/transformer/cuda_graphs.py +++ b/megatron/core/transformer/cuda_graphs.py @@ -2021,7 +2021,8 @@ def delete_cuda_graphs(self): else: graphs_not_reset += 1 layer.cuda_graphs = [] - layer.cuda_graph_manual_hooks = [] + layer.cuda_graph_manual_pre_hooks = [] + layer.cuda_graph_manual_post_hooks = [] log_on_each_pipeline_stage( logger=logger, diff --git a/megatron/core/transformer/module.py b/megatron/core/transformer/module.py index 89d9953114d..9cd23516bbe 100644 --- a/megatron/core/transformer/module.py +++ b/megatron/core/transformer/module.py @@ -187,7 +187,8 @@ def __init__(self, config: TransformerConfig, vp_stage: Optional[int] = None): # graphs. Those hooks and args are collected in this list and should be manually # triggered before CUDA Graph running. This is required to ensure the correct param # all-gather overlap with forward compute. - self.cuda_graph_manual_hooks = [] + self.cuda_graph_manual_pre_hooks = [] + self.cuda_graph_manual_post_hooks = [] def get_layer_static_inputs(self, seq_length, micro_batch_size): """ @@ -226,8 +227,9 @@ def setup_manual_hooks(self, make_forward_pre_hook_func, make_backward_post_hook self.cuda_graph_manual_post_hooks = [] # Select the modules who contain direct parameters and are covered by cudagraphs. - # Add these modules to the `cuda_graph_manual_hooks` because their hooks will not - # be automatically triggered when they go through the CUDA Graph path. + # Add these modules and params to `cuda_graph_manual_pre_hooks` and + # `cuda_graph_manual_post_hooks` because their hooks will not be + # automatically triggered when they go through the CUDA Graph path. param_modules = {} params = {} for submodule in self._get_submodules_under_cudagraphs(): @@ -285,7 +287,7 @@ def _te_cuda_graph_replay(self, *args, **kwargs): cg_index = getattr(self, 'current_microbatch', 0) % len(self.cuda_graphs) cudagraph_args, cudagraph_kwargs = self._get_te_cuda_graph_replay_args(*args, **kwargs) - for hook, hook_args in self.cuda_graph_manual_hooks: + for hook, hook_args in self.cuda_graph_manual_pre_hooks: hook(*hook_args) return self.cuda_graphs[cg_index](*cudagraph_args, **cudagraph_kwargs) From 565f1c247b882ad3e172448235c079b0339f07e1 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Wed, 21 Jan 2026 19:52:30 -0800 Subject: [PATCH 06/14] better assertion message --- megatron/core/transformer/transformer_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index dd2dbfdcdb3..4b27bc70627 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -1875,9 +1875,9 @@ def __post_init__(self): ) if self.delay_wgrad_compute: assert self.gradient_accumulation_fusion, ( - 'CUDA graph with delay_wgrad_compute is only supported when ' - 'gradient_accumulation_fusion is enabled, since gradient accumulation ' - 'implementation does not use static memory.' + 'CUDA graph with delay_wgrad_compute requires gradient_accumulation_fusion ' + 'to be enabled. This is because the default gradient accumulation does not ' + 'use static memory addresses, which breaks CUDA graph requirements.' ) # Check delay_wgrad_compute compatibility From 2691771074fc61595be3c3c6a2b1777d5f841798 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Thu, 22 Jan 2026 01:26:51 -0800 Subject: [PATCH 07/14] move get_backward_dw_params to te --- megatron/core/extensions/transformer_engine.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/megatron/core/extensions/transformer_engine.py b/megatron/core/extensions/transformer_engine.py index 8f76749a825..d823e42b0bc 100644 --- a/megatron/core/extensions/transformer_engine.py +++ b/megatron/core/extensions/transformer_engine.py @@ -915,10 +915,6 @@ def backward_dw(self): if self.config.delay_wgrad_compute: super().backward_dw() - def get_backward_dw_params(self): - """Get the parameters that need backward wgrad compute.""" - return [getattr(self, weight_name) for weight_name in self.weight_names] - class TEColumnParallelLinear(TELinear): """Wrapper for the Transformer-Engine's `Linear` layer From 2dbef10bab6528da36bbe741aed44d904b4b6a91 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Thu, 22 Jan 2026 01:34:26 -0800 Subject: [PATCH 08/14] fix comments --- .../core/models/gpt/fine_grained_callables.py | 2 +- megatron/core/transformer/cuda_graphs.py | 2 +- megatron/core/transformer/module.py | 21 +++++++++++++++---- .../core/transformer/transformer_layer.py | 11 ---------- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py index 71c5c19749c..4d74da11020 100644 --- a/megatron/core/models/gpt/fine_grained_callables.py +++ b/megatron/core/models/gpt/fine_grained_callables.py @@ -426,7 +426,7 @@ def submodule_attn_forward(node: ScheduleNode, hidden_states: torch.Tensor): ) forward_func = layer._te_cuda_graph_replay attn_backward_dw_wrapper.set_graphed_backward_dw_callable( - partial(layer.backward_dw_cudagraph, layer.current_microbatch) + partial(layer._te_cuda_graph_backward_dw_graph, layer.current_microbatch) ) else: # wrapper function that keeps consistent api with cuda graph replay diff --git a/megatron/core/transformer/cuda_graphs.py b/megatron/core/transformer/cuda_graphs.py index a02e9ccf9ce..1c366356575 100644 --- a/megatron/core/transformer/cuda_graphs.py +++ b/megatron/core/transformer/cuda_graphs.py @@ -2022,7 +2022,7 @@ def delete_cuda_graphs(self): graphs_not_reset += 1 layer.cuda_graphs = [] layer.cuda_graph_manual_pre_hooks = [] - layer.cuda_graph_manual_post_hooks = [] + layer.cuda_graph_manual_wgrad_post_hooks = [] log_on_each_pipeline_stage( logger=logger, diff --git a/megatron/core/transformer/module.py b/megatron/core/transformer/module.py index 9cd23516bbe..0a10d8db8bd 100644 --- a/megatron/core/transformer/module.py +++ b/megatron/core/transformer/module.py @@ -188,7 +188,20 @@ def __init__(self, config: TransformerConfig, vp_stage: Optional[int] = None): # triggered before CUDA Graph running. This is required to ensure the correct param # all-gather overlap with forward compute. self.cuda_graph_manual_pre_hooks = [] - self.cuda_graph_manual_post_hooks = [] + self.cuda_graph_manual_wgrad_post_hooks = [] + + def _te_cuda_graph_backward_dw_graph(self, microbatch_idx): + """ + CUDA Graph backward weight gradient computation for current layer. + """ + cg_index = microbatch_idx % len(self.cuda_graphs) + if not hasattr(self.cuda_graphs[cg_index], 'backward_dw'): + return + self.cuda_graphs[cg_index].backward_dw() + + # Trigger the backward post-hooks. + for hook in self.cuda_graph_manual_wgrad_post_hooks: + hook() def get_layer_static_inputs(self, seq_length, micro_batch_size): """ @@ -224,11 +237,11 @@ def setup_manual_hooks(self, make_forward_pre_hook_func, make_backward_post_hook covered by cudagraphs. """ self.cuda_graph_manual_pre_hooks = [] - self.cuda_graph_manual_post_hooks = [] + self.cuda_graph_manual_wgrad_post_hooks = [] # Select the modules who contain direct parameters and are covered by cudagraphs. # Add these modules and params to `cuda_graph_manual_pre_hooks` and - # `cuda_graph_manual_post_hooks` because their hooks will not be + # `cuda_graph_manual_wgrad_post_hooks` because their hooks will not be # automatically triggered when they go through the CUDA Graph path. param_modules = {} params = {} @@ -254,7 +267,7 @@ def setup_manual_hooks(self, make_forward_pre_hook_func, make_backward_post_hook self.cuda_graph_manual_pre_hooks.append((make_forward_pre_hook_func(), (module,))) if make_backward_post_hook_func is not None: for param in params.values(): - self.cuda_graph_manual_post_hooks.append(make_backward_post_hook_func(param)) + self.cuda_graph_manual_wgrad_post_hooks.append(make_backward_post_hook_func(param)) def _get_submodules_under_cudagraphs(self): """ diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index fcdbcc4884e..962e0527fed 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -1082,17 +1082,6 @@ def _should_call_local_cudagraph(self, *args, **kwargs): return True return False - def backward_dw_cudagraph(self, microbatch_idx): - """ - CUDA Graph backward weight gradient computation for this layer. - """ - cg_index = microbatch_idx % len(self.cuda_graphs) - if not hasattr(self.cuda_graphs[cg_index], 'backward_dw'): - return - self.cuda_graphs[cg_index].backward_dw() - for hook in self.cuda_graph_manual_post_hooks: - hook() - def __call__(self, *args, **kwargs): if self._should_call_local_cudagraph(*args, **kwargs): # Inference mode. From 8588fbaa33638299219115c1dfbfe31b4ba86bb1 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Thu, 22 Jan 2026 01:41:05 -0800 Subject: [PATCH 09/14] add te version guard --- megatron/core/transformer/transformer_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 4b27bc70627..9dde41c3c40 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -1874,6 +1874,9 @@ def __post_init__(self): 'supported with overlap_moe_expert_parallel_comm' ) if self.delay_wgrad_compute: + assert is_te_min_version( + "2.12.0" + ), "CUDA graph with delay_wgrad_compute requires TE version >= 2.12.0." assert self.gradient_accumulation_fusion, ( 'CUDA graph with delay_wgrad_compute requires gradient_accumulation_fusion ' 'to be enabled. This is because the default gradient accumulation does not ' From 31c19ec816baf9193748ad611f044cd556d6676b Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Thu, 22 Jan 2026 01:54:15 -0800 Subject: [PATCH 10/14] disable bias --- megatron/core/transformer/transformer_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 9dde41c3c40..51eb4672024 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -1873,7 +1873,7 @@ def __post_init__(self): 'CUDA graph scope on moe and mlp is not ' 'supported with overlap_moe_expert_parallel_comm' ) - if self.delay_wgrad_compute: + if self.delay_wgrad_compute and CudaGraphScope.attn in self.cuda_graph_scope: assert is_te_min_version( "2.12.0" ), "CUDA graph with delay_wgrad_compute requires TE version >= 2.12.0." @@ -1882,6 +1882,9 @@ def __post_init__(self): 'to be enabled. This is because the default gradient accumulation does not ' 'use static memory addresses, which breaks CUDA graph requirements.' ) + assert ( + not self.add_bias_linear and not self.add_qkv_bias + ), "CudaGraphScope.attn with delay_wgrad_compute doesn't support bias for now." # Check delay_wgrad_compute compatibility if self.delay_wgrad_compute: From 2d35b2acabc07814b6185dc2961fbbdbbd6f0fd4 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Thu, 22 Jan 2026 02:06:47 -0800 Subject: [PATCH 11/14] extend wgrad assertion scope --- megatron/core/transformer/transformer_config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 51eb4672024..3cfa68b4451 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -1873,7 +1873,12 @@ def __post_init__(self): 'CUDA graph scope on moe and mlp is not ' 'supported with overlap_moe_expert_parallel_comm' ) - if self.delay_wgrad_compute and CudaGraphScope.attn in self.cuda_graph_scope: + wgrad_in_graph_scope = CudaGraphScope.attn in self.cuda_graph_scope or ( + CudaGraphScope.moe_router in self.cuda_graph_scope + and self.moe_shared_expert_intermediate_size is not None + and not self.moe_shared_expert_overlap + ) + if self.delay_wgrad_compute and wgrad_in_graph_scope: assert is_te_min_version( "2.12.0" ), "CUDA graph with delay_wgrad_compute requires TE version >= 2.12.0." @@ -1884,7 +1889,7 @@ def __post_init__(self): ) assert ( not self.add_bias_linear and not self.add_qkv_bias - ), "CudaGraphScope.attn with delay_wgrad_compute doesn't support bias for now." + ), "CUDA graph with delay_wgrad_compute doesn't support bias for now." # Check delay_wgrad_compute compatibility if self.delay_wgrad_compute: From f7209d92a4d4ea87992cf36421bab7b72a86c6f9 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Sun, 25 Jan 2026 18:50:38 -0800 Subject: [PATCH 12/14] revert get_backward_dw_params and trigger hook after wgrad graph execution --- .../core/models/gpt/fine_grained_callables.py | 2 +- megatron/core/transformer/cuda_graphs.py | 10 +--- megatron/core/transformer/module.py | 49 +++---------------- .../core/transformer/transformer_layer.py | 9 ++++ 4 files changed, 20 insertions(+), 50 deletions(-) diff --git a/megatron/core/models/gpt/fine_grained_callables.py b/megatron/core/models/gpt/fine_grained_callables.py index 901649b0c97..5a365b015b2 100644 --- a/megatron/core/models/gpt/fine_grained_callables.py +++ b/megatron/core/models/gpt/fine_grained_callables.py @@ -424,7 +424,7 @@ def submodule_attn_forward(node: ScheduleNode, hidden_states: torch.Tensor): ) forward_func = layer._te_cuda_graph_replay attn_backward_dw_wrapper.set_graphed_backward_dw_callable( - partial(layer._te_cuda_graph_backward_dw_graph, layer.current_microbatch) + partial(layer.backward_dw_cudagraph, layer.current_microbatch) ) else: # wrapper function that keeps consistent api with cuda graph replay diff --git a/megatron/core/transformer/cuda_graphs.py b/megatron/core/transformer/cuda_graphs.py index 1c366356575..ec02555233b 100644 --- a/megatron/core/transformer/cuda_graphs.py +++ b/megatron/core/transformer/cuda_graphs.py @@ -1996,13 +1996,8 @@ def cuda_graph_set_manual_hooks(self): """ for chunk_number, layers in enumerate(self.callables_per_chunk): model_chunk = self.model[chunk_number] - make_forward_pre_hook_func = model_chunk._make_forward_pre_hook - if self.config.delay_wgrad_compute: - make_backward_post_hook_func = model_chunk._make_backward_post_hook - else: - make_backward_post_hook_func = None for layer in layers: - layer.setup_manual_hooks(make_forward_pre_hook_func, make_backward_post_hook_func) + layer.setup_manual_hooks(model_chunk._make_forward_pre_hook) def delete_cuda_graphs(self): """ @@ -2021,8 +2016,7 @@ def delete_cuda_graphs(self): else: graphs_not_reset += 1 layer.cuda_graphs = [] - layer.cuda_graph_manual_pre_hooks = [] - layer.cuda_graph_manual_wgrad_post_hooks = [] + layer.cuda_graph_manual_hooks = [] log_on_each_pipeline_stage( logger=logger, diff --git a/megatron/core/transformer/module.py b/megatron/core/transformer/module.py index 0a10d8db8bd..d68f34ffd0b 100644 --- a/megatron/core/transformer/module.py +++ b/megatron/core/transformer/module.py @@ -187,21 +187,7 @@ def __init__(self, config: TransformerConfig, vp_stage: Optional[int] = None): # graphs. Those hooks and args are collected in this list and should be manually # triggered before CUDA Graph running. This is required to ensure the correct param # all-gather overlap with forward compute. - self.cuda_graph_manual_pre_hooks = [] - self.cuda_graph_manual_wgrad_post_hooks = [] - - def _te_cuda_graph_backward_dw_graph(self, microbatch_idx): - """ - CUDA Graph backward weight gradient computation for current layer. - """ - cg_index = microbatch_idx % len(self.cuda_graphs) - if not hasattr(self.cuda_graphs[cg_index], 'backward_dw'): - return - self.cuda_graphs[cg_index].backward_dw() - - # Trigger the backward post-hooks. - for hook in self.cuda_graph_manual_wgrad_post_hooks: - hook() + self.cuda_graph_manual_hooks = [] def get_layer_static_inputs(self, seq_length, micro_batch_size): """ @@ -231,43 +217,24 @@ def get_layer_static_inputs(self, seq_length, micro_batch_size): ) return static_inputs - def setup_manual_hooks(self, make_forward_pre_hook_func, make_backward_post_hook_func): + def setup_manual_hooks(self, make_hook_func): """ Set CUDA Graph manual hooks for the submodules that contain direct parameters and are covered by cudagraphs. """ - self.cuda_graph_manual_pre_hooks = [] - self.cuda_graph_manual_wgrad_post_hooks = [] + self.cuda_graph_manual_hooks = [] # Select the modules who contain direct parameters and are covered by cudagraphs. - # Add these modules and params to `cuda_graph_manual_pre_hooks` and - # `cuda_graph_manual_wgrad_post_hooks` because their hooks will not be - # automatically triggered when they go through the CUDA Graph path. + # Add these modules to the `cuda_graph_manual_hooks` because their hooks will not + # be automatically triggered when they go through the CUDA Graph path. param_modules = {} - params = {} for submodule in self._get_submodules_under_cudagraphs(): for module in submodule.modules(): if next(module.parameters(recurse=False), None) is not None: # Module contains direct parameters. param_modules[id(module)] = module - - # Add parameters to `params` to avoid duplicate parameters. - if not hasattr(module, "register_wgrad_accumulation_and_reduce_hooks"): - continue - for param in module.parameters(recurse=False): - if hasattr(module, "get_backward_dw_params"): - for param in module.get_backward_dw_params(): - params[id(param)] = param - else: - params[id(param)] = param - - # Save forward pre-hooks and backward post-hooks. - if make_forward_pre_hook_func is not None: - for module in param_modules.values(): - self.cuda_graph_manual_pre_hooks.append((make_forward_pre_hook_func(), (module,))) - if make_backward_post_hook_func is not None: - for param in params.values(): - self.cuda_graph_manual_wgrad_post_hooks.append(make_backward_post_hook_func(param)) + for module in param_modules.values(): + self.cuda_graph_manual_hooks.append((make_hook_func(), (module,))) def _get_submodules_under_cudagraphs(self): """ @@ -300,7 +267,7 @@ def _te_cuda_graph_replay(self, *args, **kwargs): cg_index = getattr(self, 'current_microbatch', 0) % len(self.cuda_graphs) cudagraph_args, cudagraph_kwargs = self._get_te_cuda_graph_replay_args(*args, **kwargs) - for hook, hook_args in self.cuda_graph_manual_pre_hooks: + for hook, hook_args in self.cuda_graph_manual_hooks: hook(*hook_args) return self.cuda_graphs[cg_index](*cudagraph_args, **cudagraph_kwargs) diff --git a/megatron/core/transformer/transformer_layer.py b/megatron/core/transformer/transformer_layer.py index 45078cec0d9..a486b6ed3d5 100644 --- a/megatron/core/transformer/transformer_layer.py +++ b/megatron/core/transformer/transformer_layer.py @@ -1079,6 +1079,15 @@ def _should_call_local_cudagraph(self, *args, **kwargs): return True return False + def backward_dw_cudagraph(self, microbatch_idx): + """ + CUDA Graph backward weight gradient computation for this layer. + """ + cg_index = microbatch_idx % len(self.cuda_graphs) + if not hasattr(self.cuda_graphs[cg_index], 'backward_dw'): + return + self.cuda_graphs[cg_index].backward_dw() + def __call__(self, *args, **kwargs): if self._should_call_local_cudagraph(*args, **kwargs): # Inference mode. From 2e4b51016c2778607e45ed9f81683221f5b54793 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Sun, 25 Jan 2026 23:12:36 -0800 Subject: [PATCH 13/14] fix attn bias assertion scope --- megatron/core/transformer/transformer_config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index 3cfa68b4451..fae9912ab5f 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -1887,9 +1887,10 @@ def __post_init__(self): 'to be enabled. This is because the default gradient accumulation does not ' 'use static memory addresses, which breaks CUDA graph requirements.' ) - assert ( - not self.add_bias_linear and not self.add_qkv_bias - ), "CUDA graph with delay_wgrad_compute doesn't support bias for now." + if CudaGraphScope.attn in self.cuda_graph_scope: + assert ( + not self.add_bias_linear and not self.add_qkv_bias + ), "CUDA graph with delay_wgrad_compute doesn't support attn bias for now." # Check delay_wgrad_compute compatibility if self.delay_wgrad_compute: From 4a513fbd2a64ac7b38ece486e4d0a1c8cd598f91 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Mon, 26 Jan 2026 01:13:29 -0800 Subject: [PATCH 14/14] fix --- .../core/transformer/transformer_config.py | 18 ------------------ megatron/training/arguments.py | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/megatron/core/transformer/transformer_config.py b/megatron/core/transformer/transformer_config.py index fae9912ab5f..8f5462ff55b 100644 --- a/megatron/core/transformer/transformer_config.py +++ b/megatron/core/transformer/transformer_config.py @@ -1873,24 +1873,6 @@ def __post_init__(self): 'CUDA graph scope on moe and mlp is not ' 'supported with overlap_moe_expert_parallel_comm' ) - wgrad_in_graph_scope = CudaGraphScope.attn in self.cuda_graph_scope or ( - CudaGraphScope.moe_router in self.cuda_graph_scope - and self.moe_shared_expert_intermediate_size is not None - and not self.moe_shared_expert_overlap - ) - if self.delay_wgrad_compute and wgrad_in_graph_scope: - assert is_te_min_version( - "2.12.0" - ), "CUDA graph with delay_wgrad_compute requires TE version >= 2.12.0." - assert self.gradient_accumulation_fusion, ( - 'CUDA graph with delay_wgrad_compute requires gradient_accumulation_fusion ' - 'to be enabled. This is because the default gradient accumulation does not ' - 'use static memory addresses, which breaks CUDA graph requirements.' - ) - if CudaGraphScope.attn in self.cuda_graph_scope: - assert ( - not self.add_bias_linear and not self.add_qkv_bias - ), "CUDA graph with delay_wgrad_compute doesn't support attn bias for now." # Check delay_wgrad_compute compatibility if self.delay_wgrad_compute: diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index b94b5b45544..d2f5fad4b25 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1303,6 +1303,25 @@ def validate_args(args, defaults={}): assert is_te_min_version("2.8.0"), ( "overlap_grad_reduce is only supported with TE >= 2.8.0 when enabling delay_wgrad_compute" ) + wgrad_in_graph_scope = CudaGraphScope.attn in args.cuda_graph_scope or ( + CudaGraphScope.moe_router in args.cuda_graph_scope + and args.moe_shared_expert_intermediate_size is not None + and not args.moe_shared_expert_overlap + ) + if wgrad_in_graph_scope: + assert is_te_min_version( + "2.12.0" + ), "CUDA graph with delay_wgrad_compute requires TE version >= 2.12.0." + assert args.gradient_accumulation_fusion, ( + 'CUDA graph with delay_wgrad_compute requires gradient_accumulation_fusion ' + 'to be enabled. This is because the default gradient accumulation does not ' + 'use static memory addresses, which breaks CUDA graph requirements.' + ) + if CudaGraphScope.attn in args.cuda_graph_scope: + assert ( + not args.add_bias_linear and not args.add_qkv_bias + ), "CUDA graph with delay_wgrad_compute doesn't support attn bias for now." + if not args.gradient_accumulation_fusion: assert is_te_min_version("2.7.0"), ( "disabling gradient_accumulation_fusion is only supported with TE >= 2.7.0 "