Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions megatron/core/extensions/transformer_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -915,6 +915,10 @@ def backward_dw(self):
if self.config.delay_wgrad_compute:
super().backward_dw()

def get_backward_dw_params(self):
"""Get the parameters that need backward wgrad compute."""
return [getattr(self, weight_name) for weight_name in self.weight_names]
Comment thread
Wohox marked this conversation as resolved.
Resolved


class TEColumnParallelLinear(TELinear):
"""Wrapper for the Transformer-Engine's `Linear` layer
Expand Down
7 changes: 6 additions & 1 deletion megatron/core/transformer/cuda_graphs.py
Original file line number Diff line number Diff line change
Expand Up @@ -1996,8 +1996,13 @@ def cuda_graph_set_manual_hooks(self):
"""
for chunk_number, layers in enumerate(self.callables_per_chunk):
model_chunk = self.model[chunk_number]
make_forward_pre_hook_func = model_chunk._make_forward_pre_hook
if self.config.delay_wgrad_compute:
make_backward_post_hook_func = model_chunk._make_backward_post_hook
else:
make_backward_post_hook_func = None
for layer in layers:
layer.setup_manual_hooks(model_chunk._make_forward_pre_hook)
layer.setup_manual_hooks(make_forward_pre_hook_func, make_backward_post_hook_func)

def delete_cuda_graphs(self):
"""
Expand Down
26 changes: 22 additions & 4 deletions megatron/core/transformer/module.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,24 +217,42 @@ def get_layer_static_inputs(self, seq_length, micro_batch_size):
)
return static_inputs

def setup_manual_hooks(self, make_hook_func):
def setup_manual_hooks(self, make_forward_pre_hook_func, make_backward_post_hook_func):
"""
Set CUDA Graph manual hooks for the submodules that contain direct parameters and are
covered by cudagraphs.
"""
self.cuda_graph_manual_hooks = []
self.cuda_graph_manual_pre_hooks = []
self.cuda_graph_manual_post_hooks = []

# Select the modules who contain direct parameters and are covered by cudagraphs.
# Add these modules to the `cuda_graph_manual_hooks` because their hooks will not
# be automatically triggered when they go through the CUDA Graph path.
param_modules = {}
params = {}
for submodule in self._get_submodules_under_cudagraphs():
for module in submodule.modules():
if next(module.parameters(recurse=False), None) is not None:
# Module contains direct parameters.
param_modules[id(module)] = module
for module in param_modules.values():
self.cuda_graph_manual_hooks.append((make_hook_func(), (module,)))

# Add parameters to `params` to avoid duplicate parameters.
if not hasattr(module, "register_wgrad_accumulation_and_reduce_hooks"):
continue
for param in module.parameters(recurse=False):
if hasattr(module, "get_backward_dw_params"):
for param in module.get_backward_dw_params():
params[id(param)] = param
else:
params[id(param)] = param

# Save forward pre-hooks and backward post-hooks.
if make_forward_pre_hook_func is not None:
for module in param_modules.values():
self.cuda_graph_manual_pre_hooks.append((make_forward_pre_hook_func(), (module,)))
if make_backward_post_hook_func is not None:
for param in params.values():
self.cuda_graph_manual_post_hooks.append(make_backward_post_hook_func(param))

def _get_submodules_under_cudagraphs(self):
"""
Expand Down
6 changes: 6 additions & 0 deletions megatron/core/transformer/transformer_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -1873,6 +1873,12 @@ def __post_init__(self):
'CUDA graph scope on moe and mlp is not '
'supported with overlap_moe_expert_parallel_comm'
)
if self.delay_wgrad_compute:
assert self.gradient_accumulation_fusion, (
Comment thread
Wohox marked this conversation as resolved.
Resolved
'CUDA graph with delay_wgrad_compute is only supported when '
'gradient_accumulation_fusion is enabled, since gradient accumulation '
'implementation does not use static memory.'
)

# Check delay_wgrad_compute compatibility
if self.delay_wgrad_compute:
Expand Down
2 changes: 2 additions & 0 deletions megatron/core/transformer/transformer_layer.py
Original file line number Diff line number Diff line change
Expand Up @@ -1090,6 +1090,8 @@ def backward_dw_cudagraph(self, microbatch_idx):
if not hasattr(self.cuda_graphs[cg_index], 'backward_dw'):
return
self.cuda_graphs[cg_index].backward_dw()
for hook in self.cuda_graph_manual_post_hooks:
hook()

def __call__(self, *args, **kwargs):
if self._should_call_local_cudagraph(*args, **kwargs):
Expand Down