Repository navigation
[Main][feat] Support A2A Overlap for Megatron-FSDP #3797
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
523ac92
9c16462
dd16dcb
43a49bc
e2a77b9
16fd107
0d0bc43
2641d85
02d69f6
10629ab
1382745
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -265,14 +265,18 @@ def __init__( | |
| bwd_dw_callables (list): List of weight gradient functions for the layer. | ||
| extra_args (dict): Extra arguments for the node: is_moe, config. | ||
| """ | ||
| # determine whether to free input memory | ||
| # Determine whether to free input memory | ||
| config = extra_args.get("config", None) | ||
| assert config is not None, "model config must be passed to TransformerLayerNode." | ||
| is_moe = extra_args.get("is_moe", False) | ||
| num_local_experts = extra_args.get("num_local_experts", None) | ||
| free_input = should_free_input(name, is_moe, config, num_local_experts) | ||
| self.delay_wgrad_compute = extra_args.get("delay_wgrad_compute", False) | ||
|
|
||
| # Megatron-FSDP hooks setup. | ||
| self.is_layer_first_node = None | ||
| self.is_layer_last_node = None | ||
|
|
||
| super().__init__( | ||
| weak_method(self.forward_impl), | ||
| stream, | ||
|
|
@@ -287,6 +291,7 @@ def __init__( | |
| self.detached = tuple() | ||
| self.before_detached = tuple() | ||
| self.is_mtp = extra_args.get("is_mtp", False) | ||
| self.fsdp_post_backward_gradient_hooks = None | ||
|
|
||
| # Create flags to indicate first and last layer | ||
| self.is_first_layer = extra_args.get("is_first_layer", False) | ||
|
|
@@ -326,6 +331,20 @@ def backward_impl(self, outputs, output_grad): | |
| # return grads for record stream | ||
| return grads | ||
|
|
||
| def forward(self, *inputs): | ||
| """Execute forward pass and corresponding hooks.""" | ||
| output = super().forward(*inputs) | ||
| if self.is_layer_last_node: | ||
| self._post_forward_hook() | ||
| return output | ||
|
|
||
| def backward(self, *output_grad): | ||
| """Execute backward pass and corresponding hooks.""" | ||
| grads = super().backward(*output_grad) | ||
| if not self.delay_wgrad_compute and self.is_layer_first_node: | ||
| self._post_backward_hook() | ||
| return grads | ||
|
|
||
| def backward_dw(self): | ||
| """Computes the weight gradients for the transformer layer node.""" | ||
| if not self.delay_wgrad_compute: | ||
|
|
@@ -339,6 +358,36 @@ def backward_dw(self): | |
| module.backward_dw() | ||
| nvtx_range_pop(nvtx_msg) | ||
|
|
||
| # Setup delayed wgrad for Megatron FSDP by collecting gradient acc | ||
| # hooks if there is `.grad` attribute attached to param. | ||
| if self.fsdp_post_backward_gradient_hooks is None: | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can these be part of the "post_backward_hook" called later? This all seems convoluted and very FSDP specific, can this be cleaned up? In general, where possible, for shared modules generic functionality should be added that will enable FSDP to use that generic functionality to achieve the things they need to do.
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Agreed, commenting so I can find this later. |
||
| self.fsdp_post_backward_gradient_hooks = [] | ||
| for module in self.bwd_dw_callables: | ||
| for param in module.parameters(): | ||
| # Collect hook only if the gradient is generated in current | ||
| # TransformerLayerNode, because the fsdp_grad_acc hook needs | ||
| # to be executed right after `backward_dw` finishes. | ||
| # For example: Shared expert's hook should be collected in | ||
| # `attn` Node, even if the param belongs to `mlp` Node. | ||
| if ( | ||
| getattr(param, "post_wgrad_grad_acc_hook", False) | ||
| and param.requires_grad | ||
| and param.grad is not None | ||
| ): | ||
| self.fsdp_post_backward_gradient_hooks.append( | ||
| param.post_wgrad_grad_acc_hook | ||
| ) | ||
|
|
||
| # Execute gradient accumulation hooks for Megatron FSDP. | ||
| if self.fsdp_post_backward_gradient_hooks: | ||
| with torch.cuda.stream(self.stream): | ||
| for hook in self.fsdp_post_backward_gradient_hooks: | ||
| hook() | ||
|
|
||
| # Execute resharding hook for Megatron FSDP. | ||
| if self.is_layer_first_node: | ||
| self._post_backward_hook() | ||
|
|
||
| # the output grad memory is last used in wgrad compute, should be safe to release. | ||
| assert self.delay_grads_release, "output grad memory should be valid before wgrad." | ||
| if self.manual_release_grads: | ||
|
|
@@ -348,6 +397,16 @@ def backward_dw(self): | |
|
|
||
| self.bwd_dw_callables = None | ||
|
|
||
| def set_post_forward_hook(self, hook): | ||
| """Register a hook to release FSDP params after this node's forward pass.""" | ||
|
Wohox marked this conversation as resolved.
Resolved
|
||
| self.is_layer_last_node = True | ||
| self._post_forward_hook = hook | ||
|
|
||
| def set_post_backward_hook(self, hook): | ||
| """Register a hook to release FSDP params after this node's backward pass.""" | ||
| self.is_layer_first_node = True | ||
| self._post_backward_hook = hook | ||
|
|
||
| def __del__(self): | ||
| # Release reference as early as possible, this helps avoid memory leak. | ||
| self.before_detached = None | ||
|
|
@@ -380,10 +439,13 @@ def __init__(self, layer): | |
| self.layer = layer | ||
| self.graphed_backward_dw_callable = None | ||
| self.attn_dw_callable = layer.self_attention.backward_dw | ||
| self.submodules = [layer.self_attention] | ||
| if layer.is_moe_layer: | ||
| self.shared_expert_dw_callable = partial( | ||
| layer.mlp.backward_dw, routed_experts=False, shared_experts=True | ||
| ) | ||
| if layer.mlp.use_shared_expert: | ||
| self.submodules.append(layer.mlp.shared_experts) | ||
| else: | ||
| self.shared_expert_dw_callable = None | ||
| self.cuda_graph_scope = layer.config.cuda_graph_scope | ||
|
|
@@ -405,6 +467,17 @@ def set_graphed_backward_dw_callable(self, graphed_backward_dw_callable): | |
| """Store the CUDA graphed backward weight gradient callable.""" | ||
| self.graphed_backward_dw_callable = graphed_backward_dw_callable | ||
|
|
||
| def parameters(self): | ||
| """Returns an iterator over module parameters. | ||
|
|
||
| This method mimics the behavior of torch.nn.Module.parameters() by yielding | ||
| all parameters from the submodules managed by this wrapper. It is used to | ||
| collect parameters that require gradient computation during the backward pass. | ||
| """ | ||
| for module in self.submodules: | ||
| for param in module.parameters(): | ||
| yield param | ||
|
|
||
|
|
||
| def build_transformer_layer_callables(layer: TransformerLayer): | ||
| """Create callables for transformer layer nodes. | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.