From 739988f4d2174a68dcf3cf8f0bf75792e413f7c0 Mon Sep 17 00:00:00 2001 From: Deepak Narayanan Date: Fri, 16 Jan 2026 11:22:28 -0800 Subject: [PATCH 1/3] Create separate higher-precision main_grad tensor for local gradient accumulation - Add separate copy to communication buffer before collective is launched (taking care to do this with and without overlapping enabled) - Zero out extra main_grad tensors when reset() is called - Log main_grad.dtype for every parameter for debugging - Nit: Log params in right order - [From Cursor / Claude Opus 4.5] Copy from communication buffer back to .main_grad to make sure optimizer uses reduced weights in optimizer step - Scale extra main_grad if it exists if using --calculate-per-token-loss --- .../distributed/distributed_data_parallel.py | 8 +- .../distributed_data_parallel_config.py | 12 ++- .../core/distributed/param_and_grad_buffer.py | 95 ++++++++++++++++--- megatron/training/arguments.py | 3 + megatron/training/training.py | 2 + 5 files changed, 104 insertions(+), 16 deletions(-) diff --git a/megatron/core/distributed/distributed_data_parallel.py b/megatron/core/distributed/distributed_data_parallel.py index dca0a8777dc..6855c32c15a 100644 --- a/megatron/core/distributed/distributed_data_parallel.py +++ b/megatron/core/distributed/distributed_data_parallel.py @@ -120,9 +120,9 @@ def __init__( param_to_name[param] = name if getattr(param, 'allreduce', True): - dense_params.append(param) + dense_params.append((param, name)) else: - expert_parallel_params.append(param) + expert_parallel_params.append((param, name)) def _allocate_buffers_for_parameters( input_params, data_parallel_group, gradient_scaling_factor @@ -132,7 +132,7 @@ def _allocate_buffers_for_parameters( param_and_grad_dtype_to_indices = {} # Group parameters by their gradient type. - for param in input_params: + for param, param_name in input_params: assert param.requires_grad param_dtype = param.dtype @@ -147,7 +147,7 @@ def _allocate_buffers_for_parameters( grad_dtype = torch.float if self.ddp_config.grad_reduce_in_fp32 else param.dtype params = param_and_grad_dtype_to_params.get((param_dtype, grad_dtype), []) - params.append(param) + params.append((param, param_name)) param_and_grad_dtype_to_params[(param_dtype, grad_dtype)] = params # Get the index of each param among the params with same dtype, if a param is fp8, diff --git a/megatron/core/distributed/distributed_data_parallel_config.py b/megatron/core/distributed/distributed_data_parallel_config.py index 807b122b0e6..6913d12cdfe 100644 --- a/megatron/core/distributed/distributed_data_parallel_config.py +++ b/megatron/core/distributed/distributed_data_parallel_config.py @@ -1,7 +1,7 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. from dataclasses import dataclass -from typing import Optional +from typing import Optional, Tuple import torch @@ -59,6 +59,10 @@ class DistributedDataParallelConfig: over the wire (using an all-to-all to keep total communication overhead in line with the standard ring implementation) but performs accumulation locally in FP32.""" + param_name_patterns_for_fp32_local_accumulation: Tuple[str] = () + """List of param_name patterns to match against to do local gradient accumulation + in FP32. Do not specify when grad_reduce_in_fp32 is already True.""" + average_in_collective: bool = False """If true, compute average in collective directly, as opposed to dividing by the dp_size first and then computing sum in the collective.""" @@ -204,3 +208,9 @@ def __post_init__(self): "PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True is currently not supported " "with nccl_ub due to compatibility issue with torch.cuda.MemPool API." ) + + if len(self.param_name_patterns_for_fp32_local_accumulation) > 0: + assert not self.grad_reduce_in_fp32, ( + "Only need to explicitly specify param_name patterns for FP32 local accumulation " + "if .main_grads aren't already in FP32" + ) diff --git a/megatron/core/distributed/param_and_grad_buffer.py b/megatron/core/distributed/param_and_grad_buffer.py index 278a15d1a7c..6c36f119e19 100644 --- a/megatron/core/distributed/param_and_grad_buffer.py +++ b/megatron/core/distributed/param_and_grad_buffer.py @@ -1,5 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +import fnmatch import functools import logging import math @@ -7,7 +8,7 @@ from contextlib import nullcontext from enum import Enum from functools import partial -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple import torch from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors @@ -82,6 +83,8 @@ class _ParamAndGradBucket: bucket_id: Index of bucket in buffer. param_index_map: Mapping from param to (start, end, bucket_id) in the global buffer. Used to derive bucket-local offsets for param_to_index. + params_with_extra_main_grads: List of parameters in this bucket that require a + separate higher-precision main_grad tensor for local gradient accumulation. """ def __init__( @@ -94,6 +97,7 @@ def __init__( gradient_scaling_factor: float, bucket_id: int, param_index_map: Dict[torch.nn.Parameter, tuple], + params_with_extra_main_grads: List[torch.nn.Parameter], ): self.params_list = params self.params = set(params) @@ -112,6 +116,7 @@ def __init__( for param in params: global_start, global_end, _ = param_index_map[param] self.param_to_index[param] = (global_start - offset, global_end - offset) + self.params_with_extra_main_grads = params_with_extra_main_grads # Layer-wise optimizer attributes for async param gather. self.layerwise_params_list = None @@ -530,6 +535,14 @@ def start_grad_sync(self, force_all_reduce: Optional[bool] = False): self.grad_reduce_handle is None ), "Should not have multiple communication calls outstanding at once" + # Copy accumulated .main_grad into communication buffer before collective if + # .main_grad is not in .grad_data already (e.g., because we want to do local + # gradient accumulation in a higher precision). + for bucket in self.buckets: + for param in bucket.params_with_extra_main_grads: + if getattr(param, 'main_grad_copy_in_grad_buffer', None) is not None: + param.main_grad_copy_in_grad_buffer.copy_(param.main_grad) + if self.ddp_config.check_for_nan_in_grad or self.ddp_config.check_for_large_grads: self.check_grads( check_for_nan_or_inf=self.ddp_config.check_for_nan_in_grad, @@ -668,6 +681,7 @@ def finish_grad_sync(self, force_all_reduce: Optional[bool] = False): # If overlap_grad_reduce is False, start (and finish) synchronous communication call here. if not self.ddp_config.overlap_grad_reduce: self.start_grad_sync(force_all_reduce=force_all_reduce) + self._copy_back_extra_main_grads() return # If first batch, start asynchronous communication here. register_grad_ready() launches # asynchronous communication only once self.golden_per_param_grad_ready_counts is @@ -678,6 +692,7 @@ def finish_grad_sync(self, force_all_reduce: Optional[bool] = False): # communications on a separate communication stream. if self.ddp_config.num_distributed_optimizer_instances > 1: torch.cuda.current_stream().wait_stream(self.communication_stream) + self._copy_back_extra_main_grads() return assert self.grad_reduce_handle is not None, ( f"Communication call has not been issued for this bucket " @@ -686,6 +701,7 @@ def finish_grad_sync(self, force_all_reduce: Optional[bool] = False): ) self.grad_reduce_handle.wait() self.grad_reduce_handle = None + self._copy_back_extra_main_grads() def free_overlap_buffers(self): """Free GPU buffers used by overlap param gather. @@ -702,6 +718,21 @@ def free_overlap_buffers(self): bucket.layerwise_gather_list = None bucket._layerwise_src_buffer = None + def _copy_back_extra_main_grads(self): + """ + Copy reduced gradients from the communication buffer back to .main_grad for + params that have a separate higher-precision .main_grad tensor. + + This is needed because the optimizer reads from .main_grad to get the reduced + gradients, but for params with extra main_grads, .main_grad points to the local + FP32 accumulation tensor rather than the communication buffer where the reduced + gradients are stored. + """ + for bucket in self.buckets: + for param in bucket.params_with_extra_main_grads: + if getattr(param, 'main_grad_copy_in_grad_buffer', None) is not None: + param.main_grad.copy_(param.main_grad_copy_in_grad_buffer) + def register_grad_ready( self, param: torch.nn.Parameter, force_all_reduce: Optional[bool] = False ): @@ -754,7 +785,7 @@ def __init__( ddp_config: DistributedDataParallelConfig, param_dtype: torch.dtype, grad_dtype: torch.dtype, - params: List[torch.nn.Parameter], + params_with_names: List[Tuple[torch.nn.Parameter, str]], data_parallel_group: torch.distributed.ProcessGroup, bucket_size: int, param_to_name: Dict[torch.nn.Parameter, str], @@ -775,12 +806,12 @@ def __init__( self.tp_group = pg_collection.tp self.ddp_config = ddp_config - self.params = params + self.params = [param for (param, _) in params_with_names] self.param_indices = param_indices # Check that params are unique. unique_params = set() - for param in params: + for param, _ in params_with_names: assert param not in unique_params unique_params.add(param) del unique_params @@ -874,7 +905,7 @@ def _does_param_require_new_bucket(param): and self.ddp_config.use_distributed_optimizer ) - for param in params[::-1]: + for param, _ in params_with_names[::-1]: # Iterate through parameters in reverse order to roughly follow backprop order. this_numel = param.data.nelement() @@ -915,6 +946,8 @@ def _does_param_require_new_bucket(param): assert self.numel == self.numel_unpadded self.param_data = None + self.grad_data = None + self.extra_main_grads = [] if self.nccl_ub: # If nccl_ub is True, use nccl_allocator to allocate memory for param_data/grad_data. @@ -942,7 +975,9 @@ def _does_param_require_new_bucket(param): # For MXFP8 param: Create a shared buffer for param AG and grad RS for memory efficiency # The buffer is mapped to weight gradients whose dtype is either bf16 or FP32. # It can be temporarily reused by param AG. - if self.ddp_config.use_distributed_optimizer and any(is_mxfp8tensor(p) for p in params): + if self.ddp_config.use_distributed_optimizer and any( + is_mxfp8tensor(p) for p in self.params + ): self.shared_buffer = torch.zeros( self.numel, dtype=self.grad_dtype, @@ -979,9 +1014,10 @@ def _does_param_require_new_bucket(param): # Finally, map param.data and param.main_grad fields to buffers. bucket_params = [] + bucket_params_with_extra_main_grads = [] bucket_start_index = 0 cur_bucket_id = 0 - for param in params[::-1]: + for param, param_name in params_with_names[::-1]: param_start_index, param_end_index, bucket_id = self.param_index_map[param] # For MXFP8 param: # we only need to map bf16 weights (layernorm, embedding, etc) to the buffer. @@ -1003,6 +1039,30 @@ def _does_param_require_new_bucket(param): param.main_grad = self._get( param.data.shape, param_start_index, buffer_type=BufferType.GRAD ) + + # Create FP32 copy of .main_grads if necessary. + promote_main_grads_to_higher_precision = False + for param_name_pattern in ddp_config.param_name_patterns_for_fp32_local_accumulation: + if fnmatch.fnmatch(param_name, param_name_pattern) or param_name_pattern == 'all': + log_on_each_pipeline_stage( + logger, + logging.INFO, + ( + f"Matched {param_name} with '{param_name_pattern}'; promoting " + f"main_grad.type from {param.main_grad.dtype} to torch.float32!" + ), + tp_group=self.tp_group, + dp_cp_group=self.dp_cp_group, + ) + promote_main_grads_to_higher_precision = True + break + if promote_main_grads_to_higher_precision: + param.main_grad_copy_in_grad_buffer = ( + param.main_grad + ) # Slice into .grad_data buffer. + param.main_grad = torch.empty_like(param.main_grad, dtype=torch.float32) + self.extra_main_grads.append(param.main_grad) + if bucket_id != cur_bucket_id: bucket_end_index = _pad_end_of_bucket_if_needed(param_start_index) self.buckets.append( @@ -1012,14 +1072,19 @@ def _does_param_require_new_bucket(param): end_index=bucket_end_index, numel_unpadded=per_bucket_numel_unpadded[cur_bucket_id], bucket_id=cur_bucket_id, + bucket_params_with_extra_main_grads=bucket_params_with_extra_main_grads, ) ) bucket_start_index = bucket_end_index bucket_params = [] + bucket_params_with_extra_main_grads = [] assert cur_bucket_id + 1 == len(self.buckets) assert bucket_id == cur_bucket_id + 1 cur_bucket_id = bucket_id + bucket_params.append(param) + if promote_main_grads_to_higher_precision: + bucket_params_with_extra_main_grads.append(param) # Add remaining params to a new bucket. if len(bucket_params) > 0: @@ -1031,6 +1096,7 @@ def _does_param_require_new_bucket(param): end_index=bucket_end_index, numel_unpadded=per_bucket_numel_unpadded[cur_bucket_id], bucket_id=cur_bucket_id, + bucket_params_with_extra_main_grads=bucket_params_with_extra_main_grads, ) ) @@ -1041,14 +1107,15 @@ def _does_param_require_new_bucket(param): ) for index, bucket in enumerate(self.buckets): numel = 0 - for param in bucket.params: + for param in bucket.params_list: numel += param.data.nelement() log_strs.append( f"Params for bucket {index + 1} ({numel} elements, " - f"{bucket.grad_data.nelement()} padded size):" + f"{bucket.grad_data.nelement()} padded size, " + f"{len(bucket.params_with_extra_main_grads)} param(s) with extra main_grads):" ) - for param in bucket.params: - log_strs.append(f"\t{param_to_name[param]}") + for param in bucket.params_list: + log_strs.append(f"\t{param_to_name[param]} ({param.main_grad.dtype=})") log_on_each_pipeline_stage( logger, logging.INFO, @@ -1060,6 +1127,8 @@ def _does_param_require_new_bucket(param): def scale_gradients(self, scaling_factor: float) -> None: """Scale the gradient data by `scaling_factor`.""" self.grad_data *= scaling_factor + for grad in self.extra_main_grads: + grad *= scaling_factor def _get(self, shape: torch.Size, start_index: int, buffer_type: BufferType) -> torch.Tensor: """ @@ -1085,6 +1154,7 @@ def _new_bucket( end_index: int, numel_unpadded: int, bucket_id: int, + bucket_params_with_extra_main_grads: List[torch.Tensor], ) -> _ParamAndGradBucket: """ Helper function that creates a new bucket. Also updates param->bucket mapping. @@ -1115,6 +1185,7 @@ def _new_bucket( gradient_scaling_factor=self.gradient_scaling_factor, bucket_id=bucket_id, param_index_map=self.param_index_map, + params_with_extra_main_grads=bucket_params_with_extra_main_grads, ) for bucket_param in bucket_params: assert bucket_param not in self.param_to_bucket @@ -1127,6 +1198,8 @@ def reset(self): Zero out the underlying grad_buffer. """ self.grad_data.zero_() + for grad in self.extra_main_grads: + grad.zero_() def offload_to_cpu(self, move_params: bool = True, move_grads: bool = True) -> None: """ diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index d1b5c34c619..ff123da54fc 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -2660,6 +2660,9 @@ def _add_distributed_args(parser): default=False, help='If set, use a reduce-scatter implementation which sends lower-precision ' 'values over the wire (using an all-to-all to keep total communication overhead in line ' 'with the standard ring implementation) but performs accumulation locally in FP32.') + group.add_argument('--ddp-param-name-patterns-for-fp32-local-accumulation', nargs='+', default=[], + help='List of param_name patterns to match against to do local gradient accumulation ' + 'in FP32.') group.add_argument('--ddp-average-in-collective', action='store_true', default=False, help='If set, average directly in data-parallel communication collective.') group.add_argument('--overlap-param-gather', action='store_true', diff --git a/megatron/training/training.py b/megatron/training/training.py index 251d754713c..aba11d04df9 100644 --- a/megatron/training/training.py +++ b/megatron/training/training.py @@ -1457,6 +1457,8 @@ def build_model(): kwargs['bucket_size'] = args.ddp_bucket_size kwargs['pad_buckets_for_high_nccl_busbw'] = args.ddp_pad_buckets_for_high_nccl_busbw kwargs['reduce_scatter_with_fp32_accumulation'] = args.ddp_reduce_scatter_with_fp32_accumulation + kwargs['param_name_patterns_for_fp32_local_accumulation'] = \ + tuple(args.ddp_param_name_patterns_for_fp32_local_accumulation) kwargs['average_in_collective'] = args.ddp_average_in_collective # Megatron-FSDP arguments. kwargs['megatron_fsdp_main_params_dtype'] = args.megatron_fsdp_main_params_dtype From 6fdd1a1586a2fb50952cf88ad3e4ba631c088ed1 Mon Sep 17 00:00:00 2001 From: Deepak Narayanan Date: Wed, 25 Mar 2026 12:16:16 -0700 Subject: [PATCH 2/3] Add unit tests for FP32 local gradient accumulation feature Co-Authored-By: Claude Opus 4.6 --- .../distributed/test_param_and_grad_buffer.py | 177 +++++++++++++++++- 1 file changed, 176 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/distributed/test_param_and_grad_buffer.py b/tests/unit_tests/distributed/test_param_and_grad_buffer.py index 48f815531c5..06acb2603f1 100644 --- a/tests/unit_tests/distributed/test_param_and_grad_buffer.py +++ b/tests/unit_tests/distributed/test_param_and_grad_buffer.py @@ -26,14 +26,17 @@ def get_model_and_buffers( overlap_grad_reduce: bool, average_in_collective: bool, num_distributed_optimizer_instances: int = 1, + grad_reduce_in_fp32: bool = True, + param_name_patterns_for_fp32_local_accumulation: tuple = (), ): ddp_config = DistributedDataParallelConfig( - grad_reduce_in_fp32=True, + grad_reduce_in_fp32=grad_reduce_in_fp32, use_distributed_optimizer=use_distributed_optimizer, overlap_grad_reduce=overlap_grad_reduce, bucket_size=bucket_size, average_in_collective=average_in_collective, num_distributed_optimizer_instances=num_distributed_optimizer_instances, + param_name_patterns_for_fp32_local_accumulation=param_name_patterns_for_fp32_local_accumulation, ) model = TestModel( input_dim=input_dim, @@ -464,3 +467,175 @@ def test_ddp_free_overlap_buffers_delegates(self): ), "free_overlap_buffers should be called on every bucket group" Utils.destroy_model_parallel() + + +class TestFP32LocalGradAccumulation: + """Tests for the FP32 local gradient accumulation feature + (param_name_patterns_for_fp32_local_accumulation).""" + + @staticmethod + def _make_model(patterns, bucket_size=None): + """Create a DDP-wrapped model with FP32 local grad accumulation patterns.""" + return get_model_and_buffers( + input_dim=100, + output_dim=100, + num_layers=3, + bias=True, + shared_embedding=False, + bucket_size=bucket_size, + use_distributed_optimizer=False, + overlap_grad_reduce=False, + average_in_collective=False, + grad_reduce_in_fp32=False, + param_name_patterns_for_fp32_local_accumulation=patterns, + ) + + def test_config_validation_with_grad_reduce_in_fp32(self): + """param_name_patterns_for_fp32_local_accumulation and grad_reduce_in_fp32 are + mutually exclusive.""" + with pytest.raises(AssertionError): + DistributedDataParallelConfig( + grad_reduce_in_fp32=True, + param_name_patterns_for_fp32_local_accumulation=('all',), + ) + + def test_pattern_matching_creates_fp32_main_grad(self): + """Params matching patterns should get a float32 main_grad and a + main_grad_copy_in_grad_buffer; non-matching params should not.""" + Utils.initialize_model_parallel() + # Match only weight params (not bias). + model, buf, _ = self._make_model(patterns=('*.weight',)) + + for name, param in model.module.named_parameters(): + if 'weight' in name: + assert param.main_grad.dtype == torch.float32, ( + f"{name} main_grad should be float32" + ) + assert hasattr(param, 'main_grad_copy_in_grad_buffer') + assert param.main_grad_copy_in_grad_buffer is not None + # The copy in grad buffer should be in the buffer's grad dtype (bf16). + assert param.main_grad_copy_in_grad_buffer.dtype == buf.grad_dtype + else: + # Bias params should not be promoted. + assert param.main_grad.dtype == buf.grad_dtype, ( + f"{name} main_grad should remain in grad_dtype" + ) + assert getattr(param, 'main_grad_copy_in_grad_buffer', None) is None + + Utils.destroy_model_parallel() + + def test_all_pattern_matches_every_param(self): + """The 'all' pattern should match every parameter.""" + Utils.initialize_model_parallel() + model, buf, _ = self._make_model(patterns=('all',)) + + for name, param in model.module.named_parameters(): + assert param.main_grad.dtype == torch.float32, ( + f"{name} main_grad should be float32 with 'all' pattern" + ) + assert getattr(param, 'main_grad_copy_in_grad_buffer', None) is not None + + Utils.destroy_model_parallel() + + def test_bucket_tracks_params_with_extra_main_grads(self): + """Each bucket's params_with_extra_main_grads should contain exactly + the params that matched the patterns.""" + Utils.initialize_model_parallel() + model, buf, _ = self._make_model(patterns=('*.weight',)) + + promoted_params = set() + for name, param in model.module.named_parameters(): + if 'weight' in name: + promoted_params.add(param) + + bucket_promoted = set() + for bucket in buf.buckets: + for param in bucket.params_with_extra_main_grads: + bucket_promoted.add(param) + # Every param in params_with_extra_main_grads should also be in bucket.params. + assert bucket.params_with_extra_main_grads == [] or set( + bucket.params_with_extra_main_grads + ).issubset(bucket.params) + + assert bucket_promoted == promoted_params, ( + "Bucket-tracked promoted params should match the set of pattern-matched params" + ) + + Utils.destroy_model_parallel() + + def test_no_patterns_means_no_extra_main_grads(self): + """With no patterns, no params should have extra main_grads.""" + Utils.initialize_model_parallel() + _, buf, _ = self._make_model(patterns=()) + + assert len(buf.extra_main_grads) == 0 + for bucket in buf.buckets: + assert len(bucket.params_with_extra_main_grads) == 0 + + Utils.destroy_model_parallel() + + def test_reset_zeros_extra_main_grads(self): + """reset() should zero out both grad_data and all extra main_grads.""" + Utils.initialize_model_parallel() + _, buf, _ = self._make_model(patterns=('all',)) + + # Fill extra main_grads and grad_data with non-zero values. + buf.grad_data.fill_(1.0) + for grad in buf.extra_main_grads: + grad.fill_(42.0) + + buf.reset() + + assert torch.all(buf.grad_data == 0), "grad_data should be zeroed after reset" + for grad in buf.extra_main_grads: + assert torch.all(grad == 0), "extra main_grads should be zeroed after reset" + + Utils.destroy_model_parallel() + + def test_scale_gradients_scales_extra_main_grads(self): + """scale_gradients() should scale both grad_data and extra main_grads.""" + Utils.initialize_model_parallel() + _, buf, _ = self._make_model(patterns=('all',)) + + buf.grad_data.fill_(2.0) + for grad in buf.extra_main_grads: + grad.fill_(4.0) + + buf.scale_gradients(0.5) + + assert torch.allclose(buf.grad_data, torch.tensor(1.0, dtype=buf.grad_data.dtype)), ( + "grad_data should be scaled" + ) + for grad in buf.extra_main_grads: + assert torch.allclose(grad, torch.tensor(2.0, dtype=grad.dtype)), ( + "extra main_grads should be scaled" + ) + + Utils.destroy_model_parallel() + + def test_grad_sync_copies_to_and_from_comm_buffer(self): + """During grad sync, values in FP32 main_grad should be copied to the comm buffer + before the collective, and the reduced result should be copied back afterward.""" + Utils.initialize_model_parallel() + model, buf, bucket_groups = self._make_model(patterns=('all',)) + + # Simulate accumulated gradients in FP32 main_grad. + for param in model.parameters(): + param.main_grad.fill_(1.0) + + # Run grad sync (non-overlapped, so finish_grad_sync triggers start + wait). + model.finish_grad_sync() + + # After sync, main_grad should contain the reduced result (not the original 1.0, + # since the collective may have scaled / averaged). The key invariant is that + # main_grad should equal main_grad_copy_in_grad_buffer (the comm buffer slice) + # after the copy-back. + for param in model.parameters(): + if getattr(param, 'main_grad_copy_in_grad_buffer', None) is not None: + torch.testing.assert_close( + param.main_grad, + param.main_grad_copy_in_grad_buffer.float(), + msg="main_grad should equal comm buffer after grad sync copy-back", + ) + + Utils.destroy_model_parallel() From 1df3f83773700d886225bec430430d40bff3b2e2 Mon Sep 17 00:00:00 2001 From: Deepak Narayanan Date: Wed, 25 Mar 2026 12:35:03 -0700 Subject: [PATCH 3/3] Linting and code review --- .../distributed_data_parallel_config.py | 7 ++-- megatron/training/arguments.py | 5 ++- .../distributed/test_param_and_grad_buffer.py | 37 +++++++++---------- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/megatron/core/distributed/distributed_data_parallel_config.py b/megatron/core/distributed/distributed_data_parallel_config.py index 6913d12cdfe..ee592368b0c 100644 --- a/megatron/core/distributed/distributed_data_parallel_config.py +++ b/megatron/core/distributed/distributed_data_parallel_config.py @@ -59,9 +59,10 @@ class DistributedDataParallelConfig: over the wire (using an all-to-all to keep total communication overhead in line with the standard ring implementation) but performs accumulation locally in FP32.""" - param_name_patterns_for_fp32_local_accumulation: Tuple[str] = () - """List of param_name patterns to match against to do local gradient accumulation - in FP32. Do not specify when grad_reduce_in_fp32 is already True.""" + param_name_patterns_for_fp32_local_accumulation: Tuple[str, ...] = () + """List of param_name patterns (in Python's fnmatch format) to match against to do + local gradient accumulation in FP32. The special pattern 'all' matches every + parameter. Do not specify when grad_reduce_in_fp32 is already True.""" average_in_collective: bool = False """If true, compute average in collective directly, as opposed to dividing by the diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index ff123da54fc..bf9287e77ab 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -2661,8 +2661,9 @@ def _add_distributed_args(parser): 'values over the wire (using an all-to-all to keep total communication overhead in line ' 'with the standard ring implementation) but performs accumulation locally in FP32.') group.add_argument('--ddp-param-name-patterns-for-fp32-local-accumulation', nargs='+', default=[], - help='List of param_name patterns to match against to do local gradient accumulation ' - 'in FP32.') + help='List of param_name patterns (in Python\'s fnmatch format) to match against ' + 'to do local gradient accumulation in FP32. The special pattern \'all\' matches ' + 'every parameter.') group.add_argument('--ddp-average-in-collective', action='store_true', default=False, help='If set, average directly in data-parallel communication collective.') group.add_argument('--overlap-param-gather', action='store_true', diff --git a/tests/unit_tests/distributed/test_param_and_grad_buffer.py b/tests/unit_tests/distributed/test_param_and_grad_buffer.py index 06acb2603f1..5ea6640f4e1 100644 --- a/tests/unit_tests/distributed/test_param_and_grad_buffer.py +++ b/tests/unit_tests/distributed/test_param_and_grad_buffer.py @@ -495,8 +495,7 @@ def test_config_validation_with_grad_reduce_in_fp32(self): mutually exclusive.""" with pytest.raises(AssertionError): DistributedDataParallelConfig( - grad_reduce_in_fp32=True, - param_name_patterns_for_fp32_local_accumulation=('all',), + grad_reduce_in_fp32=True, param_name_patterns_for_fp32_local_accumulation=('all',) ) def test_pattern_matching_creates_fp32_main_grad(self): @@ -508,18 +507,16 @@ def test_pattern_matching_creates_fp32_main_grad(self): for name, param in model.module.named_parameters(): if 'weight' in name: - assert param.main_grad.dtype == torch.float32, ( - f"{name} main_grad should be float32" - ) + assert param.main_grad.dtype == torch.float32, f"{name} main_grad should be float32" assert hasattr(param, 'main_grad_copy_in_grad_buffer') assert param.main_grad_copy_in_grad_buffer is not None # The copy in grad buffer should be in the buffer's grad dtype (bf16). assert param.main_grad_copy_in_grad_buffer.dtype == buf.grad_dtype else: # Bias params should not be promoted. - assert param.main_grad.dtype == buf.grad_dtype, ( - f"{name} main_grad should remain in grad_dtype" - ) + assert ( + param.main_grad.dtype == buf.grad_dtype + ), f"{name} main_grad should remain in grad_dtype" assert getattr(param, 'main_grad_copy_in_grad_buffer', None) is None Utils.destroy_model_parallel() @@ -530,9 +527,9 @@ def test_all_pattern_matches_every_param(self): model, buf, _ = self._make_model(patterns=('all',)) for name, param in model.module.named_parameters(): - assert param.main_grad.dtype == torch.float32, ( - f"{name} main_grad should be float32 with 'all' pattern" - ) + assert ( + param.main_grad.dtype == torch.float32 + ), f"{name} main_grad should be float32 with 'all' pattern" assert getattr(param, 'main_grad_copy_in_grad_buffer', None) is not None Utils.destroy_model_parallel() @@ -557,9 +554,9 @@ def test_bucket_tracks_params_with_extra_main_grads(self): bucket.params_with_extra_main_grads ).issubset(bucket.params) - assert bucket_promoted == promoted_params, ( - "Bucket-tracked promoted params should match the set of pattern-matched params" - ) + assert ( + bucket_promoted == promoted_params + ), "Bucket-tracked promoted params should match the set of pattern-matched params" Utils.destroy_model_parallel() @@ -603,13 +600,13 @@ def test_scale_gradients_scales_extra_main_grads(self): buf.scale_gradients(0.5) - assert torch.allclose(buf.grad_data, torch.tensor(1.0, dtype=buf.grad_data.dtype)), ( - "grad_data should be scaled" - ) + assert torch.allclose( + buf.grad_data, torch.tensor(1.0, dtype=buf.grad_data.dtype) + ), "grad_data should be scaled" for grad in buf.extra_main_grads: - assert torch.allclose(grad, torch.tensor(2.0, dtype=grad.dtype)), ( - "extra main_grads should be scaled" - ) + assert torch.allclose( + grad, torch.tensor(2.0, dtype=grad.dtype) + ), "extra main_grads should be scaled" Utils.destroy_model_parallel()