Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions megatron/core/distributed/distributed_data_parallel.py
Original file line number Diff line number Diff line change
Expand Up @@ -120,9 +120,9 @@ def __init__(
param_to_name[param] = name

if getattr(param, 'allreduce', True):
dense_params.append(param)
dense_params.append((param, name))
else:
expert_parallel_params.append(param)
expert_parallel_params.append((param, name))

def _allocate_buffers_for_parameters(
input_params, data_parallel_group, gradient_scaling_factor
Expand All @@ -132,7 +132,7 @@ def _allocate_buffers_for_parameters(
param_and_grad_dtype_to_indices = {}

# Group parameters by their gradient type.
for param in input_params:
for param, param_name in input_params:
assert param.requires_grad

param_dtype = param.dtype
Expand All @@ -147,7 +147,7 @@ def _allocate_buffers_for_parameters(
grad_dtype = torch.float if self.ddp_config.grad_reduce_in_fp32 else param.dtype

params = param_and_grad_dtype_to_params.get((param_dtype, grad_dtype), [])
params.append(param)
params.append((param, param_name))
param_and_grad_dtype_to_params[(param_dtype, grad_dtype)] = params

# Get the index of each param among the params with same dtype, if a param is fp8,
Expand Down
12 changes: 11 additions & 1 deletion megatron/core/distributed/distributed_data_parallel_config.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.

from dataclasses import dataclass
from typing import Optional
from typing import Optional, Tuple

import torch

Expand Down Expand Up @@ -59,6 +59,10 @@ class DistributedDataParallelConfig:
over the wire (using an all-to-all to keep total communication overhead in line
with the standard ring implementation) but performs accumulation locally in FP32."""

param_name_patterns_for_fp32_local_accumulation: Tuple[str] = ()
Comment thread
deepakn94 marked this conversation as resolved.
Resolved
"""List of param_name patterns to match against to do local gradient accumulation
Comment thread
deepakn94 marked this conversation as resolved.
Resolved
in FP32. Do not specify when grad_reduce_in_fp32 is already True."""

average_in_collective: bool = False
"""If true, compute average in collective directly, as opposed to dividing by the
dp_size first and then computing sum in the collective."""
Expand Down Expand Up @@ -204,3 +208,9 @@ def __post_init__(self):
"PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True is currently not supported "
"with nccl_ub due to compatibility issue with torch.cuda.MemPool API."
)

if len(self.param_name_patterns_for_fp32_local_accumulation) > 0:
assert not self.grad_reduce_in_fp32, (
"Only need to explicitly specify param_name patterns for FP32 local accumulation "
"if .main_grads aren't already in FP32"
)
95 changes: 84 additions & 11 deletions megatron/core/distributed/param_and_grad_buffer.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.

import fnmatch
import functools
import logging
import math
import warnings
from contextlib import nullcontext
from enum import Enum
from functools import partial
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Tuple

import torch
from torch._utils import _flatten_dense_tensors, _unflatten_dense_tensors
Expand Down Expand Up @@ -82,6 +83,8 @@ class _ParamAndGradBucket:
bucket_id: Index of bucket in buffer.
param_index_map: Mapping from param to (start, end, bucket_id) in the global buffer.
Used to derive bucket-local offsets for param_to_index.
params_with_extra_main_grads: List of parameters in this bucket that require a
separate higher-precision main_grad tensor for local gradient accumulation.
"""

def __init__(
Expand All @@ -94,6 +97,7 @@ def __init__(
gradient_scaling_factor: float,
bucket_id: int,
param_index_map: Dict[torch.nn.Parameter, tuple],
params_with_extra_main_grads: List[torch.nn.Parameter],
):
self.params_list = params
self.params = set(params)
Expand All @@ -112,6 +116,7 @@ def __init__(
for param in params:
global_start, global_end, _ = param_index_map[param]
self.param_to_index[param] = (global_start - offset, global_end - offset)
self.params_with_extra_main_grads = params_with_extra_main_grads

# Layer-wise optimizer attributes for async param gather.
self.layerwise_params_list = None
Expand Down Expand Up @@ -530,6 +535,14 @@ def start_grad_sync(self, force_all_reduce: Optional[bool] = False):
self.grad_reduce_handle is None
), "Should not have multiple communication calls outstanding at once"

# Copy accumulated .main_grad into communication buffer before collective if
# .main_grad is not in .grad_data already (e.g., because we want to do local
# gradient accumulation in a higher precision).
for bucket in self.buckets:
for param in bucket.params_with_extra_main_grads:
if getattr(param, 'main_grad_copy_in_grad_buffer', None) is not None:
param.main_grad_copy_in_grad_buffer.copy_(param.main_grad)

if self.ddp_config.check_for_nan_in_grad or self.ddp_config.check_for_large_grads:
self.check_grads(
check_for_nan_or_inf=self.ddp_config.check_for_nan_in_grad,
Expand Down Expand Up @@ -668,6 +681,7 @@ def finish_grad_sync(self, force_all_reduce: Optional[bool] = False):
# If overlap_grad_reduce is False, start (and finish) synchronous communication call here.
if not self.ddp_config.overlap_grad_reduce:
self.start_grad_sync(force_all_reduce=force_all_reduce)
self._copy_back_extra_main_grads()
return
# If first batch, start asynchronous communication here. register_grad_ready() launches
# asynchronous communication only once self.golden_per_param_grad_ready_counts is
Expand All @@ -678,6 +692,7 @@ def finish_grad_sync(self, force_all_reduce: Optional[bool] = False):
# communications on a separate communication stream.
if self.ddp_config.num_distributed_optimizer_instances > 1:
torch.cuda.current_stream().wait_stream(self.communication_stream)
self._copy_back_extra_main_grads()
return
assert self.grad_reduce_handle is not None, (
f"Communication call has not been issued for this bucket "
Expand All @@ -686,6 +701,7 @@ def finish_grad_sync(self, force_all_reduce: Optional[bool] = False):
)
self.grad_reduce_handle.wait()
self.grad_reduce_handle = None
self._copy_back_extra_main_grads()

def free_overlap_buffers(self):
"""Free GPU buffers used by overlap param gather.
Expand All @@ -702,6 +718,21 @@ def free_overlap_buffers(self):
bucket.layerwise_gather_list = None
bucket._layerwise_src_buffer = None

def _copy_back_extra_main_grads(self):
"""
Copy reduced gradients from the communication buffer back to .main_grad for
params that have a separate higher-precision .main_grad tensor.

This is needed because the optimizer reads from .main_grad to get the reduced
gradients, but for params with extra main_grads, .main_grad points to the local
FP32 accumulation tensor rather than the communication buffer where the reduced
gradients are stored.
"""
for bucket in self.buckets:
for param in bucket.params_with_extra_main_grads:
if getattr(param, 'main_grad_copy_in_grad_buffer', None) is not None:
param.main_grad.copy_(param.main_grad_copy_in_grad_buffer)

def register_grad_ready(
self, param: torch.nn.Parameter, force_all_reduce: Optional[bool] = False
):
Expand Down Expand Up @@ -754,7 +785,7 @@ def __init__(
ddp_config: DistributedDataParallelConfig,
param_dtype: torch.dtype,
grad_dtype: torch.dtype,
params: List[torch.nn.Parameter],
params_with_names: List[Tuple[torch.nn.Parameter, str]],
data_parallel_group: torch.distributed.ProcessGroup,
bucket_size: int,
param_to_name: Dict[torch.nn.Parameter, str],
Expand All @@ -775,12 +806,12 @@ def __init__(
self.tp_group = pg_collection.tp

self.ddp_config = ddp_config
self.params = params
self.params = [param for (param, _) in params_with_names]
self.param_indices = param_indices

# Check that params are unique.
unique_params = set()
for param in params:
for param, _ in params_with_names:
assert param not in unique_params
unique_params.add(param)
del unique_params
Expand Down Expand Up @@ -874,7 +905,7 @@ def _does_param_require_new_bucket(param):
and self.ddp_config.use_distributed_optimizer
)

for param in params[::-1]:
for param, _ in params_with_names[::-1]:
# Iterate through parameters in reverse order to roughly follow backprop order.

this_numel = param.data.nelement()
Expand Down Expand Up @@ -915,6 +946,8 @@ def _does_param_require_new_bucket(param):
assert self.numel == self.numel_unpadded

self.param_data = None
self.grad_data = None
self.extra_main_grads = []

if self.nccl_ub:
# If nccl_ub is True, use nccl_allocator to allocate memory for param_data/grad_data.
Expand Down Expand Up @@ -942,7 +975,9 @@ def _does_param_require_new_bucket(param):
# For MXFP8 param: Create a shared buffer for param AG and grad RS for memory efficiency
# The buffer is mapped to weight gradients whose dtype is either bf16 or FP32.
# It can be temporarily reused by param AG.
if self.ddp_config.use_distributed_optimizer and any(is_mxfp8tensor(p) for p in params):
if self.ddp_config.use_distributed_optimizer and any(
is_mxfp8tensor(p) for p in self.params
):
self.shared_buffer = torch.zeros(
self.numel,
dtype=self.grad_dtype,
Expand Down Expand Up @@ -979,9 +1014,10 @@ def _does_param_require_new_bucket(param):

# Finally, map param.data and param.main_grad fields to buffers.
bucket_params = []
bucket_params_with_extra_main_grads = []
bucket_start_index = 0
cur_bucket_id = 0
for param in params[::-1]:
for param, param_name in params_with_names[::-1]:
param_start_index, param_end_index, bucket_id = self.param_index_map[param]
# For MXFP8 param:
# we only need to map bf16 weights (layernorm, embedding, etc) to the buffer.
Expand All @@ -1003,6 +1039,30 @@ def _does_param_require_new_bucket(param):
param.main_grad = self._get(
param.data.shape, param_start_index, buffer_type=BufferType.GRAD
)

# Create FP32 copy of .main_grads if necessary.
promote_main_grads_to_higher_precision = False
for param_name_pattern in ddp_config.param_name_patterns_for_fp32_local_accumulation:
if fnmatch.fnmatch(param_name, param_name_pattern) or param_name_pattern == 'all':
log_on_each_pipeline_stage(
logger,
logging.INFO,
(
f"Matched {param_name} with '{param_name_pattern}'; promoting "
f"main_grad.type from {param.main_grad.dtype} to torch.float32!"
),
tp_group=self.tp_group,
dp_cp_group=self.dp_cp_group,
)
promote_main_grads_to_higher_precision = True
break
if promote_main_grads_to_higher_precision:
param.main_grad_copy_in_grad_buffer = (
param.main_grad
) # Slice into .grad_data buffer.
param.main_grad = torch.empty_like(param.main_grad, dtype=torch.float32)
self.extra_main_grads.append(param.main_grad)

if bucket_id != cur_bucket_id:
bucket_end_index = _pad_end_of_bucket_if_needed(param_start_index)
self.buckets.append(
Expand All @@ -1012,14 +1072,19 @@ def _does_param_require_new_bucket(param):
end_index=bucket_end_index,
numel_unpadded=per_bucket_numel_unpadded[cur_bucket_id],
bucket_id=cur_bucket_id,
bucket_params_with_extra_main_grads=bucket_params_with_extra_main_grads,
)
)
bucket_start_index = bucket_end_index
bucket_params = []
bucket_params_with_extra_main_grads = []
assert cur_bucket_id + 1 == len(self.buckets)
assert bucket_id == cur_bucket_id + 1
cur_bucket_id = bucket_id

bucket_params.append(param)
if promote_main_grads_to_higher_precision:
bucket_params_with_extra_main_grads.append(param)

# Add remaining params to a new bucket.
if len(bucket_params) > 0:
Expand All @@ -1031,6 +1096,7 @@ def _does_param_require_new_bucket(param):
end_index=bucket_end_index,
numel_unpadded=per_bucket_numel_unpadded[cur_bucket_id],
bucket_id=cur_bucket_id,
bucket_params_with_extra_main_grads=bucket_params_with_extra_main_grads,
)
)

Expand All @@ -1041,14 +1107,15 @@ def _does_param_require_new_bucket(param):
)
for index, bucket in enumerate(self.buckets):
numel = 0
for param in bucket.params:
for param in bucket.params_list:
numel += param.data.nelement()
log_strs.append(
f"Params for bucket {index + 1} ({numel} elements, "
f"{bucket.grad_data.nelement()} padded size):"
f"{bucket.grad_data.nelement()} padded size, "
f"{len(bucket.params_with_extra_main_grads)} param(s) with extra main_grads):"
)
for param in bucket.params:
log_strs.append(f"\t{param_to_name[param]}")
for param in bucket.params_list:
log_strs.append(f"\t{param_to_name[param]} ({param.main_grad.dtype=})")
log_on_each_pipeline_stage(
logger,
logging.INFO,
Expand All @@ -1060,6 +1127,8 @@ def _does_param_require_new_bucket(param):
def scale_gradients(self, scaling_factor: float) -> None:
"""Scale the gradient data by `scaling_factor`."""
self.grad_data *= scaling_factor
for grad in self.extra_main_grads:
grad *= scaling_factor

def _get(self, shape: torch.Size, start_index: int, buffer_type: BufferType) -> torch.Tensor:
"""
Expand All @@ -1085,6 +1154,7 @@ def _new_bucket(
end_index: int,
numel_unpadded: int,
bucket_id: int,
bucket_params_with_extra_main_grads: List[torch.Tensor],
) -> _ParamAndGradBucket:
"""
Helper function that creates a new bucket. Also updates param->bucket mapping.
Expand Down Expand Up @@ -1115,6 +1185,7 @@ def _new_bucket(
gradient_scaling_factor=self.gradient_scaling_factor,
bucket_id=bucket_id,
param_index_map=self.param_index_map,
params_with_extra_main_grads=bucket_params_with_extra_main_grads,
)
for bucket_param in bucket_params:
assert bucket_param not in self.param_to_bucket
Expand All @@ -1127,6 +1198,8 @@ def reset(self):
Zero out the underlying grad_buffer.
"""
self.grad_data.zero_()
for grad in self.extra_main_grads:
grad.zero_()

def offload_to_cpu(self, move_params: bool = True, move_grads: bool = True) -> None:
"""
Expand Down
3 changes: 3 additions & 0 deletions megatron/training/arguments.py
Original file line number Diff line number Diff line change
Expand Up @@ -2660,6 +2660,9 @@ def _add_distributed_args(parser):
default=False, help='If set, use a reduce-scatter implementation which sends lower-precision '
'values over the wire (using an all-to-all to keep total communication overhead in line '
'with the standard ring implementation) but performs accumulation locally in FP32.')
group.add_argument('--ddp-param-name-patterns-for-fp32-local-accumulation', nargs='+', default=[],
help='List of param_name patterns to match against to do local gradient accumulation '
'in FP32.')
group.add_argument('--ddp-average-in-collective', action='store_true',
default=False, help='If set, average directly in data-parallel communication collective.')
group.add_argument('--overlap-param-gather', action='store_true',
Expand Down
2 changes: 2 additions & 0 deletions megatron/training/training.py
Original file line number Diff line number Diff line change
Expand Up @@ -1457,6 +1457,8 @@ def build_model():
kwargs['bucket_size'] = args.ddp_bucket_size
kwargs['pad_buckets_for_high_nccl_busbw'] = args.ddp_pad_buckets_for_high_nccl_busbw
kwargs['reduce_scatter_with_fp32_accumulation'] = args.ddp_reduce_scatter_with_fp32_accumulation
kwargs['param_name_patterns_for_fp32_local_accumulation'] = \
tuple(args.ddp_param_name_patterns_for_fp32_local_accumulation)
kwargs['average_in_collective'] = args.ddp_average_in_collective
# Megatron-FSDP arguments.
kwargs['megatron_fsdp_main_params_dtype'] = args.megatron_fsdp_main_params_dtype
Expand Down
Loading
Loading