Skip to content
Merged
Show file tree
Hide file tree
Changes from 10 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions megatron/core/models/mimo/optimizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import torch

from megatron.core.dist_checkpointing.mapping import ShardedObject
from megatron.core.dist_checkpointing.utils import add_prefix_for_sharding
from megatron.core.optimizer.clip_grads import clip_grad_by_total_norm_fp32
from megatron.core.optimizer.optimizer import MegatronOptimizer
from megatron.core.optimizer.optimizer_config import OptimizerConfig
Expand Down Expand Up @@ -179,6 +180,7 @@ def sharded_state_dict(self, model_sharded_state_dict, is_loading: bool = False,
_extract_param_state_sharding_type(sub_sd, name, suffix, replica_id)
_extract_grad_scaler(sub_sd, name, suffix, replica_id)

add_prefix_for_sharding(module_sd, f'mimo.{name}.')
sharded_state[name] = module_sd
else:
sharded_state[name] = {}
Expand Down
4 changes: 4 additions & 0 deletions megatron/core/models/vision/radio.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from megatron.core.transformer.spec_utils import ModuleSpec, build_module
from megatron.core.transformer.transformer_block import TransformerBlock
from megatron.core.transformer.transformer_config import TransformerConfig
from megatron.core.utils import get_tensor_model_parallel_group_if_none

# RADIO reference code: https://github.com/NVlabs/RADIO

Expand Down Expand Up @@ -211,6 +212,9 @@ def __init__(
self.ln_pre = None
self.ln_post = None
self.pg_collection = pg_collection
self.tp_group = get_tensor_model_parallel_group_if_none(
pg_collection.tp if pg_collection is not None else None
)
self.vp_stage = vp_stage
if ln_pre_impl is not None:
self.ln_pre = build_module(
Expand Down
1 change: 1 addition & 0 deletions megatron/core/ssm/gated_delta_net.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ def __init__(
self.use_qk_l2norm = use_qk_l2norm
assert pg_collection is not None, "pg_collection must be provided for GatedDeltaNet"
self.pg_collection = pg_collection
self.tp_group = pg_collection.tp
self.cp_size = self.pg_collection.cp.size()
self.tp_size = self.pg_collection.tp.size()
self.sp_size = self.tp_size if config.sequence_parallel else 1
Expand Down
1 change: 1 addition & 0 deletions megatron/core/ssm/mamba_layer.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ def __init__(
"""
super().__init__(config)
assert pg_collection is not None, "pg_collection must be provided for MambaLayer"
self.tp_group = pg_collection.tp

self.config = config
self.submodules_config = submodules
Expand Down
3 changes: 3 additions & 0 deletions megatron/core/ssm/mamba_mixer.py
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,7 @@ def __init__(
)
setattr(self.norm.weight, "tensor_model_parallel", True)
setattr(self.norm.weight, "partition_dim", 0)
self.norm.tp_group = self.pg_collection.tp
# Assume sequence parallelism: input is partitioned along d_inner and
# output is partitioned along the sequence dimension
self.out_proj = build_module(
Expand Down Expand Up @@ -1335,6 +1336,8 @@ def sharded_state_dict(self, prefix="", sharded_offsets=(), metadata=None):
"conv1d_bias": 0,
},
sharded_offsets=sharded_offsets,
tp_group=self.tp_group,
dp_cp_group=metadata["dp_cp_group"],
)
# Submodules
for name, module in self.named_children():
Expand Down
88 changes: 62 additions & 26 deletions megatron/training/checkpointing.py
Original file line number Diff line number Diff line change
Expand Up @@ -368,8 +368,20 @@ def read_metadata(tracker_filename):
return max_iter, release


def get_rng_state(ckpt_format: str, tp_group: torch.distributed.ProcessGroup, pp_group: torch.distributed.ProcessGroup) -> Union[List[Dict[str, Any]], ShardedObject]:
"""Collect rng state across data parallel ranks."""
def get_rng_state(
ckpt_format: str,
tp_group: torch.distributed.ProcessGroup,
pp_group: torch.distributed.ProcessGroup,
dp_cp_group: Optional[torch.distributed.ProcessGroup] = None,
dp_group: Optional[torch.distributed.ProcessGroup] = None,
key_prefix: str = '',
) -> Union[List[Dict[str, Any]], ShardedObject]:
"""Collect rng state across data parallel ranks.

dp_group threads the data-parallel group used for RNG gather/indexing.
dp_cp_group threads the data-parallel (with context-parallel) group used for checkpoint replica id.
key_prefix namespaces the rng ShardedObject key so disjoint grids avoid a key collision (default '').
"""
args = get_args()
rng_state = {
'random_rng_state': random.getstate(),
Expand All @@ -378,28 +390,31 @@ def get_rng_state(ckpt_format: str, tp_group: torch.distributed.ProcessGroup, pp
'cuda_rng_state': torch.cuda.get_rng_state(),
'rng_tracker_states': tensor_parallel.get_cuda_rng_tracker().get_states()}

dp_world_size = get_pg_size(dp_group) if dp_group is not None else mpu.get_data_parallel_world_size()
rng_state_list = None
if args.data_parallel_random_init and torch.distributed.is_initialized() and \
mpu.get_data_parallel_world_size() > 1:
dp_world_size > 1:
rng_state_list = \
[None for i in range(mpu.get_data_parallel_world_size())]
[None for i in range(dp_world_size)]
torch.distributed.all_gather_object(
rng_state_list,
rng_state,
group=mpu.get_data_parallel_group())
group=dp_group if dp_group is not None else mpu.get_data_parallel_group(),
)
else:
rng_state_list = [rng_state]

dp_cp_rank = get_pg_rank(dp_cp_group) if dp_cp_group is not None else mpu.get_data_parallel_rank(with_context_parallel=True)
if ckpt_format == "torch_dist":
pp_rank = get_pg_rank(pp_group)
pp_size = get_pg_size(pp_group)
tp_rank = get_pg_rank(tp_group)
tp_size = get_pg_size(tp_group)
rng_state_list = ShardedObject('rng_state', rng_state_list, (pp_size, tp_size), (pp_rank, tp_rank),
replica_id=mpu.get_data_parallel_rank(with_context_parallel=True))
rng_state_list = ShardedObject(f'{key_prefix}rng_state', rng_state_list, (pp_size, tp_size), (pp_rank, tp_rank),
replica_id=dp_cp_rank)
elif ckpt_format == "fsdp_dtensor":
pp_rank = mpu.get_pipeline_model_parallel_rank()
tp_rank = mpu.get_tensor_model_parallel_rank()
pp_rank = get_pg_rank(pp_group)
tp_rank = get_pg_rank(tp_group)
rng_state_list = {
f"({pp_rank}, {tp_rank})": rng_state_list
}
Expand Down Expand Up @@ -494,7 +509,7 @@ def save_grads(save_dir, state_dict, iteration, grad_label):

def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floating_point_operations_so_far,
checkpointing_context=None, pipeline_rank=None, expert_rank=None, tensor_rank=None, pipeline_parallel=None, expert_parallel=None, non_persistent_ckpt=False,
train_data_iterator=None, preprocess_common_state_dict_fn = None, release=False, tp_group: Optional[torch.distributed.ProcessGroup] = None, pp_group: Optional[torch.distributed.ProcessGroup] = None, dp_cp_group: Optional[torch.distributed.ProcessGroup] = None):
train_data_iterator=None, preprocess_common_state_dict_fn = None, release=False, tp_group: Optional[torch.distributed.ProcessGroup] = None, pp_group: Optional[torch.distributed.ProcessGroup] = None, dp_cp_group: Optional[torch.distributed.ProcessGroup] = None, dp_group: Optional[torch.distributed.ProcessGroup] = None, expt_dp_group: Optional[torch.distributed.ProcessGroup] = None, rng_state_key_prefix: str = ''):
"""Save a model, optimizer and optionally dataloader checkpoint.

Checkpointing context is used to persist some checkpointing state
Expand All @@ -511,6 +526,8 @@ def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floati

Args:
dp_cp_group: Data parallel + context parallel group (default: None, falls back to mpu API)
dp_group: Data parallel group (default: None, falls back to mpu API)
expt_dp_group: Expert data parallel group (default: None, falls back to mpu API)
"""
start_ckpt = time()
args = get_args()
Expand Down Expand Up @@ -557,7 +574,10 @@ def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floati
if tp_group is None and pp_group is None:
tp_group = mpu.get_tensor_model_parallel_group()
pp_group = mpu.get_pipeline_model_parallel_group()
rng_state = get_rng_state(args.ckpt_format, tp_group, pp_group)
rng_state = get_rng_state(args.ckpt_format, tp_group, pp_group,
dp_cp_group=dp_cp_group,
dp_group=dp_group,
key_prefix=rng_state_key_prefix)

# Collect rerun state across all ranks
rerun_state_machine = get_rerun_state_machine()
Expand Down Expand Up @@ -609,9 +629,9 @@ def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floati
# the dense and expert parallelism layouts disagree (e.g. TP > EP*ETP); the union
# does, with at most one rank per (tp_rank, ep_rank) inside any DP group.
if not torch.distributed.is_initialized() \
or ckpt_type != CheckpointType.LEGACY \
or mpu.get_data_parallel_rank() == 0 \
or mpu.get_expert_data_parallel_rank() == 0 \
or ckpt_type != CheckpointType.LEGACY:
or mpu.get_expert_data_parallel_rank() == 0:

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

whats the change at this line? why are we still using mpu groups here ?

if ckpt_type != CheckpointType.LEGACY:
sharded_sd_metadata = _build_sharded_state_dict_metadata(args, dp_cp_group=dp_cp_group)
if args.use_distributed_optimizer:
Expand Down Expand Up @@ -664,9 +684,9 @@ def save_checkpoint(iteration, model, optimizer, opt_param_scheduler, num_floati

if args.ckpt_fully_parallel_save:
if args.ckpt_fully_parallel_save_process_group == 'dp':
process_group = mpu.get_data_parallel_group(with_context_parallel=True)
process_group = dp_cp_group if dp_cp_group is not None else mpu.get_data_parallel_group(with_context_parallel=True)
elif args.ckpt_fully_parallel_save_process_group == 'ep_dp':
process_group = mpu.get_expert_data_parallel_group()
process_group = expt_dp_group if expt_dp_group is not None else mpu.get_expert_data_parallel_group()
save_strategy = FullyParallelSaveStrategyWrapper(save_strategy, process_group,
args.ckpt_assume_constant_structure)
# Store save strategy for future checkpoint saves
Expand Down Expand Up @@ -1631,7 +1651,7 @@ def _set_arg(arg_name, old_arg_name=None, force=False):


def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load', strict=True,
checkpointing_context=None, skip_load_to_model_and_opt=False, tp_group: Optional[torch.distributed.ProcessGroup] = None, pp_group: Optional[torch.distributed.ProcessGroup] = None, dp_cp_group: Optional[torch.distributed.ProcessGroup] = None):
checkpointing_context=None, skip_load_to_model_and_opt=False, tp_group: Optional[torch.distributed.ProcessGroup] = None, pp_group: Optional[torch.distributed.ProcessGroup] = None, dp_cp_group: Optional[torch.distributed.ProcessGroup] = None, dp_group: Optional[torch.distributed.ProcessGroup] = None, rng_state_key_prefix: str = ''):
"""Load a model checkpoint and return the iteration.
strict (bool): whether to strictly enforce that the keys in
:attr:`state_dict` of the checkpoint match the names of
Expand All @@ -1640,6 +1660,7 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load',
for :attr:`model` and :attr:`optimizer`. In case of running FSDP2 with mcore distributed
checkpointing, the tensors are already loaded in-place by `_load_base_checkpoint`.
dp_cp_group: Data parallel + context parallel group (default: None, falls back to mpu API)
dp_group: Data parallel group (default: None, falls back to mpu API)
"""
args = get_args()
load_dir = getattr(args, load_arg)
Expand Down Expand Up @@ -1716,15 +1737,18 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load',
if tp_group is None and pp_group is None:
tp_group = mpu.get_tensor_model_parallel_group()
pp_group = mpu.get_pipeline_model_parallel_group()
gen_sd_rng_state = get_rng_state(args.ckpt_format, tp_group, pp_group) # we can load the rng state
gen_sd_rng_state = get_rng_state(args.ckpt_format, tp_group, pp_group,
dp_cp_group=dp_cp_group,
dp_group=dp_group,
key_prefix=rng_state_key_prefix) # we can load the rng state
else:
ignore_rng_state = True
gen_sd_rng_state = None
if ckpt_tp_pp != run_tp_pp:
print_rank_0("{}: RNG state will be ignored".format(mismatch_msg))

if ckpt_type == CheckpointType.LOCAL:
sharded_sd_metadata = _build_sharded_state_dict_metadata(args)
sharded_sd_metadata = _build_sharded_state_dict_metadata(args, dp_cp_group=dp_cp_group)
else:
sharded_sd_metadata = dist_checkpointing.load_content_metadata(preloaded_state_dict=state_dict)
print_rank_0(f'sharded_state_dict metadata loaded from the checkpoint: {sharded_sd_metadata}')
Expand Down Expand Up @@ -1820,7 +1844,9 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load',
"optimizer": optimizer_sd,
"args": None,
"iteration": 1,
"rng_state": get_rng_state(args.ckpt_format, tp_group, pp_group),
"rng_state": get_rng_state(
args.ckpt_format, tp_group, pp_group, dp_cp_group=dp_cp_group, dp_group=dp_group
),
"checkpoint_version": None,
"opt_param_scheduler": opt_param_scheduler.state_dict(),
"num_floating_point_operations_so_far": 0,
Expand All @@ -1843,12 +1869,17 @@ def load_checkpoint(ddp_model, optimizer, opt_param_scheduler, load_arg='load',
data_iterator=None, ckpt_format=ckpt_format, force=True,
)
if not args.no_load_rng:
gen_sd_rng_state = get_rng_state(args.ckpt_format, tp_group, pp_group)
gen_sd_rng_state = get_rng_state(
args.ckpt_format, tp_group, pp_group, dp_cp_group=dp_cp_group, dp_group=dp_group
)
if not args.no_load_optim:
gen_sd_optim = optimizer
gen_sd_opt_param_scheduler = opt_param_scheduler

optim_sd_kwargs = dict(metadata=_build_sharded_state_dict_metadata(args), is_loading=True)
optim_sd_kwargs = dict(
metadata=_build_sharded_state_dict_metadata(args, dp_cp_group=dp_cp_group),
is_loading=True,
)

state_dict = generate_state_dict(
args,
Expand Down Expand Up @@ -2021,8 +2052,8 @@ def load_model_state_dict(module, state_dict, strict: bool):
if 'rng_state' in state_dict:
if args.ckpt_format == "fsdp_dtensor":
# FSDP DTensor checkpoints store rng_state in a different format.
tp_rank = mpu.get_tensor_model_parallel_rank()
pp_rank = mpu.get_pipeline_model_parallel_rank()
tp_rank = get_pg_rank(tp_group) if tp_group is not None else mpu.get_tensor_model_parallel_rank()
pp_rank = get_pg_rank(pp_group) if pp_group is not None else mpu.get_pipeline_model_parallel_rank()
if f"({pp_rank}, {tp_rank})" in state_dict['rng_state']:
rng_state = state_dict['rng_state'][f"({pp_rank}, {tp_rank})"]
else:
Expand All @@ -2033,7 +2064,8 @@ def load_model_state_dict(module, state_dict, strict: bool):

# access rng_state for data parallel rank
if args.data_parallel_random_init:
rng_state = rng_state[mpu.get_data_parallel_rank()]
dp_rank = get_pg_rank(dp_group) if dp_group is not None else mpu.get_data_parallel_rank()
rng_state = rng_state[dp_rank]
else:
rng_state = rng_state[0]
random.setstate(rng_state['random_rng_state'])
Expand Down Expand Up @@ -2071,9 +2103,13 @@ def load_model_state_dict(module, state_dict, strict: bool):
if torch.distributed.is_initialized():
torch.distributed.barrier()

_tp_r = get_pg_rank(tp_group) if tp_group is not None else mpu.get_tensor_model_parallel_rank()
_tp_w = get_pg_size(tp_group) if tp_group is not None else mpu.get_tensor_model_parallel_world_size()
_pp_r = get_pg_rank(pp_group) if pp_group is not None else mpu.get_pipeline_model_parallel_rank()
_pp_w = get_pg_size(pp_group) if pp_group is not None else mpu.get_pipeline_model_parallel_world_size()
print_rank_0(f' successfully loaded checkpoint from {load_dir} '
f'[ t {mpu.get_tensor_model_parallel_rank() + 1}/{mpu.get_tensor_model_parallel_world_size()}, '
f'p {mpu.get_pipeline_model_parallel_rank() + 1}/{mpu.get_pipeline_model_parallel_world_size()} ] '
f'[ t {_tp_r + 1}/{_tp_w}, '
f'p {_pp_r + 1}/{_pp_w} ] '
f'at iteration {iteration}')

# Additional callback for wandb (last rank)
Expand Down
Loading