From baae1010551d299783f4e4ef36c32a89832619c8 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Wed, 24 Jun 2026 09:23:35 +0800 Subject: [PATCH 1/5] support non-disopt muon with adam distopt Signed-off-by: pingtianl --- .../distributed_data_parallel_config.py | 13 ++ .../core/distributed/param_and_grad_buffer.py | 151 ++++++++++++++---- .../core/optimizer/layer_wise_optimizer.py | 38 ++++- megatron/core/optimizer/optimizer_config.py | 9 ++ megatron/training/arguments.py | 33 +++- megatron/training/training.py | 36 +++-- 6 files changed, 220 insertions(+), 60 deletions(-) diff --git a/megatron/core/distributed/distributed_data_parallel_config.py b/megatron/core/distributed/distributed_data_parallel_config.py index ed2c43a2d4f..5120ef06372 100644 --- a/megatron/core/distributed/distributed_data_parallel_config.py +++ b/megatron/core/distributed/distributed_data_parallel_config.py @@ -36,6 +36,19 @@ class DistributedDataParallelConfig: enabled. Defaults to 1, which means DistOpt is across entire DP domain. """ + use_layer_wise_param_layout: bool = True + """Layer-wise (Muon) optimizer only. When True (default), LayerWise-managed buffers use + the shard-aligned padded LayerWise param layout. When False, the compact decoupled + layout is selected instead: LayerWise-managed buffers (Muon's 2D matrix params) use a + compact no-padding DDP layout and locally disable DistributedOptimizer semantics + (all-reduce gradients, legacy whole-param ping-pong ownership, and ``allgather_params`` + param sync), while non-LayerWise buffers (embeddings, biases, layernorm) keep the + standard byte-level :class:`DistributedOptimizer` layout. This removes the persistent + ``dp_size * max(shard_load)`` LayerWise padding from the long-lived param/grad buffers, + making ``use_distributed_optimizer`` a per-buffer property. Has no effect unless a + LayerWise (Muon) distributed optimizer is in use. + """ + check_for_nan_in_grad: bool = False """ If true, check for NaNs and Infs in gradients _before_ communication collective. diff --git a/megatron/core/distributed/param_and_grad_buffer.py b/megatron/core/distributed/param_and_grad_buffer.py index 3a8357ac841..6757b0a37a4 100644 --- a/megatron/core/distributed/param_and_grad_buffer.py +++ b/megatron/core/distributed/param_and_grad_buffer.py @@ -175,17 +175,26 @@ def __init__( ddp_config: DistributedDataParallelConfig, collective_group: torch.distributed.ProcessGroup, collective_group_size: int, + use_distributed_optimizer: Optional[bool] = None, ): self.buckets = buckets self.ddp_config = ddp_config + # Effective (per-group) DistributedOptimizer flag. With the decouple-LayerWise-DDP-layout + # path, ``use_distributed_optimizer`` is a per-buffer property, so the caller + # (``partition_buckets``) passes the value derived from this group's buffers. When None, + # fall back to the model-level ddp_config knob. + if use_distributed_optimizer is None: + use_distributed_optimizer = self.ddp_config.use_distributed_optimizer + self.use_distributed_optimizer = use_distributed_optimizer + # overlap_param_gather covers the layer-wise optimizer case, which sets # overlap_param_gather=True without use_distributed_optimizer. - if self.ddp_config.use_distributed_optimizer or self.ddp_config.overlap_param_gather: + if self.use_distributed_optimizer or self.ddp_config.overlap_param_gather: self.intra_distributed_optimizer_instance_group = collective_group self.intra_distributed_optimizer_instance_size = collective_group_size self.intra_distributed_optimizer_instance_rank = collective_group.rank() - if not self.ddp_config.use_distributed_optimizer: + if not self.use_distributed_optimizer: self.data_parallel_group = collective_group # State for bookkeeping: params is the set of parameters this bucket group is @@ -212,14 +221,12 @@ def __init__( not self.ddp_config.reduce_scatter_with_fp32_accumulation ), "RS w/ FP32 accumulation not supported with num_distributed_optimizer_instances > 1" - reduction_collective = ( - "reduce-scatter" if self.ddp_config.use_distributed_optimizer else "all-reduce" - ) + reduction_collective = "reduce-scatter" if self.use_distributed_optimizer else "all-reduce" log_single_rank( logger, logging.INFO, f"Using {reduction_collective} for gradient reductions because " - f"{self.ddp_config.use_distributed_optimizer=}", + f"{self.use_distributed_optimizer=}", ) global dist_reduce_scatter_func @@ -363,7 +370,7 @@ def start_param_sync(self, force_sync: bool = False): """ # overlap_param_gather covers the layer-wise optimizer case, which sets # overlap_param_gather=True without use_distributed_optimizer. - assert self.ddp_config.use_distributed_optimizer or self.ddp_config.overlap_param_gather + assert self.use_distributed_optimizer or self.ddp_config.overlap_param_gather if force_sync: if self.param_gather_handle is not None: @@ -376,7 +383,7 @@ def start_param_sync(self, force_sync: bool = False): async_op = self.ddp_config.overlap_param_gather and not force_sync - if not self.ddp_config.use_distributed_optimizer: + if not self.use_distributed_optimizer: # Legacy layer-wise optimizer path: use all_gather for variable-size # param gather. Once all layerwise call sites set # ddp_config.use_distributed_optimizer=True, this branch can be removed. @@ -530,7 +537,7 @@ def finish_param_sync(self, skip_next_bucket_dispatch: bool = False): else: self.next_param_gather_bucket_group.start_param_sync() - if not self.ddp_config.use_distributed_optimizer: + if not self.use_distributed_optimizer: for bucket in self.buckets: if bucket.layerwise_gather_list is None: continue @@ -642,7 +649,7 @@ def start_grad_sync(self, force_all_reduce: Optional[bool] = False): else: stream_context = nullcontext() - if self.ddp_config.use_distributed_optimizer: + if self.use_distributed_optimizer: communication_group = self.intra_distributed_optimizer_instance_group else: communication_group = self.data_parallel_group @@ -651,7 +658,7 @@ def start_grad_sync(self, force_all_reduce: Optional[bool] = False): grad_reduce_handle = None with stream_context, _coalescing_manager(communication_group, async_ops=async_op) as cm: for idx, bucket in enumerate(self.buckets): - if self.ddp_config.use_distributed_optimizer and not force_all_reduce: + if self.use_distributed_optimizer and not force_all_reduce: if self.cached_grad_buffer_shard_list[idx] is None: self.cached_grad_buffer_shard_list[idx] = shard_buffer( bucket.grad_data, self.intra_distributed_optimizer_instance_size @@ -677,7 +684,7 @@ def start_grad_sync(self, force_all_reduce: Optional[bool] = False): # With multiple DistOpt instances, we need to all-reduce across instances. if ( - self.ddp_config.use_distributed_optimizer + self.use_distributed_optimizer and self.ddp_config.num_distributed_optimizer_instances > 1 ): assert self.inter_distributed_optimizer_instance_group is not None @@ -1006,6 +1013,22 @@ def __init__( self.gradient_scaling_factor = gradient_scaling_factor self.nccl_ub = nccl_ub + # Effective (per-buffer) DistributedOptimizer flag. ``use_distributed_optimizer`` is a + # model-level ddp_config knob, but a LayerWise (Muon) optimizer with the padded LayerWise + # param layout disabled (``use_layer_wise_param_layout=False``) turns it into a per-buffer + # property: LayerWise-managed (Muon 2D matrix) buffers locally disable DistOpt semantics + # (compact no-padding layout, all-reduce gradients, legacy whole-param ping-pong ownership + # + allgather_params), while sibling non-LayerWise buffers (embeddings, biases, layernorm) + # keep the standard byte-level DistributedOptimizer path. + self._is_layer_wise_buffer = bool( + self.params and getattr(self.params[0], "is_managed_by_layer_wise_optimizer", False) + ) + self.use_distributed_optimizer = self.ddp_config.use_distributed_optimizer + if self._is_layer_wise_buffer and not getattr( + self.ddp_config, "use_layer_wise_param_layout", True + ): + self.use_distributed_optimizer = False + # Data structures to store underlying buckets and relevant indexing data. self.buckets = [] self.param_to_bucket = {} # Param -> bucket mapping. @@ -1053,9 +1076,29 @@ def __init__( # nvfp4_packed_numel_unpadded is already set by _compute_nvfp4_packed_layout. assert self.numel_unpadded <= self.numel + + # Diagnostic: log persistent buffer size vs. unpadded payload so the cost of any + # optimizer-driven padding (e.g. the LayerWise shard-aligned ``dp_size * max(shard_load)`` + # layout) is visible per buffer. Emit at INFO only when it is interesting — a + # LayerWise-managed buffer or one that actually carries padding — and DEBUG otherwise, so + # ordinary (zero-padding) buffers do not spam non-experimental runs. + _padding = self.numel - self.numel_unpadded + _pad_frac = _padding / max(self.numel_unpadded, 1) + log_on_each_pipeline_stage( + logger, + logging.INFO if (self._is_layer_wise_buffer or _padding > 0) else logging.DEBUG, + f"ParamAndGradBuffer layout: param_dtype={self.param_dtype} " + f"grad_dtype={self.grad_dtype} dp_world_size={self.data_parallel_world_size} " + f"layerwise={self._is_layer_wise_buffer} distopt={self.use_distributed_optimizer} " + f"numel={self.numel} numel_unpadded={self.numel_unpadded} " + f"padding={_padding} ({_pad_frac:.1%})", + tp_group=self.tp_group, + dp_cp_group=self.dp_cp_group, + ) + if self.has_nvfp4_params: assert self.nvfp4_packed_numel_unpadded <= self.nvfp4_packed_numel - if self.ddp_config.use_distributed_optimizer: + if self.use_distributed_optimizer: assert self.numel % self.data_parallel_world_size == 0 if self.has_nvfp4_params: assert self.nvfp4_packed_numel % self.data_parallel_world_size == 0 @@ -1094,9 +1137,7 @@ def __init__( # For MXFP8 param: Create a shared buffer for param AG and grad RS for memory efficiency # The buffer is mapped to weight gradients whose dtype is either bf16 or FP32. # It can be temporarily reused by param AG. - if self.ddp_config.use_distributed_optimizer and any( - is_mxfp8tensor(p) for p in self.params - ): + if self.use_distributed_optimizer and any(is_mxfp8tensor(p) for p in self.params): self.shared_buffer = torch.zeros( self.numel, dtype=self.grad_dtype, @@ -1113,7 +1154,7 @@ def __init__( self.grad_data = self.shared_buffer else: # Only re-map param tensors if using distributed optimizer. - if self.ddp_config.use_distributed_optimizer: + if self.use_distributed_optimizer: numel = self.nvfp4_packed_numel if self.has_nvfp4_params else self.numel self.param_data = torch.zeros( numel, @@ -1309,12 +1350,12 @@ def _compute_nvfp4_packed_layout(self, params_with_names): """ def _pad_start_of_param(param_start_index: int) -> int: - if self.ddp_config.use_distributed_optimizer: + if self.use_distributed_optimizer: return pad_param_start(param_start_index) return param_start_index def _pad_end_of_bucket(bucket_end_index: int) -> int: - if self.ddp_config.use_distributed_optimizer: + if self.use_distributed_optimizer: return pad_bucket_end( bucket_end_index, self.data_parallel_world_size, @@ -1424,7 +1465,7 @@ def _new_bucket( # Assert that indices are correctly padded (if needed), and that bucket # position is same as originally computed. - if self.ddp_config.use_distributed_optimizer: + if self.use_distributed_optimizer: assert start_index % self.data_parallel_world_size == 0 assert end_index % self.data_parallel_world_size == 0 assert (start_index, end_index) == self.bucket_indices[bucket_id] @@ -1565,22 +1606,72 @@ def partition_buckets( assert fp8_buffer is None fp8_buffer = buffer - # Case 1: Put all buckets into a single bucket group if force_single_bucket_group is True. + # A bucket group performs a single collective type (reduce-scatter for DistOpt buffers, + # all-reduce otherwise), so buckets merged into one group must agree on the effective + # per-buffer ``use_distributed_optimizer``. The decoupled LayerWise layout + # (``use_layer_wise_param_layout=False``) gives LayerWise (Muon) buffers + # ``use_distributed_optimizer=False`` while sibling buffers keep True; the no-fp8 Case 2 below + # keeps every bucket in its own group so they never mix, but the merging Cases 1/3 must assert + # consistency. + _ddp_config = buffers[0].ddp_config + _decouple = not getattr(_ddp_config, "use_layer_wise_param_layout", True) + + def _bucket_distopt(bucket): + """This bucket's effective ``use_distributed_optimizer``.""" + is_lw = bool( + bucket.params_list + and getattr(bucket.params_list[0], "is_managed_by_layer_wise_optimizer", False) + ) + if _decouple and is_lw: + return False + return _ddp_config.use_distributed_optimizer + + def _merged_use_distributed_optimizer(merge_buckets): + values = {_bucket_distopt(bucket) for bucket in merge_buckets} + assert len(values) == 1, ( + "Cannot merge buckets with differing effective use_distributed_optimizer into one " + "bucket group. This happens when the decoupled LayerWise layout " + "(use_layer_wise_param_layout=False) mixes LayerWise (all-reduce) and non-LayerWise " + "(reduce-scatter) buffers under a merging bucketing strategy (e.g. the fp8 merge " + "path). Disable bucket merging for the decoupled LayerWise path." + ) + return values.pop() + + # Case 1: Put all buckets into a single bucket group if force_single_bucket_group is True + # (e.g. disable_bucketing / non-first VPP chunks). A bucket group performs a single + # collective type, so when the decoupled LayerWise layout (use_layer_wise_param_layout=False) + # mixes LayerWise (all-reduce, non-DistOpt) and non-LayerWise (reduce-scatter, DistOpt) + # buffers in one chunk, we cannot + # merge them into a single group. Split by the effective per-bucket use_distributed_optimizer + # instead, preserving order. When all buckets agree (the non-decoupled case) this collapses + # to exactly one group, identical to the previous behavior. if force_single_bucket_group: - buckets = [] ddp_config = buffers[0].ddp_config data_parallel_group = buffers[0].data_parallel_group data_parallel_world_size = buffers[0].data_parallel_world_size + ordered_distopt_values = [] + buckets_by_distopt = {} for buffer in buffers: assert ddp_config == buffer.ddp_config assert data_parallel_group == buffer.data_parallel_group assert data_parallel_world_size == buffer.data_parallel_world_size - buckets.extend(buffer.buckets) - - bucket_group = _ParamAndGradBucketGroup( - buckets, ddp_config, data_parallel_group, data_parallel_world_size - ) - return [bucket_group] + for bucket in buffer.buckets: + distopt = _bucket_distopt(bucket) + if distopt not in buckets_by_distopt: + buckets_by_distopt[distopt] = [] + ordered_distopt_values.append(distopt) + buckets_by_distopt[distopt].append(bucket) + + return [ + _ParamAndGradBucketGroup( + buckets_by_distopt[distopt], + ddp_config, + data_parallel_group, + data_parallel_world_size, + use_distributed_optimizer=distopt, + ) + for distopt in ordered_distopt_values + ] if fp8_buffer is None: # Case 2: When there is no fp8 buffer in the input buffers, let each bucket group have @@ -1594,6 +1685,7 @@ def partition_buckets( buffer.ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, + use_distributed_optimizer=buffer.use_distributed_optimizer, ) ) return bucket_groups @@ -1620,6 +1712,7 @@ def partition_buckets( buffer.ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, + use_distributed_optimizer=_bucket_distopt(bucket), ) ) if non_fp8_buckets: @@ -1630,6 +1723,7 @@ def partition_buckets( buffer.ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, + use_distributed_optimizer=_bucket_distopt(non_fp8_bucket), ) ) @@ -1645,6 +1739,7 @@ def partition_buckets( buffer.ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, + use_distributed_optimizer=_merged_use_distributed_optimizer(group_buckets), ) ) return bucket_groups diff --git a/megatron/core/optimizer/layer_wise_optimizer.py b/megatron/core/optimizer/layer_wise_optimizer.py index 606525f8097..e9e85008aff 100644 --- a/megatron/core/optimizer/layer_wise_optimizer.py +++ b/megatron/core/optimizer/layer_wise_optimizer.py @@ -317,8 +317,15 @@ def compute_full_param_layout( :class:`FullParamLayout` with a :class:`PerBufferParamLayout` per buffer group. """ # Avoid a circular import: DistributedOptimizer imports LayerWise indirectly. + from ..distributed.param_and_grad_buffer import _compute_default_per_buffer_param_layout from .distrib_optimizer import DistributedOptimizer + # Decoupled layout (use_layer_wise_param_layout=False): LayerWise (Muon) buffers use a + # compact no-padding DDP layout (and locally disable DistributedOptimizer semantics in + # DDP), so they must NOT receive the shard-aligned ``dp_size * max(shard_load)`` padded + # layout here. Non-LayerWise buffers keep DistOpt's byte-level layout regardless. + decouple_ddp_layout = not getattr(ddp_config, 'use_layer_wise_param_layout', True) + buffer_groups = group_params_for_buffers(params, ddp_config.grad_reduce_in_fp32) layouts = {} for buffer_key, (group_params, param_indices) in buffer_groups.items(): @@ -334,6 +341,15 @@ def compute_full_param_layout( # Dispatch per buffer: LayerWise (Muon) params get the shard-aligned # layout; non-LayerWise params (e.g. Adam-managed embeddings, biases) # get DistOpt's byte-level layout. + if buffer_key.is_managed_by_layer_wise_optimizer and decouple_ddp_layout: + # Compact no-padding layout (DDP treats this buffer as non-DistOpt). Attach + # param_indices so DDP's layout/grouping consistency check passes. + per_buffer_layout = _compute_default_per_buffer_param_layout( + group_params, bucket_size + ) + per_buffer_layout.param_indices = param_indices + layouts[buffer_key] = per_buffer_layout + continue if buffer_key.is_managed_by_layer_wise_optimizer: compute_per_buffer_layout = ( LayerWiseDistributedOptimizer._compute_per_buffer_param_layout @@ -366,8 +382,15 @@ def __init__( self.pg_collection = pg_collection + # Decoupled layout (use_layer_wise_param_layout=False): LayerWise (Muon) buffers use a + # compact no-padding DDP layout that does NOT encode whole-param shard ownership, and DDP + # treats them as non-DistOpt (all-reduce gradients). Ownership must therefore use the + # legacy whole-param ping-pong assignment and param sync must use ``allgather_params``, + # even though the model chunks carry a (compact) full_param_layout for DDP's benefit. + self.decouple_ddp_layout = not getattr(config, 'use_layer_wise_param_layout', True) + full_param_layouts = None - if model_chunks is not None: + if model_chunks is not None and not self.decouple_ddp_layout: full_param_layouts = [ chunk.full_param_layout for chunk in model_chunks @@ -375,12 +398,13 @@ def __init__( ] or None self.shard_params(optimizers, full_param_layouts) - # When a full_param_layout is available, ddp_config.use_distributed_optimizer - # is True and model params are views into the DDP param buffer. After the - # optimizer step copies updated fp32 main params → bf16 model params, the - # buffer is already up-to-date in-place. We can use DDP's buffer-based - # all-gather (start_param_sync) instead of the flatten/unflatten allgather_params - # path. + # When a full_param_layout is available (and we are not decoupling), + # ddp_config.use_distributed_optimizer is True and model params are views into the + # DDP param buffer. After the optimizer step copies updated fp32 main params → bf16 + # model params, the buffer is already up-to-date in-place. We can use DDP's + # buffer-based all-gather (start_param_sync) instead of the flatten/unflatten + # allgather_params path. In the decouple path, Muon buffers are non-DistOpt and own + # whole params via ping-pong, so we use the legacy allgather_params path instead. self.use_buffer_param_sync = full_param_layouts is not None # Set up overlap param gather using DDP bucket infrastructure. diff --git a/megatron/core/optimizer/optimizer_config.py b/megatron/core/optimizer/optimizer_config.py index 35bf1e3ceaf..d30288d0745 100644 --- a/megatron/core/optimizer/optimizer_config.py +++ b/megatron/core/optimizer/optimizer_config.py @@ -323,6 +323,15 @@ class OptimizerConfig: arguments layer sets this flag and resets ``use_distributed_optimizer`` to False so that the standard distributed-optimizer path is not triggered.""" + use_layer_wise_param_layout: bool = True + """Layer-wise (Muon) optimizer only. When True (default), LayerWise-managed buffers use + the shard-aligned padded LayerWise param layout. When False, the compact decoupled layout + is selected: LayerWise-managed (Muon) buffers use a compact no-padding DDP layout with + all-reduce gradients and legacy whole-param ping-pong ownership + ``allgather_params`` + param sync, while non-LayerWise buffers keep the standard byte-level + :class:`DistributedOptimizer` layout. Removes the persistent ``dp_size * max(shard_load)`` + LayerWise padding. Mirrors ``DistributedDataParallelConfig.use_layer_wise_param_layout``.""" + overlap_param_gather: bool = False """If true, overlap param all-gather with forward compute. This argument is intended to have the same value as the "overlap_param_gather" argument diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 5125c5abc92..2e7c7635f41 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1893,6 +1893,20 @@ def validate_args(args, defaults={}): "torch_dist", ], "Emerging optimizer supports torch and torch_dist checkpoint format." + if args.use_layer_wise_distributed_optimizer: + assert not args.fp8_param_gather and not getattr(args, 'fp4_param_gather', False), ( + "Layer-wise (Muon) distributed optimizer does not support FP8/FP4 parameter gather " + "(fp8_param_gather / fp4_param_gather). Use fp8_param_gather=False (e.g. blockwise/" + "MXFP8 compute with parameters persisted in bf16)." + ) + if not args.use_layer_wise_param_layout: + assert args.num_distributed_optimizer_instances == 1, ( + "--no-use-layer-wise-param-layout (decoupled compact LayerWise DDP layout) requires " + "num_distributed_optimizer_instances == 1: the non-DistOpt LayerWise (Muon) buffers " + "only all-reduce within a single optimizer instance, so partial DistOpt (>1 " + "instance) would under-reduce Muon gradients across the full data-parallel domain." + ) + # Make sure all functionality that requires Gloo process groups is disabled. if not args.use_gloo_process_groups: if args.use_distributed_optimizer: @@ -4207,14 +4221,17 @@ def _add_distributed_args(parser): '--no-use-layer-wise-param-layout', action='store_false', dest='use_layer_wise_param_layout', - help='Opt out of the precomputed LayerWise param layout. When set, ' - 'falls back to the legacy LayerWise ping-pong path: all params ' - '(including non-Muon embeddings, biases, layernorm) live in a single ' - 'LayerWise buffer and the optimizer uses the allgather_params() codepath. ' - 'The default (precomputed layout) routes non-Muon params through a ' - 'separate DistributedOptimizer with byte-level sharding, which is faster ' - 'and uses less padding but produces different bf16 reduction ordering ' - 'and so will not match legacy-path loss curves bit-for-bit.', + help='Opt out of the precomputed shard-aligned (padded) LayerWise param layout. ' + 'With an emerging (Muon) optimizer under --use-distributed-optimizer, this selects ' + 'the experimental compact, decoupled DDP layout: LayerWise-managed (Muon 2D matrix) ' + 'buffers use a compact no-padding DDP layout and locally disable DistributedOptimizer ' + 'semantics (all-reduce gradients + legacy whole-param ping-pong ownership + ' + 'allgather_params param sync), while sibling non-LayerWise buffers (embeddings, ' + 'biases, layernorm) keep the standard byte-level DistributedOptimizer. This removes ' + 'the persistent dp_size * max(shard_load) LayerWise padding from the long-lived ' + 'buffers. Incompatible with FP8 parameter gather (fp8_param_gather=True). Produces ' + 'different bf16 reduction ordering from the default padded layout, so will not match ' + 'its loss curves bit-for-bit.', ) return parser diff --git a/megatron/training/training.py b/megatron/training/training.py index ae6216c260b..f65a50f34cc 100644 --- a/megatron/training/training.py +++ b/megatron/training/training.py @@ -1745,7 +1745,6 @@ def wrap_model_chunks_with_ddp( ddp_config, *, use_layer_wise_distributed_optimizer=False, - use_layer_wise_param_layout=True, DP=DDP, pg_collection=None, bucket_sizes=None, @@ -1756,12 +1755,13 @@ def wrap_model_chunks_with_ddp( Centralises the DDP-wrapping wiring shared between :func:`get_model` and unit tests. - For ``use_layer_wise_distributed_optimizer=True`` and ``use_layer_wise_param_layout=True``: - forces ``ddp_config.use_distributed_optimizer=True`` (mutated in place; needed - for reduce-scatter), and computes per-chunk shard-aligned layouts via - :meth:`LayerWiseDistributedOptimizer.compute_full_param_layout`. With - ``use_layer_wise_param_layout=False``, no layout is supplied and LayerWise falls back - to its legacy ``allgather_params`` sync path. + For ``use_layer_wise_distributed_optimizer=True``: forces + ``ddp_config.use_distributed_optimizer=True`` (mutated in place; needed for reduce-scatter) + and computes per-chunk layouts via + :meth:`LayerWiseDistributedOptimizer.compute_full_param_layout`. That method picks the + padded shard-aligned LayerWise layout or the compact decoupled layout per + ``ddp_config.use_layer_wise_param_layout`` (True → padded, False → compact, with LayerWise + buffers treated as non-DistOpt and synced via legacy ``allgather_params``). For non-layerwise with ``ddp_config.use_distributed_optimizer=True``: computes per-chunk byte-level layouts via @@ -1777,11 +1777,9 @@ def wrap_model_chunks_with_ddp( model_chunks: List of model chunks to wrap (un-DDP-wrapped). config: :class:`TransformerConfig`. ddp_config: :class:`DistributedDataParallelConfig`. Mutated in place when - ``use_layer_wise_distributed_optimizer=True`` and ``use_layer_wise_param_layout=True``. + ``use_layer_wise_distributed_optimizer=True``. Its ``use_layer_wise_param_layout`` + field selects the padded (default) vs compact decoupled LayerWise layout. use_layer_wise_distributed_optimizer: Whether the layerwise wiring runs. - use_layer_wise_param_layout: When ``use_layer_wise_distributed_optimizer=True``, - controls whether to compute and supply a shard-aligned param layout - to DDP. ``False`` keeps LayerWise on its legacy sync path. DP: The DDP class to construct (``DistributedDataParallel`` or an FSDP variant). pg_collection: Optional :class:`ProcessGroupCollection`. When provided, @@ -1804,13 +1802,18 @@ def wrap_model_chunks_with_ddp( # Compute per-chunk layouts (DDP only). per_chunk_layouts = [None] * n if DP is DDP: - if use_layer_wise_distributed_optimizer and use_layer_wise_param_layout: + if use_layer_wise_distributed_optimizer: + # LayerWise (Muon) optimizer. Force use_distributed_optimizer=True so sibling + # non-LayerWise buffers (embeddings, biases, layernorm) shard with the byte-level + # DistributedOptimizer layout, and tag params so DDP buffer grouping routes + # LayerWise-managed matrices (Muon's Newton-Schulz domain) to a separate buffer. + # The padded-vs-compact LayerWise layout decision is made inside + # compute_full_param_layout / _ParamAndGradBuffer from use_layer_wise_param_layout: + # with the default padded layout LayerWise buffers stay DistOpt; with + # --no-use-layer-wise-param-layout they get the compact no-padding layout and the + # per-buffer override flips use_distributed_optimizer off for them. ddp_config.use_distributed_optimizer = True compute_layout = LayerWiseDistributedOptimizer.compute_full_param_layout - # Tag params so DDP buffer grouping routes LayerWise-managed matrices - # (Muon's Newton-Schulz domain) to a shard-aligned buffer and routes - # everything else (embeddings, biases, layernorm) to a separate - # DistOpt-style buffer. tag_params_for_buffer_routing(model_chunks) elif not use_layer_wise_distributed_optimizer and ddp_config.use_distributed_optimizer: compute_layout = DistributedOptimizer.compute_full_param_layout @@ -2053,7 +2056,6 @@ def build_model(): use_layer_wise_distributed_optimizer=getattr( args, 'use_layer_wise_distributed_optimizer', False ), - use_layer_wise_param_layout=getattr(args, 'use_layer_wise_param_layout', True), DP=DP, pg_collection=pg_collection if args.use_megatron_fsdp else None, bucket_sizes=per_chunk_bucket_sizes, From b21a44ae115867cb4e19aa2b94130f7c729e77d9 Mon Sep 17 00:00:00 2001 From: pingtianl Date: Thu, 25 Jun 2026 19:30:11 -0700 Subject: [PATCH 2/5] Address review: compact layout default + single-source-of-truth per-buffer distopt Two changes from PR #5388 review (FDecaYed): 1. Make the compact decoupled LayerWise layout the DEFAULT. Replace the `--no-use-layer-wise-param-layout` (store_false) flag with `--use-layer-wise-param-layout` (store_true, default False), and flip the DistributedDataParallelConfig.use_layer_wise_param_layout default to False. The padded LayerWise layout is now opt-in (its dp_size*max(shard_load) padding is a large memory overhead and the reduce-scatter perf gain is not always significant). validate_args message updated to the new flag name. 2. Drop the separate `use_distributed_optimizer` argument threaded into _ParamAndGradBucketGroup. Instead, each _ParamAndGradBuffer bakes its per-buffer DistOpt decision into its own ddp_config ONCE via dataclasses.replace (a LayerWise/Muon buffer on the compact decoupled layout disables DistributedOptimizer; siblings keep the model-level setting). buffer.ddp_config is then the single source of truth and bucket groups simply inherit it in partition_buckets -- no per-site threading or per-call replace. Behavior-preserving. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: pingtianl --- .../distributed_data_parallel_config.py | 15 +-- .../core/distributed/param_and_grad_buffer.py | 96 +++++++++---------- .../core/optimizer/layer_wise_optimizer.py | 22 ++--- megatron/core/optimizer/optimizer_config.py | 4 +- megatron/training/arguments.py | 25 +++-- megatron/training/training.py | 6 +- .../model_config.yaml | 1 - .../model_config.yaml | 1 - 8 files changed, 72 insertions(+), 98 deletions(-) diff --git a/megatron/core/distributed/distributed_data_parallel_config.py b/megatron/core/distributed/distributed_data_parallel_config.py index 5120ef06372..56ec9e89539 100644 --- a/megatron/core/distributed/distributed_data_parallel_config.py +++ b/megatron/core/distributed/distributed_data_parallel_config.py @@ -36,17 +36,10 @@ class DistributedDataParallelConfig: enabled. Defaults to 1, which means DistOpt is across entire DP domain. """ - use_layer_wise_param_layout: bool = True - """Layer-wise (Muon) optimizer only. When True (default), LayerWise-managed buffers use - the shard-aligned padded LayerWise param layout. When False, the compact decoupled - layout is selected instead: LayerWise-managed buffers (Muon's 2D matrix params) use a - compact no-padding DDP layout and locally disable DistributedOptimizer semantics - (all-reduce gradients, legacy whole-param ping-pong ownership, and ``allgather_params`` - param sync), while non-LayerWise buffers (embeddings, biases, layernorm) keep the - standard byte-level :class:`DistributedOptimizer` layout. This removes the persistent - ``dp_size * max(shard_load)`` LayerWise padding from the long-lived param/grad buffers, - making ``use_distributed_optimizer`` a per-buffer property. Has no effect unless a - LayerWise (Muon) distributed optimizer is in use. + use_layer_wise_param_layout: bool = False + """Layer-wise (Muon) optimizer only. When True, LayerWise-managed buffers use + the shard-aligned padded LayerWise param layout. When False (default), the compact + decoupled layout is selected instead. """ check_for_nan_in_grad: bool = False diff --git a/megatron/core/distributed/param_and_grad_buffer.py b/megatron/core/distributed/param_and_grad_buffer.py index 6757b0a37a4..329a673259a 100644 --- a/megatron/core/distributed/param_and_grad_buffer.py +++ b/megatron/core/distributed/param_and_grad_buffer.py @@ -1,5 +1,6 @@ # Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. +import dataclasses import fnmatch import functools import logging @@ -175,26 +176,17 @@ def __init__( ddp_config: DistributedDataParallelConfig, collective_group: torch.distributed.ProcessGroup, collective_group_size: int, - use_distributed_optimizer: Optional[bool] = None, ): self.buckets = buckets self.ddp_config = ddp_config - # Effective (per-group) DistributedOptimizer flag. With the decouple-LayerWise-DDP-layout - # path, ``use_distributed_optimizer`` is a per-buffer property, so the caller - # (``partition_buckets``) passes the value derived from this group's buffers. When None, - # fall back to the model-level ddp_config knob. - if use_distributed_optimizer is None: - use_distributed_optimizer = self.ddp_config.use_distributed_optimizer - self.use_distributed_optimizer = use_distributed_optimizer - # overlap_param_gather covers the layer-wise optimizer case, which sets # overlap_param_gather=True without use_distributed_optimizer. - if self.use_distributed_optimizer or self.ddp_config.overlap_param_gather: + if self.ddp_config.use_distributed_optimizer or self.ddp_config.overlap_param_gather: self.intra_distributed_optimizer_instance_group = collective_group self.intra_distributed_optimizer_instance_size = collective_group_size self.intra_distributed_optimizer_instance_rank = collective_group.rank() - if not self.use_distributed_optimizer: + if not self.ddp_config.use_distributed_optimizer: self.data_parallel_group = collective_group # State for bookkeeping: params is the set of parameters this bucket group is @@ -221,12 +213,14 @@ def __init__( not self.ddp_config.reduce_scatter_with_fp32_accumulation ), "RS w/ FP32 accumulation not supported with num_distributed_optimizer_instances > 1" - reduction_collective = "reduce-scatter" if self.use_distributed_optimizer else "all-reduce" + reduction_collective = ( + "reduce-scatter" if self.ddp_config.use_distributed_optimizer else "all-reduce" + ) log_single_rank( logger, logging.INFO, f"Using {reduction_collective} for gradient reductions because " - f"{self.use_distributed_optimizer=}", + f"{self.ddp_config.use_distributed_optimizer=}", ) global dist_reduce_scatter_func @@ -370,7 +364,7 @@ def start_param_sync(self, force_sync: bool = False): """ # overlap_param_gather covers the layer-wise optimizer case, which sets # overlap_param_gather=True without use_distributed_optimizer. - assert self.use_distributed_optimizer or self.ddp_config.overlap_param_gather + assert self.ddp_config.use_distributed_optimizer or self.ddp_config.overlap_param_gather if force_sync: if self.param_gather_handle is not None: @@ -383,7 +377,7 @@ def start_param_sync(self, force_sync: bool = False): async_op = self.ddp_config.overlap_param_gather and not force_sync - if not self.use_distributed_optimizer: + if not self.ddp_config.use_distributed_optimizer: # Legacy layer-wise optimizer path: use all_gather for variable-size # param gather. Once all layerwise call sites set # ddp_config.use_distributed_optimizer=True, this branch can be removed. @@ -537,7 +531,7 @@ def finish_param_sync(self, skip_next_bucket_dispatch: bool = False): else: self.next_param_gather_bucket_group.start_param_sync() - if not self.use_distributed_optimizer: + if not self.ddp_config.use_distributed_optimizer: for bucket in self.buckets: if bucket.layerwise_gather_list is None: continue @@ -649,7 +643,7 @@ def start_grad_sync(self, force_all_reduce: Optional[bool] = False): else: stream_context = nullcontext() - if self.use_distributed_optimizer: + if self.ddp_config.use_distributed_optimizer: communication_group = self.intra_distributed_optimizer_instance_group else: communication_group = self.data_parallel_group @@ -658,7 +652,7 @@ def start_grad_sync(self, force_all_reduce: Optional[bool] = False): grad_reduce_handle = None with stream_context, _coalescing_manager(communication_group, async_ops=async_op) as cm: for idx, bucket in enumerate(self.buckets): - if self.use_distributed_optimizer and not force_all_reduce: + if self.ddp_config.use_distributed_optimizer and not force_all_reduce: if self.cached_grad_buffer_shard_list[idx] is None: self.cached_grad_buffer_shard_list[idx] = shard_buffer( bucket.grad_data, self.intra_distributed_optimizer_instance_size @@ -684,7 +678,7 @@ def start_grad_sync(self, force_all_reduce: Optional[bool] = False): # With multiple DistOpt instances, we need to all-reduce across instances. if ( - self.use_distributed_optimizer + self.ddp_config.use_distributed_optimizer and self.ddp_config.num_distributed_optimizer_instances > 1 ): assert self.inter_distributed_optimizer_instance_group is not None @@ -1013,21 +1007,16 @@ def __init__( self.gradient_scaling_factor = gradient_scaling_factor self.nccl_ub = nccl_ub - # Effective (per-buffer) DistributedOptimizer flag. ``use_distributed_optimizer`` is a - # model-level ddp_config knob, but a LayerWise (Muon) optimizer with the padded LayerWise - # param layout disabled (``use_layer_wise_param_layout=False``) turns it into a per-buffer - # property: LayerWise-managed (Muon 2D matrix) buffers locally disable DistOpt semantics - # (compact no-padding layout, all-reduce gradients, legacy whole-param ping-pong ownership - # + allgather_params), while sibling non-LayerWise buffers (embeddings, biases, layernorm) - # keep the standard byte-level DistributedOptimizer path. self._is_layer_wise_buffer = bool( self.params and getattr(self.params[0], "is_managed_by_layer_wise_optimizer", False) ) - self.use_distributed_optimizer = self.ddp_config.use_distributed_optimizer + # Bake the per-buffer DistOpt decision into this buffer's ddp_config (single source of + # truth; bucket groups inherit it): a LayerWise (Muon) buffer on the compact decoupled + # layout disables DistributedOptimizer, while sibling buffers keep the model-level setting. if self._is_layer_wise_buffer and not getattr( self.ddp_config, "use_layer_wise_param_layout", True ): - self.use_distributed_optimizer = False + self.ddp_config = dataclasses.replace(self.ddp_config, use_distributed_optimizer=False) # Data structures to store underlying buckets and relevant indexing data. self.buckets = [] @@ -1089,7 +1078,8 @@ def __init__( logging.INFO if (self._is_layer_wise_buffer or _padding > 0) else logging.DEBUG, f"ParamAndGradBuffer layout: param_dtype={self.param_dtype} " f"grad_dtype={self.grad_dtype} dp_world_size={self.data_parallel_world_size} " - f"layerwise={self._is_layer_wise_buffer} distopt={self.use_distributed_optimizer} " + f"layerwise={self._is_layer_wise_buffer} " + f"distopt={self.ddp_config.use_distributed_optimizer} " f"numel={self.numel} numel_unpadded={self.numel_unpadded} " f"padding={_padding} ({_pad_frac:.1%})", tp_group=self.tp_group, @@ -1098,7 +1088,7 @@ def __init__( if self.has_nvfp4_params: assert self.nvfp4_packed_numel_unpadded <= self.nvfp4_packed_numel - if self.use_distributed_optimizer: + if self.ddp_config.use_distributed_optimizer: assert self.numel % self.data_parallel_world_size == 0 if self.has_nvfp4_params: assert self.nvfp4_packed_numel % self.data_parallel_world_size == 0 @@ -1137,7 +1127,9 @@ def __init__( # For MXFP8 param: Create a shared buffer for param AG and grad RS for memory efficiency # The buffer is mapped to weight gradients whose dtype is either bf16 or FP32. # It can be temporarily reused by param AG. - if self.use_distributed_optimizer and any(is_mxfp8tensor(p) for p in self.params): + if self.ddp_config.use_distributed_optimizer and any( + is_mxfp8tensor(p) for p in self.params + ): self.shared_buffer = torch.zeros( self.numel, dtype=self.grad_dtype, @@ -1154,7 +1146,7 @@ def __init__( self.grad_data = self.shared_buffer else: # Only re-map param tensors if using distributed optimizer. - if self.use_distributed_optimizer: + if self.ddp_config.use_distributed_optimizer: numel = self.nvfp4_packed_numel if self.has_nvfp4_params else self.numel self.param_data = torch.zeros( numel, @@ -1350,12 +1342,12 @@ def _compute_nvfp4_packed_layout(self, params_with_names): """ def _pad_start_of_param(param_start_index: int) -> int: - if self.use_distributed_optimizer: + if self.ddp_config.use_distributed_optimizer: return pad_param_start(param_start_index) return param_start_index def _pad_end_of_bucket(bucket_end_index: int) -> int: - if self.use_distributed_optimizer: + if self.ddp_config.use_distributed_optimizer: return pad_bucket_end( bucket_end_index, self.data_parallel_world_size, @@ -1465,7 +1457,7 @@ def _new_bucket( # Assert that indices are correctly padded (if needed), and that bucket # position is same as originally computed. - if self.use_distributed_optimizer: + if self.ddp_config.use_distributed_optimizer: assert start_index % self.data_parallel_world_size == 0 assert end_index % self.data_parallel_world_size == 0 assert (start_index, end_index) == self.bucket_indices[bucket_id] @@ -1646,17 +1638,18 @@ def _merged_use_distributed_optimizer(merge_buckets): # instead, preserving order. When all buckets agree (the non-decoupled case) this collapses # to exactly one group, identical to the previous behavior. if force_single_bucket_group: - ddp_config = buffers[0].ddp_config data_parallel_group = buffers[0].data_parallel_group data_parallel_world_size = buffers[0].data_parallel_world_size ordered_distopt_values = [] buckets_by_distopt = {} + # buffer.ddp_config already carries the per-buffer use_distributed_optimizer. + ddp_config_by_distopt = {} for buffer in buffers: - assert ddp_config == buffer.ddp_config assert data_parallel_group == buffer.data_parallel_group assert data_parallel_world_size == buffer.data_parallel_world_size + distopt = buffer.ddp_config.use_distributed_optimizer + ddp_config_by_distopt.setdefault(distopt, buffer.ddp_config) for bucket in buffer.buckets: - distopt = _bucket_distopt(bucket) if distopt not in buckets_by_distopt: buckets_by_distopt[distopt] = [] ordered_distopt_values.append(distopt) @@ -1665,10 +1658,9 @@ def _merged_use_distributed_optimizer(merge_buckets): return [ _ParamAndGradBucketGroup( buckets_by_distopt[distopt], - ddp_config, + ddp_config_by_distopt[distopt], data_parallel_group, data_parallel_world_size, - use_distributed_optimizer=distopt, ) for distopt in ordered_distopt_values ] @@ -1685,17 +1677,17 @@ def _merged_use_distributed_optimizer(merge_buckets): buffer.ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, - use_distributed_optimizer=buffer.use_distributed_optimizer, ) ) return bucket_groups else: # Case 3: When using fp8 params, merge all non-fp8 buckets into the last fp8 bucket group. - non_fp8_buckets = [] + # Track each non-fp8 bucket with its buffer's (authoritative) ddp_config. + non_fp8_buckets = [] # list of (bucket, ddp_config) for buffer in buffers: if buffer.param_dtype != torch.uint8: for bucket in buffer.buckets: - non_fp8_buckets.append(bucket) + non_fp8_buckets.append((bucket, buffer.ddp_config)) bucket_groups = [] for bucket in fp8_buffer.buckets: @@ -1709,37 +1701,39 @@ def _merged_use_distributed_optimizer(merge_buckets): bucket_groups.append( _ParamAndGradBucketGroup( [bucket], - buffer.ddp_config, + fp8_buffer.ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, - use_distributed_optimizer=_bucket_distopt(bucket), ) ) if non_fp8_buckets: - for non_fp8_bucket in non_fp8_buckets: + for non_fp8_bucket, non_fp8_ddp_config in non_fp8_buckets: bucket_groups.append( _ParamAndGradBucketGroup( [non_fp8_bucket], - buffer.ddp_config, + non_fp8_ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, - use_distributed_optimizer=_bucket_distopt(non_fp8_bucket), ) ) continue # Skip the default bucket group creation below else: - group_buckets = [bucket] + non_fp8_buckets + group_buckets = [bucket] + [b for b, _ in non_fp8_buckets] else: # The first N-1 bucket groups. group_buckets = [bucket] + # Merged buckets must share the fp8 group's effective use_distributed_optimizer. + assert ( + _merged_use_distributed_optimizer(group_buckets) + == fp8_buffer.ddp_config.use_distributed_optimizer + ) bucket_groups.append( _ParamAndGradBucketGroup( group_buckets, - buffer.ddp_config, + fp8_buffer.ddp_config, buffer.data_parallel_group, buffer.data_parallel_world_size, - use_distributed_optimizer=_merged_use_distributed_optimizer(group_buckets), ) ) return bucket_groups diff --git a/megatron/core/optimizer/layer_wise_optimizer.py b/megatron/core/optimizer/layer_wise_optimizer.py index e9e85008aff..8d47d6a1793 100644 --- a/megatron/core/optimizer/layer_wise_optimizer.py +++ b/megatron/core/optimizer/layer_wise_optimizer.py @@ -324,7 +324,7 @@ def compute_full_param_layout( # compact no-padding DDP layout (and locally disable DistributedOptimizer semantics in # DDP), so they must NOT receive the shard-aligned ``dp_size * max(shard_load)`` padded # layout here. Non-LayerWise buffers keep DistOpt's byte-level layout regardless. - decouple_ddp_layout = not getattr(ddp_config, 'use_layer_wise_param_layout', True) + decouple_ddp_layout = not ddp_config.use_layer_wise_param_layout buffer_groups = group_params_for_buffers(params, ddp_config.grad_reduce_in_fp32) layouts = {} @@ -381,13 +381,7 @@ def __init__( """ self.pg_collection = pg_collection - - # Decoupled layout (use_layer_wise_param_layout=False): LayerWise (Muon) buffers use a - # compact no-padding DDP layout that does NOT encode whole-param shard ownership, and DDP - # treats them as non-DistOpt (all-reduce gradients). Ownership must therefore use the - # legacy whole-param ping-pong assignment and param sync must use ``allgather_params``, - # even though the model chunks carry a (compact) full_param_layout for DDP's benefit. - self.decouple_ddp_layout = not getattr(config, 'use_layer_wise_param_layout', True) + self.decouple_ddp_layout = not config.use_layer_wise_param_layout full_param_layouts = None if model_chunks is not None and not self.decouple_ddp_layout: @@ -398,12 +392,12 @@ def __init__( ] or None self.shard_params(optimizers, full_param_layouts) - # When a full_param_layout is available (and we are not decoupling), - # ddp_config.use_distributed_optimizer is True and model params are views into the - # DDP param buffer. After the optimizer step copies updated fp32 main params → bf16 - # model params, the buffer is already up-to-date in-place. We can use DDP's - # buffer-based all-gather (start_param_sync) instead of the flatten/unflatten - # allgather_params path. In the decouple path, Muon buffers are non-DistOpt and own + # When a full_param_layout is available (no decoupling), use_distributed_optimizer + # is True and model params are views into the DDP param buffer. After the + # optimizer step copies updated fp32 main params → bf16 model params, the + # buffer is already up-to-date in-place. We can use DDP's buffer-based + # all-gather (start_param_sync) instead of the flatten/unflatten allgather_params + # In the decouple path, Muon buffers are non-DistOpt and own # whole params via ping-pong, so we use the legacy allgather_params path instead. self.use_buffer_param_sync = full_param_layouts is not None diff --git a/megatron/core/optimizer/optimizer_config.py b/megatron/core/optimizer/optimizer_config.py index d30288d0745..f06920e0f85 100644 --- a/megatron/core/optimizer/optimizer_config.py +++ b/megatron/core/optimizer/optimizer_config.py @@ -328,9 +328,7 @@ class OptimizerConfig: the shard-aligned padded LayerWise param layout. When False, the compact decoupled layout is selected: LayerWise-managed (Muon) buffers use a compact no-padding DDP layout with all-reduce gradients and legacy whole-param ping-pong ownership + ``allgather_params`` - param sync, while non-LayerWise buffers keep the standard byte-level - :class:`DistributedOptimizer` layout. Removes the persistent ``dp_size * max(shard_load)`` - LayerWise padding. Mirrors ``DistributedDataParallelConfig.use_layer_wise_param_layout``.""" + param sync.""" overlap_param_gather: bool = False """If true, overlap param all-gather with forward compute. diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 2e7c7635f41..f082674935c 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -1901,7 +1901,8 @@ def validate_args(args, defaults={}): ) if not args.use_layer_wise_param_layout: assert args.num_distributed_optimizer_instances == 1, ( - "--no-use-layer-wise-param-layout (decoupled compact LayerWise DDP layout) requires " + "the decoupled compact LayerWise DDP layout (the default; pass " + "--use-layer-wise-param-layout for the padded layout) requires " "num_distributed_optimizer_instances == 1: the non-DistOpt LayerWise (Muon) buffers " "only all-reduce within a single optimizer instance, so partial DistOpt (>1 " "instance) would under-reduce Muon gradients across the full data-parallel domain." @@ -4218,20 +4219,16 @@ def _add_distributed_args(parser): Setting WORLD_SIZE and RANK to the specific values for target distribtued scale.', ) group.add_argument( - '--no-use-layer-wise-param-layout', - action='store_false', + '--use-layer-wise-param-layout', + action='store_true', dest='use_layer_wise_param_layout', - help='Opt out of the precomputed shard-aligned (padded) LayerWise param layout. ' - 'With an emerging (Muon) optimizer under --use-distributed-optimizer, this selects ' - 'the experimental compact, decoupled DDP layout: LayerWise-managed (Muon 2D matrix) ' - 'buffers use a compact no-padding DDP layout and locally disable DistributedOptimizer ' - 'semantics (all-reduce gradients + legacy whole-param ping-pong ownership + ' - 'allgather_params param sync), while sibling non-LayerWise buffers (embeddings, ' - 'biases, layernorm) keep the standard byte-level DistributedOptimizer. This removes ' - 'the persistent dp_size * max(shard_load) LayerWise padding from the long-lived ' - 'buffers. Incompatible with FP8 parameter gather (fp8_param_gather=True). Produces ' - 'different bf16 reduction ordering from the default padded layout, so will not match ' - 'its loss curves bit-for-bit.', + default=False, + help='Opt INTO the padded shard-aligned LayerWise param layout. The default is the compact ' + 'decoupled layout, where LayerWise (Muon 2D) buffers use a no-padding DDP layout and locally ' + 'disable DistributedOptimizer (all-reduce grads + whole-param ping-pong + allgather_params), ' + 'while sibling buffers keep the byte-level DistributedOptimizer; this avoids the persistent ' + 'dp_size * max(shard_load) padding. Pass this flag to restore the padded layout (e.g. for ' + 'bit-for-bit comparison; it uses a different bf16 reduction ordering).', ) return parser diff --git a/megatron/training/training.py b/megatron/training/training.py index f65a50f34cc..b258abd713a 100644 --- a/megatron/training/training.py +++ b/megatron/training/training.py @@ -1809,9 +1809,9 @@ def wrap_model_chunks_with_ddp( # LayerWise-managed matrices (Muon's Newton-Schulz domain) to a separate buffer. # The padded-vs-compact LayerWise layout decision is made inside # compute_full_param_layout / _ParamAndGradBuffer from use_layer_wise_param_layout: - # with the default padded layout LayerWise buffers stay DistOpt; with - # --no-use-layer-wise-param-layout they get the compact no-padding layout and the - # per-buffer override flips use_distributed_optimizer off for them. + # by default (compact) LayerWise buffers get the no-padding layout and the per-buffer + # override flips use_distributed_optimizer off for them; with + # --use-layer-wise-param-layout they stay on the padded DistOpt layout. ddp_config.use_distributed_optimizer = True compute_layout = LayerWiseDistributedOptimizer.compute_full_param_layout tag_params_for_buffer_routing(model_chunks) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/model_config.yaml index fffad86a016..028074bd34f 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/model_config.yaml @@ -66,5 +66,4 @@ MODEL_ARGS: --async-save: true --use-persistent-ckpt-worker: true --use-distributed-optimizer: true - --no-use-layer-wise-param-layout: true TEST_TYPE: ckpt-resume diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/model_config.yaml b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/model_config.yaml index 874e7dccf8d..40ac94eed9b 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/model_config.yaml +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/model_config.yaml @@ -66,5 +66,4 @@ MODEL_ARGS: --async-save: true --use-persistent-ckpt-worker: true --use-distributed-optimizer: true - --no-use-layer-wise-param-layout: true TEST_TYPE: ckpt-resume From 88ace457b091c65ab5a7fc6fc837979004b78829 Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Mon, 29 Jun 2026 11:57:55 +0800 Subject: [PATCH 3/5] Default use_layer_wise_param_layout to the compact decoupled layout Flip OptimizerConfig.use_layer_wise_param_layout from True to False so the compact decoupled LayerWise DDP layout is the default, and update the docstring so the (default) annotation tracks the False branch. Signed-off-by: Pingtian Li --- megatron/core/optimizer/optimizer_config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/megatron/core/optimizer/optimizer_config.py b/megatron/core/optimizer/optimizer_config.py index f06920e0f85..1048220953a 100644 --- a/megatron/core/optimizer/optimizer_config.py +++ b/megatron/core/optimizer/optimizer_config.py @@ -323,10 +323,10 @@ class OptimizerConfig: arguments layer sets this flag and resets ``use_distributed_optimizer`` to False so that the standard distributed-optimizer path is not triggered.""" - use_layer_wise_param_layout: bool = True - """Layer-wise (Muon) optimizer only. When True (default), LayerWise-managed buffers use - the shard-aligned padded LayerWise param layout. When False, the compact decoupled layout - is selected: LayerWise-managed (Muon) buffers use a compact no-padding DDP layout with + use_layer_wise_param_layout: bool = False + """Layer-wise (Muon) optimizer only. When True, LayerWise-managed buffers use + the shard-aligned padded LayerWise param layout. When False (default), the compact decoupled + layout is selected: LayerWise-managed (Muon) buffers use a compact no-padding DDP layout with all-reduce gradients and legacy whole-param ping-pong ownership + ``allgather_params`` param sync.""" From e93f1fafcc1e142f4e48634e69422950f94407b1 Mon Sep 17 00:00:00 2001 From: pingtianl Date: Mon, 29 Jun 2026 20:09:38 -0700 Subject: [PATCH 4/5] ci(moe): update GB200 goldens for dist_dist_muon ckpt-resume (padded layout) Regenerate golden_values_dev_dgx_gb200.json for gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon and its _1node variant. Removing --no-use-layer-wise-param-layout switches these tests from the compact decoupled LayerWise layout back to the default padded LayerWise layout, which changes the deterministic trajectory. Values regenerated in the GB200 CI container (mcore-pyt-dev-dgx_gb200, torch nv26.04) on GB200; an unchanged sibling test reproduced its committed golden bitwise, confirming CI fidelity. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: Pingtian Li --- .../golden_values_dev_dgx_gb200.json | 988 +++++++++--------- .../golden_values_dev_dgx_gb200.json | 976 ++++++++--------- 2 files changed, 982 insertions(+), 982 deletions(-) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json index 7af7158c2f7..10d1ac067a5 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_gb200.json @@ -6,104 +6,104 @@ "values": { "1": 10.94223, "2": 10.91037, - "3": 10.92695, - "4": 10.91755, - "5": 10.92109, - "6": 10.92714, - "7": 10.92204, - "8": 10.91988, - "9": 10.93259, - "10": 10.92694, - "11": 10.91813, - "12": 10.93449, - "13": 10.91939, - "14": 10.90997, - "15": 10.91823, - "16": 10.92119, - "17": 10.93417, - "18": 10.91659, - "19": 10.92828, - "20": 10.90269, - "21": 10.92486, - "22": 10.92321, - "23": 10.92998, - "24": 10.9038, - "25": 10.91386, - "26": 10.915, - "27": 10.93016, - "28": 10.91934, - "29": 10.92036, - "30": 10.91169, - "31": 10.90623, - "32": 10.91809, - "33": 10.91676, - "34": 10.90345, - "35": 10.9267, - "36": 10.90712, - "37": 10.90571, - "38": 10.91714, - "39": 10.90958, - "40": 10.90392, - "41": 10.91141, - "42": 10.91504, - "43": 10.90745, - "44": 10.91699, - "45": 10.91067, - "46": 10.90596, - "47": 10.90033, - "48": 10.89363, - "49": 10.90514, - "50": 10.89275, - "51": 10.89862, - "52": 10.89719, - "53": 10.90118, - "54": 10.89132, - "55": 10.9069, - "56": 10.88484, - "57": 10.88827, - "58": 10.87969, - "59": 10.8783, - "60": 10.86783, - "61": 10.86994, - "62": 10.86303, - "63": 10.87019, - "64": 10.86276, - "65": 10.86493, - "66": 10.86769, - "67": 10.85571, - "68": 10.84807, - "69": 10.85373, - "70": 10.85849, - "71": 10.85049, - "72": 10.84976, - "73": 10.83918, - "74": 10.83438, - "75": 10.84061, - "76": 10.82898, - "77": 10.83019, - "78": 10.81284, - "79": 10.82943, - "80": 10.83362, - "81": 10.81529, - "82": 10.81451, - "83": 10.80445, - "84": 10.78089, - "85": 10.7817, - "86": 10.79887, - "87": 10.79855, - "88": 10.80194, - "89": 10.78447, - "90": 10.78272, - "91": 10.78271, - "92": 10.78221, - "93": 10.75837, - "94": 10.76705, - "95": 10.76515, - "96": 10.73834, - "97": 10.73022, - "98": 10.75364, - "99": 10.76033, - "100": 10.7384 + "3": 10.92689, + "4": 10.91732, + "5": 10.92073, + "6": 10.92693, + "7": 10.92235, + "8": 10.91976, + "9": 10.93279, + "10": 10.92687, + "11": 10.91753, + "12": 10.93487, + "13": 10.91926, + "14": 10.90963, + "15": 10.91832, + "16": 10.92073, + "17": 10.93462, + "18": 10.91619, + "19": 10.92847, + "20": 10.90222, + "21": 10.92602, + "22": 10.92393, + "23": 10.92966, + "24": 10.90446, + "25": 10.91393, + "26": 10.91419, + "27": 10.92945, + "28": 10.91949, + "29": 10.9202, + "30": 10.91177, + "31": 10.90639, + "32": 10.91783, + "33": 10.91662, + "34": 10.90316, + "35": 10.92657, + "36": 10.90721, + "37": 10.90658, + "38": 10.91677, + "39": 10.90981, + "40": 10.90353, + "41": 10.91122, + "42": 10.91511, + "43": 10.90851, + "44": 10.91655, + "45": 10.91027, + "46": 10.90623, + "47": 10.9001, + "48": 10.89382, + "49": 10.9048, + "50": 10.89278, + "51": 10.89905, + "52": 10.89767, + "53": 10.90084, + "54": 10.8923, + "55": 10.90663, + "56": 10.8838, + "57": 10.88866, + "58": 10.87959, + "59": 10.87852, + "60": 10.86726, + "61": 10.86983, + "62": 10.86335, + "63": 10.87053, + "64": 10.86169, + "65": 10.86492, + "66": 10.8677, + "67": 10.85567, + "68": 10.84804, + "69": 10.85315, + "70": 10.85755, + "71": 10.85001, + "72": 10.84963, + "73": 10.83886, + "74": 10.83548, + "75": 10.84083, + "76": 10.82944, + "77": 10.83025, + "78": 10.81245, + "79": 10.82976, + "80": 10.83318, + "81": 10.81584, + "82": 10.81392, + "83": 10.80503, + "84": 10.78078, + "85": 10.78168, + "86": 10.79809, + "87": 10.79897, + "88": 10.80131, + "89": 10.78499, + "90": 10.78261, + "91": 10.78255, + "92": 10.78214, + "93": 10.75855, + "94": 10.7675, + "95": 10.76539, + "96": 10.73837, + "97": 10.73075, + "98": 10.75333, + "99": 10.75986, + "100": 10.73815 } }, "num-zeros": { @@ -113,104 +113,104 @@ "values": { "1": 1356.0, "2": 1146.0, - "3": 1276.0, - "4": 1214.0, - "5": 1190.0, + "3": 1336.0, + "4": 1201.0, + "5": 1176.0, "6": 1254.0, - "7": 1581.0, - "8": 1285.0, - "9": 1275.0, - "10": 1217.0, - "11": 1227.0, - "12": 1262.0, - "13": 1425.0, - "14": 1372.0, - "15": 1183.0, - "16": 1341.0, - "17": 1345.0, - "18": 1224.0, - "19": 1225.0, - "20": 1299.0, - "21": 1435.0, - "22": 1354.0, - "23": 1274.0, - "24": 1502.0, - "25": 1248.0, - "26": 1341.0, - "27": 1293.0, - "28": 1242.0, - "29": 1388.0, - "30": 1307.0, - "31": 1409.0, - "32": 1409.0, - "33": 1439.0, - "34": 1432.0, - "35": 1352.0, - "36": 1292.0, - "37": 1366.0, - "38": 1251.0, - "39": 1226.0, - "40": 1241.0, - "41": 1208.0, - "42": 1100.0, - "43": 1377.0, - "44": 1300.0, - "45": 1284.0, - "46": 1293.0, - "47": 1397.0, - "48": 1385.0, - "49": 1384.0, - "50": 1202.0, - "51": 1278.0, - "52": 1275.0, - "53": 1235.0, - "54": 1250.0, - "55": 1273.0, - "56": 1204.0, - "57": 1272.0, - "58": 1313.0, - "59": 1297.0, - "60": 1343.0, - "61": 1174.0, - "62": 1403.0, - "63": 1220.0, - "64": 1393.0, - "65": 1390.0, - "66": 1237.0, - "67": 1366.0, - "68": 1145.0, - "69": 1318.0, - "70": 1226.0, - "71": 1188.0, - "72": 1354.0, - "73": 1307.0, - "74": 1336.0, - "75": 1331.0, - "76": 1431.0, - "77": 1310.0, - "78": 1227.0, - "79": 1221.0, - "80": 1173.0, - "81": 1321.0, - "82": 1184.0, - "83": 1277.0, - "84": 1232.0, - "85": 1253.0, - "86": 1259.0, - "87": 1196.0, - "88": 1342.0, - "89": 1161.0, - "90": 1181.0, - "91": 1207.0, - "92": 1138.0, - "93": 1288.0, - "94": 1366.0, - "95": 1196.0, - "96": 1334.0, - "97": 1252.0, - "98": 1196.0, - "99": 1296.0, - "100": 1356.0 + "7": 1565.0, + "8": 1319.0, + "9": 1252.0, + "10": 1209.0, + "11": 1224.0, + "12": 1339.0, + "13": 1413.0, + "14": 1413.0, + "15": 1156.0, + "16": 1302.0, + "17": 1368.0, + "18": 1235.0, + "19": 1169.0, + "20": 1312.0, + "21": 1432.0, + "22": 1355.0, + "23": 1247.0, + "24": 1520.0, + "25": 1251.0, + "26": 1377.0, + "27": 1370.0, + "28": 1213.0, + "29": 1314.0, + "30": 1352.0, + "31": 1420.0, + "32": 1488.0, + "33": 1447.0, + "34": 1463.0, + "35": 1305.0, + "36": 1250.0, + "37": 1426.0, + "38": 1328.0, + "39": 1258.0, + "40": 1371.0, + "41": 1251.0, + "42": 1068.0, + "43": 1387.0, + "44": 1344.0, + "45": 1357.0, + "46": 1257.0, + "47": 1422.0, + "48": 1412.0, + "49": 1420.0, + "50": 1264.0, + "51": 1272.0, + "52": 1272.0, + "53": 1285.0, + "54": 1212.0, + "55": 1232.0, + "56": 1234.0, + "57": 1258.0, + "58": 1309.0, + "59": 1277.0, + "60": 1362.0, + "61": 1187.0, + "62": 1351.0, + "63": 1268.0, + "64": 1299.0, + "65": 1452.0, + "66": 1272.0, + "67": 1394.0, + "68": 1194.0, + "69": 1387.0, + "70": 1266.0, + "71": 1180.0, + "72": 1315.0, + "73": 1324.0, + "74": 1293.0, + "75": 1315.0, + "76": 1425.0, + "77": 1336.0, + "78": 1156.0, + "79": 1284.0, + "80": 1195.0, + "81": 1320.0, + "82": 1098.0, + "83": 1281.0, + "84": 1267.0, + "85": 1217.0, + "86": 1231.0, + "87": 1194.0, + "88": 1376.0, + "89": 1163.0, + "90": 1173.0, + "91": 1215.0, + "92": 1101.0, + "93": 1305.0, + "94": 1364.0, + "95": 1112.0, + "96": 1279.0, + "97": 1283.0, + "98": 1287.0, + "99": 1223.0, + "100": 1403.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 993736704.0, - "2": 993747968.0, - "3": 993740800.0, - "4": 993725952.0, - "5": 993734144.0, - "6": 993718272.0, - "7": 993741312.0, - "8": 993709056.0, - "9": 993735168.0, - "10": 993729536.0, - "11": 993708544.0, - "12": 993701888.0, - "13": 993785856.0, - "14": 993799168.0, - "15": 993735168.0, - "16": 993737728.0, - "17": 993748480.0, - "18": 993705984.0, - "19": 993765888.0, - "20": 993732096.0, - "21": 993782784.0, - "22": 993733120.0, - "23": 993708544.0, - "24": 993744896.0, - "25": 993698304.0, - "26": 993810432.0, - "27": 993738752.0, - "28": 993734144.0, - "29": 993771008.0, - "30": 993692672.0, - "31": 993728000.0, - "32": 993732608.0, - "33": 993720320.0, - "34": 993783296.0, - "35": 993718272.0, - "36": 993655808.0, - "37": 993730560.0, - "38": 993759744.0, - "39": 993702912.0, - "40": 993736704.0, - "41": 993739264.0, - "42": 993735168.0, - "43": 993736192.0, - "44": 993706496.0, - "45": 993702912.0, - "46": 993764352.0, - "47": 993757696.0, - "48": 993702912.0, - "49": 993731072.0, - "50": 993682944.0, - "51": 993666560.0, - "52": 993688064.0, - "53": 993730048.0, - "54": 993729536.0, - "55": 993711616.0, - "56": 993721344.0, - "57": 993754112.0, - "58": 993792512.0, - "59": 993699840.0, - "60": 993713152.0, - "61": 993680896.0, - "62": 993712128.0, - "63": 993723904.0, - "64": 993727488.0, - "65": 993713152.0, - "66": 993652736.0, - "67": 993728000.0, - "68": 993741824.0, - "69": 993719808.0, - "70": 993671168.0, - "71": 993694720.0, - "72": 993763840.0, - "73": 993697792.0, - "74": 993695232.0, - "75": 993733632.0, - "76": 993709568.0, - "77": 993723904.0, - "78": 993714688.0, - "79": 993704448.0, - "80": 993724416.0, - "81": 993707520.0, - "82": 993683456.0, - "83": 993739264.0, - "84": 993730560.0, - "85": 993741824.0, - "86": 993717248.0, - "87": 993612800.0, - "88": 993669120.0, - "89": 993677824.0, - "90": 993700352.0, - "91": 993731072.0, - "92": 993679872.0, - "93": 993647616.0, - "94": 993690112.0, - "95": 993677824.0, - "96": 993651712.0, - "97": 993678336.0, - "98": 993742848.0, - "99": 993701888.0, - "100": 993733632.0 + "1": 725680128.0, + "2": 725691392.0, + "3": 725684224.0, + "4": 725668864.0, + "5": 725675520.0, + "6": 725662720.0, + "7": 725684736.0, + "8": 725651456.0, + "9": 725678080.0, + "10": 725672448.0, + "11": 725651456.0, + "12": 725644800.0, + "13": 725731328.0, + "14": 725742080.0, + "15": 725679616.0, + "16": 725680128.0, + "17": 725691904.0, + "18": 725650432.0, + "19": 725709824.0, + "20": 725673984.0, + "21": 725726208.0, + "22": 725676544.0, + "23": 725652480.0, + "24": 725687296.0, + "25": 725640704.0, + "26": 725754368.0, + "27": 725682688.0, + "28": 725678592.0, + "29": 725713408.0, + "30": 725637632.0, + "31": 725675008.0, + "32": 725677056.0, + "33": 725663744.0, + "34": 725725696.0, + "35": 725662720.0, + "36": 725598208.0, + "37": 725675520.0, + "38": 725702656.0, + "39": 725645824.0, + "40": 725681152.0, + "41": 725683712.0, + "42": 725679104.0, + "43": 725680640.0, + "44": 725647872.0, + "45": 725645312.0, + "46": 725708288.0, + "47": 725701632.0, + "48": 725649408.0, + "49": 725671936.0, + "50": 725624832.0, + "51": 725610496.0, + "52": 725630976.0, + "53": 725675008.0, + "54": 725673984.0, + "55": 725656576.0, + "56": 725663232.0, + "57": 725699072.0, + "58": 725733888.0, + "59": 725643264.0, + "60": 725654016.0, + "61": 725626368.0, + "62": 725657600.0, + "63": 725666816.0, + "64": 725671936.0, + "65": 725657088.0, + "66": 725595648.0, + "67": 725673984.0, + "68": 725685248.0, + "69": 725663744.0, + "70": 725615104.0, + "71": 725638656.0, + "72": 725706752.0, + "73": 725640704.0, + "74": 725637120.0, + "75": 725676032.0, + "76": 725652992.0, + "77": 725665792.0, + "78": 725658112.0, + "79": 725648384.0, + "80": 725668864.0, + "81": 725651968.0, + "82": 725625344.0, + "83": 725683712.0, + "84": 725675008.0, + "85": 725683200.0, + "86": 725659648.0, + "87": 725555712.0, + "88": 725612544.0, + "89": 725619712.0, + "90": 725642240.0, + "91": 725673984.0, + "92": 725622272.0, + "93": 725592064.0, + "94": 725631488.0, + "95": 725619200.0, + "96": 725594624.0, + "97": 725621760.0, + "98": 725686272.0, + "99": 725646848.0, + "100": 725678080.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3034706432.0, - "2": 3331357184.0, - "3": 3331357184.0, - "4": 3331357184.0, - "5": 3331357184.0, - "6": 3331357184.0, - "7": 3331357184.0, - "8": 3331357184.0, - "9": 3331357184.0, - "10": 3331357184.0, - "11": 3331357184.0, - "12": 3331357184.0, - "13": 3357274624.0, - "14": 3362478592.0, - "15": 3362478592.0, - "16": 3362478592.0, - "17": 3362478592.0, - "18": 3362478592.0, - "19": 3362478592.0, - "20": 3362478592.0, - "21": 3362478592.0, - "22": 3362478592.0, - "23": 3362478592.0, - "24": 3362478592.0, - "25": 3362478592.0, - "26": 3375542784.0, - "27": 3375542784.0, - "28": 3375542784.0, - "29": 3375542784.0, - "30": 3375542784.0, - "31": 3375542784.0, - "32": 3375542784.0, - "33": 3375542784.0, - "34": 3375542784.0, - "35": 3375542784.0, - "36": 3375542784.0, - "37": 3375542784.0, - "38": 3375542784.0, - "39": 3375542784.0, - "40": 3375542784.0, - "41": 3375542784.0, - "42": 3375542784.0, - "43": 3375542784.0, - "44": 3375542784.0, - "45": 3375542784.0, - "46": 3375542784.0, - "47": 3375542784.0, - "48": 3375542784.0, - "49": 3375542784.0, - "50": 3375542784.0, - "51": 3375542784.0, - "52": 3375542784.0, - "53": 3375542784.0, - "54": 3375542784.0, - "55": 3375542784.0, - "56": 3375542784.0, - "57": 3375542784.0, - "58": 3375542784.0, - "59": 3375542784.0, - "60": 3375542784.0, - "61": 3375542784.0, - "62": 3375542784.0, - "63": 3375542784.0, - "64": 3375542784.0, - "65": 3375542784.0, - "66": 3375542784.0, - "67": 3375542784.0, - "68": 3375542784.0, - "69": 3375542784.0, - "70": 3375542784.0, - "71": 3375542784.0, - "72": 3375542784.0, - "73": 3375542784.0, - "74": 3375542784.0, - "75": 3375542784.0, - "76": 3375542784.0, - "77": 3375542784.0, - "78": 3375542784.0, - "79": 3375542784.0, - "80": 3375542784.0, - "81": 3375542784.0, - "82": 3375542784.0, - "83": 3375542784.0, - "84": 3375542784.0, - "85": 3375542784.0, - "86": 3375542784.0, - "87": 3375542784.0, - "88": 3375542784.0, - "89": 3375542784.0, - "90": 3375542784.0, - "91": 3375542784.0, - "92": 3375542784.0, - "93": 3375542784.0, - "94": 3375542784.0, - "95": 3375542784.0, - "96": 3375542784.0, - "97": 3375542784.0, - "98": 3375542784.0, - "99": 3375542784.0, - "100": 3375542784.0 + "1": 2941569536.0, + "2": 3065461248.0, + "3": 3065461248.0, + "4": 3065461248.0, + "5": 3065461248.0, + "6": 3065461248.0, + "7": 3065461248.0, + "8": 3065461248.0, + "9": 3065461248.0, + "10": 3065461248.0, + "11": 3065461248.0, + "12": 3065461248.0, + "13": 3086917120.0, + "14": 3094382592.0, + "15": 3094382592.0, + "16": 3094382592.0, + "17": 3094382592.0, + "18": 3094382592.0, + "19": 3094382592.0, + "20": 3094382592.0, + "21": 3094382592.0, + "22": 3094382592.0, + "23": 3094382592.0, + "24": 3094382592.0, + "25": 3094382592.0, + "26": 3104519168.0, + "27": 3104519168.0, + "28": 3104519168.0, + "29": 3104519168.0, + "30": 3104519168.0, + "31": 3104519168.0, + "32": 3104519168.0, + "33": 3104519168.0, + "34": 3104519168.0, + "35": 3104519168.0, + "36": 3104519168.0, + "37": 3104519168.0, + "38": 3104519168.0, + "39": 3104519168.0, + "40": 3104519168.0, + "41": 3104519168.0, + "42": 3104519168.0, + "43": 3104519168.0, + "44": 3104519168.0, + "45": 3104519168.0, + "46": 3104519168.0, + "47": 3104519168.0, + "48": 3104519168.0, + "49": 3104519168.0, + "50": 3104519168.0, + "51": 3104519168.0, + "52": 3104519168.0, + "53": 3104519168.0, + "54": 3104519168.0, + "55": 3104519168.0, + "56": 3104519168.0, + "57": 3104519168.0, + "58": 3104519168.0, + "59": 3104519168.0, + "60": 3104519168.0, + "61": 3104519168.0, + "62": 3104519168.0, + "63": 3104519168.0, + "64": 3104519168.0, + "65": 3104519168.0, + "66": 3104519168.0, + "67": 3104519168.0, + "68": 3104519168.0, + "69": 3104519168.0, + "70": 3104519168.0, + "71": 3104519168.0, + "72": 3104519168.0, + "73": 3104519168.0, + "74": 3104519168.0, + "75": 3104519168.0, + "76": 3104519168.0, + "77": 3104519168.0, + "78": 3104519168.0, + "79": 3104519168.0, + "80": 3104519168.0, + "81": 3104519168.0, + "82": 3104519168.0, + "83": 3104519168.0, + "84": 3104519168.0, + "85": 3104519168.0, + "86": 3104519168.0, + "87": 3104519168.0, + "88": 3104519168.0, + "89": 3104519168.0, + "90": 3104519168.0, + "91": 3104519168.0, + "92": 3104519168.0, + "93": 3104519168.0, + "94": 3104519168.0, + "95": 3104519168.0, + "96": 3104519168.0, + "97": 3104519168.0, + "98": 3104519168.0, + "99": 3104519168.0, + "100": 3104519168.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 8.49271, - "3": 0.25688, - "4": 0.22124, - "5": 0.21824, - "6": 0.21083, - "7": 0.20712, - "8": 0.19956, - "9": 0.19815, - "10": 0.1926, - "11": 0.19351, - "12": 0.19256, - "13": 0.19281, - "14": 0.19127, - "15": 0.19154, - "16": 0.18548, - "17": 0.19195, - "18": 0.1983, - "19": 0.25528, - "20": 0.22052, - "21": 0.28285, - "22": 0.2177, - "23": 0.21469, - "24": 0.21162, - "25": 0.23116, - "26": 0.18774, - "27": 0.18611, - "28": 0.19088, - "29": 0.18301, - "30": 0.18331, - "31": 0.18247, - "32": 0.18431, - "33": 0.19054, - "34": 0.18773, - "35": 0.18089, - "36": 0.18111, - "37": 0.18092, - "38": 0.18475, - "39": 0.18645, - "40": 0.18212, - "41": 0.18174, - "42": 0.18552, - "43": 0.18528, - "44": 0.17967, - "45": 0.18292, - "46": 0.18062, - "47": 0.1801, - "48": 0.18752, - "49": 0.18553, - "50": 0.18368, - "51": 0.45252, - "52": 0.22591, - "53": 0.1812, - "54": 0.18363, - "55": 0.18724, - "56": 0.18336, - "57": 0.18169, - "58": 0.18373, - "59": 0.18345, - "60": 0.18069, - "61": 0.18631, - "62": 0.18358, - "63": 0.18134, - "64": 0.18375, - "65": 0.18154, - "66": 0.18594, - "67": 0.18109, - "68": 0.18792, - "69": 0.1783, - "70": 0.18347, - "71": 0.18189, - "72": 0.18531, - "73": 0.18202, - "74": 0.18041, - "75": 0.1821, - "76": 0.17952, - "77": 0.18016, - "78": 0.18287, - "79": 0.18241, - "80": 0.17827, - "81": 0.18204, - "82": 0.1888, - "83": 0.18477, - "84": 0.18326, - "85": 0.1834, - "86": 0.18213, - "87": 0.18585, - "88": 0.18155, - "89": 0.17993, - "90": 0.17981, - "91": 0.18275, - "92": 0.18312, - "93": 0.18, - "94": 0.18267, - "95": 0.18972, - "96": 0.18008, - "97": 0.18222, - "98": 0.1815, - "99": 0.17947, - "100": 0.18358 + "2": 6.63782, + "3": 0.2637, + "4": 0.21996, + "5": 0.21553, + "6": 0.20374, + "7": 0.20661, + "8": 0.19818, + "9": 0.19749, + "10": 0.19173, + "11": 0.1987, + "12": 0.19018, + "13": 0.19597, + "14": 0.1884, + "15": 0.18307, + "16": 0.18702, + "17": 0.19775, + "18": 0.189, + "19": 0.18368, + "20": 0.19309, + "21": 0.18792, + "22": 0.1846, + "23": 0.18766, + "24": 0.1799, + "25": 0.18234, + "26": 0.18186, + "27": 0.18022, + "28": 0.17842, + "29": 0.17978, + "30": 0.18366, + "31": 0.18753, + "32": 0.17992, + "33": 0.1828, + "34": 0.18372, + "35": 0.18094, + "36": 0.18109, + "37": 0.1815, + "38": 0.18114, + "39": 0.18388, + "40": 0.18476, + "41": 0.18114, + "42": 0.17837, + "43": 0.18035, + "44": 0.18161, + "45": 0.17889, + "46": 0.18536, + "47": 0.18128, + "48": 0.17933, + "49": 0.1776, + "50": 0.18398, + "51": 0.33933, + "52": 0.23357, + "53": 0.1844, + "54": 0.18338, + "55": 0.18004, + "56": 0.17914, + "57": 0.18225, + "58": 0.18225, + "59": 0.18599, + "60": 0.18651, + "61": 0.181, + "62": 0.17833, + "63": 0.18309, + "64": 0.17884, + "65": 0.18492, + "66": 0.18734, + "67": 0.17967, + "68": 0.18534, + "69": 0.18128, + "70": 0.18586, + "71": 0.17897, + "72": 0.17948, + "73": 0.18341, + "74": 0.17965, + "75": 0.18156, + "76": 0.17884, + "77": 0.17944, + "78": 0.18049, + "79": 0.18043, + "80": 0.18024, + "81": 0.18795, + "82": 0.1778, + "83": 0.1784, + "84": 0.17957, + "85": 0.18215, + "86": 0.17901, + "87": 0.18436, + "88": 0.17959, + "89": 0.17723, + "90": 0.18082, + "91": 0.1816, + "92": 0.17714, + "93": 0.17902, + "94": 0.17945, + "95": 0.18029, + "96": 0.18047, + "97": 0.1838, + "98": 0.18041, + "99": 0.17903, + "100": 0.1838 } } } \ No newline at end of file diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json index c647bd3eeed..096ebe8293a 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon_1node/golden_values_dev_dgx_gb200.json @@ -9,101 +9,101 @@ "3": 10.92451, "4": 10.91716, "5": 10.92714, - "6": 10.92064, - "7": 10.91647, - "8": 10.91788, - "9": 10.93809, - "10": 10.924, - "11": 10.91584, - "12": 10.92558, - "13": 10.9341, - "14": 10.91552, - "15": 10.92865, - "16": 10.91861, - "17": 10.92663, - "18": 10.91342, - "19": 10.93055, - "20": 10.89988, - "21": 10.94059, - "22": 10.92549, - "23": 10.92966, - "24": 10.90596, - "25": 10.91497, - "26": 10.92625, - "27": 10.92807, - "28": 10.92038, - "29": 10.92592, - "30": 10.91434, - "31": 10.91031, - "32": 10.919, - "33": 10.92014, - "34": 10.89998, - "35": 10.91944, - "36": 10.90693, - "37": 10.91287, - "38": 10.92352, - "39": 10.91196, - "40": 10.91094, - "41": 10.91323, - "42": 10.90919, - "43": 10.90589, - "44": 10.90761, - "45": 10.89741, - "46": 10.90636, - "47": 10.90203, - "48": 10.88495, - "49": 10.90015, - "50": 10.89905, - "51": 10.89814, - "52": 10.90393, - "53": 10.90158, - "54": 10.88808, - "55": 10.88643, - "56": 10.88868, - "57": 10.88748, - "58": 10.88859, - "59": 10.88263, - "60": 10.87391, - "61": 10.87367, - "62": 10.86781, - "63": 10.87607, - "64": 10.85882, - "65": 10.86026, + "6": 10.92082, + "7": 10.91663, + "8": 10.91792, + "9": 10.93806, + "10": 10.92409, + "11": 10.91561, + "12": 10.92548, + "13": 10.93464, + "14": 10.91533, + "15": 10.92895, + "16": 10.91827, + "17": 10.92717, + "18": 10.91285, + "19": 10.93046, + "20": 10.90022, + "21": 10.94036, + "22": 10.92611, + "23": 10.93029, + "24": 10.90595, + "25": 10.91505, + "26": 10.92637, + "27": 10.92801, + "28": 10.91982, + "29": 10.92581, + "30": 10.9142, + "31": 10.91018, + "32": 10.91867, + "33": 10.92008, + "34": 10.90026, + "35": 10.9194, + "36": 10.90793, + "37": 10.91236, + "38": 10.92378, + "39": 10.91222, + "40": 10.91008, + "41": 10.9127, + "42": 10.90877, + "43": 10.90581, + "44": 10.90741, + "45": 10.89766, + "46": 10.90697, + "47": 10.90174, + "48": 10.88466, + "49": 10.89961, + "50": 10.89919, + "51": 10.89771, + "52": 10.90377, + "53": 10.90167, + "54": 10.88817, + "55": 10.88647, + "56": 10.88848, + "57": 10.88745, + "58": 10.88844, + "59": 10.88238, + "60": 10.87385, + "61": 10.87332, + "62": 10.86837, + "63": 10.87616, + "64": 10.85944, + "65": 10.86068, "66": 10.85712, - "67": 10.86113, - "68": 10.86032, - "69": 10.84979, - "70": 10.86738, - "71": 10.8492, - "72": 10.84019, - "73": 10.84868, - "74": 10.83869, - "75": 10.83621, - "76": 10.83434, - "77": 10.82938, - "78": 10.82319, - "79": 10.82846, - "80": 10.82987, - "81": 10.81761, - "82": 10.81704, - "83": 10.80437, - "84": 10.78281, - "85": 10.78319, - "86": 10.80029, - "87": 10.7956, - "88": 10.79539, - "89": 10.77706, - "90": 10.78274, - "91": 10.79015, - "92": 10.78298, - "93": 10.75507, - "94": 10.7607, - "95": 10.77273, - "96": 10.7355, - "97": 10.7421, - "98": 10.74705, - "99": 10.76264, - "100": 10.74262 + "67": 10.86049, + "68": 10.85997, + "69": 10.84934, + "70": 10.86718, + "71": 10.84951, + "72": 10.84053, + "73": 10.84865, + "74": 10.839, + "75": 10.8362, + "76": 10.83533, + "77": 10.82903, + "78": 10.82316, + "79": 10.82914, + "80": 10.82949, + "81": 10.81781, + "82": 10.81639, + "83": 10.80445, + "84": 10.78278, + "85": 10.78317, + "86": 10.80042, + "87": 10.79572, + "88": 10.79471, + "89": 10.77715, + "90": 10.78299, + "91": 10.78951, + "92": 10.78289, + "93": 10.75484, + "94": 10.76066, + "95": 10.77299, + "96": 10.73627, + "97": 10.74163, + "98": 10.74724, + "99": 10.76314, + "100": 10.74261 } }, "num-zeros": { @@ -116,101 +116,101 @@ "3": 49283.0, "4": 48797.0, "5": 50222.0, - "6": 50071.0, - "7": 51516.0, - "8": 49687.0, - "9": 50960.0, - "10": 52404.0, - "11": 50454.0, - "12": 48350.0, - "13": 52171.0, - "14": 49951.0, - "15": 48080.0, - "16": 49110.0, - "17": 50288.0, - "18": 52118.0, - "19": 51099.0, - "20": 52646.0, - "21": 49816.0, - "22": 49955.0, - "23": 53512.0, - "24": 51380.0, - "25": 48547.0, - "26": 52530.0, - "27": 51993.0, - "28": 48540.0, - "29": 50032.0, - "30": 51384.0, - "31": 53098.0, - "32": 49324.0, - "33": 51925.0, - "34": 50079.0, - "35": 51371.0, - "36": 51207.0, - "37": 49665.0, - "38": 50936.0, - "39": 50632.0, - "40": 50968.0, - "41": 48043.0, - "42": 50241.0, - "43": 50548.0, - "44": 47518.0, - "45": 54359.0, - "46": 47844.0, - "47": 50697.0, - "48": 50607.0, - "49": 54927.0, - "50": 51129.0, - "51": 48773.0, - "52": 50817.0, - "53": 49252.0, - "54": 48794.0, - "55": 49077.0, - "56": 48641.0, - "57": 48141.0, - "58": 52709.0, - "59": 50075.0, - "60": 49760.0, - "61": 47761.0, - "62": 50557.0, - "63": 48459.0, - "64": 56353.0, - "65": 51179.0, - "66": 48545.0, - "67": 51570.0, - "68": 49653.0, - "69": 54155.0, - "70": 49808.0, - "71": 49534.0, - "72": 52109.0, - "73": 52558.0, - "74": 49904.0, - "75": 50500.0, - "76": 52020.0, - "77": 49771.0, - "78": 49836.0, - "79": 48741.0, - "80": 52069.0, - "81": 50079.0, - "82": 46782.0, - "83": 52982.0, - "84": 48744.0, - "85": 50660.0, - "86": 50050.0, - "87": 46875.0, - "88": 48017.0, - "89": 48621.0, - "90": 51995.0, - "91": 50317.0, - "92": 50374.0, - "93": 50702.0, - "94": 51589.0, - "95": 48646.0, - "96": 50031.0, - "97": 49732.0, - "98": 48573.0, - "99": 48599.0, - "100": 47055.0 + "6": 49880.0, + "7": 51305.0, + "8": 49585.0, + "9": 51395.0, + "10": 52160.0, + "11": 50152.0, + "12": 48416.0, + "13": 51965.0, + "14": 49872.0, + "15": 47840.0, + "16": 49596.0, + "17": 49562.0, + "18": 51773.0, + "19": 51110.0, + "20": 52591.0, + "21": 49859.0, + "22": 49840.0, + "23": 52787.0, + "24": 51093.0, + "25": 48890.0, + "26": 52427.0, + "27": 51657.0, + "28": 48305.0, + "29": 50151.0, + "30": 50908.0, + "31": 53105.0, + "32": 49357.0, + "33": 52338.0, + "34": 50300.0, + "35": 51106.0, + "36": 51162.0, + "37": 49478.0, + "38": 51342.0, + "39": 50394.0, + "40": 50590.0, + "41": 48412.0, + "42": 49788.0, + "43": 51138.0, + "44": 47535.0, + "45": 54510.0, + "46": 47948.0, + "47": 50330.0, + "48": 50385.0, + "49": 54543.0, + "50": 50668.0, + "51": 49291.0, + "52": 51142.0, + "53": 49747.0, + "54": 49028.0, + "55": 49564.0, + "56": 48657.0, + "57": 48452.0, + "58": 52702.0, + "59": 50738.0, + "60": 49582.0, + "61": 47206.0, + "62": 50707.0, + "63": 48522.0, + "64": 56483.0, + "65": 50859.0, + "66": 49141.0, + "67": 51934.0, + "68": 50075.0, + "69": 54455.0, + "70": 50164.0, + "71": 49636.0, + "72": 52313.0, + "73": 52698.0, + "74": 49574.0, + "75": 50329.0, + "76": 51772.0, + "77": 49786.0, + "78": 50069.0, + "79": 49183.0, + "80": 51814.0, + "81": 49927.0, + "82": 46857.0, + "83": 52872.0, + "84": 48633.0, + "85": 50653.0, + "86": 49778.0, + "87": 46803.0, + "88": 48116.0, + "89": 49166.0, + "90": 52060.0, + "91": 50704.0, + "92": 50076.0, + "93": 50655.0, + "94": 51681.0, + "95": 48597.0, + "96": 50305.0, + "97": 50392.0, + "98": 48304.0, + "99": 49152.0, + "100": 46788.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 870735872.0, - "2": 870754816.0, - "3": 870738944.0, - "4": 870736896.0, - "5": 870747136.0, - "6": 870734848.0, - "7": 870729728.0, - "8": 870758400.0, - "9": 870706688.0, - "10": 870708224.0, - "11": 870691840.0, - "12": 870722560.0, - "13": 870722048.0, - "14": 870712320.0, - "15": 870742016.0, - "16": 870731264.0, - "17": 870742528.0, - "18": 870718976.0, - "19": 870715904.0, - "20": 870726656.0, - "21": 870749696.0, - "22": 870685696.0, - "23": 870727168.0, - "24": 870740992.0, - "25": 870705664.0, - "26": 870743040.0, - "27": 870745600.0, - "28": 870710272.0, - "29": 870727680.0, - "30": 870737408.0, - "31": 870700544.0, - "32": 870722048.0, - "33": 870707712.0, - "34": 870750208.0, - "35": 870678016.0, - "36": 870753792.0, - "37": 870719488.0, - "38": 870744064.0, - "39": 870709760.0, - "40": 870759424.0, - "41": 870726656.0, - "42": 870705664.0, - "43": 870706176.0, - "44": 870716416.0, - "45": 870730752.0, - "46": 870681600.0, - "47": 870704640.0, - "48": 870693888.0, - "49": 870726144.0, - "50": 870706688.0, - "51": 870676480.0, - "52": 870724096.0, - "53": 870680576.0, - "54": 870741504.0, - "55": 870686208.0, - "56": 870737920.0, - "57": 870706176.0, - "58": 870713856.0, - "59": 870715904.0, - "60": 870680576.0, - "61": 870662656.0, - "62": 870695936.0, - "63": 870699008.0, - "64": 870713344.0, - "65": 870717952.0, - "66": 870641152.0, - "67": 870729216.0, - "68": 870673920.0, - "69": 870702080.0, - "70": 870668288.0, - "71": 870687744.0, - "72": 870685184.0, - "73": 870686208.0, - "74": 870650368.0, - "75": 870676992.0, - "76": 870678016.0, - "77": 870716928.0, - "78": 870710784.0, - "79": 870678528.0, - "80": 870626816.0, - "81": 870669312.0, - "82": 870644736.0, - "83": 870636032.0, - "84": 870677504.0, - "85": 870650880.0, - "86": 870644736.0, - "87": 870636032.0, - "88": 870648832.0, - "89": 870634496.0, - "90": 870674432.0, - "91": 870637056.0, - "92": 870639104.0, - "93": 870653952.0, - "94": 870635520.0, - "95": 870631424.0, - "96": 870637056.0, - "97": 870630912.0, - "98": 870680064.0, - "99": 870621184.0, - "100": 870622720.0 + "1": 950326784.0, + "2": 950345728.0, + "3": 950329856.0, + "4": 950327808.0, + "5": 950338048.0, + "6": 950325760.0, + "7": 950320640.0, + "8": 950348800.0, + "9": 950297600.0, + "10": 950299136.0, + "11": 950282240.0, + "12": 950311424.0, + "13": 950314496.0, + "14": 950304256.0, + "15": 950332928.0, + "16": 950321152.0, + "17": 950333952.0, + "18": 950308864.0, + "19": 950306816.0, + "20": 950318592.0, + "21": 950339584.0, + "22": 950276608.0, + "23": 950317568.0, + "24": 950333952.0, + "25": 950294528.0, + "26": 950332928.0, + "27": 950337536.0, + "28": 950300672.0, + "29": 950319104.0, + "30": 950330368.0, + "31": 950289920.0, + "32": 950311424.0, + "33": 950298112.0, + "34": 950341120.0, + "35": 950269952.0, + "36": 950342656.0, + "37": 950311936.0, + "38": 950336512.0, + "39": 950299136.0, + "40": 950349824.0, + "41": 950313472.0, + "42": 950295040.0, + "43": 950297088.0, + "44": 950306304.0, + "45": 950322688.0, + "46": 950270976.0, + "47": 950295040.0, + "48": 950284288.0, + "49": 950316032.0, + "50": 950298112.0, + "51": 950267392.0, + "52": 950315008.0, + "53": 950272512.0, + "54": 950329856.0, + "55": 950277120.0, + "56": 950328832.0, + "57": 950299136.0, + "58": 950304256.0, + "59": 950306816.0, + "60": 950271488.0, + "61": 950252544.0, + "62": 950286336.0, + "63": 950288896.0, + "64": 950307328.0, + "65": 950311424.0, + "66": 950231040.0, + "67": 950319104.0, + "68": 950263808.0, + "69": 950292992.0, + "70": 950259200.0, + "71": 950279680.0, + "72": 950278144.0, + "73": 950275072.0, + "74": 950241280.0, + "75": 950266368.0, + "76": 950269952.0, + "77": 950306304.0, + "78": 950301696.0, + "79": 950270464.0, + "80": 950216192.0, + "81": 950262272.0, + "82": 950234624.0, + "83": 950227456.0, + "84": 950269440.0, + "85": 950242816.0, + "86": 950237184.0, + "87": 950224896.0, + "88": 950240768.0, + "89": 950225408.0, + "90": 950263808.0, + "91": 950225920.0, + "92": 950230016.0, + "93": 950243328.0, + "94": 950227456.0, + "95": 950223360.0, + "96": 950230016.0, + "97": 950221824.0, + "98": 950270976.0, + "99": 950209536.0, + "100": 950214656.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3338426368.0, - "2": 3490031616.0, - "3": 3490031616.0, - "4": 3490031616.0, - "5": 3490031616.0, - "6": 3490031616.0, - "7": 3490031616.0, - "8": 3490031616.0, - "9": 3490031616.0, - "10": 3490031616.0, - "11": 3490031616.0, - "12": 3490031616.0, - "13": 3490031616.0, - "14": 3490031616.0, - "15": 3490031616.0, - "16": 3490031616.0, - "17": 3490031616.0, - "18": 3490031616.0, - "19": 3490031616.0, - "20": 3490031616.0, - "21": 3490031616.0, - "22": 3490031616.0, - "23": 3490031616.0, - "24": 3490031616.0, - "25": 3490031616.0, - "26": 3490031616.0, - "27": 3490031616.0, - "28": 3490031616.0, - "29": 3490031616.0, - "30": 3490031616.0, - "31": 3490031616.0, - "32": 3490031616.0, - "33": 3490031616.0, - "34": 3490031616.0, - "35": 3490031616.0, - "36": 3490031616.0, - "37": 3490031616.0, - "38": 3490031616.0, - "39": 3490031616.0, - "40": 3490031616.0, - "41": 3490031616.0, - "42": 3490031616.0, - "43": 3490031616.0, - "44": 3490031616.0, - "45": 3490031616.0, - "46": 3490031616.0, - "47": 3490031616.0, - "48": 3490031616.0, - "49": 3490031616.0, - "50": 3490031616.0, - "51": 3490031616.0, - "52": 3490031616.0, - "53": 3490031616.0, - "54": 3490031616.0, - "55": 3490031616.0, - "56": 3490031616.0, - "57": 3490031616.0, - "58": 3490031616.0, - "59": 3490031616.0, - "60": 3490031616.0, - "61": 3490031616.0, - "62": 3490031616.0, - "63": 3490031616.0, - "64": 3490031616.0, - "65": 3490031616.0, - "66": 3490031616.0, - "67": 3490031616.0, - "68": 3490031616.0, - "69": 3490031616.0, - "70": 3490031616.0, - "71": 3490031616.0, - "72": 3490031616.0, - "73": 3490031616.0, - "74": 3490031616.0, - "75": 3490031616.0, - "76": 3490031616.0, - "77": 3490031616.0, - "78": 3490031616.0, - "79": 3490031616.0, - "80": 3490031616.0, - "81": 3490031616.0, - "82": 3490031616.0, - "83": 3490031616.0, - "84": 3490031616.0, - "85": 3490031616.0, - "86": 3490031616.0, - "87": 3490031616.0, - "88": 3490031616.0, - "89": 3490031616.0, - "90": 3490031616.0, - "91": 3490031616.0, - "92": 3490031616.0, - "93": 3490031616.0, - "94": 3490031616.0, - "95": 3490031616.0, - "96": 3490031616.0, - "97": 3490031616.0, - "98": 3490031616.0, - "99": 3490031616.0, - "100": 3490031616.0 + "1": 3365262336.0, + "2": 3569029632.0, + "3": 3569029632.0, + "4": 3569029632.0, + "5": 3569029632.0, + "6": 3569029632.0, + "7": 3569029632.0, + "8": 3569029632.0, + "9": 3569029632.0, + "10": 3569029632.0, + "11": 3569029632.0, + "12": 3569029632.0, + "13": 3569029632.0, + "14": 3570864128.0, + "15": 3570864128.0, + "16": 3570864128.0, + "17": 3570864128.0, + "18": 3570864128.0, + "19": 3570864128.0, + "20": 3570864128.0, + "21": 3570864128.0, + "22": 3570864128.0, + "23": 3570864128.0, + "24": 3570864128.0, + "25": 3570864128.0, + "26": 3570864128.0, + "27": 3570864128.0, + "28": 3570864128.0, + "29": 3570864128.0, + "30": 3570864128.0, + "31": 3570864128.0, + "32": 3570864128.0, + "33": 3570864128.0, + "34": 3570864128.0, + "35": 3570864128.0, + "36": 3570864128.0, + "37": 3570864128.0, + "38": 3570864128.0, + "39": 3570864128.0, + "40": 3570864128.0, + "41": 3570864128.0, + "42": 3570864128.0, + "43": 3570864128.0, + "44": 3570864128.0, + "45": 3570864128.0, + "46": 3570864128.0, + "47": 3570864128.0, + "48": 3570864128.0, + "49": 3570864128.0, + "50": 3570864128.0, + "51": 3570864128.0, + "52": 3570864128.0, + "53": 3570864128.0, + "54": 3570864128.0, + "55": 3570864128.0, + "56": 3570864128.0, + "57": 3570864128.0, + "58": 3570864128.0, + "59": 3570864128.0, + "60": 3570864128.0, + "61": 3570864128.0, + "62": 3570864128.0, + "63": 3570864128.0, + "64": 3570864128.0, + "65": 3570864128.0, + "66": 3570864128.0, + "67": 3570864128.0, + "68": 3570864128.0, + "69": 3570864128.0, + "70": 3570864128.0, + "71": 3570864128.0, + "72": 3570864128.0, + "73": 3570864128.0, + "74": 3570864128.0, + "75": 3570864128.0, + "76": 3570864128.0, + "77": 3570864128.0, + "78": 3570864128.0, + "79": 3570864128.0, + "80": 3570864128.0, + "81": 3570864128.0, + "82": 3570864128.0, + "83": 3570864128.0, + "84": 3570864128.0, + "85": 3570864128.0, + "86": 3570864128.0, + "87": 3570864128.0, + "88": 3570864128.0, + "89": 3570864128.0, + "90": 3570864128.0, + "91": 3570864128.0, + "92": 3570864128.0, + "93": 3570864128.0, + "94": 3570864128.0, + "95": 3570864128.0, + "96": 3570864128.0, + "97": 3570864128.0, + "98": 3570864128.0, + "99": 3570864128.0, + "100": 3570864128.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 5.83302, - "3": 0.39835, - "4": 0.38075, - "5": 0.37076, - "6": 0.4933, - "7": 0.74612, - "8": 0.78858, - "9": 0.57808, - "10": 0.35052, - "11": 1.14199, - "12": 0.35131, - "13": 0.43605, - "14": 1.01433, - "15": 0.34635, - "16": 0.8652, - "17": 0.4883, - "18": 0.78111, - "19": 0.34941, - "20": 0.34539, - "21": 0.80911, - "22": 0.63224, - "23": 0.59018, - "24": 0.35026, - "25": 0.48482, - "26": 0.69507, - "27": 0.35024, - "28": 0.62934, - "29": 0.63054, - "30": 0.35463, - "31": 0.36418, - "32": 0.92689, - "33": 0.55794, - "34": 0.61214, - "35": 0.34834, - "36": 0.48957, - "37": 0.54155, - "38": 0.37385, - "39": 0.57673, - "40": 0.61101, - "41": 0.56321, - "42": 0.72541, - "43": 0.37383, - "44": 0.87492, - "45": 0.53872, - "46": 0.61778, - "47": 0.55835, - "48": 0.69866, - "49": 0.35686, - "50": 0.63177, - "51": 0.41131, - "52": 0.42473, - "53": 0.35518, - "54": 0.72158, - "55": 0.58026, - "56": 0.75569, - "57": 0.4147, - "58": 0.35309, - "59": 0.78616, - "60": 0.66808, - "61": 0.35136, - "62": 0.563, - "63": 0.48971, - "64": 0.75453, - "65": 0.48762, - "66": 0.71564, - "67": 0.58774, - "68": 0.46593, - "69": 0.47007, - "70": 0.96668, - "71": 0.34914, - "72": 0.50835, - "73": 0.56066, - "74": 0.41042, - "75": 0.93675, - "76": 0.39087, - "77": 0.41741, - "78": 1.30294, - "79": 0.3568, - "80": 0.61012, - "81": 0.63075, - "82": 0.50353, - "83": 0.67509, - "84": 0.65086, - "85": 0.35856, - "86": 0.35432, - "87": 0.78386, - "88": 0.90597, - "89": 0.54771, - "90": 0.48869, - "91": 0.51287, - "92": 0.6772, - "93": 0.35448, - "94": 0.5901, - "95": 0.68505, - "96": 0.36981, - "97": 0.61461, - "98": 0.6597, - "99": 1.25701, - "100": 0.35321 + "2": 6.19239, + "3": 0.47385, + "4": 0.37831, + "5": 0.36091, + "6": 0.35992, + "7": 0.36793, + "8": 0.36078, + "9": 0.34798, + "10": 0.35404, + "11": 0.34712, + "12": 0.34699, + "13": 0.34572, + "14": 0.36365, + "15": 0.34823, + "16": 0.35768, + "17": 0.34538, + "18": 0.34613, + "19": 0.3421, + "20": 0.34749, + "21": 0.33764, + "22": 0.34602, + "23": 0.34215, + "24": 0.35222, + "25": 0.34527, + "26": 0.34599, + "27": 0.34559, + "28": 0.34705, + "29": 0.34548, + "30": 0.34632, + "31": 0.34645, + "32": 0.34627, + "33": 0.34678, + "34": 0.34491, + "35": 0.3427, + "36": 0.34243, + "37": 0.34774, + "38": 0.33936, + "39": 0.33981, + "40": 0.33543, + "41": 0.337, + "42": 0.34058, + "43": 0.35363, + "44": 0.34148, + "45": 0.33316, + "46": 0.34036, + "47": 0.34181, + "48": 0.34974, + "49": 0.34527, + "50": 0.3475, + "51": 0.46059, + "52": 0.40915, + "53": 0.35025, + "54": 0.34838, + "55": 0.35212, + "56": 0.33684, + "57": 0.33111, + "58": 0.3411, + "59": 0.34188, + "60": 0.34145, + "61": 0.33891, + "62": 0.34637, + "63": 0.34043, + "64": 0.34574, + "65": 0.34353, + "66": 0.34903, + "67": 0.34419, + "68": 0.34427, + "69": 0.35255, + "70": 0.33543, + "71": 0.33899, + "72": 0.34082, + "73": 0.33675, + "74": 0.34071, + "75": 0.34145, + "76": 0.34236, + "77": 0.34129, + "78": 0.34221, + "79": 0.34582, + "80": 0.35397, + "81": 0.34483, + "82": 0.34025, + "83": 0.34453, + "84": 0.3498, + "85": 0.3464, + "86": 0.34814, + "87": 0.33846, + "88": 0.35242, + "89": 0.34859, + "90": 0.34886, + "91": 0.3512, + "92": 0.34096, + "93": 0.3547, + "94": 0.35398, + "95": 0.35146, + "96": 0.34486, + "97": 0.33662, + "98": 0.33647, + "99": 0.34145, + "100": 0.3431 } } } \ No newline at end of file From f61f1814f3023096b2320772e83558cf85824cfe Mon Sep 17 00:00:00 2001 From: Pingtian Li Date: Tue, 30 Jun 2026 05:25:58 -0700 Subject: [PATCH 5/5] Refresh dist_dist_muon dev_dgx_h100 golden for compact LayerWise layout The compact decoupled LayerWise DDP layout (now the default) routes non-Muon (Adam) params through a real DistributedOptimizer (reduce-scatter) instead of the legacy in-LayerWise all-reduce path, shifting lm-loss/num-zeros within tolerance and lowering peak memory. Regenerated on dgx_h100 with the common_pile CI dataset; verified deterministic + resume-consistency pass. Signed-off-by: Pingtian Li --- .../golden_values_dev_dgx_h100.json | 984 +++++++++--------- 1 file changed, 492 insertions(+), 492 deletions(-) diff --git a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json index bc80b167114..4f344c3f7be 100644 --- a/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json +++ b/tests/functional_tests/test_cases/moe/gpt3_moe_mcore_te_ep8_resume_torch_dist_dist_muon/golden_values_dev_dgx_h100.json @@ -7,103 +7,103 @@ "1": 10.90111, "2": 10.8957, "3": 10.90636, - "4": 10.90606, - "5": 10.91349, - "6": 10.89747, - "7": 10.89834, - "8": 10.91366, - "9": 10.89559, - "10": 10.90375, - "11": 10.90014, - "12": 10.91887, - "13": 10.88393, - "14": 10.87584, - "15": 10.91035, - "16": 10.90325, - "17": 10.88717, - "18": 10.89416, - "19": 10.89487, - "20": 10.89092, - "21": 10.88636, - "22": 10.90341, - "23": 10.91256, - "24": 10.88547, - "25": 10.90164, - "26": 10.88848, - "27": 10.89705, - "28": 10.88168, - "29": 10.88997, - "30": 10.91097, - "31": 10.89128, - "32": 10.88933, - "33": 10.89772, - "34": 10.87339, - "35": 10.89595, - "36": 10.90751, - "37": 10.87112, - "38": 10.87839, - "39": 10.88579, - "40": 10.89247, - "41": 10.88364, - "42": 10.89496, - "43": 10.88058, - "44": 10.88492, - "45": 10.88361, - "46": 10.88582, - "47": 10.88565, - "48": 10.86424, - "49": 10.8759, - "50": 10.88259, - "51": 10.89305, - "52": 10.87123, - "53": 10.85905, - "54": 10.8702, - "55": 10.86807, - "56": 10.87324, - "57": 10.84465, - "58": 10.8563, - "59": 10.84628, - "60": 10.84343, + "4": 10.90598, + "5": 10.91366, + "6": 10.89735, + "7": 10.89759, + "8": 10.91393, + "9": 10.89588, + "10": 10.90417, + "11": 10.90065, + "12": 10.91872, + "13": 10.88457, + "14": 10.87494, + "15": 10.91013, + "16": 10.90284, + "17": 10.88653, + "18": 10.89377, + "19": 10.89514, + "20": 10.89099, + "21": 10.88603, + "22": 10.90381, + "23": 10.91251, + "24": 10.88559, + "25": 10.90137, + "26": 10.88911, + "27": 10.89679, + "28": 10.88224, + "29": 10.89011, + "30": 10.91096, + "31": 10.89182, + "32": 10.88859, + "33": 10.89729, + "34": 10.87331, + "35": 10.89645, + "36": 10.90777, + "37": 10.87098, + "38": 10.87883, + "39": 10.88605, + "40": 10.89265, + "41": 10.88411, + "42": 10.89528, + "43": 10.88027, + "44": 10.8848, + "45": 10.88278, + "46": 10.88616, + "47": 10.88534, + "48": 10.8635, + "49": 10.8757, + "50": 10.88243, + "51": 10.89306, + "52": 10.87094, + "53": 10.85923, + "54": 10.87102, + "55": 10.86779, + "56": 10.87345, + "57": 10.84429, + "58": 10.85659, + "59": 10.84665, + "60": 10.84309, "61": 10.8604, - "62": 10.85675, - "63": 10.86117, - "64": 10.83852, - "65": 10.82804, - "66": 10.84584, - "67": 10.82846, - "68": 10.83007, - "69": 10.81894, - "70": 10.82797, - "71": 10.82677, - "72": 10.8088, - "73": 10.80753, - "74": 10.80659, - "75": 10.81472, - "76": 10.81086, - "77": 10.80817, - "78": 10.79255, - "79": 10.80218, - "80": 10.7879, - "81": 10.79487, - "82": 10.79556, + "62": 10.85696, + "63": 10.861, + "64": 10.83786, + "65": 10.8282, + "66": 10.84643, + "67": 10.82884, + "68": 10.8303, + "69": 10.81908, + "70": 10.82819, + "71": 10.82683, + "72": 10.80767, + "73": 10.80754, + "74": 10.80646, + "75": 10.81452, + "76": 10.8117, + "77": 10.80784, + "78": 10.79243, + "79": 10.80273, + "80": 10.78868, + "81": 10.7952, + "82": 10.7961, "83": 10.78629, - "84": 10.75658, - "85": 10.76241, - "86": 10.77699, - "87": 10.79624, - "88": 10.77534, - "89": 10.77559, - "90": 10.76414, - "91": 10.74033, - "92": 10.76012, - "93": 10.74683, - "94": 10.73435, - "95": 10.75231, - "96": 10.72344, - "97": 10.71428, - "98": 10.72631, - "99": 10.74637, - "100": 10.69592 + "84": 10.75682, + "85": 10.76204, + "86": 10.77629, + "87": 10.79603, + "88": 10.77505, + "89": 10.77618, + "90": 10.76497, + "91": 10.74028, + "92": 10.76001, + "93": 10.74646, + "94": 10.73433, + "95": 10.75254, + "96": 10.72303, + "97": 10.71502, + "98": 10.72618, + "99": 10.74667, + "100": 10.69552 } }, "num-zeros": { @@ -113,104 +113,104 @@ "values": { "1": 1133.0, "2": 1120.0, - "3": 1371.0, - "4": 1297.0, - "5": 1158.0, - "6": 1283.0, - "7": 1334.0, - "8": 1122.0, - "9": 1269.0, - "10": 1134.0, - "11": 1184.0, - "12": 1223.0, - "13": 1340.0, - "14": 1232.0, - "15": 1234.0, - "16": 1179.0, - "17": 1212.0, - "18": 1083.0, - "19": 1187.0, - "20": 1174.0, - "21": 1196.0, - "22": 1227.0, - "23": 1318.0, - "24": 1113.0, - "25": 1191.0, - "26": 1243.0, - "27": 1343.0, - "28": 1251.0, - "29": 1186.0, - "30": 1221.0, - "31": 1205.0, - "32": 1269.0, - "33": 1254.0, - "34": 1196.0, - "35": 1308.0, - "36": 1277.0, - "37": 1196.0, - "38": 1321.0, - "39": 1407.0, - "40": 1299.0, - "41": 1284.0, - "42": 1170.0, - "43": 1170.0, - "44": 1252.0, - "45": 1154.0, - "46": 1403.0, - "47": 1148.0, - "48": 1176.0, - "49": 1389.0, - "50": 1329.0, - "51": 1209.0, - "52": 1224.0, - "53": 1276.0, - "54": 1198.0, - "55": 1179.0, - "56": 1166.0, - "57": 1168.0, - "58": 1318.0, - "59": 1047.0, - "60": 1118.0, - "61": 1192.0, - "62": 1108.0, - "63": 1225.0, - "64": 1256.0, - "65": 1221.0, - "66": 1228.0, - "67": 1193.0, - "68": 1279.0, - "69": 1141.0, - "70": 1248.0, - "71": 1165.0, - "72": 1230.0, - "73": 1236.0, - "74": 1336.0, - "75": 1346.0, - "76": 1259.0, - "77": 1205.0, - "78": 1217.0, - "79": 1298.0, - "80": 1228.0, - "81": 1335.0, - "82": 1198.0, - "83": 1246.0, - "84": 1152.0, - "85": 1250.0, - "86": 1287.0, - "87": 1111.0, - "88": 1169.0, - "89": 1119.0, - "90": 1402.0, - "91": 1297.0, - "92": 1194.0, - "93": 1271.0, - "94": 1409.0, - "95": 980.0, - "96": 1170.0, - "97": 1212.0, - "98": 1325.0, - "99": 1125.0, - "100": 1206.0 + "3": 1414.0, + "4": 1257.0, + "5": 1266.0, + "6": 1247.0, + "7": 1357.0, + "8": 1194.0, + "9": 1248.0, + "10": 1206.0, + "11": 1219.0, + "12": 1193.0, + "13": 1294.0, + "14": 1288.0, + "15": 1298.0, + "16": 1209.0, + "17": 1244.0, + "18": 1136.0, + "19": 1198.0, + "20": 1085.0, + "21": 1197.0, + "22": 1187.0, + "23": 1220.0, + "24": 1225.0, + "25": 1268.0, + "26": 1205.0, + "27": 1289.0, + "28": 1176.0, + "29": 1189.0, + "30": 1225.0, + "31": 1233.0, + "32": 1292.0, + "33": 1269.0, + "34": 1189.0, + "35": 1271.0, + "36": 1241.0, + "37": 1241.0, + "38": 1227.0, + "39": 1355.0, + "40": 1353.0, + "41": 1259.0, + "42": 1137.0, + "43": 1197.0, + "44": 1207.0, + "45": 1080.0, + "46": 1422.0, + "47": 1201.0, + "48": 1127.0, + "49": 1349.0, + "50": 1358.0, + "51": 1239.0, + "52": 1195.0, + "53": 1259.0, + "54": 1169.0, + "55": 1200.0, + "56": 1192.0, + "57": 1182.0, + "58": 1299.0, + "59": 1090.0, + "60": 1184.0, + "61": 1187.0, + "62": 1167.0, + "63": 1223.0, + "64": 1321.0, + "65": 1168.0, + "66": 1214.0, + "67": 1244.0, + "68": 1231.0, + "69": 1154.0, + "70": 1182.0, + "71": 1205.0, + "72": 1241.0, + "73": 1231.0, + "74": 1324.0, + "75": 1395.0, + "76": 1239.0, + "77": 1196.0, + "78": 1211.0, + "79": 1374.0, + "80": 1252.0, + "81": 1303.0, + "82": 1135.0, + "83": 1227.0, + "84": 1146.0, + "85": 1175.0, + "86": 1355.0, + "87": 1142.0, + "88": 1187.0, + "89": 1139.0, + "90": 1372.0, + "91": 1296.0, + "92": 1171.0, + "93": 1300.0, + "94": 1299.0, + "95": 1062.0, + "96": 1222.0, + "97": 1205.0, + "98": 1340.0, + "99": 1189.0, + "100": 1217.0 } }, "mem-allocated-bytes": { @@ -218,106 +218,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 994063360.0, - "2": 994054144.0, - "3": 994029056.0, - "4": 994010624.0, - "5": 994059776.0, - "6": 994042368.0, - "7": 993970176.0, - "8": 994006528.0, - "9": 994008064.0, - "10": 994039808.0, - "11": 994015232.0, - "12": 994024960.0, - "13": 994029056.0, - "14": 994030592.0, - "15": 994007040.0, - "16": 993992704.0, - "17": 994001408.0, - "18": 994003968.0, - "19": 994032640.0, - "20": 993997312.0, - "21": 994052608.0, - "22": 994075648.0, - "23": 994049024.0, - "24": 993964032.0, - "25": 994045440.0, - "26": 994076672.0, - "27": 994028544.0, - "28": 993984512.0, - "29": 994056704.0, - "30": 993990656.0, - "31": 993996800.0, - "32": 994003456.0, - "33": 994011136.0, - "34": 993989120.0, - "35": 993974784.0, - "36": 994040832.0, - "37": 993988608.0, - "38": 994046976.0, - "39": 994054144.0, - "40": 994001408.0, - "41": 994021888.0, - "42": 993964544.0, - "43": 994032640.0, - "44": 994014208.0, - "45": 993989120.0, - "46": 994052096.0, - "47": 994011136.0, - "48": 994040320.0, - "49": 993998848.0, - "50": 994044928.0, - "51": 994030592.0, - "52": 994003968.0, - "53": 993953280.0, - "54": 994051072.0, - "55": 994039808.0, - "56": 994038784.0, - "57": 994098688.0, - "58": 994035200.0, - "59": 994040832.0, - "60": 994019328.0, - "61": 993988608.0, - "62": 993980928.0, - "63": 993987584.0, - "64": 994015744.0, - "65": 994023424.0, - "66": 994037248.0, - "67": 994018816.0, - "68": 994020352.0, - "69": 993985024.0, - "70": 994067456.0, - "71": 994047488.0, - "72": 994044416.0, - "73": 993973760.0, - "74": 994015744.0, - "75": 994060800.0, - "76": 994017792.0, - "77": 994060288.0, - "78": 994006016.0, - "79": 994072064.0, - "80": 993993728.0, - "81": 994046464.0, - "82": 993994752.0, - "83": 994054656.0, - "84": 994024448.0, - "85": 994032128.0, - "86": 994016256.0, - "87": 994030080.0, - "88": 994010112.0, - "89": 994001408.0, - "90": 994024960.0, - "91": 994011136.0, - "92": 994009600.0, - "93": 994040320.0, - "94": 994015232.0, - "95": 994036736.0, - "96": 994028544.0, - "97": 993997312.0, - "98": 994035712.0, - "99": 994051584.0, - "100": 994083328.0 + "1": 725830656.0, + "2": 725821440.0, + "3": 725796352.0, + "4": 725776384.0, + "5": 725827072.0, + "6": 725809664.0, + "7": 725737472.0, + "8": 725773312.0, + "9": 725776384.0, + "10": 725808128.0, + "11": 725783552.0, + "12": 725791744.0, + "13": 725797376.0, + "14": 725796864.0, + "15": 725773824.0, + "16": 725762560.0, + "17": 725769728.0, + "18": 725770752.0, + "19": 725798912.0, + "20": 725764096.0, + "21": 725817344.0, + "22": 725843968.0, + "23": 725814272.0, + "24": 725733376.0, + "25": 725812224.0, + "26": 725844480.0, + "27": 725797376.0, + "28": 725750272.0, + "29": 725824000.0, + "30": 725757440.0, + "31": 725763584.0, + "32": 725772800.0, + "33": 725777408.0, + "34": 725756416.0, + "35": 725742080.0, + "36": 725807104.0, + "37": 725754880.0, + "38": 725814272.0, + "39": 725819904.0, + "40": 725768192.0, + "41": 725787648.0, + "42": 725731840.0, + "43": 725800448.0, + "44": 725784064.0, + "45": 725755904.0, + "46": 725822464.0, + "47": 725777920.0, + "48": 725807616.0, + "49": 725767168.0, + "50": 725813760.0, + "51": 725799936.0, + "52": 725770240.0, + "53": 725720576.0, + "54": 725817856.0, + "55": 725807616.0, + "56": 725807104.0, + "57": 725866496.0, + "58": 725803008.0, + "59": 725806592.0, + "60": 725786112.0, + "61": 725756416.0, + "62": 725747200.0, + "63": 725754880.0, + "64": 725783040.0, + "65": 725789696.0, + "66": 725805568.0, + "67": 725786624.0, + "68": 725787136.0, + "69": 725753344.0, + "70": 725838336.0, + "71": 725813248.0, + "72": 725811712.0, + "73": 725742080.0, + "74": 725780992.0, + "75": 725829632.0, + "76": 725785600.0, + "77": 725826560.0, + "78": 725773312.0, + "79": 725839872.0, + "80": 725761024.0, + "81": 725814272.0, + "82": 725759488.0, + "83": 725821440.0, + "84": 725791744.0, + "85": 725798400.0, + "86": 725781504.0, + "87": 725797376.0, + "88": 725780992.0, + "89": 725770240.0, + "90": 725792256.0, + "91": 725776896.0, + "92": 725778432.0, + "93": 725806592.0, + "94": 725779968.0, + "95": 725801984.0, + "96": 725794304.0, + "97": 725764608.0, + "98": 725803008.0, + "99": 725818880.0, + "100": 725849088.0 } }, "mem-max-allocated-bytes": { @@ -325,106 +325,106 @@ "end_step": 100, "step_interval": 1, "values": { - "1": 3205999104.0, - "2": 3490994688.0, - "3": 3490994688.0, - "4": 3490994688.0, - "5": 3498078208.0, - "6": 3498078208.0, - "7": 3498078208.0, - "8": 3498078208.0, - "9": 3498078208.0, - "10": 3498078208.0, - "11": 3498078208.0, - "12": 3498078208.0, - "13": 3498078208.0, - "14": 3498078208.0, - "15": 3498078208.0, - "16": 3498078208.0, - "17": 3498078208.0, - "18": 3498078208.0, - "19": 3498078208.0, - "20": 3498078208.0, - "21": 3498078208.0, - "22": 3507077632.0, - "23": 3507077632.0, - "24": 3507077632.0, - "25": 3507077632.0, - "26": 3510859264.0, - "27": 3510859264.0, - "28": 3510859264.0, - "29": 3510859264.0, - "30": 3510859264.0, - "31": 3510859264.0, - "32": 3510859264.0, - "33": 3510859264.0, - "34": 3510859264.0, - "35": 3510859264.0, - "36": 3510859264.0, - "37": 3510859264.0, - "38": 3510859264.0, - "39": 3510859264.0, - "40": 3510859264.0, - "41": 3510859264.0, - "42": 3510859264.0, - "43": 3510859264.0, - "44": 3510859264.0, - "45": 3510859264.0, - "46": 3510859264.0, - "47": 3510859264.0, - "48": 3510859264.0, - "49": 3510859264.0, - "50": 3510859264.0, - "51": 3510859264.0, - "52": 3510859264.0, - "53": 3510859264.0, - "54": 3510859264.0, - "55": 3510859264.0, - "56": 3510859264.0, - "57": 3526288384.0, - "58": 3526288384.0, - "59": 3526288384.0, - "60": 3526288384.0, - "61": 3526288384.0, - "62": 3526288384.0, - "63": 3526288384.0, - "64": 3526288384.0, - "65": 3526288384.0, - "66": 3526288384.0, - "67": 3526288384.0, - "68": 3526288384.0, - "69": 3526288384.0, - "70": 3526288384.0, - "71": 3526288384.0, - "72": 3526288384.0, - "73": 3526288384.0, - "74": 3526288384.0, - "75": 3526288384.0, - "76": 3526288384.0, - "77": 3526288384.0, - "78": 3526288384.0, - "79": 3526288384.0, - "80": 3526288384.0, - "81": 3526288384.0, - "82": 3526288384.0, - "83": 3526288384.0, - "84": 3526288384.0, - "85": 3526288384.0, - "86": 3526288384.0, - "87": 3526288384.0, - "88": 3526288384.0, - "89": 3526288384.0, - "90": 3526288384.0, - "91": 3526288384.0, - "92": 3526288384.0, - "93": 3526288384.0, - "94": 3526288384.0, - "95": 3526288384.0, - "96": 3526288384.0, - "97": 3526288384.0, - "98": 3526288384.0, - "99": 3526288384.0, - "100": 3526288384.0 + "1": 3117981184.0, + "2": 3223878144.0, + "3": 3223878144.0, + "4": 3223878144.0, + "5": 3225959424.0, + "6": 3225959424.0, + "7": 3225959424.0, + "8": 3225959424.0, + "9": 3225959424.0, + "10": 3225959424.0, + "11": 3225959424.0, + "12": 3225959424.0, + "13": 3225959424.0, + "14": 3225959424.0, + "15": 3225959424.0, + "16": 3225959424.0, + "17": 3225959424.0, + "18": 3225959424.0, + "19": 3225959424.0, + "20": 3225959424.0, + "21": 3225959424.0, + "22": 3236540928.0, + "23": 3236540928.0, + "24": 3236540928.0, + "25": 3236540928.0, + "26": 3238976512.0, + "27": 3238976512.0, + "28": 3238976512.0, + "29": 3238976512.0, + "30": 3238976512.0, + "31": 3238976512.0, + "32": 3238976512.0, + "33": 3238976512.0, + "34": 3238976512.0, + "35": 3238976512.0, + "36": 3238976512.0, + "37": 3238976512.0, + "38": 3238976512.0, + "39": 3238976512.0, + "40": 3238976512.0, + "41": 3238976512.0, + "42": 3238976512.0, + "43": 3238976512.0, + "44": 3238976512.0, + "45": 3238976512.0, + "46": 3238976512.0, + "47": 3238976512.0, + "48": 3238976512.0, + "49": 3238976512.0, + "50": 3238976512.0, + "51": 3238976512.0, + "52": 3238976512.0, + "53": 3238976512.0, + "54": 3238976512.0, + "55": 3238976512.0, + "56": 3238976512.0, + "57": 3260471808.0, + "58": 3260471808.0, + "59": 3260471808.0, + "60": 3260471808.0, + "61": 3260471808.0, + "62": 3260471808.0, + "63": 3260471808.0, + "64": 3260471808.0, + "65": 3260471808.0, + "66": 3260471808.0, + "67": 3260471808.0, + "68": 3260471808.0, + "69": 3260471808.0, + "70": 3260471808.0, + "71": 3260471808.0, + "72": 3260471808.0, + "73": 3260471808.0, + "74": 3260471808.0, + "75": 3260471808.0, + "76": 3260471808.0, + "77": 3260471808.0, + "78": 3260471808.0, + "79": 3260471808.0, + "80": 3260471808.0, + "81": 3260471808.0, + "82": 3260471808.0, + "83": 3260471808.0, + "84": 3260471808.0, + "85": 3260471808.0, + "86": 3260471808.0, + "87": 3260471808.0, + "88": 3260471808.0, + "89": 3260471808.0, + "90": 3260471808.0, + "91": 3260471808.0, + "92": 3260471808.0, + "93": 3260471808.0, + "94": 3260471808.0, + "95": 3260471808.0, + "96": 3260471808.0, + "97": 3260471808.0, + "98": 3260471808.0, + "99": 3260471808.0, + "100": 3260471808.0 } }, "iteration-time": { @@ -433,105 +433,105 @@ "step_interval": 1, "values": { "1": "nan", - "2": 7.24019, - "3": 0.23582, - "4": 0.2146, - "5": 0.22942, - "6": 0.21484, - "7": 0.21536, - "8": 0.22845, - "9": 0.21127, - "10": 0.21126, - "11": 0.213, - "12": 0.20896, - "13": 0.21279, - "14": 0.20309, - "15": 0.21429, - "16": 0.19442, - "17": 0.20188, - "18": 0.20506, - "19": 0.20244, - "20": 0.20349, - "21": 0.19509, - "22": 0.20011, - "23": 0.19574, - "24": 0.19884, - "25": 0.20575, - "26": 0.19346, - "27": 0.18966, - "28": 0.20276, - "29": 0.20359, - "30": 0.20204, - "31": 0.19889, - "32": 0.20168, - "33": 0.20239, - "34": 0.19255, - "35": 0.19644, - "36": 0.18899, - "37": 0.19166, - "38": 0.19653, - "39": 0.1882, - "40": 0.19935, - "41": 0.1924, - "42": 0.19212, - "43": 0.19745, - "44": 0.1908, - "45": 0.20082, - "46": 0.18806, - "47": 0.19186, - "48": 0.19998, - "49": 0.19241, - "50": 0.18985, - "51": 0.25778, - "52": 0.2161, - "53": 0.19159, - "54": 0.19056, - "55": 0.18596, - "56": 0.18386, - "57": 0.19519, - "58": 0.19298, - "59": 0.1916, - "60": 0.18867, - "61": 0.19145, - "62": 0.19734, - "63": 0.18907, - "64": 0.18626, - "65": 0.18071, - "66": 0.18447, - "67": 0.18932, - "68": 0.18954, - "69": 0.19211, - "70": 0.18494, - "71": 0.18733, - "72": 0.18788, - "73": 0.19702, - "74": 0.19368, - "75": 0.19226, - "76": 0.18585, - "77": 0.20408, - "78": 0.19362, - "79": 0.18545, - "80": 0.18915, - "81": 0.20149, - "82": 0.18881, - "83": 0.18833, - "84": 0.19607, - "85": 0.18814, - "86": 0.19212, - "87": 0.19213, - "88": 0.18924, - "89": 0.18653, - "90": 0.18266, - "91": 0.19803, - "92": 0.18712, - "93": 0.18152, - "94": 0.19216, - "95": 0.18948, - "96": 0.18961, - "97": 0.19462, - "98": 0.19427, - "99": 0.1846, - "100": 0.19363 + "2": 7.94511, + "3": 0.21327, + "4": 0.18417, + "5": 0.18823, + "6": 0.18012, + "7": 0.18411, + "8": 0.18711, + "9": 0.17729, + "10": 0.17165, + "11": 0.17139, + "12": 0.18817, + "13": 0.17105, + "14": 0.17064, + "15": 0.17296, + "16": 0.17618, + "17": 0.16878, + "18": 0.17209, + "19": 0.16241, + "20": 0.15652, + "21": 0.15077, + "22": 0.15825, + "23": 0.1679, + "24": 0.15619, + "25": 0.16777, + "26": 0.15394, + "27": 0.16067, + "28": 0.15617, + "29": 0.16213, + "30": 0.16019, + "31": 0.15836, + "32": 0.16351, + "33": 0.17052, + "34": 0.1655, + "35": 0.16399, + "36": 0.15842, + "37": 0.14989, + "38": 0.1633, + "39": 0.1544, + "40": 0.15916, + "41": 0.15259, + "42": 0.1648, + "43": 0.1624, + "44": 0.1578, + "45": 0.15971, + "46": 0.15183, + "47": 0.1703, + "48": 0.15489, + "49": 0.16006, + "50": 0.15498, + "51": 0.20709, + "52": 0.16388, + "53": 0.16604, + "54": 0.1814, + "55": 0.15256, + "56": 0.1609, + "57": 0.1561, + "58": 0.15544, + "59": 0.14701, + "60": 0.16423, + "61": 0.15458, + "62": 0.15475, + "63": 0.15252, + "64": 0.16413, + "65": 0.16324, + "66": 0.14677, + "67": 0.15157, + "68": 0.1518, + "69": 0.16411, + "70": 0.1479, + "71": 0.15353, + "72": 0.15465, + "73": 0.16625, + "74": 0.1603, + "75": 0.14847, + "76": 0.15699, + "77": 0.15335, + "78": 0.15846, + "79": 0.15407, + "80": 0.15636, + "81": 0.16016, + "82": 0.16572, + "83": 0.15044, + "84": 0.15398, + "85": 0.16425, + "86": 0.15458, + "87": 0.16024, + "88": 0.15916, + "89": 0.15625, + "90": 0.15512, + "91": 0.16701, + "92": 0.15537, + "93": 0.16374, + "94": 0.15183, + "95": 0.15954, + "96": 0.15835, + "97": 0.16558, + "98": 0.1447, + "99": 0.15948, + "100": 0.15963 } } } \ No newline at end of file